From ad6b902b8462adb568d799c69a74b60d69574d0c Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Sep 2022 09:20:40 +0100 Subject: [PATCH 01/20] Refactor appservices component (#2687) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR refactors the app services component. It makes the following changes: * Each appservice now gets its own NATS JetStream consumer * The appservice database is now removed entirely, since we just use JetStream as a data source instead * The entire component is now much simpler and we deleted lots of lines of code 💅 The result is that it should be much lighter and hopefully much more performant. --- appservice/README.md | 10 - appservice/appservice.go | 55 +-- appservice/consumers/roomserver.go | 330 +++++++++++------- appservice/query/query.go | 2 +- appservice/storage/interface.go | 30 -- .../postgres/appservice_events_table.go | 256 -------------- appservice/storage/postgres/storage.go | 115 ------ .../storage/postgres/txn_id_counter_table.go | 53 --- .../sqlite3/appservice_events_table.go | 267 -------------- appservice/storage/sqlite3/storage.go | 114 ------ .../storage/sqlite3/txn_id_counter_table.go | 82 ----- appservice/storage/storage.go | 40 --- appservice/storage/storage_wasm.go | 34 -- appservice/types/types.go | 64 ---- appservice/workers/transaction_scheduler.go | 236 ------------- build/gobind-pinecone/monolith.go | 1 - build/gobind-yggdrasil/monolith.go | 1 - cmd/dendrite-demo-pinecone/main.go | 1 - cmd/dendrite-demo-yggdrasil/main.go | 1 - cmd/generate-config/main.go | 1 - dendrite-sample.polylith.yaml | 7 - setup/config/config_appservice.go | 9 - setup/jetstream/helpers.go | 35 +- setup/jetstream/nats.go | 1 + test/testrig/base.go | 1 - userapi/internal/api.go | 3 +- 26 files changed, 231 insertions(+), 1518 deletions(-) delete mode 100644 appservice/README.md delete mode 100644 appservice/storage/interface.go delete mode 100644 appservice/storage/postgres/appservice_events_table.go delete mode 100644 appservice/storage/postgres/storage.go delete mode 100644 appservice/storage/postgres/txn_id_counter_table.go delete mode 100644 appservice/storage/sqlite3/appservice_events_table.go delete mode 100644 appservice/storage/sqlite3/storage.go delete mode 100644 appservice/storage/sqlite3/txn_id_counter_table.go delete mode 100644 appservice/storage/storage.go delete mode 100644 appservice/storage/storage_wasm.go delete mode 100644 appservice/types/types.go delete mode 100644 appservice/workers/transaction_scheduler.go diff --git a/appservice/README.md b/appservice/README.md deleted file mode 100644 index d75557448..000000000 --- a/appservice/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Application Service - -This component interfaces with external [Application -Services](https://matrix.org/docs/spec/application_service/unstable.html). -This includes any HTTP endpoints that application services call, as well as talking -to any HTTP endpoints that application services provide themselves. - -## Consumers - -This component consumes and filters events from the Roomserver Kafka stream, passing on any necessary events to subscribing application services. \ No newline at end of file diff --git a/appservice/appservice.go b/appservice/appservice.go index 8fe1b2fc4..9000adb1d 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -18,7 +18,6 @@ import ( "context" "crypto/tls" "net/http" - "sync" "time" "github.com/gorilla/mux" @@ -28,9 +27,6 @@ import ( "github.com/matrix-org/dendrite/appservice/consumers" "github.com/matrix-org/dendrite/appservice/inthttp" "github.com/matrix-org/dendrite/appservice/query" - "github.com/matrix-org/dendrite/appservice/storage" - "github.com/matrix-org/dendrite/appservice/types" - "github.com/matrix-org/dendrite/appservice/workers" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" @@ -59,57 +55,40 @@ func NewInternalAPI( Proxy: http.ProxyFromEnvironment, }, } - js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) + // Create appserivce query API with an HTTP client that will be used for all + // outbound and inbound requests (inbound only for the internal API) + appserviceQueryAPI := &query.AppServiceQueryAPI{ + HTTPClient: client, + Cfg: &base.Cfg.AppServiceAPI, + } - // Create a connection to the appservice postgres DB - appserviceDB, err := storage.NewDatabase(base, &base.Cfg.AppServiceAPI.Database) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to appservice db") + if len(base.Cfg.Derived.ApplicationServices) == 0 { + return appserviceQueryAPI } // Wrap application services in a type that relates the application service and // a sync.Cond object that can be used to notify workers when there are new // events to be sent out. - workerStates := make([]types.ApplicationServiceWorkerState, len(base.Cfg.Derived.ApplicationServices)) - for i, appservice := range base.Cfg.Derived.ApplicationServices { - m := sync.Mutex{} - ws := types.ApplicationServiceWorkerState{ - AppService: appservice, - Cond: sync.NewCond(&m), - } - workerStates[i] = ws - + for _, appservice := range base.Cfg.Derived.ApplicationServices { // Create bot account for this AS if it doesn't already exist - if err = generateAppServiceAccount(userAPI, appservice); err != nil { + if err := generateAppServiceAccount(userAPI, appservice); err != nil { logrus.WithFields(logrus.Fields{ "appservice": appservice.ID, }).WithError(err).Panicf("failed to generate bot account for appservice") } } - // Create appserivce query API with an HTTP client that will be used for all - // outbound and inbound requests (inbound only for the internal API) - appserviceQueryAPI := &query.AppServiceQueryAPI{ - HTTPClient: client, - Cfg: base.Cfg, - } - // Only consume if we actually have ASes to track, else we'll just chew cycles needlessly. // We can't add ASes at runtime so this is safe to do. - if len(workerStates) > 0 { - consumer := consumers.NewOutputRoomEventConsumer( - base.ProcessContext, base.Cfg, js, appserviceDB, - rsAPI, workerStates, - ) - if err := consumer.Start(); err != nil { - logrus.WithError(err).Panicf("failed to start appservice roomserver consumer") - } + js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) + consumer := consumers.NewOutputRoomEventConsumer( + base.ProcessContext, &base.Cfg.AppServiceAPI, + client, js, rsAPI, + ) + if err := consumer.Start(); err != nil { + logrus.WithError(err).Panicf("failed to start appservice roomserver consumer") } - // Create application service transaction workers - if err := workers.SetupTransactionWorkers(client, appserviceDB, workerStates); err != nil { - logrus.WithError(err).Panicf("failed to start app service transaction workers") - } return appserviceQueryAPI } diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 21b52bc3c..a30944e75 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -15,14 +15,18 @@ package consumers import ( + "bytes" "context" "encoding/json" + "fmt" + "math" + "net/http" + "net/url" + "time" "github.com/matrix-org/gomatrixserverlib" "github.com/nats-io/nats.go" - "github.com/matrix-org/dendrite/appservice/storage" - "github.com/matrix-org/dendrite/appservice/types" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" @@ -33,178 +37,192 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - ctx context.Context - jetstream nats.JetStreamContext - durable string - topic string - asDB storage.Database - rsAPI api.AppserviceRoomserverAPI - serverName string - workerStates []types.ApplicationServiceWorkerState + ctx context.Context + cfg *config.AppServiceAPI + client *http.Client + jetstream nats.JetStreamContext + topic string + rsAPI api.AppserviceRoomserverAPI +} + +type appserviceState struct { + *config.ApplicationService + backoff int } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call // Start() to begin consuming from room servers. func NewOutputRoomEventConsumer( process *process.ProcessContext, - cfg *config.Dendrite, + cfg *config.AppServiceAPI, + client *http.Client, js nats.JetStreamContext, - appserviceDB storage.Database, rsAPI api.AppserviceRoomserverAPI, - workerStates []types.ApplicationServiceWorkerState, ) *OutputRoomEventConsumer { return &OutputRoomEventConsumer{ - ctx: process.Context(), - jetstream: js, - durable: cfg.Global.JetStream.Durable("AppserviceRoomserverConsumer"), - topic: cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent), - asDB: appserviceDB, - rsAPI: rsAPI, - serverName: string(cfg.Global.ServerName), - workerStates: workerStates, + ctx: process.Context(), + cfg: cfg, + client: client, + jetstream: js, + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent), + rsAPI: rsAPI, } } // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return jetstream.JetStreamConsumer( - s.ctx, s.jetstream, s.topic, s.durable, 1, - s.onMessage, nats.DeliverAll(), nats.ManualAck(), - ) + for _, as := range s.cfg.Derived.ApplicationServices { + appsvc := as + state := &appserviceState{ + ApplicationService: &appsvc, + } + token := jetstream.Tokenise(as.ID) + if err := jetstream.JetStreamConsumer( + s.ctx, s.jetstream, s.topic, + s.cfg.Matrix.JetStream.Durable("Appservice_"+token), + 50, // maximum number of events to send in a single transaction + func(ctx context.Context, msgs []*nats.Msg) bool { + return s.onMessage(ctx, state, msgs) + }, + nats.DeliverNew(), nats.ManualAck(), + ); err != nil { + return fmt.Errorf("failed to create %q consumer: %w", token, err) + } + } + return nil } // onMessage is called when the appservice component receives a new event from // the room server output log. -func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool { - msg := msgs[0] // Guaranteed to exist if onMessage is called - // Parse out the event JSON - var output api.OutputEvent - if err := json.Unmarshal(msg.Data, &output); err != nil { - // If the message was invalid, log it and move on to the next message in the stream - log.WithError(err).Errorf("roomserver output log: message parse failure") - return true - } - - log.WithFields(log.Fields{ - "type": output.Type, - }).Debug("Got a message in OutputRoomEventConsumer") - - events := []*gomatrixserverlib.HeaderedEvent{} - if output.Type == api.OutputTypeNewRoomEvent && output.NewRoomEvent != nil { - newEventID := output.NewRoomEvent.Event.EventID() - events = append(events, output.NewRoomEvent.Event) - if len(output.NewRoomEvent.AddsStateEventIDs) > 0 { - eventsReq := &api.QueryEventsByIDRequest{ - EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)), - } - eventsRes := &api.QueryEventsByIDResponse{} - for _, eventID := range output.NewRoomEvent.AddsStateEventIDs { - if eventID != newEventID { - eventsReq.EventIDs = append(eventsReq.EventIDs, eventID) - } - } - if len(eventsReq.EventIDs) > 0 { - if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil { - log.WithError(err).Errorf("s.rsAPI.QueryEventsByID failed") - return false - } - events = append(events, eventsRes.Events...) - } +func (s *OutputRoomEventConsumer) onMessage( + ctx context.Context, state *appserviceState, msgs []*nats.Msg, +) bool { + log.WithField("appservice", state.ID).Tracef("Appservice worker received %d message(s) from roomserver", len(msgs)) + events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(msgs)) + for _, msg := range msgs { + // Parse out the event JSON + var output api.OutputEvent + if err := json.Unmarshal(msg.Data, &output); err != nil { + // If the message was invalid, log it and move on to the next message in the stream + log.WithField("appservice", state.ID).WithError(err).Errorf("Appservice failed to parse message, ignoring") + continue } - } else if output.Type == api.OutputTypeNewInviteEvent && output.NewInviteEvent != nil { - events = append(events, output.NewInviteEvent.Event) - } else { - log.WithFields(log.Fields{ - "type": output.Type, - }).Debug("appservice OutputRoomEventConsumer ignoring event", string(msg.Data)) + switch output.Type { + case api.OutputTypeNewRoomEvent: + if output.NewRoomEvent == nil || !s.appserviceIsInterestedInEvent(ctx, output.NewRoomEvent.Event, state.ApplicationService) { + continue + } + events = append(events, output.NewRoomEvent.Event) + if len(output.NewRoomEvent.AddsStateEventIDs) > 0 { + newEventID := output.NewRoomEvent.Event.EventID() + eventsReq := &api.QueryEventsByIDRequest{ + EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)), + } + eventsRes := &api.QueryEventsByIDResponse{} + for _, eventID := range output.NewRoomEvent.AddsStateEventIDs { + if eventID != newEventID { + eventsReq.EventIDs = append(eventsReq.EventIDs, eventID) + } + } + if len(eventsReq.EventIDs) > 0 { + if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil { + log.WithError(err).Errorf("s.rsAPI.QueryEventsByID failed") + return false + } + events = append(events, eventsRes.Events...) + } + } + + case api.OutputTypeNewInviteEvent: + if output.NewInviteEvent == nil { + continue + } + events = append(events, output.NewInviteEvent.Event) + + default: + continue + } + } + + // If there are no events selected for sending then we should + // ack the messages so that we don't get sent them again in the + // future. + if len(events) == 0 { return true } - // Send event to any relevant application services - if err := s.filterRoomserverEvents(context.TODO(), events); err != nil { - log.WithError(err).Errorf("roomserver output log: filter error") - return true - } - - return true + // Send event to any relevant application services. If we hit + // an error here, return false, so that we negatively ack. + log.WithField("appservice", state.ID).Debugf("Appservice worker sending %d events(s) from roomserver", len(events)) + return s.sendEvents(ctx, state, events) == nil } -// filterRoomserverEvents takes in events and decides whether any of them need -// to be passed on to an external application service. It does this by checking -// each namespace of each registered application service, and if there is a -// match, adds the event to the queue for events to be sent to a particular -// application service. -func (s *OutputRoomEventConsumer) filterRoomserverEvents( - ctx context.Context, +// sendEvents passes events to the appservice by using the transactions +// endpoint. It will block for the backoff period if necessary. +func (s *OutputRoomEventConsumer) sendEvents( + ctx context.Context, state *appserviceState, events []*gomatrixserverlib.HeaderedEvent, ) error { - for _, ws := range s.workerStates { - for _, event := range events { - // Check if this event is interesting to this application service - if s.appserviceIsInterestedInEvent(ctx, event, ws.AppService) { - // Queue this event to be sent off to the application service - if err := s.asDB.StoreEvent(ctx, ws.AppService.ID, event); err != nil { - log.WithError(err).Warn("failed to insert incoming event into appservices database") - return err - } else { - // Tell our worker to send out new messages by updating remaining message - // count and waking them up with a broadcast - ws.NotifyNewEvents() - } - } - } + // Create the transaction body. + transaction, err := json.Marshal( + gomatrixserverlib.ApplicationServiceTransaction{ + Events: gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatAll), + }, + ) + if err != nil { + return err } + // TODO: We should probably be more intelligent and pick something not + // in the control of the event. A NATS timestamp header or something maybe. + txnID := events[0].Event.OriginServerTS() + + // Send the transaction to the appservice. + // https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid + address := fmt.Sprintf("%s/transactions/%d?access_token=%s", state.URL, txnID, url.QueryEscape(state.HSToken)) + req, err := http.NewRequestWithContext(ctx, "PUT", address, bytes.NewBuffer(transaction)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := s.client.Do(req) + if err != nil { + return state.backoffAndPause(err) + } + + // If the response was fine then we can clear any backoffs in place and + // report that everything was OK. Otherwise, back off for a while. + switch resp.StatusCode { + case http.StatusOK: + state.backoff = 0 + default: + return state.backoffAndPause(fmt.Errorf("received HTTP status code %d from appservice", resp.StatusCode)) + } return nil } -// appserviceJoinedAtEvent returns a boolean depending on whether a given -// appservice has membership at the time a given event was created. -func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice config.ApplicationService) bool { - // TODO: This is only checking the current room state, not the state at - // the event in question. Pretty sure this is what Synapse does too, but - // until we have a lighter way of checking the state before the event that - // doesn't involve state res, then this is probably OK. - membershipReq := &api.QueryMembershipsForRoomRequest{ - RoomID: event.RoomID(), - JoinedOnly: true, +// backoff pauses the calling goroutine for a 2^some backoff exponent seconds +func (s *appserviceState) backoffAndPause(err error) error { + if s.backoff < 6 { + s.backoff++ } - membershipRes := &api.QueryMembershipsForRoomResponse{} - - // XXX: This could potentially race if the state for the event is not known yet - // e.g. the event came over federation but we do not have the full state persisted. - if err := s.rsAPI.QueryMembershipsForRoom(ctx, membershipReq, membershipRes); err == nil { - for _, ev := range membershipRes.JoinEvents { - var membership gomatrixserverlib.MemberContent - if err = json.Unmarshal(ev.Content, &membership); err != nil || ev.StateKey == nil { - continue - } - if appservice.IsInterestedInUserID(*ev.StateKey) { - return true - } - } - } else { - log.WithFields(log.Fields{ - "room_id": event.RoomID(), - }).WithError(err).Errorf("Unable to get membership for room") - } - return false + duration := time.Second * time.Duration(math.Pow(2, float64(s.backoff))) + log.WithField("appservice", s.ID).WithError(err).Errorf("Unable to send transaction to appservice, backing off for %s", duration.String()) + time.Sleep(duration) + return err } // appserviceIsInterestedInEvent returns a boolean depending on whether a given // event falls within one of a given application service's namespaces. // // TODO: This should be cached, see https://github.com/matrix-org/dendrite/issues/1682 -func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice config.ApplicationService) bool { - // No reason to queue events if they'll never be sent to the application - // service - if appservice.URL == "" { +func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice *config.ApplicationService) bool { + switch { + case appservice.URL == "": return false - } - - // Check Room ID and Sender of the event - if appservice.IsInterestedInUserID(event.Sender()) || - appservice.IsInterestedInRoomID(event.RoomID()) { + case appservice.IsInterestedInUserID(event.Sender()): + return true + case appservice.IsInterestedInRoomID(event.RoomID()): return true } @@ -225,10 +243,52 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont } } else { log.WithFields(log.Fields{ - "room_id": event.RoomID(), + "appservice": appservice.ID, + "room_id": event.RoomID(), }).WithError(err).Errorf("Unable to get aliases for room") } // Check if any of the members in the room match the appservice return s.appserviceJoinedAtEvent(ctx, event, appservice) } + +// appserviceJoinedAtEvent returns a boolean depending on whether a given +// appservice has membership at the time a given event was created. +func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice *config.ApplicationService) bool { + // TODO: This is only checking the current room state, not the state at + // the event in question. Pretty sure this is what Synapse does too, but + // until we have a lighter way of checking the state before the event that + // doesn't involve state res, then this is probably OK. + membershipReq := &api.QueryMembershipsForRoomRequest{ + RoomID: event.RoomID(), + JoinedOnly: true, + } + membershipRes := &api.QueryMembershipsForRoomResponse{} + + // XXX: This could potentially race if the state for the event is not known yet + // e.g. the event came over federation but we do not have the full state persisted. + if err := s.rsAPI.QueryMembershipsForRoom(ctx, membershipReq, membershipRes); err == nil { + for _, ev := range membershipRes.JoinEvents { + switch { + case ev.StateKey == nil: + continue + case ev.Type != gomatrixserverlib.MRoomMember: + continue + } + var membership gomatrixserverlib.MemberContent + err = json.Unmarshal(ev.Content, &membership) + switch { + case err != nil: + continue + case membership.Membership == gomatrixserverlib.Join: + return true + } + } + } else { + log.WithFields(log.Fields{ + "appservice": appservice.ID, + "room_id": event.RoomID(), + }).WithError(err).Errorf("Unable to get membership for room") + } + return false +} diff --git a/appservice/query/query.go b/appservice/query/query.go index dacd3caa8..53b34cb18 100644 --- a/appservice/query/query.go +++ b/appservice/query/query.go @@ -33,7 +33,7 @@ const userIDExistsPath = "/users/" // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI type AppServiceQueryAPI struct { HTTPClient *http.Client - Cfg *config.Dendrite + Cfg *config.AppServiceAPI } // RoomAliasExists performs a request to '/room/{roomAlias}' on all known diff --git a/appservice/storage/interface.go b/appservice/storage/interface.go deleted file mode 100644 index 25d35af6c..000000000 --- a/appservice/storage/interface.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "context" - - "github.com/matrix-org/gomatrixserverlib" -) - -type Database interface { - StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.HeaderedEvent) error - GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error) - CountEventsWithAppServiceID(ctx context.Context, appServiceID string) (int, error) - UpdateTxnIDForEvents(ctx context.Context, appserviceID string, maxID, txnID int) error - RemoveEventsBeforeAndIncludingID(ctx context.Context, appserviceID string, eventTableID int) error - GetLatestTxnID(ctx context.Context) (int, error) -} diff --git a/appservice/storage/postgres/appservice_events_table.go b/appservice/storage/postgres/appservice_events_table.go deleted file mode 100644 index a95be6b8a..000000000 --- a/appservice/storage/postgres/appservice_events_table.go +++ /dev/null @@ -1,256 +0,0 @@ -// Copyright 2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "context" - "database/sql" - "encoding/json" - "time" - - "github.com/matrix-org/gomatrixserverlib" - log "github.com/sirupsen/logrus" -) - -const appserviceEventsSchema = ` --- Stores events to be sent to application services -CREATE TABLE IF NOT EXISTS appservice_events ( - -- An auto-incrementing id unique to each event in the table - id BIGSERIAL NOT NULL PRIMARY KEY, - -- The ID of the application service the event will be sent to - as_id TEXT NOT NULL, - -- JSON representation of the event - headered_event_json TEXT NOT NULL, - -- The ID of the transaction that this event is a part of - txn_id BIGINT NOT NULL -); - -CREATE INDEX IF NOT EXISTS appservice_events_as_id ON appservice_events(as_id); -` - -const selectEventsByApplicationServiceIDSQL = "" + - "SELECT id, headered_event_json, txn_id " + - "FROM appservice_events WHERE as_id = $1 ORDER BY txn_id DESC, id ASC" - -const countEventsByApplicationServiceIDSQL = "" + - "SELECT COUNT(id) FROM appservice_events WHERE as_id = $1" - -const insertEventSQL = "" + - "INSERT INTO appservice_events(as_id, headered_event_json, txn_id) " + - "VALUES ($1, $2, $3)" - -const updateTxnIDForEventsSQL = "" + - "UPDATE appservice_events SET txn_id = $1 WHERE as_id = $2 AND id <= $3" - -const deleteEventsBeforeAndIncludingIDSQL = "" + - "DELETE FROM appservice_events WHERE as_id = $1 AND id <= $2" - -const ( - // A transaction ID number that no transaction should ever have. Used for - // checking again the default value. - invalidTxnID = -2 -) - -type eventsStatements struct { - selectEventsByApplicationServiceIDStmt *sql.Stmt - countEventsByApplicationServiceIDStmt *sql.Stmt - insertEventStmt *sql.Stmt - updateTxnIDForEventsStmt *sql.Stmt - deleteEventsBeforeAndIncludingIDStmt *sql.Stmt -} - -func (s *eventsStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(appserviceEventsSchema) - if err != nil { - return - } - - if s.selectEventsByApplicationServiceIDStmt, err = db.Prepare(selectEventsByApplicationServiceIDSQL); err != nil { - return - } - if s.countEventsByApplicationServiceIDStmt, err = db.Prepare(countEventsByApplicationServiceIDSQL); err != nil { - return - } - if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil { - return - } - if s.updateTxnIDForEventsStmt, err = db.Prepare(updateTxnIDForEventsSQL); err != nil { - return - } - if s.deleteEventsBeforeAndIncludingIDStmt, err = db.Prepare(deleteEventsBeforeAndIncludingIDSQL); err != nil { - return - } - - return -} - -// selectEventsByApplicationServiceID takes in an application service ID and -// returns a slice of events that need to be sent to that application service, -// as well as an int later used to remove these same events from the database -// once successfully sent to an application service. -func (s *eventsStatements) selectEventsByApplicationServiceID( - ctx context.Context, - applicationServiceID string, - limit int, -) ( - txnID, maxID int, - events []gomatrixserverlib.HeaderedEvent, - eventsRemaining bool, - err error, -) { - defer func() { - if err != nil { - log.WithFields(log.Fields{ - "appservice": applicationServiceID, - }).WithError(err).Fatalf("appservice unable to select new events to send") - } - }() - // Retrieve events from the database. Unsuccessfully sent events first - eventRows, err := s.selectEventsByApplicationServiceIDStmt.QueryContext(ctx, applicationServiceID) - if err != nil { - return - } - defer checkNamedErr(eventRows.Close, &err) - events, maxID, txnID, eventsRemaining, err = retrieveEvents(eventRows, limit) - if err != nil { - return - } - - return -} - -// checkNamedErr calls fn and overwrite err if it was nil and fn returned non-nil -func checkNamedErr(fn func() error, err *error) { - if e := fn(); e != nil && *err == nil { - *err = e - } -} - -func retrieveEvents(eventRows *sql.Rows, limit int) (events []gomatrixserverlib.HeaderedEvent, maxID, txnID int, eventsRemaining bool, err error) { - // Get current time for use in calculating event age - nowMilli := time.Now().UnixNano() / int64(time.Millisecond) - - // Iterate through each row and store event contents - // If txn_id changes dramatically, we've switched from collecting old events to - // new ones. Send back those events first. - lastTxnID := invalidTxnID - for eventsProcessed := 0; eventRows.Next(); { - var event gomatrixserverlib.HeaderedEvent - var eventJSON []byte - var id int - err = eventRows.Scan( - &id, - &eventJSON, - &txnID, - ) - if err != nil { - return nil, 0, 0, false, err - } - - // Unmarshal eventJSON - if err = json.Unmarshal(eventJSON, &event); err != nil { - return nil, 0, 0, false, err - } - - // If txnID has changed on this event from the previous event, then we've - // reached the end of a transaction's events. Return only those events. - if lastTxnID > invalidTxnID && lastTxnID != txnID { - return events, maxID, lastTxnID, true, nil - } - lastTxnID = txnID - - // Limit events that aren't part of an old transaction - if txnID == -1 { - // Return if we've hit the limit - if eventsProcessed++; eventsProcessed > limit { - return events, maxID, lastTxnID, true, nil - } - } - - if id > maxID { - maxID = id - } - - // Portion of the event that is unsigned due to rapid change - // TODO: Consider removing age as not many app services use it - if err = event.SetUnsignedField("age", nowMilli-int64(event.OriginServerTS())); err != nil { - return nil, 0, 0, false, err - } - - events = append(events, event) - } - - return -} - -// countEventsByApplicationServiceID inserts an event mapped to its corresponding application service -// IDs into the db. -func (s *eventsStatements) countEventsByApplicationServiceID( - ctx context.Context, - appServiceID string, -) (int, error) { - var count int - err := s.countEventsByApplicationServiceIDStmt.QueryRowContext(ctx, appServiceID).Scan(&count) - if err != nil && err != sql.ErrNoRows { - return 0, err - } - - return count, nil -} - -// insertEvent inserts an event mapped to its corresponding application service -// IDs into the db. -func (s *eventsStatements) insertEvent( - ctx context.Context, - appServiceID string, - event *gomatrixserverlib.HeaderedEvent, -) (err error) { - // Convert event to JSON before inserting - eventJSON, err := json.Marshal(event) - if err != nil { - return err - } - - _, err = s.insertEventStmt.ExecContext( - ctx, - appServiceID, - eventJSON, - -1, // No transaction ID yet - ) - return -} - -// updateTxnIDForEvents sets the transactionID for a collection of events. Done -// before sending them to an AppService. Referenced before sending to make sure -// we aren't constructing multiple transactions with the same events. -func (s *eventsStatements) updateTxnIDForEvents( - ctx context.Context, - appserviceID string, - maxID, txnID int, -) (err error) { - _, err = s.updateTxnIDForEventsStmt.ExecContext(ctx, txnID, appserviceID, maxID) - return -} - -// deleteEventsBeforeAndIncludingID removes events matching given IDs from the database. -func (s *eventsStatements) deleteEventsBeforeAndIncludingID( - ctx context.Context, - appserviceID string, - eventTableID int, -) (err error) { - _, err = s.deleteEventsBeforeAndIncludingIDStmt.ExecContext(ctx, appserviceID, eventTableID) - return -} diff --git a/appservice/storage/postgres/storage.go b/appservice/storage/postgres/storage.go deleted file mode 100644 index a4c04b2cc..000000000 --- a/appservice/storage/postgres/storage.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "context" - "database/sql" - - // Import postgres database driver - _ "github.com/lib/pq" - "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/gomatrixserverlib" -) - -// Database stores events intended to be later sent to application services -type Database struct { - events eventsStatements - txnID txnStatements - db *sql.DB - writer sqlutil.Writer -} - -// NewDatabase opens a new database -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*Database, error) { - var result Database - var err error - if result.db, result.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil { - return nil, err - } - if err = result.prepare(); err != nil { - return nil, err - } - return &result, nil -} - -func (d *Database) prepare() error { - if err := d.events.prepare(d.db); err != nil { - return err - } - - return d.txnID.prepare(d.db) -} - -// StoreEvent takes in a gomatrixserverlib.HeaderedEvent and stores it in the database -// for a transaction worker to pull and later send to an application service. -func (d *Database) StoreEvent( - ctx context.Context, - appServiceID string, - event *gomatrixserverlib.HeaderedEvent, -) error { - return d.events.insertEvent(ctx, appServiceID, event) -} - -// GetEventsWithAppServiceID returns a slice of events and their IDs intended to -// be sent to an application service given its ID. -func (d *Database) GetEventsWithAppServiceID( - ctx context.Context, - appServiceID string, - limit int, -) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error) { - return d.events.selectEventsByApplicationServiceID(ctx, appServiceID, limit) -} - -// CountEventsWithAppServiceID returns the number of events destined for an -// application service given its ID. -func (d *Database) CountEventsWithAppServiceID( - ctx context.Context, - appServiceID string, -) (int, error) { - return d.events.countEventsByApplicationServiceID(ctx, appServiceID) -} - -// UpdateTxnIDForEvents takes in an application service ID and a -// and stores them in the DB, unless the pair already exists, in -// which case it updates them. -func (d *Database) UpdateTxnIDForEvents( - ctx context.Context, - appserviceID string, - maxID, txnID int, -) error { - return d.events.updateTxnIDForEvents(ctx, appserviceID, maxID, txnID) -} - -// RemoveEventsBeforeAndIncludingID removes all events from the database that -// are less than or equal to a given maximum ID. IDs here are implemented as a -// serial, thus this should always delete events in chronological order. -func (d *Database) RemoveEventsBeforeAndIncludingID( - ctx context.Context, - appserviceID string, - eventTableID int, -) error { - return d.events.deleteEventsBeforeAndIncludingID(ctx, appserviceID, eventTableID) -} - -// GetLatestTxnID returns the latest available transaction id -func (d *Database) GetLatestTxnID( - ctx context.Context, -) (int, error) { - return d.txnID.selectTxnID(ctx) -} diff --git a/appservice/storage/postgres/txn_id_counter_table.go b/appservice/storage/postgres/txn_id_counter_table.go deleted file mode 100644 index a96a0e360..000000000 --- a/appservice/storage/postgres/txn_id_counter_table.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "context" - "database/sql" -) - -const txnIDSchema = ` --- Keeps a count of the current transaction ID -CREATE SEQUENCE IF NOT EXISTS txn_id_counter START 1; -` - -const selectTxnIDSQL = "SELECT nextval('txn_id_counter')" - -type txnStatements struct { - selectTxnIDStmt *sql.Stmt -} - -func (s *txnStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(txnIDSchema) - if err != nil { - return - } - - if s.selectTxnIDStmt, err = db.Prepare(selectTxnIDSQL); err != nil { - return - } - - return -} - -// selectTxnID selects the latest ascending transaction ID -func (s *txnStatements) selectTxnID( - ctx context.Context, -) (txnID int, err error) { - err = s.selectTxnIDStmt.QueryRowContext(ctx).Scan(&txnID) - return -} diff --git a/appservice/storage/sqlite3/appservice_events_table.go b/appservice/storage/sqlite3/appservice_events_table.go deleted file mode 100644 index 34b4859ea..000000000 --- a/appservice/storage/sqlite3/appservice_events_table.go +++ /dev/null @@ -1,267 +0,0 @@ -// Copyright 2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "context" - "database/sql" - "encoding/json" - "time" - - "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/gomatrixserverlib" - log "github.com/sirupsen/logrus" -) - -const appserviceEventsSchema = ` --- Stores events to be sent to application services -CREATE TABLE IF NOT EXISTS appservice_events ( - -- An auto-incrementing id unique to each event in the table - id INTEGER PRIMARY KEY AUTOINCREMENT, - -- The ID of the application service the event will be sent to - as_id TEXT NOT NULL, - -- JSON representation of the event - headered_event_json TEXT NOT NULL, - -- The ID of the transaction that this event is a part of - txn_id INTEGER NOT NULL -); - -CREATE INDEX IF NOT EXISTS appservice_events_as_id ON appservice_events(as_id); -` - -const selectEventsByApplicationServiceIDSQL = "" + - "SELECT id, headered_event_json, txn_id " + - "FROM appservice_events WHERE as_id = $1 ORDER BY txn_id DESC, id ASC" - -const countEventsByApplicationServiceIDSQL = "" + - "SELECT COUNT(id) FROM appservice_events WHERE as_id = $1" - -const insertEventSQL = "" + - "INSERT INTO appservice_events(as_id, headered_event_json, txn_id) " + - "VALUES ($1, $2, $3)" - -const updateTxnIDForEventsSQL = "" + - "UPDATE appservice_events SET txn_id = $1 WHERE as_id = $2 AND id <= $3" - -const deleteEventsBeforeAndIncludingIDSQL = "" + - "DELETE FROM appservice_events WHERE as_id = $1 AND id <= $2" - -const ( - // A transaction ID number that no transaction should ever have. Used for - // checking again the default value. - invalidTxnID = -2 -) - -type eventsStatements struct { - db *sql.DB - writer sqlutil.Writer - selectEventsByApplicationServiceIDStmt *sql.Stmt - countEventsByApplicationServiceIDStmt *sql.Stmt - insertEventStmt *sql.Stmt - updateTxnIDForEventsStmt *sql.Stmt - deleteEventsBeforeAndIncludingIDStmt *sql.Stmt -} - -func (s *eventsStatements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) { - s.db = db - s.writer = writer - _, err = db.Exec(appserviceEventsSchema) - if err != nil { - return - } - - if s.selectEventsByApplicationServiceIDStmt, err = db.Prepare(selectEventsByApplicationServiceIDSQL); err != nil { - return - } - if s.countEventsByApplicationServiceIDStmt, err = db.Prepare(countEventsByApplicationServiceIDSQL); err != nil { - return - } - if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil { - return - } - if s.updateTxnIDForEventsStmt, err = db.Prepare(updateTxnIDForEventsSQL); err != nil { - return - } - if s.deleteEventsBeforeAndIncludingIDStmt, err = db.Prepare(deleteEventsBeforeAndIncludingIDSQL); err != nil { - return - } - - return -} - -// selectEventsByApplicationServiceID takes in an application service ID and -// returns a slice of events that need to be sent to that application service, -// as well as an int later used to remove these same events from the database -// once successfully sent to an application service. -func (s *eventsStatements) selectEventsByApplicationServiceID( - ctx context.Context, - applicationServiceID string, - limit int, -) ( - txnID, maxID int, - events []gomatrixserverlib.HeaderedEvent, - eventsRemaining bool, - err error, -) { - defer func() { - if err != nil { - log.WithFields(log.Fields{ - "appservice": applicationServiceID, - }).WithError(err).Fatalf("appservice unable to select new events to send") - } - }() - // Retrieve events from the database. Unsuccessfully sent events first - eventRows, err := s.selectEventsByApplicationServiceIDStmt.QueryContext(ctx, applicationServiceID) - if err != nil { - return - } - defer checkNamedErr(eventRows.Close, &err) - events, maxID, txnID, eventsRemaining, err = retrieveEvents(eventRows, limit) - if err != nil { - return - } - - return -} - -// checkNamedErr calls fn and overwrite err if it was nil and fn returned non-nil -func checkNamedErr(fn func() error, err *error) { - if e := fn(); e != nil && *err == nil { - *err = e - } -} - -func retrieveEvents(eventRows *sql.Rows, limit int) (events []gomatrixserverlib.HeaderedEvent, maxID, txnID int, eventsRemaining bool, err error) { - // Get current time for use in calculating event age - nowMilli := time.Now().UnixNano() / int64(time.Millisecond) - - // Iterate through each row and store event contents - // If txn_id changes dramatically, we've switched from collecting old events to - // new ones. Send back those events first. - lastTxnID := invalidTxnID - for eventsProcessed := 0; eventRows.Next(); { - var event gomatrixserverlib.HeaderedEvent - var eventJSON []byte - var id int - err = eventRows.Scan( - &id, - &eventJSON, - &txnID, - ) - if err != nil { - return nil, 0, 0, false, err - } - - // Unmarshal eventJSON - if err = json.Unmarshal(eventJSON, &event); err != nil { - return nil, 0, 0, false, err - } - - // If txnID has changed on this event from the previous event, then we've - // reached the end of a transaction's events. Return only those events. - if lastTxnID > invalidTxnID && lastTxnID != txnID { - return events, maxID, lastTxnID, true, nil - } - lastTxnID = txnID - - // Limit events that aren't part of an old transaction - if txnID == -1 { - // Return if we've hit the limit - if eventsProcessed++; eventsProcessed > limit { - return events, maxID, lastTxnID, true, nil - } - } - - if id > maxID { - maxID = id - } - - // Portion of the event that is unsigned due to rapid change - // TODO: Consider removing age as not many app services use it - if err = event.SetUnsignedField("age", nowMilli-int64(event.OriginServerTS())); err != nil { - return nil, 0, 0, false, err - } - - events = append(events, event) - } - - return -} - -// countEventsByApplicationServiceID inserts an event mapped to its corresponding application service -// IDs into the db. -func (s *eventsStatements) countEventsByApplicationServiceID( - ctx context.Context, - appServiceID string, -) (int, error) { - var count int - err := s.countEventsByApplicationServiceIDStmt.QueryRowContext(ctx, appServiceID).Scan(&count) - if err != nil && err != sql.ErrNoRows { - return 0, err - } - - return count, nil -} - -// insertEvent inserts an event mapped to its corresponding application service -// IDs into the db. -func (s *eventsStatements) insertEvent( - ctx context.Context, - appServiceID string, - event *gomatrixserverlib.HeaderedEvent, -) (err error) { - // Convert event to JSON before inserting - eventJSON, err := json.Marshal(event) - if err != nil { - return err - } - - return s.writer.Do(s.db, nil, func(txn *sql.Tx) error { - _, err := s.insertEventStmt.ExecContext( - ctx, - appServiceID, - eventJSON, - -1, // No transaction ID yet - ) - return err - }) -} - -// updateTxnIDForEvents sets the transactionID for a collection of events. Done -// before sending them to an AppService. Referenced before sending to make sure -// we aren't constructing multiple transactions with the same events. -func (s *eventsStatements) updateTxnIDForEvents( - ctx context.Context, - appserviceID string, - maxID, txnID int, -) (err error) { - return s.writer.Do(s.db, nil, func(txn *sql.Tx) error { - _, err := s.updateTxnIDForEventsStmt.ExecContext(ctx, txnID, appserviceID, maxID) - return err - }) -} - -// deleteEventsBeforeAndIncludingID removes events matching given IDs from the database. -func (s *eventsStatements) deleteEventsBeforeAndIncludingID( - ctx context.Context, - appserviceID string, - eventTableID int, -) (err error) { - return s.writer.Do(s.db, nil, func(txn *sql.Tx) error { - _, err := s.deleteEventsBeforeAndIncludingIDStmt.ExecContext(ctx, appserviceID, eventTableID) - return err - }) -} diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go deleted file mode 100644 index ad62b3628..000000000 --- a/appservice/storage/sqlite3/storage.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "context" - "database/sql" - - // Import SQLite database driver - "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/gomatrixserverlib" -) - -// Database stores events intended to be later sent to application services -type Database struct { - events eventsStatements - txnID txnStatements - db *sql.DB - writer sqlutil.Writer -} - -// NewDatabase opens a new database -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*Database, error) { - var result Database - var err error - if result.db, result.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()); err != nil { - return nil, err - } - if err = result.prepare(); err != nil { - return nil, err - } - return &result, nil -} - -func (d *Database) prepare() error { - if err := d.events.prepare(d.db, d.writer); err != nil { - return err - } - - return d.txnID.prepare(d.db, d.writer) -} - -// StoreEvent takes in a gomatrixserverlib.HeaderedEvent and stores it in the database -// for a transaction worker to pull and later send to an application service. -func (d *Database) StoreEvent( - ctx context.Context, - appServiceID string, - event *gomatrixserverlib.HeaderedEvent, -) error { - return d.events.insertEvent(ctx, appServiceID, event) -} - -// GetEventsWithAppServiceID returns a slice of events and their IDs intended to -// be sent to an application service given its ID. -func (d *Database) GetEventsWithAppServiceID( - ctx context.Context, - appServiceID string, - limit int, -) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error) { - return d.events.selectEventsByApplicationServiceID(ctx, appServiceID, limit) -} - -// CountEventsWithAppServiceID returns the number of events destined for an -// application service given its ID. -func (d *Database) CountEventsWithAppServiceID( - ctx context.Context, - appServiceID string, -) (int, error) { - return d.events.countEventsByApplicationServiceID(ctx, appServiceID) -} - -// UpdateTxnIDForEvents takes in an application service ID and a -// and stores them in the DB, unless the pair already exists, in -// which case it updates them. -func (d *Database) UpdateTxnIDForEvents( - ctx context.Context, - appserviceID string, - maxID, txnID int, -) error { - return d.events.updateTxnIDForEvents(ctx, appserviceID, maxID, txnID) -} - -// RemoveEventsBeforeAndIncludingID removes all events from the database that -// are less than or equal to a given maximum ID. IDs here are implemented as a -// serial, thus this should always delete events in chronological order. -func (d *Database) RemoveEventsBeforeAndIncludingID( - ctx context.Context, - appserviceID string, - eventTableID int, -) error { - return d.events.deleteEventsBeforeAndIncludingID(ctx, appserviceID, eventTableID) -} - -// GetLatestTxnID returns the latest available transaction id -func (d *Database) GetLatestTxnID( - ctx context.Context, -) (int, error) { - return d.txnID.selectTxnID(ctx) -} diff --git a/appservice/storage/sqlite3/txn_id_counter_table.go b/appservice/storage/sqlite3/txn_id_counter_table.go deleted file mode 100644 index f2e902f98..000000000 --- a/appservice/storage/sqlite3/txn_id_counter_table.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "context" - "database/sql" - - "github.com/matrix-org/dendrite/internal/sqlutil" -) - -const txnIDSchema = ` --- Keeps a count of the current transaction ID -CREATE TABLE IF NOT EXISTS appservice_counters ( - name TEXT PRIMARY KEY NOT NULL, - last_id INTEGER DEFAULT 1 -); -INSERT OR IGNORE INTO appservice_counters (name, last_id) VALUES('txn_id', 1); -` - -const selectTxnIDSQL = ` - SELECT last_id FROM appservice_counters WHERE name='txn_id' -` - -const updateTxnIDSQL = ` - UPDATE appservice_counters SET last_id=last_id+1 WHERE name='txn_id' -` - -type txnStatements struct { - db *sql.DB - writer sqlutil.Writer - selectTxnIDStmt *sql.Stmt - updateTxnIDStmt *sql.Stmt -} - -func (s *txnStatements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) { - s.db = db - s.writer = writer - _, err = db.Exec(txnIDSchema) - if err != nil { - return - } - - if s.selectTxnIDStmt, err = db.Prepare(selectTxnIDSQL); err != nil { - return - } - - if s.updateTxnIDStmt, err = db.Prepare(updateTxnIDSQL); err != nil { - return - } - - return -} - -// selectTxnID selects the latest ascending transaction ID -func (s *txnStatements) selectTxnID( - ctx context.Context, -) (txnID int, err error) { - err = s.writer.Do(s.db, nil, func(txn *sql.Tx) error { - err := s.selectTxnIDStmt.QueryRowContext(ctx).Scan(&txnID) - if err != nil { - return err - } - - _, err = s.updateTxnIDStmt.ExecContext(ctx) - return err - }) - return -} diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go deleted file mode 100644 index 89d5e0cc2..000000000 --- a/appservice/storage/storage.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build !wasm -// +build !wasm - -package storage - -import ( - "fmt" - - "github.com/matrix-org/dendrite/appservice/storage/postgres" - "github.com/matrix-org/dendrite/appservice/storage/sqlite3" - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) -// and sets DB connection parameters -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) { - switch { - case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(base, dbProperties) - case dbProperties.ConnectionString.IsPostgres(): - return postgres.NewDatabase(base, dbProperties) - default: - return nil, fmt.Errorf("unexpected database type") - } -} diff --git a/appservice/storage/storage_wasm.go b/appservice/storage/storage_wasm.go deleted file mode 100644 index 230254598..000000000 --- a/appservice/storage/storage_wasm.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "fmt" - - "github.com/matrix-org/dendrite/appservice/storage/sqlite3" - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) { - switch { - case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(base, dbProperties) - case dbProperties.ConnectionString.IsPostgres(): - return nil, fmt.Errorf("can't use Postgres implementation") - default: - return nil, fmt.Errorf("unexpected database type") - } -} diff --git a/appservice/types/types.go b/appservice/types/types.go deleted file mode 100644 index 098face62..000000000 --- a/appservice/types/types.go +++ /dev/null @@ -1,64 +0,0 @@ -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package types - -import ( - "sync" - - "github.com/matrix-org/dendrite/setup/config" -) - -const ( - // AppServiceDeviceID is the AS dummy device ID - AppServiceDeviceID = "AS_Device" -) - -// ApplicationServiceWorkerState is a type that couples an application service, -// a lockable condition as well as some other state variables, allowing the -// roomserver to notify appservice workers when there are events ready to send -// externally to application services. -type ApplicationServiceWorkerState struct { - AppService config.ApplicationService - Cond *sync.Cond - // Events ready to be sent - EventsReady bool - // Backoff exponent (2^x secs). Max 6, aka 64s. - Backoff int -} - -// NotifyNewEvents wakes up all waiting goroutines, notifying that events remain -// in the event queue for this application service worker. -func (a *ApplicationServiceWorkerState) NotifyNewEvents() { - a.Cond.L.Lock() - a.EventsReady = true - a.Cond.Broadcast() - a.Cond.L.Unlock() -} - -// FinishEventProcessing marks all events of this worker as being sent to the -// application service. -func (a *ApplicationServiceWorkerState) FinishEventProcessing() { - a.Cond.L.Lock() - a.EventsReady = false - a.Cond.L.Unlock() -} - -// WaitForNewEvents causes the calling goroutine to wait on the worker state's -// condition for a broadcast or similar wakeup, if there are no events ready. -func (a *ApplicationServiceWorkerState) WaitForNewEvents() { - a.Cond.L.Lock() - if !a.EventsReady { - a.Cond.Wait() - } - a.Cond.L.Unlock() -} diff --git a/appservice/workers/transaction_scheduler.go b/appservice/workers/transaction_scheduler.go deleted file mode 100644 index 4dab00bd7..000000000 --- a/appservice/workers/transaction_scheduler.go +++ /dev/null @@ -1,236 +0,0 @@ -// Copyright 2018 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package workers - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "math" - "net/http" - "net/url" - "time" - - "github.com/matrix-org/dendrite/appservice/storage" - "github.com/matrix-org/dendrite/appservice/types" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/gomatrixserverlib" - log "github.com/sirupsen/logrus" -) - -var ( - // Maximum size of events sent in each transaction. - transactionBatchSize = 50 -) - -// SetupTransactionWorkers spawns a separate goroutine for each application -// service. Each of these "workers" handle taking all events intended for their -// app service, batch them up into a single transaction (up to a max transaction -// size), then send that off to the AS's /transactions/{txnID} endpoint. It also -// handles exponentially backing off in case the AS isn't currently available. -func SetupTransactionWorkers( - client *http.Client, - appserviceDB storage.Database, - workerStates []types.ApplicationServiceWorkerState, -) error { - // Create a worker that handles transmitting events to a single homeserver - for _, workerState := range workerStates { - // Don't create a worker if this AS doesn't want to receive events - if workerState.AppService.URL != "" { - go worker(client, appserviceDB, workerState) - } - } - return nil -} - -// worker is a goroutine that sends any queued events to the application service -// it is given. -func worker(client *http.Client, db storage.Database, ws types.ApplicationServiceWorkerState) { - log.WithFields(log.Fields{ - "appservice": ws.AppService.ID, - }).Info("Starting application service") - ctx := context.Background() - - // Initial check for any leftover events to send from last time - eventCount, err := db.CountEventsWithAppServiceID(ctx, ws.AppService.ID) - if err != nil { - log.WithFields(log.Fields{ - "appservice": ws.AppService.ID, - }).WithError(err).Fatal("appservice worker unable to read queued events from DB") - return - } - if eventCount > 0 { - ws.NotifyNewEvents() - } - - // Loop forever and keep waiting for more events to send - for { - // Wait for more events if we've sent all the events in the database - ws.WaitForNewEvents() - - // Batch events up into a transaction - transactionJSON, txnID, maxEventID, eventsRemaining, err := createTransaction(ctx, db, ws.AppService.ID) - if err != nil { - log.WithFields(log.Fields{ - "appservice": ws.AppService.ID, - }).WithError(err).Fatal("appservice worker unable to create transaction") - - return - } - - // Send the events off to the application service - // Backoff if the application service does not respond - err = send(client, ws.AppService, txnID, transactionJSON) - if err != nil { - log.WithFields(log.Fields{ - "appservice": ws.AppService.ID, - }).WithError(err).Error("unable to send event") - // Backoff - backoff(&ws, err) - continue - } - - // We sent successfully, hooray! - ws.Backoff = 0 - - // Transactions have a maximum event size, so there may still be some events - // left over to send. Keep sending until none are left - if !eventsRemaining { - ws.FinishEventProcessing() - } - - // Remove sent events from the DB - err = db.RemoveEventsBeforeAndIncludingID(ctx, ws.AppService.ID, maxEventID) - if err != nil { - log.WithFields(log.Fields{ - "appservice": ws.AppService.ID, - }).WithError(err).Fatal("unable to remove appservice events from the database") - return - } - } -} - -// backoff pauses the calling goroutine for a 2^some backoff exponent seconds -func backoff(ws *types.ApplicationServiceWorkerState, err error) { - // Calculate how long to backoff for - backoffDuration := time.Duration(math.Pow(2, float64(ws.Backoff))) - backoffSeconds := time.Second * backoffDuration - - log.WithFields(log.Fields{ - "appservice": ws.AppService.ID, - }).WithError(err).Warnf("unable to send transactions successfully, backing off for %ds", - backoffDuration) - - ws.Backoff++ - if ws.Backoff > 6 { - ws.Backoff = 6 - } - - // Backoff - time.Sleep(backoffSeconds) -} - -// createTransaction takes in a slice of AS events, stores them in an AS -// transaction, and JSON-encodes the results. -func createTransaction( - ctx context.Context, - db storage.Database, - appserviceID string, -) ( - transactionJSON []byte, - txnID, maxID int, - eventsRemaining bool, - err error, -) { - // Retrieve the latest events from the DB (will return old events if they weren't successfully sent) - txnID, maxID, events, eventsRemaining, err := db.GetEventsWithAppServiceID(ctx, appserviceID, transactionBatchSize) - if err != nil { - log.WithFields(log.Fields{ - "appservice": appserviceID, - }).WithError(err).Fatalf("appservice worker unable to read queued events from DB") - - return - } - - // Check if these events do not already have a transaction ID - if txnID == -1 { - // If not, grab next available ID from the DB - txnID, err = db.GetLatestTxnID(ctx) - if err != nil { - return nil, 0, 0, false, err - } - - // Mark new events with current transactionID - if err = db.UpdateTxnIDForEvents(ctx, appserviceID, maxID, txnID); err != nil { - return nil, 0, 0, false, err - } - } - - var ev []*gomatrixserverlib.HeaderedEvent - for i := range events { - ev = append(ev, &events[i]) - } - - // Create a transaction and store the events inside - transaction := gomatrixserverlib.ApplicationServiceTransaction{ - Events: gomatrixserverlib.HeaderedToClientEvents(ev, gomatrixserverlib.FormatAll), - } - - transactionJSON, err = json.Marshal(transaction) - if err != nil { - return - } - - return -} - -// send sends events to an application service. Returns an error if an OK was not -// received back from the application service or the request timed out. -func send( - client *http.Client, - appservice config.ApplicationService, - txnID int, - transaction []byte, -) (err error) { - // PUT a transaction to our AS - // https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid - address := fmt.Sprintf("%s/transactions/%d?access_token=%s", appservice.URL, txnID, url.QueryEscape(appservice.HSToken)) - req, err := http.NewRequest("PUT", address, bytes.NewBuffer(transaction)) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json") - resp, err := client.Do(req) - if err != nil { - return err - } - defer checkNamedErr(resp.Body.Close, &err) - - // Check the AS received the events correctly - if resp.StatusCode != http.StatusOK { - // TODO: Handle non-200 error codes from application services - return fmt.Errorf("non-OK status code %d returned from AS", resp.StatusCode) - } - - return nil -} - -// checkNamedErr calls fn and overwrite err if it was nil and fn returned non-nil -func checkNamedErr(fn func() error, err *error) { - if e := fn(); e != nil && *err == nil { - *err = e - } -} diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index f3895ae23..60e817212 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -255,7 +255,6 @@ func (m *DendriteMonolith) Start() { cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-roomserver.db", m.StorageDirectory, prefix)) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-keyserver.db", m.StorageDirectory, prefix)) cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-federationsender.db", m.StorageDirectory, prefix)) - cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-appservice.db", m.StorageDirectory, prefix)) cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory)) cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory)) cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 99b180c81..4432a3a45 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -94,7 +94,6 @@ func (m *DendriteMonolith) Start() { cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-roomserver.db", m.StorageDirectory)) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-keyserver.db", m.StorageDirectory)) cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-federationsender.db", m.StorageDirectory)) - cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-appservice.db", m.StorageDirectory)) cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory)) cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory)) cfg.ClientAPI.RegistrationDisabled = false diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index b16cfec6a..52c9dc8eb 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -124,7 +124,6 @@ func main() { cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName)) cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName)) - cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName)) cfg.ClientAPI.RegistrationDisabled = false diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 619720d6c..086baa264 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -86,7 +86,6 @@ func main() { cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName)) cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName)) - cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) cfg.MSCs.MSCs = []string{"msc2836"} cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName)) cfg.ClientAPI.RegistrationDisabled = false diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index 1c585d916..6ae3ff9c9 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -24,7 +24,6 @@ func main() { cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName) } if *dbURI != "" { - cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(*dbURI) cfg.FederationAPI.Database.ConnectionString = config.DataSource(*dbURI) cfg.KeyServer.Database.ConnectionString = config.DataSource(*dbURI) cfg.MSCs.Database.ConnectionString = config.DataSource(*dbURI) diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml index 856b4ab22..3caf91434 100644 --- a/dendrite-sample.polylith.yaml +++ b/dendrite-sample.polylith.yaml @@ -132,13 +132,6 @@ app_service_api: listen: http://[::]:7777 # The listen address for incoming API requests connect: http://app_service_api:7777 # The connect address for other components to use - # Database configuration for this component. - database: - connection_string: postgresql://username:password@hostname/dendrite_appservice?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - # Disable the validation of TLS certificates of appservices. This is # not recommended in production since it may allow appservice traffic # to be sent to an insecure endpoint. diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go index b8f99a612..9c3771272 100644 --- a/setup/config/config_appservice.go +++ b/setup/config/config_appservice.go @@ -31,8 +31,6 @@ type AppServiceAPI struct { InternalAPI InternalAPIOptions `yaml:"internal_api"` - Database DatabaseOptions `yaml:"database"` - // DisableTLSValidation disables the validation of X.509 TLS certs // on appservice endpoints. This is not recommended in production! DisableTLSValidation bool `yaml:"disable_tls_validation"` @@ -43,16 +41,9 @@ type AppServiceAPI struct { func (c *AppServiceAPI) Defaults(generate bool) { c.InternalAPI.Listen = "http://localhost:7777" c.InternalAPI.Connect = "http://localhost:7777" - c.Database.Defaults(5) - if generate { - c.Database.ConnectionString = "file:appservice.db" - } } func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { - if c.Matrix.DatabaseOptions.ConnectionString == "" { - checkNotEmpty(configErrs, "app_service_api.database.connection_string", string(c.Database.ConnectionString)) - } if isMonolith { // polylith required configs below return } diff --git a/setup/jetstream/helpers.go b/setup/jetstream/helpers.go index f47637c69..1ec860b04 100644 --- a/setup/jetstream/helpers.go +++ b/setup/jetstream/helpers.go @@ -34,14 +34,6 @@ func JetStreamConsumer( } }() - // If the batch size is greater than 1, we will want to acknowledge all - // received messages in the batch. Below we will send an acknowledgement - // for the most recent message in the batch and AckAll will ensure that - // all messages that came before it are also acknowledged implicitly. - if batch > 1 { - opts = append(opts, nats.AckAll()) - } - name := durable + "Pull" sub, err := js.PullSubscribe(subj, name, opts...) if err != nil { @@ -89,21 +81,26 @@ func JetStreamConsumer( if len(msgs) < 1 { continue } - msg := msgs[len(msgs)-1] // most recent message, in case of AckAll - if err = msg.InProgress(nats.Context(ctx)); err != nil { - logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.InProgress: %w", err)) - sentry.CaptureException(err) - continue + for _, msg := range msgs { + if err = msg.InProgress(nats.Context(ctx)); err != nil { + logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.InProgress: %w", err)) + sentry.CaptureException(err) + continue + } } if f(ctx, msgs) { - if err = msg.AckSync(nats.Context(ctx)); err != nil { - logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.AckSync: %w", err)) - sentry.CaptureException(err) + for _, msg := range msgs { + if err = msg.AckSync(nats.Context(ctx)); err != nil { + logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.AckSync: %w", err)) + sentry.CaptureException(err) + } } } else { - if err = msg.Nak(nats.Context(ctx)); err != nil { - logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.Nak: %w", err)) - sentry.CaptureException(err) + for _, msg := range msgs { + if err = msg.Nak(nats.Context(ctx)); err != nil { + logrus.WithContext(ctx).WithField("subject", subj).Warn(fmt.Errorf("msg.Nak: %w", err)) + sentry.CaptureException(err) + } } } } diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go index 051d55a35..3660e91e3 100644 --- a/setup/jetstream/nats.go +++ b/setup/jetstream/nats.go @@ -183,6 +183,7 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc OutputReceiptEvent: {"SyncAPIEDUServerReceiptConsumer", "FederationAPIEDUServerConsumer"}, OutputSendToDeviceEvent: {"SyncAPIEDUServerSendToDeviceConsumer", "FederationAPIEDUServerConsumer"}, OutputTypingEvent: {"SyncAPIEDUServerTypingConsumer", "FederationAPIEDUServerConsumer"}, + OutputRoomEvent: {"AppserviceRoomserverConsumer"}, } { streamName := cfg.Matrix.JetStream.Prefixed(stream) for _, consumer := range consumers { diff --git a/test/testrig/base.go b/test/testrig/base.go index d13c43129..33230921c 100644 --- a/test/testrig/base.go +++ b/test/testrig/base.go @@ -57,7 +57,6 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f return base.NewBaseDendrite(&cfg, "Test", base.DisableMetrics), func() { // cleanup db files. This risks getting out of sync as we add more database strings :( dbFiles := []config.DataSource{ - cfg.AppServiceAPI.Database.ConnectionString, cfg.FederationAPI.Database.ConnectionString, cfg.KeyServer.Database.ConnectionString, cfg.MSCs.Database.ConnectionString, diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 6ba469327..2f28ee906 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -28,7 +28,6 @@ import ( "github.com/sirupsen/logrus" "golang.org/x/crypto/bcrypt" - "github.com/matrix-org/dendrite/appservice/types" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/pushrules" @@ -454,7 +453,7 @@ func (a *UserInternalAPI) queryAppServiceToken(ctx context.Context, token, appSe // Create a dummy device for AS user dev := api.Device{ // Use AS dummy device ID - ID: types.AppServiceDeviceID, + ID: "AS_Device", // AS dummy device has AS's token. AccessToken: token, AppserviceID: appService.ID, From 51d229b02505df770ed891ec8ab20843bc2b1d1d Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Sep 2022 14:15:41 +0100 Subject: [PATCH 02/20] Configuration tweaks (#2567) This makes the following changes: * The various `Defaults` functions are now responsible for setting sane defaults if `generate` is specified, rather than hiding them in `generate-config` * Some configuration options have been marked as `omitempty` so that they don't appear in generated configs unnecessarily (monolith-specific vs. polylith-specific options) * A new option `-polylith` has been added to `generate-config` to create a config that makes sense for polylith deployments (i.e. including the internal/external API listeners and per-component database sections) * A new option `-normalise` has been added to `generate-config` to take an existing file and add any missing options and/or defaults --- build/gobind-pinecone/monolith.go | 5 +- build/gobind-yggdrasil/monolith.go | 5 +- build/scripts/ComplementPostgres.Dockerfile | 7 +- clientapi/routing/register_test.go | 5 +- cmd/dendrite-demo-pinecone/main.go | 5 +- cmd/dendrite-demo-yggdrasil/main.go | 5 +- cmd/generate-config/main.go | 150 +++++++++----------- federationapi/federationapi_keys_test.go | 5 +- federationapi/federationapi_test.go | 5 +- setup/config/config.go | 33 +++-- setup/config/config_appservice.go | 10 +- setup/config/config_clientapi.go | 16 ++- setup/config/config_federationapi.go | 45 ++++-- setup/config/config_global.go | 32 +++-- setup/config/config_jetstream.go | 4 +- setup/config/config_keyserver.go | 26 ++-- setup/config/config_mediaapi.go | 47 ++++-- setup/config/config_mscs.go | 17 ++- setup/config/config_roomserver.go | 26 ++-- setup/config/config_syncapi.go | 30 ++-- setup/config/config_userapi.go | 26 ++-- setup/mscs/msc2836/msc2836_test.go | 5 +- test/testrig/base.go | 25 +++- 23 files changed, 322 insertions(+), 212 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 60e817212..44eab4cd7 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -243,7 +243,10 @@ func (m *DendriteMonolith) Start() { prefix := hex.EncodeToString(pk) cfg := &config.Dendrite{} - cfg.Defaults(true) + cfg.Defaults(config.DefaultOpts{ + Generate: true, + Monolithic: true, + }) cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk)) cfg.Global.PrivateKey = sk cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 4432a3a45..896f3d14f 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -82,7 +82,10 @@ func (m *DendriteMonolith) Start() { m.YggdrasilNode = ygg cfg := &config.Dendrite{} - cfg.Defaults(true) + cfg.Defaults(config.DefaultOpts{ + Generate: true, + Monolithic: true, + }) cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName()) cfg.Global.PrivateKey = ygg.PrivateKey() cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) diff --git a/build/scripts/ComplementPostgres.Dockerfile b/build/scripts/ComplementPostgres.Dockerfile index 699540120..785090b0b 100644 --- a/build/scripts/ComplementPostgres.Dockerfile +++ b/build/scripts/ComplementPostgres.Dockerfile @@ -46,9 +46,8 @@ EXPOSE 8008 8448 # At runtime, generate TLS cert based on the CA now mounted at /ca # At runtime, replace the SERVER_NAME with what we are told CMD /build/run_postgres.sh && ./generate-keys --keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \ - ./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \ - # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password, bump max_conns - sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \ - sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \ + ./generate-config -server $SERVER_NAME --ci --db postgresql://postgres@localhost/postgres?sslmode=disable > dendrite.yaml && \ + # Bump max_open_conns up here in the global database config + sed -i 's/max_open_conns:.*$/max_open_conns: 1990/g' dendrite.yaml && \ cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} \ No newline at end of file diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index 0507116f1..85846c7d6 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -181,7 +181,10 @@ func TestValidationOfApplicationServices(t *testing.T) { // Set up a config fakeConfig := &config.Dendrite{} - fakeConfig.Defaults(true) + fakeConfig.Defaults(config.DefaultOpts{ + Generate: true, + Monolithic: true, + }) fakeConfig.Global.ServerName = "localhost" fakeConfig.ClientAPI.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService} diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 52c9dc8eb..83b52132a 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -115,7 +115,10 @@ func main() { panic("failed to load PEM key: " + err.Error()) } } - cfg.Defaults(true) + cfg.Defaults(config.DefaultOpts{ + Generate: true, + Monolithic: true, + }) cfg.Global.PrivateKey = sk cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName)) cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 086baa264..d2fddf8bc 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -78,7 +78,10 @@ func main() { if configFlagSet { cfg = setup.ParseFlags(true) } else { - cfg.Defaults(true) + cfg.Defaults(config.DefaultOpts{ + Generate: true, + Monolithic: true, + }) cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName)) cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index 6ae3ff9c9..c24e8153e 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -3,6 +3,7 @@ package main import ( "flag" "fmt" + "path/filepath" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" @@ -11,90 +12,81 @@ import ( ) func main() { - defaultsForCI := flag.Bool("ci", false, "sane defaults for CI testing") + defaultsForCI := flag.Bool("ci", false, "Populate the configuration with sane defaults for use in CI") serverName := flag.String("server", "", "The domain name of the server if not 'localhost'") - dbURI := flag.String("db", "", "The DB URI to use for all components if not SQLite files") + dbURI := flag.String("db", "", "The DB URI to use for all components (PostgreSQL only)") + dirPath := flag.String("dir", "./", "The folder to use for paths (like SQLite databases, media storage)") + normalise := flag.String("normalise", "", "Normalise an existing configuration file by adding new/missing options and defaults") + polylith := flag.Bool("polylith", false, "Generate a config that makes sense for polylith deployments") flag.Parse() - cfg := &config.Dendrite{ - Version: config.Version, - } - cfg.Defaults(true) - if *serverName != "" { - cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName) - } - if *dbURI != "" { - cfg.FederationAPI.Database.ConnectionString = config.DataSource(*dbURI) - cfg.KeyServer.Database.ConnectionString = config.DataSource(*dbURI) - cfg.MSCs.Database.ConnectionString = config.DataSource(*dbURI) - cfg.MediaAPI.Database.ConnectionString = config.DataSource(*dbURI) - cfg.RoomServer.Database.ConnectionString = config.DataSource(*dbURI) - cfg.SyncAPI.Database.ConnectionString = config.DataSource(*dbURI) - cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(*dbURI) - } - cfg.Global.TrustedIDServers = []string{ - "matrix.org", - "vector.im", - } - cfg.Logging = []config.LogrusHook{ - { - Type: "file", - Level: "info", - Params: map[string]interface{}{ - "path": "/var/log/dendrite", - }, - }, - } - cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{ - { - ServerName: "matrix.org", - Keys: []config.KeyPerspectiveTrustKey{ - { - KeyID: "ed25519:auto", - PublicKey: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw", - }, - { - KeyID: "ed25519:a_RXGa", - PublicKey: "l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ", + var cfg *config.Dendrite + if *normalise == "" { + cfg = &config.Dendrite{ + Version: config.Version, + } + cfg.Defaults(config.DefaultOpts{ + Generate: true, + Monolithic: !*polylith, + }) + if *serverName != "" { + cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName) + } + uri := config.DataSource(*dbURI) + if *polylith || uri.IsSQLite() || uri == "" { + for name, db := range map[string]*config.DatabaseOptions{ + "federationapi": &cfg.FederationAPI.Database, + "keyserver": &cfg.KeyServer.Database, + "mscs": &cfg.MSCs.Database, + "mediaapi": &cfg.MediaAPI.Database, + "roomserver": &cfg.RoomServer.Database, + "syncapi": &cfg.SyncAPI.Database, + "userapi": &cfg.UserAPI.AccountDatabase, + } { + if uri == "" { + path := filepath.Join(*dirPath, fmt.Sprintf("dendrite_%s.db", name)) + db.ConnectionString = config.DataSource(fmt.Sprintf("file:%s", path)) + } else { + db.ConnectionString = uri + } + } + } else { + cfg.Global.DatabaseOptions.ConnectionString = uri + } + cfg.Logging = []config.LogrusHook{ + { + Type: "file", + Level: "info", + Params: map[string]interface{}{ + "path": filepath.Join(*dirPath, "log"), }, }, - }, - } - cfg.MediaAPI.ThumbnailSizes = []config.ThumbnailSize{ - { - Width: 32, - Height: 32, - ResizeMethod: "crop", - }, - { - Width: 96, - Height: 96, - ResizeMethod: "crop", - }, - { - Width: 640, - Height: 480, - ResizeMethod: "scale", - }, - } - - if *defaultsForCI { - cfg.AppServiceAPI.DisableTLSValidation = true - cfg.ClientAPI.RateLimiting.Enabled = false - cfg.FederationAPI.DisableTLSValidation = false - // don't hit matrix.org when running tests!!! - cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{} - cfg.MSCs.MSCs = []string{"msc2836", "msc2946", "msc2444", "msc2753"} - cfg.Logging[0].Level = "trace" - cfg.Logging[0].Type = "std" - cfg.UserAPI.BCryptCost = bcrypt.MinCost - cfg.Global.JetStream.InMemory = true - cfg.ClientAPI.RegistrationDisabled = false - cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true - cfg.ClientAPI.RegistrationSharedSecret = "complement" - cfg.Global.Presence = config.PresenceOptions{ - EnableInbound: true, - EnableOutbound: true, + } + if *defaultsForCI { + cfg.AppServiceAPI.DisableTLSValidation = true + cfg.ClientAPI.RateLimiting.Enabled = false + cfg.FederationAPI.DisableTLSValidation = false + // don't hit matrix.org when running tests!!! + cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{} + cfg.MediaAPI.BasePath = config.Path(filepath.Join(*dirPath, "media")) + cfg.MSCs.MSCs = []string{"msc2836", "msc2946", "msc2444", "msc2753"} + cfg.Logging[0].Level = "trace" + cfg.Logging[0].Type = "std" + cfg.UserAPI.BCryptCost = bcrypt.MinCost + cfg.Global.JetStream.InMemory = true + cfg.Global.JetStream.StoragePath = config.Path(*dirPath) + cfg.ClientAPI.RegistrationDisabled = false + cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true + cfg.ClientAPI.RegistrationSharedSecret = "complement" + cfg.Global.Presence = config.PresenceOptions{ + EnableInbound: true, + EnableOutbound: true, + } + } + } else { + var err error + if cfg, err = config.Load(*normalise, !*polylith); err != nil { + panic(err) } } diff --git a/federationapi/federationapi_keys_test.go b/federationapi/federationapi_keys_test.go index 9c3446222..4469da35b 100644 --- a/federationapi/federationapi_keys_test.go +++ b/federationapi/federationapi_keys_test.go @@ -75,7 +75,10 @@ func TestMain(m *testing.M) { // Draw up just enough Dendrite config for the server key // API to work. cfg := &config.Dendrite{} - cfg.Defaults(true) + cfg.Defaults(config.DefaultOpts{ + Generate: true, + Monolithic: true, + }) cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name) cfg.Global.PrivateKey = testPriv cfg.Global.JetStream.InMemory = true diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index bdcb9f57c..15f7a6840 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -263,7 +263,10 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) { func TestRoomsV3URLEscapeDoNot404(t *testing.T) { _, privKey, _ := ed25519.GenerateKey(nil) cfg := &config.Dendrite{} - cfg.Defaults(true) + cfg.Defaults(config.DefaultOpts{ + Generate: true, + Monolithic: true, + }) cfg.Global.KeyID = gomatrixserverlib.KeyID("ed25519:auto") cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost") cfg.Global.PrivateKey = privKey diff --git a/setup/config/config.go b/setup/config/config.go index cc9c04470..5a618d671 100644 --- a/setup/config/config.go +++ b/setup/config/config.go @@ -211,7 +211,10 @@ func loadConfig( monolithic bool, ) (*Dendrite, error) { var c Dendrite - c.Defaults(false) + c.Defaults(DefaultOpts{ + Generate: false, + Monolithic: monolithic, + }) c.IsMonolith = monolithic var err error @@ -295,21 +298,25 @@ func (config *Dendrite) Derive() error { return nil } +type DefaultOpts struct { + Generate bool + Monolithic bool +} + // SetDefaults sets default config values if they are not explicitly set. -func (c *Dendrite) Defaults(generate bool) { +func (c *Dendrite) Defaults(opts DefaultOpts) { c.Version = Version - c.Global.Defaults(generate) - c.ClientAPI.Defaults(generate) - c.FederationAPI.Defaults(generate) - c.KeyServer.Defaults(generate) - c.MediaAPI.Defaults(generate) - c.RoomServer.Defaults(generate) - c.SyncAPI.Defaults(generate) - c.UserAPI.Defaults(generate) - c.AppServiceAPI.Defaults(generate) - c.MSCs.Defaults(generate) - + c.Global.Defaults(opts) + c.ClientAPI.Defaults(opts) + c.FederationAPI.Defaults(opts) + c.KeyServer.Defaults(opts) + c.MediaAPI.Defaults(opts) + c.RoomServer.Defaults(opts) + c.SyncAPI.Defaults(opts) + c.UserAPI.Defaults(opts) + c.AppServiceAPI.Defaults(opts) + c.MSCs.Defaults(opts) c.Wiring() } diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go index 9c3771272..bd21826fe 100644 --- a/setup/config/config_appservice.go +++ b/setup/config/config_appservice.go @@ -29,7 +29,7 @@ type AppServiceAPI struct { Matrix *Global `yaml:"-"` Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit - InternalAPI InternalAPIOptions `yaml:"internal_api"` + InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` // DisableTLSValidation disables the validation of X.509 TLS certs // on appservice endpoints. This is not recommended in production! @@ -38,9 +38,11 @@ type AppServiceAPI struct { ConfigFiles []string `yaml:"config_files"` } -func (c *AppServiceAPI) Defaults(generate bool) { - c.InternalAPI.Listen = "http://localhost:7777" - c.InternalAPI.Connect = "http://localhost:7777" +func (c *AppServiceAPI) Defaults(opts DefaultOpts) { + if !opts.Monolithic { + c.InternalAPI.Listen = "http://localhost:7777" + c.InternalAPI.Connect = "http://localhost:7777" + } } func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go index ecf8f6bd5..56f4b3f92 100644 --- a/setup/config/config_clientapi.go +++ b/setup/config/config_clientapi.go @@ -9,8 +9,8 @@ type ClientAPI struct { Matrix *Global `yaml:"-"` Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit - InternalAPI InternalAPIOptions `yaml:"internal_api"` - ExternalAPI ExternalAPIOptions `yaml:"external_api"` + InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` + ExternalAPI ExternalAPIOptions `yaml:"external_api,omitempty"` // If set disables new users from registering (except via shared // secrets) @@ -48,13 +48,15 @@ type ClientAPI struct { // Rate-limiting options RateLimiting RateLimiting `yaml:"rate_limiting"` - MSCs *MSCs `yaml:"mscs"` + MSCs *MSCs `yaml:"-"` } -func (c *ClientAPI) Defaults(generate bool) { - c.InternalAPI.Listen = "http://localhost:7771" - c.InternalAPI.Connect = "http://localhost:7771" - c.ExternalAPI.Listen = "http://[::]:8071" +func (c *ClientAPI) Defaults(opts DefaultOpts) { + if !opts.Monolithic { + c.InternalAPI.Listen = "http://localhost:7771" + c.InternalAPI.Connect = "http://localhost:7771" + c.ExternalAPI.Listen = "http://[::]:8071" + } c.RegistrationSharedSecret = "" c.RecaptchaPublicKey = "" c.RecaptchaPrivateKey = "" diff --git a/setup/config/config_federationapi.go b/setup/config/config_federationapi.go index a7a515fda..d3a4b2b67 100644 --- a/setup/config/config_federationapi.go +++ b/setup/config/config_federationapi.go @@ -5,12 +5,12 @@ import "github.com/matrix-org/gomatrixserverlib" type FederationAPI struct { Matrix *Global `yaml:"-"` - InternalAPI InternalAPIOptions `yaml:"internal_api"` - ExternalAPI ExternalAPIOptions `yaml:"external_api"` + InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` + ExternalAPI ExternalAPIOptions `yaml:"external_api,omitempty"` // The database stores information used by the federation destination queues to // send transactions to remote servers. - Database DatabaseOptions `yaml:"database"` + Database DatabaseOptions `yaml:"database,omitempty"` // Federation failure threshold. How many consecutive failures that we should // tolerate when sending federation requests to a specific server. The backoff @@ -30,25 +30,44 @@ type FederationAPI struct { PreferDirectFetch bool `yaml:"prefer_direct_fetch"` } -func (c *FederationAPI) Defaults(generate bool) { - c.InternalAPI.Listen = "http://localhost:7772" - c.InternalAPI.Connect = "http://localhost:7772" - c.ExternalAPI.Listen = "http://[::]:8072" +func (c *FederationAPI) Defaults(opts DefaultOpts) { + if !opts.Monolithic { + c.InternalAPI.Listen = "http://localhost:7772" + c.InternalAPI.Connect = "http://localhost:7772" + c.ExternalAPI.Listen = "http://[::]:8072" + c.Database.Defaults(10) + } c.FederationMaxRetries = 16 c.DisableTLSValidation = false - c.Database.Defaults(10) - if generate { - c.Database.ConnectionString = "file:federationapi.db" + if opts.Generate { + c.KeyPerspectives = KeyPerspectives{ + { + ServerName: "matrix.org", + Keys: []KeyPerspectiveTrustKey{ + { + KeyID: "ed25519:auto", + PublicKey: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw", + }, + { + KeyID: "ed25519:a_RXGa", + PublicKey: "l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ", + }, + }, + }, + } + if !opts.Monolithic { + c.Database.ConnectionString = "file:federationapi.db" + } } } func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { - if c.Matrix.DatabaseOptions.ConnectionString == "" { - checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString)) - } if isMonolith { // polylith required configs below return } + if c.Matrix.DatabaseOptions.ConnectionString == "" { + checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString)) + } checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen)) checkURL(configErrs, "federation_api.internal_api.listen", string(c.InternalAPI.Listen)) checkURL(configErrs, "federation_api.internal_api.connect", string(c.InternalAPI.Connect)) diff --git a/setup/config/config_global.go b/setup/config/config_global.go index d4e54e203..acc608dd7 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -41,7 +41,7 @@ type Global struct { // connections will be used instead. This way we don't have to manage connection // counts on a per-component basis, but can instead do it for the entire monolith. // In a polylith deployment, this will be ignored. - DatabaseOptions DatabaseOptions `yaml:"database"` + DatabaseOptions DatabaseOptions `yaml:"database,omitempty"` // The server name to delegate server-server communications to, with optional port WellKnownServerName string `yaml:"well_known_server_name"` @@ -83,22 +83,28 @@ type Global struct { Cache Cache `yaml:"cache"` } -func (c *Global) Defaults(generate bool) { - if generate { +func (c *Global) Defaults(opts DefaultOpts) { + if opts.Generate { c.ServerName = "localhost" c.PrivateKeyPath = "matrix_key.pem" _, c.PrivateKey, _ = ed25519.GenerateKey(rand.New(rand.NewSource(0))) c.KeyID = "ed25519:auto" + c.TrustedIDServers = []string{ + "matrix.org", + "vector.im", + } } c.KeyValidityPeriod = time.Hour * 24 * 7 - - c.JetStream.Defaults(generate) - c.Metrics.Defaults(generate) + if opts.Monolithic { + c.DatabaseOptions.Defaults(90) + } + c.JetStream.Defaults(opts) + c.Metrics.Defaults(opts) c.DNSCache.Defaults() c.Sentry.Defaults() - c.ServerNotices.Defaults(generate) + c.ServerNotices.Defaults(opts) c.ReportStats.Defaults() - c.Cache.Defaults(generate) + c.Cache.Defaults() } func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) { @@ -142,9 +148,9 @@ type Metrics struct { } `yaml:"basic_auth"` } -func (c *Metrics) Defaults(generate bool) { +func (c *Metrics) Defaults(opts DefaultOpts) { c.Enabled = false - if generate { + if opts.Generate { c.BasicAuth.Username = "metrics" c.BasicAuth.Password = "metrics" } @@ -166,8 +172,8 @@ type ServerNotices struct { RoomName string `yaml:"room_name"` } -func (c *ServerNotices) Defaults(generate bool) { - if generate { +func (c *ServerNotices) Defaults(opts DefaultOpts) { + if opts.Generate { c.Enabled = true c.LocalPart = "_server" c.DisplayName = "Server Alert" @@ -183,7 +189,7 @@ type Cache struct { MaxAge time.Duration `yaml:"max_age"` } -func (c *Cache) Defaults(generate bool) { +func (c *Cache) Defaults() { c.EstimatedMaxSize = 1024 * 1024 * 1024 // 1GB c.MaxAge = time.Hour } diff --git a/setup/config/config_jetstream.go b/setup/config/config_jetstream.go index a7827597e..ef8bf014b 100644 --- a/setup/config/config_jetstream.go +++ b/setup/config/config_jetstream.go @@ -31,10 +31,10 @@ func (c *JetStream) Durable(name string) string { return c.Prefixed(name) } -func (c *JetStream) Defaults(generate bool) { +func (c *JetStream) Defaults(opts DefaultOpts) { c.Addresses = []string{} c.TopicPrefix = "Dendrite" - if generate { + if opts.Generate { c.StoragePath = Path("./") c.NoLog = true c.DisableTLSValidation = true diff --git a/setup/config/config_keyserver.go b/setup/config/config_keyserver.go index 5f2f22c8a..dca9ca9f5 100644 --- a/setup/config/config_keyserver.go +++ b/setup/config/config_keyserver.go @@ -3,27 +3,31 @@ package config type KeyServer struct { Matrix *Global `yaml:"-"` - InternalAPI InternalAPIOptions `yaml:"internal_api"` + InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` - Database DatabaseOptions `yaml:"database"` + Database DatabaseOptions `yaml:"database,omitempty"` } -func (c *KeyServer) Defaults(generate bool) { - c.InternalAPI.Listen = "http://localhost:7779" - c.InternalAPI.Connect = "http://localhost:7779" - c.Database.Defaults(10) - if generate { - c.Database.ConnectionString = "file:keyserver.db" +func (c *KeyServer) Defaults(opts DefaultOpts) { + if !opts.Monolithic { + c.InternalAPI.Listen = "http://localhost:7779" + c.InternalAPI.Connect = "http://localhost:7779" + c.Database.Defaults(10) + } + if opts.Generate { + if !opts.Monolithic { + c.Database.ConnectionString = "file:keyserver.db" + } } } func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) { - if c.Matrix.DatabaseOptions.ConnectionString == "" { - checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString)) - } if isMonolith { // polylith required configs below return } + if c.Matrix.DatabaseOptions.ConnectionString == "" { + checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString)) + } checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen)) checkURL(configErrs, "key_server.internal_api.connect", string(c.InternalAPI.Connect)) } diff --git a/setup/config/config_mediaapi.go b/setup/config/config_mediaapi.go index 9717aa59e..53a8219eb 100644 --- a/setup/config/config_mediaapi.go +++ b/setup/config/config_mediaapi.go @@ -7,12 +7,12 @@ import ( type MediaAPI struct { Matrix *Global `yaml:"-"` - InternalAPI InternalAPIOptions `yaml:"internal_api"` - ExternalAPI ExternalAPIOptions `yaml:"external_api"` + InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` + ExternalAPI ExternalAPIOptions `yaml:"external_api,omitempty"` // The MediaAPI database stores information about files uploaded and downloaded // by local users. It is only accessed by the MediaAPI. - Database DatabaseOptions `yaml:"database"` + Database DatabaseOptions `yaml:"database,omitempty"` // The base path to where the media files will be stored. May be relative or absolute. BasePath Path `yaml:"base_path"` @@ -38,23 +38,41 @@ type MediaAPI struct { // DefaultMaxFileSizeBytes defines the default file size allowed in transfers var DefaultMaxFileSizeBytes = FileSizeBytes(10485760) -func (c *MediaAPI) Defaults(generate bool) { - c.InternalAPI.Listen = "http://localhost:7774" - c.InternalAPI.Connect = "http://localhost:7774" - c.ExternalAPI.Listen = "http://[::]:8074" +func (c *MediaAPI) Defaults(opts DefaultOpts) { + if !opts.Monolithic { + c.InternalAPI.Listen = "http://localhost:7774" + c.InternalAPI.Connect = "http://localhost:7774" + c.ExternalAPI.Listen = "http://[::]:8074" + c.Database.Defaults(5) + } c.MaxFileSizeBytes = DefaultMaxFileSizeBytes c.MaxThumbnailGenerators = 10 - c.Database.Defaults(5) - if generate { - c.Database.ConnectionString = "file:mediaapi.db" + if opts.Generate { + c.ThumbnailSizes = []ThumbnailSize{ + { + Width: 32, + Height: 32, + ResizeMethod: "crop", + }, + { + Width: 96, + Height: 96, + ResizeMethod: "crop", + }, + { + Width: 640, + Height: 480, + ResizeMethod: "scale", + }, + } + if !opts.Monolithic { + c.Database.ConnectionString = "file:mediaapi.db" + } c.BasePath = "./media_store" } } func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { - if c.Matrix.DatabaseOptions.ConnectionString == "" { - checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString)) - } checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath)) checkPositive(configErrs, "media_api.max_file_size_bytes", int64(c.MaxFileSizeBytes)) checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators)) @@ -66,6 +84,9 @@ func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { if isMonolith { // polylith required configs below return } + if c.Matrix.DatabaseOptions.ConnectionString == "" { + checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString)) + } checkURL(configErrs, "media_api.internal_api.listen", string(c.InternalAPI.Listen)) checkURL(configErrs, "media_api.internal_api.connect", string(c.InternalAPI.Connect)) checkURL(configErrs, "media_api.external_api.listen", string(c.ExternalAPI.Listen)) diff --git a/setup/config/config_mscs.go b/setup/config/config_mscs.go index b992f7152..6d5ff39a5 100644 --- a/setup/config/config_mscs.go +++ b/setup/config/config_mscs.go @@ -10,13 +10,17 @@ type MSCs struct { // 'msc2946': Spaces Summary - https://github.com/matrix-org/matrix-doc/pull/2946 MSCs []string `yaml:"mscs"` - Database DatabaseOptions `yaml:"database"` + Database DatabaseOptions `yaml:"database,omitempty"` } -func (c *MSCs) Defaults(generate bool) { - c.Database.Defaults(5) - if generate { - c.Database.ConnectionString = "file:mscs.db" +func (c *MSCs) Defaults(opts DefaultOpts) { + if !opts.Monolithic { + c.Database.Defaults(5) + } + if opts.Generate { + if !opts.Monolithic { + c.Database.ConnectionString = "file:mscs.db" + } } } @@ -31,6 +35,9 @@ func (c *MSCs) Enabled(msc string) bool { } func (c *MSCs) Verify(configErrs *ConfigErrors, isMonolith bool) { + if isMonolith { // polylith required configs below + return + } if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "mscs.database.connection_string", string(c.Database.ConnectionString)) } diff --git a/setup/config/config_roomserver.go b/setup/config/config_roomserver.go index bd6aa1167..5e3b7f2ec 100644 --- a/setup/config/config_roomserver.go +++ b/setup/config/config_roomserver.go @@ -3,27 +3,31 @@ package config type RoomServer struct { Matrix *Global `yaml:"-"` - InternalAPI InternalAPIOptions `yaml:"internal_api"` + InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` - Database DatabaseOptions `yaml:"database"` + Database DatabaseOptions `yaml:"database,omitempty"` } -func (c *RoomServer) Defaults(generate bool) { - c.InternalAPI.Listen = "http://localhost:7770" - c.InternalAPI.Connect = "http://localhost:7770" - c.Database.Defaults(10) - if generate { - c.Database.ConnectionString = "file:roomserver.db" +func (c *RoomServer) Defaults(opts DefaultOpts) { + if !opts.Monolithic { + c.InternalAPI.Listen = "http://localhost:7770" + c.InternalAPI.Connect = "http://localhost:7770" + c.Database.Defaults(20) + } + if opts.Generate { + if !opts.Monolithic { + c.Database.ConnectionString = "file:roomserver.db" + } } } func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) { - if c.Matrix.DatabaseOptions.ConnectionString == "" { - checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString)) - } if isMonolith { // polylith required configs below return } + if c.Matrix.DatabaseOptions.ConnectionString == "" { + checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString)) + } checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen)) checkURL(configErrs, "room_server.internal_ap.connect", string(c.InternalAPI.Connect)) } diff --git a/setup/config/config_syncapi.go b/setup/config/config_syncapi.go index 7d5e3808a..4e302ae09 100644 --- a/setup/config/config_syncapi.go +++ b/setup/config/config_syncapi.go @@ -3,31 +3,35 @@ package config type SyncAPI struct { Matrix *Global `yaml:"-"` - InternalAPI InternalAPIOptions `yaml:"internal_api"` - ExternalAPI ExternalAPIOptions `yaml:"external_api"` + InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` + ExternalAPI ExternalAPIOptions `yaml:"external_api,omitempty"` - Database DatabaseOptions `yaml:"database"` + Database DatabaseOptions `yaml:"database,omitempty"` RealIPHeader string `yaml:"real_ip_header"` } -func (c *SyncAPI) Defaults(generate bool) { - c.InternalAPI.Listen = "http://localhost:7773" - c.InternalAPI.Connect = "http://localhost:7773" - c.ExternalAPI.Listen = "http://localhost:8073" - c.Database.Defaults(10) - if generate { - c.Database.ConnectionString = "file:syncapi.db" +func (c *SyncAPI) Defaults(opts DefaultOpts) { + if !opts.Monolithic { + c.InternalAPI.Listen = "http://localhost:7773" + c.InternalAPI.Connect = "http://localhost:7773" + c.ExternalAPI.Listen = "http://localhost:8073" + c.Database.Defaults(20) + } + if opts.Generate { + if !opts.Monolithic { + c.Database.ConnectionString = "file:syncapi.db" + } } } func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { - if c.Matrix.DatabaseOptions.ConnectionString == "" { - checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString)) - } if isMonolith { // polylith required configs below return } + if c.Matrix.DatabaseOptions.ConnectionString == "" { + checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString)) + } checkURL(configErrs, "sync_api.internal_api.listen", string(c.InternalAPI.Listen)) checkURL(configErrs, "sync_api.internal_api.connect", string(c.InternalAPI.Connect)) checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen)) diff --git a/setup/config/config_userapi.go b/setup/config/config_userapi.go index d1e2b7fe1..97a6d738b 100644 --- a/setup/config/config_userapi.go +++ b/setup/config/config_userapi.go @@ -5,7 +5,7 @@ import "golang.org/x/crypto/bcrypt" type UserAPI struct { Matrix *Global `yaml:"-"` - InternalAPI InternalAPIOptions `yaml:"internal_api"` + InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` // The cost when hashing passwords. BCryptCost int `yaml:"bcrypt_cost"` @@ -18,30 +18,34 @@ type UserAPI struct { // The Account database stores the login details and account information // for local users. It is accessed by the UserAPI. - AccountDatabase DatabaseOptions `yaml:"account_database"` + AccountDatabase DatabaseOptions `yaml:"account_database,omitempty"` } const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes -func (c *UserAPI) Defaults(generate bool) { - c.InternalAPI.Listen = "http://localhost:7781" - c.InternalAPI.Connect = "http://localhost:7781" +func (c *UserAPI) Defaults(opts DefaultOpts) { + if !opts.Monolithic { + c.InternalAPI.Listen = "http://localhost:7781" + c.InternalAPI.Connect = "http://localhost:7781" + c.AccountDatabase.Defaults(10) + } c.BCryptCost = bcrypt.DefaultCost c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS - c.AccountDatabase.Defaults(10) - if generate { - c.AccountDatabase.ConnectionString = "file:userapi_accounts.db" + if opts.Generate { + if !opts.Monolithic { + c.AccountDatabase.ConnectionString = "file:userapi_accounts.db" + } } } func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { checkPositive(configErrs, "user_api.openid_token_lifetime_ms", c.OpenIDTokenLifetimeMS) - if c.Matrix.DatabaseOptions.ConnectionString == "" { - checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString)) - } if isMonolith { // polylith required configs below return } + if c.Matrix.DatabaseOptions.ConnectionString == "" { + checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString)) + } checkURL(configErrs, "user_api.internal_api.listen", string(c.InternalAPI.Listen)) checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect)) } diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go index 3e9d90a1f..0388fcc53 100644 --- a/setup/mscs/msc2836/msc2836_test.go +++ b/setup/mscs/msc2836/msc2836_test.go @@ -547,7 +547,10 @@ func (r *testRoomserverAPI) QueryMembershipForUser(ctx context.Context, req *roo func injectEvents(t *testing.T, userAPI userapi.UserInternalAPI, rsAPI roomserver.RoomserverInternalAPI, events []*gomatrixserverlib.HeaderedEvent) *mux.Router { t.Helper() cfg := &config.Dendrite{} - cfg.Defaults(true) + cfg.Defaults(config.DefaultOpts{ + Generate: true, + Monolithic: true, + }) cfg.Global.ServerName = "localhost" cfg.MSCs.Database.ConnectionString = "file:msc2836_test.db" cfg.MSCs.MSCs = []string{"msc2836"} diff --git a/test/testrig/base.go b/test/testrig/base.go index 33230921c..9747ea600 100644 --- a/test/testrig/base.go +++ b/test/testrig/base.go @@ -30,12 +30,21 @@ import ( func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, func()) { var cfg config.Dendrite - cfg.Defaults(false) + cfg.Defaults(config.DefaultOpts{ + Generate: false, + Monolithic: true, + }) cfg.Global.JetStream.InMemory = true switch dbType { case test.DBTypePostgres: - cfg.Global.Defaults(true) // autogen a signing key - cfg.MediaAPI.Defaults(true) // autogen a media path + cfg.Global.Defaults(config.DefaultOpts{ // autogen a signing key + Generate: true, + Monolithic: true, + }) + cfg.MediaAPI.Defaults(config.DefaultOpts{ // autogen a media path + Generate: true, + Monolithic: true, + }) cfg.Global.ServerName = "test" // use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use // the file system event with InMemory=true :( @@ -49,7 +58,10 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f } return base.NewBaseDendrite(&cfg, "Test", base.DisableMetrics), close case test.DBTypeSQLite: - cfg.Defaults(true) // sets a sqlite db per component + cfg.Defaults(config.DefaultOpts{ + Generate: true, + Monolithic: false, // because we need a database per component + }) cfg.Global.ServerName = "test" // use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use // the file system event with InMemory=true :( @@ -82,7 +94,10 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f func Base(cfg *config.Dendrite) (*base.BaseDendrite, nats.JetStreamContext, *nats.Conn) { if cfg == nil { cfg = &config.Dendrite{} - cfg.Defaults(true) + cfg.Defaults(config.DefaultOpts{ + Generate: true, + Monolithic: true, + }) } cfg.Global.JetStream.InMemory = true base := base.NewBaseDendrite(cfg, "Tests") From 304acd7adc4929f64af930d0b5801e33ba101193 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Sep 2022 14:49:42 +0100 Subject: [PATCH 03/20] Version 0.9.6 (#2689) Changelog and version bump. --- CHANGES.md | 20 ++++++++++++++++++++ internal/version.go | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 0f57bffcb..7d0575663 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,25 @@ # Changelog +## Dendrite 0.9.6 (2022-09-01) + +### Features + +* The appservice API has been refactored for improved performance and stability + * The appservice database has been deprecated, as the roomserver output stream is now used as the data source instead +* The `generate-config` tool has been updated to support additional scenarios, i.e. for CI configuration generation and generating both monolith and polylith skeleton config files + +### Fixes + +* The username length check has been fixed on new account creation +* The length of the `type`, `sender`, `state_key` and `room_id` fields in events are now verified by number of codepoints rather than bytes, fixing the "Cat Overflow" bug +* UTF-16 surrogate handling in the canonical JSON implementation has been fixed +* A race condition when starting the keyserver has been fixed +* A race condition when configuring HTTP servers and routing at startup has been fixed +* A bug where the incorrect limit was used for lazy-loading memberships has been fixed +* The number of push notifications will now be sent to the push gateway +* A missing index causing slow performance on the sync API send-to-device table has been added (contributed by [PiotrKozimor](https://github.com/PiotrKozimor)) +* Event auth will now correctly check for the existence of the `"creator"` field in create events + ## Dendrite 0.9.5 (2022-08-25) ### Fixes diff --git a/internal/version.go b/internal/version.go index 108b8ab0f..c9a9f3af5 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 9 - VersionPatch = 5 + VersionPatch = 6 VersionTag = "" // example: "rc1" ) From fea869b41fc2a9f4db70ef664442a856c095f7c7 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Sep 2022 17:12:27 +0100 Subject: [PATCH 04/20] Update P2P demos --- build/gobind-pinecone/monolith.go | 43 ++++++---- build/gobind-yggdrasil/monolith.go | 42 +++++++++- cmd/dendrite-demo-yggdrasil/main.go | 59 +++++++++++--- cmd/dendrite-demo-yggdrasil/yggconn/node.go | 48 +++++------ go.mod | 19 +++-- go.sum | 88 +++++++++------------ 6 files changed, 176 insertions(+), 123 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 44eab4cd7..b51e97786 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -22,10 +22,10 @@ import ( "encoding/hex" "fmt" "io" - "io/ioutil" "net" "net/http" "os" + "path/filepath" "strings" "sync" "time" @@ -45,6 +45,7 @@ import ( "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/userapi" userapiAPI "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -204,27 +205,37 @@ func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, e // nolint:gocyclo func (m *DendriteMonolith) Start() { - var err error var sk ed25519.PrivateKey var pk ed25519.PublicKey - keyfile := fmt.Sprintf("%s/p2p.key", m.StorageDirectory) - if _, err = os.Stat(keyfile); os.IsNotExist(err) { - if pk, sk, err = ed25519.GenerateKey(nil); err != nil { - panic(err) + + keyfile := filepath.Join(m.StorageDirectory, "p2p.pem") + if _, err := os.Stat(keyfile); os.IsNotExist(err) { + oldkeyfile := filepath.Join(m.StorageDirectory, "p2p.key") + if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) { + if err = test.NewMatrixKey(keyfile); err != nil { + panic("failed to generate a new PEM key: " + err.Error()) + } + if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil { + panic("failed to load PEM key: " + err.Error()) + } + } else { + if sk, err = os.ReadFile(oldkeyfile); err != nil { + panic("failed to read the old private key: " + err.Error()) + } + if len(sk) != ed25519.PrivateKeySize { + panic("the private key is not long enough") + } + if err = test.SaveMatrixKey(keyfile, sk); err != nil { + panic("failed to convert the private key to PEM format: " + err.Error()) + } } - if err = ioutil.WriteFile(keyfile, sk, 0644); err != nil { - panic(err) + } else { + if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil { + panic("failed to load PEM key: " + err.Error()) } - } else if err == nil { - if sk, err = ioutil.ReadFile(keyfile); err != nil { - panic(err) - } - if len(sk) != ed25519.PrivateKeySize { - panic("the private key is not long enough") - } - pk = sk.Public().(ed25519.PublicKey) } + var err error m.listener, err = net.Listen("tcp", "localhost:65432") if err != nil { panic(err) diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 896f3d14f..c671ae098 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -2,10 +2,14 @@ package gobind import ( "context" + "crypto/ed25519" "crypto/tls" + "encoding/hex" "fmt" "net" "net/http" + "os" + "path/filepath" "time" "github.com/gorilla/mux" @@ -22,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -63,19 +68,50 @@ func (m *DendriteMonolith) DisconnectMulticastPeers() { } func (m *DendriteMonolith) Start() { + var pk ed25519.PublicKey + var sk ed25519.PrivateKey + m.logger = logrus.Logger{ Out: BindLogger{}, } m.logger.SetOutput(BindLogger{}) logrus.SetOutput(BindLogger{}) + keyfile := filepath.Join(m.StorageDirectory, "p2p.pem") + if _, err := os.Stat(keyfile); os.IsNotExist(err) { + oldkeyfile := filepath.Join(m.StorageDirectory, "p2p.key") + if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) { + if err = test.NewMatrixKey(keyfile); err != nil { + panic("failed to generate a new PEM key: " + err.Error()) + } + if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil { + panic("failed to load PEM key: " + err.Error()) + } + } else { + if sk, err = os.ReadFile(oldkeyfile); err != nil { + panic("failed to read the old private key: " + err.Error()) + } + if len(sk) != ed25519.PrivateKeySize { + panic("the private key is not long enough") + } + if err := test.SaveMatrixKey(keyfile, sk); err != nil { + panic("failed to convert the private key to PEM format: " + err.Error()) + } + } + } else { + var err error + if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil { + panic("failed to load PEM key: " + err.Error()) + } + } + var err error m.listener, err = net.Listen("tcp", "localhost:65432") if err != nil { panic(err) } - ygg, err := yggconn.Setup("dendrite", m.StorageDirectory, "") + ygg, err := yggconn.Setup(sk, "dendrite", m.StorageDirectory, "", "") if err != nil { panic(err) } @@ -86,8 +122,8 @@ func (m *DendriteMonolith) Start() { Generate: true, Monolithic: true, }) - cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName()) - cfg.Global.PrivateKey = ygg.PrivateKey() + cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk)) + cfg.Global.PrivateKey = sk cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", m.StorageDirectory)) cfg.Global.JetStream.InMemory = true diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index d2fddf8bc..7445b6bfe 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -16,7 +16,9 @@ package main import ( "context" + "crypto/ed25519" "crypto/tls" + "encoding/hex" "flag" "fmt" "net" @@ -42,6 +44,7 @@ import ( "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/mscs" + "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/userapi" "github.com/sirupsen/logrus" @@ -49,19 +52,18 @@ import ( ) var ( - instanceName = flag.String("name", "dendrite-p2p-ygg", "the name of this P2P demo instance") - instancePort = flag.Int("port", 8008, "the port that the client API will listen on") - instancePeer = flag.String("peer", "", "an internet Yggdrasil peer to connect to") + instanceName = flag.String("name", "dendrite-p2p-ygg", "the name of this P2P demo instance") + instancePort = flag.Int("port", 8008, "the port that the client API will listen on") + instancePeer = flag.String("peer", "", "the static Yggdrasil peers to connect to, comma separated-list") + instanceListen = flag.String("listen", "tcp://:0", "the port Yggdrasil peers can connect to") ) func main() { flag.Parse() internal.SetupPprof() - ygg, err := yggconn.Setup(*instanceName, ".", *instancePeer) - if err != nil { - panic(err) - } + var pk ed25519.PublicKey + var sk ed25519.PrivateKey // iterate through the cli args and check if the config flag was set configFlagSet := false @@ -74,6 +76,34 @@ func main() { cfg := &config.Dendrite{} + keyfile := *instanceName + ".pem" + if _, err := os.Stat(keyfile); os.IsNotExist(err) { + oldkeyfile := *instanceName + ".key" + if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) { + if err = test.NewMatrixKey(keyfile); err != nil { + panic("failed to generate a new PEM key: " + err.Error()) + } + if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil { + panic("failed to load PEM key: " + err.Error()) + } + } else { + if sk, err = os.ReadFile(oldkeyfile); err != nil { + panic("failed to read the old private key: " + err.Error()) + } + if len(sk) != ed25519.PrivateKeySize { + panic("the private key is not long enough") + } + if err := test.SaveMatrixKey(keyfile, sk); err != nil { + panic("failed to convert the private key to PEM format: " + err.Error()) + } + } + } else { + var err error + if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil { + panic("failed to load PEM key: " + err.Error()) + } + } + // use custom config if config flag is set if configFlagSet { cfg = setup.ParseFlags(true) @@ -82,6 +112,7 @@ func main() { Generate: true, Monolithic: true, }) + cfg.Global.PrivateKey = sk cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName)) cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) @@ -93,19 +124,23 @@ func main() { cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName)) cfg.ClientAPI.RegistrationDisabled = false cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true - if err = cfg.Derive(); err != nil { + if err := cfg.Derive(); err != nil { panic(err) } } - // always override ServerName, PrivateKey and KeyID - cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName()) - cfg.Global.PrivateKey = ygg.PrivateKey() - cfg.Global.KeyID = signing.KeyID + pk = sk.Public().(ed25519.PublicKey) + cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk)) + cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) base := base.NewBaseDendrite(cfg, "Monolith") defer base.Close() // nolint: errcheck + ygg, err := yggconn.Setup(sk, *instanceName, ".", *instancePeer, *instanceListen) + if err != nil { + panic(err) + } + federation := ygg.CreateFederationClient(base) serverKeyAPI := &signing.YggdrasilKeys{} diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go index ff3c73ec8..83b4cdf9e 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go @@ -18,15 +18,13 @@ import ( "context" "crypto/ed25519" "encoding/hex" - "encoding/json" "fmt" - "log" "net" - "os" "strings" "github.com/matrix-org/gomatrixserverlib" "github.com/neilalexander/utp" + "github.com/sirupsen/logrus" ironwoodtypes "github.com/Arceliar/ironwood/types" yggdrasilconfig "github.com/yggdrasil-network/yggdrasil-go/src/config" @@ -57,48 +55,38 @@ func (n *Node) DialerContext(ctx context.Context, _, address string) (net.Conn, return n.utpSocket.DialAddrContext(ctx, pk) } -func Setup(instanceName, storageDirectory, peerURI string) (*Node, error) { +func Setup(sk ed25519.PrivateKey, instanceName, storageDirectory, peerURI, listenURI string) (*Node, error) { n := &Node{ core: &yggdrasilcore.Core{}, config: yggdrasildefaults.GenerateConfig(), multicast: &yggdrasilmulticast.Multicast{}, - log: gologme.New(os.Stdout, "YGG ", log.Flags()), + log: gologme.New(logrus.StandardLogger().Writer(), "", 0), incoming: make(chan net.Conn), } - yggfile := fmt.Sprintf("%s/%s-yggdrasil.conf", storageDirectory, instanceName) - if _, err := os.Stat(yggfile); !os.IsNotExist(err) { - yggconf, e := os.ReadFile(yggfile) - if e != nil { - panic(err) - } - if err := json.Unmarshal([]byte(yggconf), &n.config); err != nil { - panic(err) - } + options := []yggdrasilcore.SetupOption{ + yggdrasilcore.AdminListenAddress("none"), + } + if listenURI != "" { + options = append(options, yggdrasilcore.ListenAddress(listenURI)) } - - n.config.Peers = []string{} if peerURI != "" { - n.config.Peers = append(n.config.Peers, peerURI) + for _, uri := range strings.Split(peerURI, ",") { + options = append(options, yggdrasilcore.Peer{ + URI: uri, + }) + } } - n.config.AdminListen = "none" - j, err := json.MarshalIndent(n.config, "", " ") - if err != nil { + var err error + if n.core, err = yggdrasilcore.New(sk, options...); err != nil { panic(err) } - if e := os.WriteFile(yggfile, j, 0600); e != nil { - n.log.Printf("Couldn't write private key to file '%s': %s\n", yggfile, e) - } - n.log.EnableLevel("error") n.log.EnableLevel("warn") n.log.EnableLevel("info") - if err = n.core.Start(n.config, n.log); err != nil { - panic(err) - } - n.utpSocket, err = utp.NewSocketFromPacketConnNoClose(n.core) - if err != nil { + n.core.SetLogger(n.log) + if n.utpSocket, err = utp.NewSocketFromPacketConnNoClose(n.core); err != nil { panic(err) } if err = n.multicast.Init(n.core, n.config, n.log, nil); err != nil { @@ -108,7 +96,7 @@ func Setup(instanceName, storageDirectory, peerURI string) (*Node, error) { panic(err) } - n.log.Println("Public key:", n.core.PublicKey()) + n.log.Printf("Public key: %x", n.core.PublicKey()) go n.listenFromYgg() return n, nil diff --git a/go.mod b/go.mod index 8bf8f454d..67a7771ec 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/matrix-org/dendrite require ( - github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98 + github.com/Arceliar/ironwood v0.0.0-20220409035209-b7f71f05435a github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/MFAshby/stdemuxerhook v1.0.0 @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2 - github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9 + github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.13 github.com/nats-io/nats-server/v2 v2.8.5-0.20220811224153-d8d25d9b0b1c @@ -40,12 +40,12 @@ require ( github.com/tidwall/sjson v1.2.4 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible - github.com/yggdrasil-network/yggdrasil-go v0.4.3 + github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c go.uber.org/atomic v1.9.0 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9 - golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd - golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e + golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105 + golang.org/x/net v0.0.0-20220728211354-c7608f3a8462 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 @@ -64,7 +64,7 @@ require ( github.com/docker/go-units v0.4.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/frankban/quicktest v1.14.3 // indirect - github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect @@ -87,7 +87,7 @@ require ( github.com/nats-io/nkeys v0.3.0 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/nxadm/tail v1.4.8 // indirect - github.com/onsi/ginkgo v1.16.4 // indirect + github.com/onsi/ginkgo v1.16.5 // indirect github.com/onsi/gomega v1.17.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect @@ -98,12 +98,11 @@ require ( github.com/stretchr/objx v0.2.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect - golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect + golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect - golang.org/x/tools v0.1.10 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + golang.org/x/tools v0.1.12 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/macaroon.v2 v2.1.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect diff --git a/go.sum b/go.sum index 8b8baabcf..dcfed5eaa 100644 --- a/go.sum +++ b/go.sum @@ -38,9 +38,8 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/Arceliar/ironwood v0.0.0-20211125050254-8951369625d0/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk= -github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98 h1:PsaZb47k7WB1V+AlGpb+W7SM+ZOhp16vVevg5gl9YkU= -github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk= +github.com/Arceliar/ironwood v0.0.0-20220409035209-b7f71f05435a h1:yfbnOyqPcx2gi5cFIJ2rlPz5M6rFPHT/c8FgZmFjCdc= +github.com/Arceliar/ironwood v0.0.0-20220409035209-b7f71f05435a/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk= github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3ugUxx2EnnWmgba1kCqPkd4Gk1yQ= github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= @@ -59,10 +58,8 @@ github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0 github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= -github.com/RyanCarrier/dijkstra v1.0.0/go.mod h1:5agGUBNEtUAGIANmbw09fuO3a2htPEkc1jNH01qxCWA= +github.com/RyanCarrier/dijkstra v1.1.0/go.mod h1:5agGUBNEtUAGIANmbw09fuO3a2htPEkc1jNH01qxCWA= github.com/RyanCarrier/dijkstra-1 v0.0.0-20170512020943-0e5801a26345/go.mod h1:OK4EvWJ441LQqGzed5NGB6vKBAE34n3z7iayPcEwr30= -github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= -github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/albertorestifo/dijkstra v0.0.0-20160910063646-aba76f725f72/go.mod h1:o+JdB7VetTHjLhU0N57x18B9voDBQe0paApdEAEoEfw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -98,7 +95,6 @@ github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cb github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= -github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -132,8 +128,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= @@ -142,8 +136,9 @@ github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03D github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo= github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -218,7 +213,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U= github.com/gologme/log v1.3.0 h1:l781G4dE+pbigClDSDzSaaYKtiueHCILUa/qSDsmHAo= github.com/gologme/log v1.3.0/go.mod h1:yKT+DvIPdDdDoPtqFrFxheooyVmoqi0BAsw+erN3wA4= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -260,7 +254,6 @@ github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORR github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -269,10 +262,8 @@ github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= @@ -324,8 +315,6 @@ github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lucas-clemente/quic-go v0.28.1 h1:Uo0lvVxWg5la9gflIF9lwa39ONq85Xq2D91YNEIslzU= github.com/lucas-clemente/quic-go v0.28.1/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= -github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ= -github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ= @@ -345,17 +334,13 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2 h1:esbNn9hg//tAStA6TogatAJAursw23A+yfVRQsdiv70= github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= -github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9 h1:ed8yvWhTLk7+sNeK/eOZRTvESFTOHDRevoRoyeqPtvY= -github.com/matrix-org/pinecone v0.0.0-20220803093810-b7a830c08fb9/go.mod h1:P4MqPf+u83OPulPJ+XTbSDbbWrdFYNY4LZ/B1PIduFE= +github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed h1:YMcCnrmTbT5M1LtTiagiFFaj9vEgvC6iVEzWsIb0tQQ= +github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.13 h1:1tj15ngiFfcZzii7yd82foL+ks+ouQcj8j/TPq3fk1I= github.com/mattn/go-sqlite3 v1.14.13/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattomatic/dijkstra v0.0.0-20130617153013-6f6d134eb237/go.mod h1:UOnLAUmVG5paym8pD3C4B9BQylUDC2vXFJJpT7JrlEA= @@ -368,7 +353,6 @@ github.com/miekg/dns v1.1.49 h1:qe0mQU3Z/XpFeE+AEBo2rqaS1IPBJ3anmqZ4XiZJVG8= github.com/miekg/dns v1.1.49/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -410,8 +394,9 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= @@ -459,8 +444,6 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -535,16 +518,17 @@ github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49u github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/yggdrasil-network/yggdrasil-go v0.4.3 h1:LNS7kNpKzFlxQ9xmD5tfmMEvzwa+utBoD6pV9t2a8q4= -github.com/yggdrasil-network/yggdrasil-go v0.4.3/go.mod h1:A1/8kOQT7vzBxlkQtLf1KzJR0cbfL/2zjOCiYOAdjjo= +github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c h1:/cTmA6pV2Z20BT/FGSmnb5BmJ8eRbDP0HbCB5IO1aKw= +github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c/go.mod h1:cIwhYwX9yT9Bcei59O0oOBSaj+kQP+9aVQUMWHh5R00= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -569,6 +553,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -603,10 +588,8 @@ golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20220112015953-858099ff7816/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= -golang.org/x/mobile v0.0.0-20220325161704-447654d348e3/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= -golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd h1:x1GptNaTtxPAlTVIAJk61fuXg0y17h09DTxyb+VNC/k= -golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= +golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105 h1:3vUV5x5+3LfQbgk7paCM6INOaJG9xXQbn79xoNkwfIk= +golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -614,8 +597,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -657,11 +640,12 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210927181540-4e4d966f7476/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211011170408-caeb26a5c8c0/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220728211354-c7608f3a8462 h1:UreQrH7DbFXSi9ZFox6FNT3WBooWmdANpU+IfkT1T4I= +golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -682,6 +666,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -730,12 +715,10 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -743,13 +726,17 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220315194320-039c03cc5b86/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220730100132-1609e554cd39/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80= golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -822,19 +809,16 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.zx2c4.com/wireguard v0.0.0-20210927201915-bb745b2ea326/go.mod h1:SDoazCvdy7RDjBPNEMBwrXhomlmtG7svs8mgwWEqtVI= -golang.zx2c4.com/wireguard v0.0.0-20211012062646-82d2aa87aa62/go.mod h1:id8Oh3eCCmpj9uVGWVjsUAl6UPX5ysMLzu6QxJU2UOU= -golang.zx2c4.com/wireguard v0.0.0-20211017052713-f87e87af0d9a/go.mod h1:id8Oh3eCCmpj9uVGWVjsUAl6UPX5ysMLzu6QxJU2UOU= -golang.zx2c4.com/wireguard/windows v0.4.12/go.mod h1:PW4y+d9oY83XU9rRwRwrJDwEMuhVjMxu2gfD1cfzS7w= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= +golang.zx2c4.com/wireguard v0.0.0-20220703234212-c31a7b1ab478/go.mod h1:bVQfyl2sCM/QIIGHpWbFGfHPuDvqnCNkT6MQLTCjO/U= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= From 350a5e5393503780c6ab0a8983daae5301e845e2 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 2 Sep 2022 16:03:33 +0200 Subject: [PATCH 05/20] Only send (invite) events to the AS if it's interested in it (#2692) This should hopefully fix #2691 (and maybe also #2690) --- appservice/consumers/roomserver.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index a30944e75..a4bcfa7d9 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -134,7 +134,7 @@ func (s *OutputRoomEventConsumer) onMessage( } case api.OutputTypeNewInviteEvent: - if output.NewInviteEvent == nil { + if output.NewInviteEvent == nil || !s.appserviceIsInterestedInEvent(ctx, output.NewInviteEvent.Event, state.ApplicationService) { continue } events = append(events, output.NewInviteEvent.Event) @@ -281,7 +281,9 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e case err != nil: continue case membership.Membership == gomatrixserverlib.Join: - return true + if appservice.IsInterestedInUserID(*ev.StateKey) { + return true + } } } } else { From d1f87e63f1b744825f40e59b8202933608480a70 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Sep 2022 13:16:14 +0100 Subject: [PATCH 06/20] Move `SetLatestEvents` call --- roomserver/internal/input/input_latest_events.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index d6efad79d..ec6626720 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -178,6 +178,10 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { u.newStateNID = u.oldStateNID } + if err = u.updater.SetLatestEvents(u.roomInfo.RoomNID, u.latest, u.stateAtEvent.EventNID, u.newStateNID); err != nil { + return fmt.Errorf("u.updater.SetLatestEvents: %w", err) + } + update, err := u.makeOutputNewRoomEvent() if err != nil { return fmt.Errorf("u.makeOutputNewRoomEvent: %w", err) @@ -196,10 +200,6 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { return fmt.Errorf("u.api.WriteOutputEvents: %w", err) } - if err = u.updater.SetLatestEvents(u.roomInfo.RoomNID, u.latest, u.stateAtEvent.EventNID, u.newStateNID); err != nil { - return fmt.Errorf("u.updater.SetLatestEvents: %w", err) - } - if err = u.updater.MarkEventAsSent(u.stateAtEvent.EventNID); err != nil { return fmt.Errorf("u.updater.MarkEventAsSent: %w", err) } From 1c1d09abd4009e3fcc76b71ad158fb16e5e776cc Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Sep 2022 14:17:04 +0100 Subject: [PATCH 07/20] Optimise `resolve-state` tool --- cmd/resolve-state/main.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go index 6852ab4fb..f8bb130c7 100644 --- a/cmd/resolve-state/main.go +++ b/cmd/resolve-state/main.go @@ -76,9 +76,14 @@ func main() { panic(err) } - var eventNIDs []types.EventNID + eventNIDMap := map[types.EventNID]struct{}{} for _, entry := range append(removed, added...) { - eventNIDs = append(eventNIDs, entry.EventNID) + eventNIDMap[entry.EventNID] = struct{}{} + } + + eventNIDs := make([]types.EventNID, 0, len(eventNIDMap)) + for eventNID := range eventNIDMap { + eventNIDs = append(eventNIDs, eventNID) } var eventEntries []types.Event @@ -129,12 +134,17 @@ func main() { stateEntries = append(stateEntries, entries...) } - var eventNIDs []types.EventNID + eventNIDMap := map[types.EventNID]struct{}{} for _, entry := range stateEntries { - eventNIDs = append(eventNIDs, entry.EventNID) + eventNIDMap[entry.EventNID] = struct{}{} } - fmt.Println("Fetching", len(eventNIDs), "state events") + eventNIDs := make([]types.EventNID, 0, len(eventNIDMap)) + for eventNID := range eventNIDMap { + eventNIDs = append(eventNIDs, eventNID) + } + + fmt.Println("Fetching", len(eventNIDMap), "state events") eventEntries, err := roomserverDB.Events(ctx, eventNIDs) if err != nil { panic(err) From bea3dbe77bac078f07927df54c63f145df8a0caf Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Sep 2022 17:00:54 +0100 Subject: [PATCH 08/20] State and auth tweaks (update to matrix-org/gomatrixserverlib#335) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 67a7771ec..b7f1920ca 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2 + github.com/matrix-org/gomatrixserverlib v0.0.0-20220905160024-206bfc07ea29 github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.13 diff --git a/go.sum b/go.sum index dcfed5eaa..3212c5d70 100644 --- a/go.sum +++ b/go.sum @@ -332,8 +332,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2 h1:esbNn9hg//tAStA6TogatAJAursw23A+yfVRQsdiv70= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220830164018-c71e518537a2/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220905160024-206bfc07ea29 h1:55lVTCUNuq/WvTH6usVwrliqnxbziOjOCHeQs1tcSeg= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220905160024-206bfc07ea29/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed h1:YMcCnrmTbT5M1LtTiagiFFaj9vEgvC6iVEzWsIb0tQQ= github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From ecee5f10f49487f0b6628fea065914ea3e685a79 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Sep 2022 17:08:54 +0100 Subject: [PATCH 09/20] Tweak logging for detected state resets --- roomserver/internal/input/input_latest_events.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index ec6626720..7a4105ab8 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -275,7 +275,7 @@ func (u *latestEventsUpdater) latestState() error { return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err) } - if removed := len(u.removed) - len(u.added); removed > 0 { + if removed := len(u.removed) - len(u.added); !u.rewritesState && removed > 0 { logrus.WithFields(logrus.Fields{ "event_id": u.event.EventID(), "room_id": u.event.RoomID(), @@ -283,7 +283,7 @@ func (u *latestEventsUpdater) latestState() error { "new_state_nid": u.newStateNID, "old_latest": u.oldLatest.EventIDs(), "new_latest": u.latest.EventIDs(), - }).Errorf("Unexpected state deletion (removing %d events)", removed) + }).Warnf("State reset detected (removing %d events)", removed) } // Also work out the state before the event removes and the event From cd22ba22b01fe5197bf8e4630c8414dd413225af Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Sep 2022 17:25:11 +0100 Subject: [PATCH 10/20] Improve Sentry reporting --- internal/httputil/httpapi.go | 1 + roomserver/internal/input/input.go | 6 ++++++ roomserver/internal/input/input_latest_events.go | 10 ++++++++++ 3 files changed, 17 insertions(+) diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go index e0436c60a..ab8319bbb 100644 --- a/internal/httputil/httpapi.go +++ b/internal/httputil/httpapi.go @@ -60,6 +60,7 @@ func MakeAuthAPI( // add the user to Sentry, if enabled hub := sentry.GetHubFromContext(req.Context()) if hub != nil { + hub.Scope().SetTag("username", device.UserID) hub.Scope().SetTag("user_id", device.UserID) hub.Scope().SetTag("device_id", device.ID) } diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go index 429cc4bd2..a8a3e0248 100644 --- a/roomserver/internal/input/input.go +++ b/roomserver/internal/input/input.go @@ -188,6 +188,9 @@ func (w *worker) _next() { // Look up what the next event is that's waiting to be processed. ctx, cancel := context.WithTimeout(w.r.ProcessContext.Context(), time.Minute) defer cancel() + if scope := sentry.CurrentHub().Scope(); scope != nil { + scope.SetTag("room_id", w.roomID) + } msgs, err := w.subscription.Fetch(1, nats.Context(ctx)) switch err { case nil: @@ -239,6 +242,9 @@ func (w *worker) _next() { return } + if scope := sentry.CurrentHub().Scope(); scope != nil { + scope.SetTag("event_id", inputRoomEvent.Event.EventID()) + } roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Inc() defer roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Dec() diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index 7a4105ab8..7e42db1b7 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + "github.com/getsentry/sentry-go" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/opentracing/opentracing-go" @@ -284,6 +285,15 @@ func (u *latestEventsUpdater) latestState() error { "old_latest": u.oldLatest.EventIDs(), "new_latest": u.latest.EventIDs(), }).Warnf("State reset detected (removing %d events)", removed) + sentry.WithScope(func(scope *sentry.Scope) { + scope.SetLevel("warning") + scope.SetTag("event_id", u.event.EventID()) + scope.SetTag("old_state_nid", fmt.Sprintf("%d", u.oldStateNID)) + scope.SetTag("new_state_nid", fmt.Sprintf("%d", u.newStateNID)) + scope.SetTag("old_latest", u.oldLatest.EventIDs()) + scope.SetTag("new_latest", u.latest.EventIDs()) + sentry.CaptureMessage("State reset detected") + }) } // Also work out the state before the event removes and the event From 5992b4c7ed2779e12c8b9da7bc96b2b6398cf04c Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Sep 2022 17:35:43 +0100 Subject: [PATCH 11/20] Set Sentry user where known --- internal/httputil/httpapi.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go index ab8319bbb..5612d78c1 100644 --- a/internal/httputil/httpapi.go +++ b/internal/httputil/httpapi.go @@ -60,7 +60,10 @@ func MakeAuthAPI( // add the user to Sentry, if enabled hub := sentry.GetHubFromContext(req.Context()) if hub != nil { - hub.Scope().SetTag("username", device.UserID) + hub.Scope().SetUser(sentry.User{ + Username: device.UserID, + IPAddress: req.RemoteAddr, + }) hub.Scope().SetTag("user_id", device.UserID) hub.Scope().SetTag("device_id", device.ID) } From fd7661f69cfecb426f0170a27a4e414b178a1cdf Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Sep 2022 17:37:17 +0100 Subject: [PATCH 12/20] Tweak Sentry again --- internal/httputil/httpapi.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go index 5612d78c1..36dcaf453 100644 --- a/internal/httputil/httpapi.go +++ b/internal/httputil/httpapi.go @@ -61,8 +61,7 @@ func MakeAuthAPI( hub := sentry.GetHubFromContext(req.Context()) if hub != nil { hub.Scope().SetUser(sentry.User{ - Username: device.UserID, - IPAddress: req.RemoteAddr, + Username: device.UserID, }) hub.Scope().SetTag("user_id", device.UserID) hub.Scope().SetTag("device_id", device.ID) From 70cd9a902c84f15817b191999340d003ccbc41eb Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 6 Sep 2022 17:19:21 +0100 Subject: [PATCH 13/20] State resolution fixes in `applyAndAuthEvent` (update to matrix-org/gomatrixserverlib#336) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b7f1920ca..623e8f397 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220905160024-206bfc07ea29 + github.com/matrix-org/gomatrixserverlib v0.0.0-20220906161857-baa46d493267 github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.13 diff --git a/go.sum b/go.sum index 3212c5d70..d30bc39f3 100644 --- a/go.sum +++ b/go.sum @@ -332,8 +332,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220905160024-206bfc07ea29 h1:55lVTCUNuq/WvTH6usVwrliqnxbziOjOCHeQs1tcSeg= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220905160024-206bfc07ea29/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220906161857-baa46d493267 h1:+Ix9XC5ah7cofDaEFHWrupBQWUQEqz9bOkHcZvhzw2s= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220906161857-baa46d493267/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed h1:YMcCnrmTbT5M1LtTiagiFFaj9vEgvC6iVEzWsIb0tQQ= github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From 489ccc1c600836fda8b66b783e1f510e67c3666d Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 6 Sep 2022 17:25:17 +0100 Subject: [PATCH 14/20] Stronger default power levels --- internal/eventutil/eventcontent.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/eventutil/eventcontent.go b/internal/eventutil/eventcontent.go index 4ecb5fb56..e3c80f1d0 100644 --- a/internal/eventutil/eventcontent.go +++ b/internal/eventutil/eventcontent.go @@ -53,6 +53,9 @@ func InitialPowerLevelsContent(roomCreator string) (c gomatrixserverlib.PowerLev "m.room.history_visibility": 100, "m.room.canonical_alias": 50, "m.room.avatar": 50, + "m.room.tombstone": 100, + "m.room.encryption": 100, + "m.room.server_acl": 100, } c.Users = map[string]int64{roomCreator: 100} return c From 847032df36086ff6f383bcc4c332c391e86eaec8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 7 Sep 2022 09:11:36 +0100 Subject: [PATCH 15/20] Update to matrix-org/gomatrixserverlib@637a173 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 623e8f397..c737e467c 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220906161857-baa46d493267 + github.com/matrix-org/gomatrixserverlib v0.0.0-20220907081047-637a173a3661 github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.13 diff --git a/go.sum b/go.sum index d30bc39f3..0b331befe 100644 --- a/go.sum +++ b/go.sum @@ -332,8 +332,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220906161857-baa46d493267 h1:+Ix9XC5ah7cofDaEFHWrupBQWUQEqz9bOkHcZvhzw2s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220906161857-baa46d493267/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220907081047-637a173a3661 h1:dww9rH0HVfAO9JOBD1nxq26GHKbEw07thAJTu1DrAQs= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220907081047-637a173a3661/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed h1:YMcCnrmTbT5M1LtTiagiFFaj9vEgvC6iVEzWsIb0tQQ= github.com/matrix-org/pinecone v0.0.0-20220901133433-565beccfebed/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From 440eb0f3a2e5aecddebe3e093d5740ce5c4505c2 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 7 Sep 2022 11:44:27 +0200 Subject: [PATCH 16/20] Handle errors differently in the `DeviceListUpdater` (#2695) `If a device list update goes missing, the server resyncs on the next one` was failing because a previous test would receive a `waitTime` of 1h, resulting in the test timing out. This now tries to handle the returned errors differently, e.g. by using the default `waitTime` of 2s. Also doesn't try further users in the list, if one of the errors would cause a longer `waitTime`. --- federationapi/storage/shared/storage_edus.go | 6 +-- internal/httputil/http.go | 6 +-- keyserver/internal/device_list_update.go | 48 +++++++++++++++----- sytest-whitelist | 3 +- 4 files changed, 44 insertions(+), 19 deletions(-) diff --git a/federationapi/storage/shared/storage_edus.go b/federationapi/storage/shared/storage_edus.go index ce9632ed3..e0c740c11 100644 --- a/federationapi/storage/shared/storage_edus.go +++ b/federationapi/storage/shared/storage_edus.go @@ -53,9 +53,9 @@ func (d *Database) AssociateEDUWithDestination( // Keep EDUs for at least x minutes before deleting them expiresAt = gomatrixserverlib.AsTimestamp(time.Now().Add(duration)) } - // We forcibly set m.direct_to_device events to 0, as we always want them - // to be delivered. (required for E2EE) - if eduType == gomatrixserverlib.MDirectToDevice { + // We forcibly set m.direct_to_device and m.device_list_update events + // to 0, as we always want them to be delivered. (required for E2EE) + if eduType == gomatrixserverlib.MDirectToDevice || eduType == gomatrixserverlib.MDeviceListUpdate { expiresAt = 0 } return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { diff --git a/internal/httputil/http.go b/internal/httputil/http.go index 1e07ee33c..ad26de512 100644 --- a/internal/httputil/http.go +++ b/internal/httputil/http.go @@ -24,7 +24,7 @@ import ( "net/url" "strings" - opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" ) @@ -81,8 +81,8 @@ func PostJSON[reqtype, restype any, errtype error]( return fmt.Errorf("HTTP %d from %s (no response body)", res.StatusCode, apiURL) } var reserr errtype - if err = json.Unmarshal(body, reserr); err != nil { - return fmt.Errorf("HTTP %d from %s", res.StatusCode, apiURL) + if err = json.Unmarshal(body, &reserr); err != nil { + return fmt.Errorf("HTTP %d from %s - %w", res.StatusCode, apiURL, err) } return reserr } diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go index 304b67b23..8530f9078 100644 --- a/keyserver/internal/device_list_update.go +++ b/keyserver/internal/device_list_update.go @@ -19,9 +19,11 @@ import ( "encoding/json" "fmt" "hash/fnv" + "net" "sync" "time" + "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" @@ -388,6 +390,8 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam return waitTime, true } failCount := 0 + +userLoop: for _, userID := range userIDs { if ctx.Err() != nil { // we've timed out, give up and go to the back of the queue to let another server be processed. @@ -397,19 +401,35 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam res, err := u.fedClient.GetUserDevices(ctx, serverName, userID) if err != nil { failCount += 1 - fcerr, ok := err.(*fedsenderapi.FederationClientError) - if ok { - if fcerr.RetryAfter > 0 { - waitTime = fcerr.RetryAfter - } else if fcerr.Blacklisted { + switch e := err.(type) { + case *fedsenderapi.FederationClientError: + if e.RetryAfter > 0 { + waitTime = e.RetryAfter + break userLoop + } else if e.Blacklisted { waitTime = time.Hour * 8 - } else { - // For all other errors (DNS resolution, network etc.) wait 1 hour. - waitTime = time.Hour + break userLoop } - } else { - waitTime = time.Hour - logger.WithError(err).WithField("user_id", userID).Debug("GetUserDevices returned unknown error type") + case net.Error: + // Use the default waitTime, if it's a timeout. + // It probably doesn't make sense to try further users. + if !e.Timeout() { + waitTime = time.Minute * 10 + logrus.WithError(e).Error("GetUserDevices returned net.Error") + break userLoop + } + case gomatrix.HTTPError: + // The remote server returned an error, give it some time to recover + if e.Code >= 500 { + waitTime = time.Minute * 10 + logrus.WithError(e).Error("GetUserDevices returned gomatrix.HTTPError") + break userLoop + } + default: + // Something else failed + waitTime = time.Minute * 10 + logger.WithError(err).WithField("user_id", userID).Debugf("GetUserDevices returned unknown error type: %T", err) + break userLoop } continue } @@ -437,7 +457,11 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam } } if failCount > 0 { - logger.WithField("total", len(userIDs)).WithField("failed", failCount).WithField("wait", waitTime).Warn("Failed to query device keys for some users") + logger.WithFields(logrus.Fields{ + "total": len(userIDs), + "failed": failCount, + "skipped": len(userIDs) - failCount, + }).Warn("Failed to query device keys for some users") } for _, userID := range userIDs { // always clear the channel to unblock Update calls regardless of success/failure diff --git a/sytest-whitelist b/sytest-whitelist index 5c8896b99..fc10bd3cc 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -741,4 +741,5 @@ Newly joined room includes presence in incremental sync User in private room doesn't appear in user directory User joining then leaving public room appears and dissappears from directory User in remote room doesn't appear in user directory after server left room -User in shared private room does appear in user directory until leave \ No newline at end of file +User in shared private room does appear in user directory until leave +If a device list update goes missing, the server resyncs on the next one \ No newline at end of file From 2cfcfddeccb435709824e1beb8d6731eff7c8b86 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 7 Sep 2022 11:45:12 +0200 Subject: [PATCH 17/20] Add a SigningKeyUpdate producer (#2697) This adds a new stream for signing key updates, this should ensure we don't lose any updates over federation. --- federationapi/federationapi.go | 7 +- federationapi/producers/syncapi.go | 20 ++++- federationapi/routing/send.go | 47 ++-------- keyserver/consumers/signingkeyupdate.go | 110 ++++++++++++++++++++++++ keyserver/keyserver.go | 10 ++- setup/jetstream/streams.go | 6 ++ 6 files changed, 153 insertions(+), 47 deletions(-) create mode 100644 keyserver/consumers/signingkeyupdate.go diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index ff01b1952..4a13c9d9b 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -18,6 +18,8 @@ import ( "time" "github.com/gorilla/mux" + "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/federationapi/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/consumers" @@ -33,10 +35,10 @@ import ( "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/jetstream" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/sirupsen/logrus" + + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/federationapi/routing" - "github.com/matrix-org/gomatrixserverlib" ) // AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions @@ -66,6 +68,7 @@ func AddPublicRoutes( TopicTypingEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), TopicPresenceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent), TopicDeviceListUpdate: cfg.Matrix.JetStream.Prefixed(jetstream.InputDeviceListUpdate), + TopicSigningKeyUpdate: cfg.Matrix.JetStream.Prefixed(jetstream.InputSigningKeyUpdate), ServerName: cfg.Matrix.ServerName, UserAPI: userAPI, } diff --git a/federationapi/producers/syncapi.go b/federationapi/producers/syncapi.go index 43dd08dd8..0825bccbb 100644 --- a/federationapi/producers/syncapi.go +++ b/federationapi/producers/syncapi.go @@ -21,12 +21,13 @@ import ( "strconv" "time" - "github.com/matrix-org/dendrite/setup/jetstream" - "github.com/matrix-org/dendrite/syncapi/types" - userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/nats-io/nats.go" log "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" ) // SyncAPIProducer produces events for the sync API server to consume @@ -36,6 +37,7 @@ type SyncAPIProducer struct { TopicTypingEvent string TopicPresenceEvent string TopicDeviceListUpdate string + TopicSigningKeyUpdate string JetStream nats.JetStreamContext ServerName gomatrixserverlib.ServerName UserAPI userapi.UserInternalAPI @@ -178,3 +180,15 @@ func (p *SyncAPIProducer) SendDeviceListUpdate( _, err = p.JetStream.PublishMsg(m, nats.Context(ctx)) return err } + +func (p *SyncAPIProducer) SendSigningKeyUpdate( + ctx context.Context, data gomatrixserverlib.RawJSON, origin gomatrixserverlib.ServerName, +) (err error) { + m := nats.NewMsg(p.TopicSigningKeyUpdate) + m.Header.Set("origin", string(origin)) + m.Data = data + + log.Debugf("Sending signing key update") + _, err = p.JetStream.PublishMsg(m, nats.Context(ctx)) + return err +} diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 87b6fa33e..3d9319964 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -22,6 +22,11 @@ import ( "sync" "time" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/clientapi/jsonerror" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/producers" @@ -31,10 +36,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" syncTypes "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" ) const ( @@ -391,7 +392,7 @@ func (t *txnReq) processEDUs(ctx context.Context) { } } case types.MSigningKeyUpdate: - if err := t.processSigningKeyUpdate(ctx, e); err != nil { + if err := t.producer.SendSigningKeyUpdate(ctx, e.Content, t.Origin); err != nil { logrus.WithError(err).Errorf("Failed to process signing key update") } case gomatrixserverlib.MPresence: @@ -431,42 +432,6 @@ func (t *txnReq) processPresence(ctx context.Context, e gomatrixserverlib.EDU) e return nil } -func (t *txnReq) processSigningKeyUpdate(ctx context.Context, e gomatrixserverlib.EDU) error { - var updatePayload keyapi.CrossSigningKeyUpdate - if err := json.Unmarshal(e.Content, &updatePayload); err != nil { - util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{ - "user_id": updatePayload.UserID, - }).Debug("Failed to unmarshal signing key update") - return err - } - if _, serverName, err := gomatrixserverlib.SplitID('@', updatePayload.UserID); err != nil { - return nil - } else if serverName == t.ourServerName { - return nil - } else if serverName != t.Origin { - return nil - } - keys := gomatrixserverlib.CrossSigningKeys{} - if updatePayload.MasterKey != nil { - keys.MasterKey = *updatePayload.MasterKey - } - if updatePayload.SelfSigningKey != nil { - keys.SelfSigningKey = *updatePayload.SelfSigningKey - } - uploadReq := &keyapi.PerformUploadDeviceKeysRequest{ - CrossSigningKeys: keys, - UserID: updatePayload.UserID, - } - uploadRes := &keyapi.PerformUploadDeviceKeysResponse{} - if err := t.keyAPI.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes); err != nil { - return err - } - if uploadRes.Error != nil { - return uploadRes.Error - } - return nil -} - // processReceiptEvent sends receipt events to JetStream func (t *txnReq) processReceiptEvent(ctx context.Context, userID, roomID, receiptType string, diff --git a/keyserver/consumers/signingkeyupdate.go b/keyserver/consumers/signingkeyupdate.go new file mode 100644 index 000000000..366e259b4 --- /dev/null +++ b/keyserver/consumers/signingkeyupdate.go @@ -0,0 +1,110 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "context" + "encoding/json" + + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + "github.com/sirupsen/logrus" + + keyapi "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/keyserver/internal" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" +) + +// SigningKeyUpdateConsumer consumes signing key updates that came in over federation. +type SigningKeyUpdateConsumer struct { + ctx context.Context + jetstream nats.JetStreamContext + durable string + topic string + keyAPI *internal.KeyInternalAPI + cfg *config.KeyServer +} + +// NewSigningKeyUpdateConsumer creates a new SigningKeyUpdateConsumer. Call Start() to begin consuming from key servers. +func NewSigningKeyUpdateConsumer( + process *process.ProcessContext, + cfg *config.KeyServer, + js nats.JetStreamContext, + keyAPI *internal.KeyInternalAPI, +) *SigningKeyUpdateConsumer { + return &SigningKeyUpdateConsumer{ + ctx: process.Context(), + jetstream: js, + durable: cfg.Matrix.JetStream.Prefixed("KeyServerSigningKeyConsumer"), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.InputSigningKeyUpdate), + keyAPI: keyAPI, + cfg: cfg, + } +} + +// Start consuming from key servers +func (t *SigningKeyUpdateConsumer) Start() error { + return jetstream.JetStreamConsumer( + t.ctx, t.jetstream, t.topic, t.durable, 1, + t.onMessage, nats.DeliverAll(), nats.ManualAck(), + ) +} + +// onMessage is called in response to a message received on the +// signing key update events topic from the key server. +func (t *SigningKeyUpdateConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool { + msg := msgs[0] // Guaranteed to exist if onMessage is called + var updatePayload keyapi.CrossSigningKeyUpdate + if err := json.Unmarshal(msg.Data, &updatePayload); err != nil { + logrus.WithError(err).Errorf("Failed to read from signing key update input topic") + return true + } + origin := gomatrixserverlib.ServerName(msg.Header.Get("origin")) + if _, serverName, err := gomatrixserverlib.SplitID('@', updatePayload.UserID); err != nil { + logrus.WithError(err).Error("failed to split user id") + return true + } else if serverName == t.cfg.Matrix.ServerName { + logrus.Warn("dropping device key update from ourself") + return true + } else if serverName != origin { + logrus.Warnf("dropping device key update, %s != %s", serverName, origin) + return true + } + + keys := gomatrixserverlib.CrossSigningKeys{} + if updatePayload.MasterKey != nil { + keys.MasterKey = *updatePayload.MasterKey + } + if updatePayload.SelfSigningKey != nil { + keys.SelfSigningKey = *updatePayload.SelfSigningKey + } + uploadReq := &keyapi.PerformUploadDeviceKeysRequest{ + CrossSigningKeys: keys, + UserID: updatePayload.UserID, + } + uploadRes := &keyapi.PerformUploadDeviceKeysResponse{} + if err := t.keyAPI.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes); err != nil { + logrus.WithError(err).Error("failed to upload device keys") + return false + } + if uploadRes.Error != nil { + logrus.WithError(uploadRes.Error).Error("failed to upload device keys") + return true + } + + return true +} diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index 75cdaedb4..5124b777e 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -16,6 +16,8 @@ package keyserver import ( "github.com/gorilla/mux" + "github.com/sirupsen/logrus" + fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/keyserver/consumers" @@ -26,7 +28,6 @@ import ( "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" - "github.com/sirupsen/logrus" ) // AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions @@ -72,5 +73,12 @@ func NewInternalAPI( logrus.WithError(err).Panic("failed to start device list consumer") } + sigConsumer := consumers.NewSigningKeyUpdateConsumer( + base.ProcessContext, cfg, js, ap, + ) + if err := sigConsumer.Start(); err != nil { + logrus.WithError(err).Panic("failed to start signing key consumer") + } + return ap } diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go index 110808b1b..a06a12662 100644 --- a/setup/jetstream/streams.go +++ b/setup/jetstream/streams.go @@ -17,6 +17,7 @@ const ( var ( InputRoomEvent = "InputRoomEvent" InputDeviceListUpdate = "InputDeviceListUpdate" + InputSigningKeyUpdate = "InputSigningKeyUpdate" OutputRoomEvent = "OutputRoomEvent" OutputSendToDeviceEvent = "OutputSendToDeviceEvent" OutputKeyChangeEvent = "OutputKeyChangeEvent" @@ -51,6 +52,11 @@ var streams = []*nats.StreamConfig{ Retention: nats.InterestPolicy, Storage: nats.FileStorage, }, + { + Name: InputSigningKeyUpdate, + Retention: nats.InterestPolicy, + Storage: nats.FileStorage, + }, { Name: OutputRoomEvent, Retention: nats.InterestPolicy, From 4e352390b6827870a2fb19e01c59bd5c267dd7e0 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Wed, 7 Sep 2022 12:13:02 +0200 Subject: [PATCH 18/20] Re-add waitTime if we're not blacklisted and no RetryAfter was specified. --- keyserver/internal/device_list_update.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go index 8530f9078..b70d6a5e1 100644 --- a/keyserver/internal/device_list_update.go +++ b/keyserver/internal/device_list_update.go @@ -405,11 +405,12 @@ userLoop: case *fedsenderapi.FederationClientError: if e.RetryAfter > 0 { waitTime = e.RetryAfter - break userLoop } else if e.Blacklisted { waitTime = time.Hour * 8 - break userLoop + } else { + waitTime = time.Hour } + break userLoop case net.Error: // Use the default waitTime, if it's a timeout. // It probably doesn't make sense to try further users. From 7e8c605f98458f829fc914c3476cf4999e317bcd Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 7 Sep 2022 12:16:04 +0200 Subject: [PATCH 19/20] Avoid unneeded JSON operations (#2698) We were `json.Unmarshal`ing the EDU and `json.Marshal`ing right before sending the EDU to the stream. Those are now removed and the consumer does `json.Unmarshal` once. --- federationapi/producers/syncapi.go | 10 ++----- federationapi/routing/send.go | 22 ++------------ keyserver/consumers/devicelistupdate.go | 38 ++++++++++++++++--------- 3 files changed, 31 insertions(+), 39 deletions(-) diff --git a/federationapi/producers/syncapi.go b/federationapi/producers/syncapi.go index 0825bccbb..86c8c10a3 100644 --- a/federationapi/producers/syncapi.go +++ b/federationapi/producers/syncapi.go @@ -167,15 +167,11 @@ func (p *SyncAPIProducer) SendPresence( } func (p *SyncAPIProducer) SendDeviceListUpdate( - ctx context.Context, deviceListUpdate *gomatrixserverlib.DeviceListUpdateEvent, + ctx context.Context, deviceListUpdate gomatrixserverlib.RawJSON, origin string, ) (err error) { m := nats.NewMsg(p.TopicDeviceListUpdate) - m.Header.Set(jetstream.UserID, deviceListUpdate.UserID) - m.Data, err = json.Marshal(deviceListUpdate) - if err != nil { - return fmt.Errorf("json.Marshal: %w", err) - } - + m.Header.Set("origin", origin) + m.Data = deviceListUpdate log.Debugf("Sending device list update: %+v", m.Header) _, err = p.JetStream.PublishMsg(m, nats.Context(ctx)) return err diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 3d9319964..a9714c65a 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -359,7 +359,9 @@ func (t *txnReq) processEDUs(ctx context.Context) { } } case gomatrixserverlib.MDeviceListUpdate: - t.processDeviceListUpdate(ctx, e) + if err := t.producer.SendDeviceListUpdate(ctx, e.Content, e.Origin); err != nil { + util.GetLogger(ctx).WithError(err).Error("failed to InputDeviceListUpdate") + } case gomatrixserverlib.MReceipt: // https://matrix.org/docs/spec/server_server/r0.1.4#receipts payload := map[string]types.FederationReceiptMRead{} @@ -454,21 +456,3 @@ func (t *txnReq) processReceiptEvent(ctx context.Context, return nil } - -func (t *txnReq) processDeviceListUpdate(ctx context.Context, e gomatrixserverlib.EDU) { - var payload gomatrixserverlib.DeviceListUpdateEvent - if err := json.Unmarshal(e.Content, &payload); err != nil { - util.GetLogger(ctx).WithError(err).Error("Failed to unmarshal device list update event") - return - } - if _, serverName, err := gomatrixserverlib.SplitID('@', payload.UserID); err != nil { - return - } else if serverName == t.ourServerName { - return - } else if serverName != t.Origin { - return - } - if err := t.producer.SendDeviceListUpdate(ctx, &payload); err != nil { - util.GetLogger(ctx).WithError(err).WithField("user_id", payload.UserID).Error("failed to InputDeviceListUpdate") - } -} diff --git a/keyserver/consumers/devicelistupdate.go b/keyserver/consumers/devicelistupdate.go index d15f94267..575e41281 100644 --- a/keyserver/consumers/devicelistupdate.go +++ b/keyserver/consumers/devicelistupdate.go @@ -18,22 +18,24 @@ import ( "context" "encoding/json" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/keyserver/internal" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" - "github.com/matrix-org/gomatrixserverlib" - "github.com/nats-io/nats.go" - "github.com/sirupsen/logrus" ) // DeviceListUpdateConsumer consumes device list updates that came in over federation. type DeviceListUpdateConsumer struct { - ctx context.Context - jetstream nats.JetStreamContext - durable string - topic string - updater *internal.DeviceListUpdater + ctx context.Context + jetstream nats.JetStreamContext + durable string + topic string + updater *internal.DeviceListUpdater + serverName gomatrixserverlib.ServerName } // NewDeviceListUpdateConsumer creates a new DeviceListConsumer. Call Start() to begin consuming from key servers. @@ -44,11 +46,12 @@ func NewDeviceListUpdateConsumer( updater *internal.DeviceListUpdater, ) *DeviceListUpdateConsumer { return &DeviceListUpdateConsumer{ - ctx: process.Context(), - jetstream: js, - durable: cfg.Matrix.JetStream.Prefixed("KeyServerInputDeviceListConsumer"), - topic: cfg.Matrix.JetStream.Prefixed(jetstream.InputDeviceListUpdate), - updater: updater, + ctx: process.Context(), + jetstream: js, + durable: cfg.Matrix.JetStream.Prefixed("KeyServerInputDeviceListConsumer"), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.InputDeviceListUpdate), + updater: updater, + serverName: cfg.Matrix.ServerName, } } @@ -69,6 +72,15 @@ func (t *DeviceListUpdateConsumer) onMessage(ctx context.Context, msgs []*nats.M logrus.WithError(err).Errorf("Failed to read from device list update input topic") return true } + origin := gomatrixserverlib.ServerName(msg.Header.Get("origin")) + if _, serverName, err := gomatrixserverlib.SplitID('@', m.UserID); err != nil { + return true + } else if serverName == t.serverName { + return true + } else if serverName != origin { + return true + } + err := t.updater.Update(ctx, m) if err != nil { logrus.WithFields(logrus.Fields{ From 0d697f675484cfd990af640aad6fa54c44ecd208 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 7 Sep 2022 16:14:09 +0200 Subject: [PATCH 20/20] Add HTTP status code to FederationClientError (#2699) Also ensures we wait on more HTTP status codes. --- federationapi/api/api.go | 4 +++- federationapi/inthttp/server.go | 10 ++++++++-- keyserver/internal/device_list_update.go | 21 +++++++++++++-------- sytest-blacklist | 3 +++ sytest-whitelist | 1 - 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/federationapi/api/api.go b/federationapi/api/api.go index 292ed55ad..f25538784 100644 --- a/federationapi/api/api.go +++ b/federationapi/api/api.go @@ -5,9 +5,10 @@ import ( "fmt" "time" - "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" + + "github.com/matrix-org/dendrite/federationapi/types" ) // FederationInternalAPI is used to query information from the federation sender. @@ -108,6 +109,7 @@ type FederationClientError struct { Err string RetryAfter time.Duration Blacklisted bool + Code int // HTTP Status code from the remote server } func (e FederationClientError) Error() string { diff --git a/federationapi/inthttp/server.go b/federationapi/inthttp/server.go index a8b829a71..58ea9ddc6 100644 --- a/federationapi/inthttp/server.go +++ b/federationapi/inthttp/server.go @@ -6,10 +6,12 @@ import ( "net/http" "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/federationapi/api" - "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + + "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/internal/httputil" ) // AddRoutes adds the FederationInternalAPI handlers to the http.ServeMux. @@ -229,6 +231,10 @@ func federationClientError(err error) error { return &ferr case *api.FederationClientError: return ferr + case gomatrix.HTTPError: + return &api.FederationClientError{ + Code: ferr.Code, + } default: return &api.FederationClientError{ Err: err.Error(), diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go index b70d6a5e1..41534fe82 100644 --- a/keyserver/internal/device_list_update.go +++ b/keyserver/internal/device_list_update.go @@ -407,10 +407,13 @@ userLoop: waitTime = e.RetryAfter } else if e.Blacklisted { waitTime = time.Hour * 8 - } else { + break userLoop + } else if e.Code >= 300 { + // We didn't get a real FederationClientError (e.g. in polylith mode, where gomatrix.HTTPError + // are "converted" to FederationClientError), but we probably shouldn't hit them every $waitTime seconds. waitTime = time.Hour + break userLoop } - break userLoop case net.Error: // Use the default waitTime, if it's a timeout. // It probably doesn't make sense to try further users. @@ -420,9 +423,10 @@ userLoop: break userLoop } case gomatrix.HTTPError: - // The remote server returned an error, give it some time to recover - if e.Code >= 500 { - waitTime = time.Minute * 10 + // The remote server returned an error, give it some time to recover. + // This is to avoid spamming remote servers, which may not be Matrix servers anymore. + if e.Code >= 300 { + waitTime = time.Hour logrus.WithError(e).Error("GetUserDevices returned gomatrix.HTTPError") break userLoop } @@ -459,9 +463,10 @@ userLoop: } if failCount > 0 { logger.WithFields(logrus.Fields{ - "total": len(userIDs), - "failed": failCount, - "skipped": len(userIDs) - failCount, + "total": len(userIDs), + "failed": failCount, + "skipped": len(userIDs) - failCount, + "waittime": waitTime, }).Warn("Failed to query device keys for some users") } for _, userID := range userIDs { diff --git a/sytest-blacklist b/sytest-blacklist index bcc345f6e..5b2e973a6 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -49,3 +49,6 @@ Notifications can be viewed with GET /notifications If remote user leaves room we no longer receive device updates Guest users can join guest_access rooms + +# This will fail in HTTP API mode, so blacklisted for now +If a device list update goes missing, the server resyncs on the next one \ No newline at end of file diff --git a/sytest-whitelist b/sytest-whitelist index fc10bd3cc..31940b884 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -742,4 +742,3 @@ User in private room doesn't appear in user directory User joining then leaving public room appears and dissappears from directory User in remote room doesn't appear in user directory after server left room User in shared private room does appear in user directory until leave -If a device list update goes missing, the server resyncs on the next one \ No newline at end of file