From 02597f15f0b60dbea49f8540c87981f7967d5509 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 6 Jun 2022 08:56:06 +0100 Subject: [PATCH 01/25] Fix panic in `QueryRestrictedJoinAllowed` --- roomserver/internal/query/query.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 6d306fb09..da1b32530 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -778,11 +778,18 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.Query } else if !allowRestrictedJoins { return nil } + // Start off by populating the "resident" flag in the response. If we + // come across any rooms in the request that are missing, we will unset + // the flag. + res.Resident = true // Get the join rules to work out if the join rule is "restricted". joinRulesEvent, err := r.DB.GetStateEvent(ctx, req.RoomID, gomatrixserverlib.MRoomJoinRules, "") if err != nil { return fmt.Errorf("r.DB.GetStateEvent: %w", err) } + if joinRulesEvent == nil { + return nil + } var joinRules gomatrixserverlib.JoinRuleContent if err = json.Unmarshal(joinRulesEvent.Content(), &joinRules); err != nil { return fmt.Errorf("json.Unmarshal: %w", err) @@ -792,10 +799,6 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.Query if !res.Restricted { return nil } - // Start off by populating the "resident" flag in the response. If we - // come across any rooms in the request that are missing, we will unset - // the flag. - res.Resident = true // If the user is already invited to the room then the join is allowed // but we don't specify an authorised via user, since the event auth // will allow the join anyway. From 2cb609c428bbff81634edbc02315861660c1bc25 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 6 Jun 2022 15:18:02 +0100 Subject: [PATCH 02/25] Room upgrade tweaks Squashed commit of the following: commit 7a1568c716866594af6d0b1d561c58c96de29b20 Author: Neil Alexander Date: Mon Jun 6 15:17:49 2022 +0100 Make errors more useful commit 64befe7c9a901b00650442171660c2dc4ea575fa Author: Neil Alexander Date: Mon Jun 6 15:02:40 2022 +0100 Tweak ordering a bit --- .../internal/perform/perform_upgrade.go | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/roomserver/internal/perform/perform_upgrade.go b/roomserver/internal/perform/perform_upgrade.go index fcd19b936..8d975209a 100644 --- a/roomserver/internal/perform/perform_upgrade.go +++ b/roomserver/internal/perform/perform_upgrade.go @@ -105,13 +105,13 @@ func (r *Upgrader) performRoomUpgrade( return "", pErr } - // 5. Send the tombstone event to the old room (must do this before we set the new canonical_alias) - if pErr = r.sendHeaderedEvent(ctx, tombstoneEvent); pErr != nil { + // Send the setup events to the new room + if pErr = r.sendInitialEvents(ctx, evTime, userID, newRoomID, string(req.RoomVersion), eventsToMake); pErr != nil { return "", pErr } - // Send the setup events to the new room - if pErr = r.sendInitialEvents(ctx, evTime, userID, newRoomID, string(req.RoomVersion), eventsToMake); pErr != nil { + // 5. Send the tombstone event to the old room + if pErr = r.sendHeaderedEvent(ctx, tombstoneEvent); pErr != nil { return "", pErr } @@ -147,7 +147,7 @@ func (r *Upgrader) getRoomPowerLevels(ctx context.Context, roomID string) (*goma if err != nil { util.GetLogger(ctx).WithError(err).Error() return nil, &api.PerformError{ - Msg: "powerLevel event was not actually a power level event", + Msg: "Power level event was invalid or malformed", } } return powerLevelContent, nil @@ -198,7 +198,7 @@ func moveLocalAliases(ctx context.Context, aliasRes := api.GetAliasesForRoomIDResponse{} if err = URSAPI.GetAliasesForRoomID(ctx, &aliasReq, &aliasRes); err != nil { return &api.PerformError{ - Msg: "Could not get aliases for old room", + Msg: fmt.Sprintf("Failed to get old room aliases: %s", err), } } @@ -207,7 +207,7 @@ func moveLocalAliases(ctx context.Context, removeAliasRes := api.RemoveRoomAliasResponse{} if err = URSAPI.RemoveRoomAlias(ctx, &removeAliasReq, &removeAliasRes); err != nil { return &api.PerformError{ - Msg: "api.RemoveRoomAlias failed", + Msg: fmt.Sprintf("Failed to remove old room alias: %s", err), } } @@ -215,7 +215,7 @@ func moveLocalAliases(ctx context.Context, setAliasRes := api.SetRoomAliasResponse{} if err = URSAPI.SetRoomAlias(ctx, &setAliasReq, &setAliasRes); err != nil { return &api.PerformError{ - Msg: "api.SetRoomAlias failed", + Msg: fmt.Sprintf("Failed to set new room alias: %s", err), } } } @@ -509,7 +509,7 @@ func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, user err = builder.SetContent(e.Content) if err != nil { return &api.PerformError{ - Msg: "builder.SetContent failed", + Msg: fmt.Sprintf("Failed to set content of new %q event: %s", builder.Type, err), } } if i > 0 { @@ -519,13 +519,13 @@ func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, user event, err = r.buildEvent(&builder, &authEvents, evTime, gomatrixserverlib.RoomVersion(newVersion)) if err != nil { return &api.PerformError{ - Msg: "buildEvent failed", + Msg: fmt.Sprintf("Failed to build new %q event: %s", builder.Type, err), } } if err = gomatrixserverlib.Allowed(event, &authEvents); err != nil { return &api.PerformError{ - Msg: "gomatrixserverlib.Allowed failed", + Msg: fmt.Sprintf("Failed to auth new %q event: %s", builder.Type, err), } } @@ -534,7 +534,7 @@ func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, user err = authEvents.AddEvent(event) if err != nil { return &api.PerformError{ - Msg: "authEvents.AddEvent failed", + Msg: fmt.Sprintf("Failed to add new %q event to auth set: %s", builder.Type, err), } } } @@ -550,7 +550,7 @@ func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, user } if err = api.SendInputRoomEvents(ctx, r.URSAPI, inputs, false); err != nil { return &api.PerformError{ - Msg: "api.SendInputRoomEvents failed", + Msg: fmt.Sprintf("Failed to send new room %q to roomserver: %s", newRoomID, err), } } return nil @@ -582,7 +582,7 @@ func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, user err := builder.SetContent(event.Content) if err != nil { return nil, &api.PerformError{ - Msg: "builder.SetContent failed", + Msg: fmt.Sprintf("Failed to set new %q event content: %s", builder.Type, err), } } var queryRes api.QueryLatestEventsAndStateResponse @@ -607,7 +607,7 @@ func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, user } } else if err != nil { return nil, &api.PerformError{ - Msg: "eventutil.BuildEvent failed", + Msg: fmt.Sprintf("Failed to build new %q event: %s", builder.Type, err), } } // check to see if this user can perform this operation @@ -619,7 +619,7 @@ func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, user if err = gomatrixserverlib.Allowed(headeredEvent.Event, &provider); err != nil { return nil, &api.PerformError{ Code: api.PerformErrorNotAllowed, - Msg: err.Error(), // TODO: Is this error string comprehensible to the client? + Msg: fmt.Sprintf("Failed to auth new %q event: %s", builder.Type, err), // TODO: Is this error string comprehensible to the client? } } @@ -676,7 +676,7 @@ func (r *Upgrader) sendHeaderedEvent( }) if err := api.SendInputRoomEvents(ctx, r.URSAPI, inputs, false); err != nil { return &api.PerformError{ - Msg: "api.SendInputRoomEvents failed", + Msg: fmt.Sprintf("Failed to send new %q event to roomserver: %s", headeredEvent.Type(), err), } } @@ -703,7 +703,7 @@ func (r *Upgrader) buildEvent( r.Cfg.Matrix.PrivateKey, roomVersion, ) if err != nil { - return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %w", builder.Type, err) + return nil, err } return event, nil } From 0d7020fbaf97fb3275697033a7c3c2e6cf6eeacd Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 6 Jun 2022 17:27:50 +0100 Subject: [PATCH 03/25] Send tombstone to other servers when upgrading rooms --- roomserver/internal/perform/perform_upgrade.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/roomserver/internal/perform/perform_upgrade.go b/roomserver/internal/perform/perform_upgrade.go index 8d975209a..393d7dd14 100644 --- a/roomserver/internal/perform/perform_upgrade.go +++ b/roomserver/internal/perform/perform_upgrade.go @@ -111,7 +111,7 @@ func (r *Upgrader) performRoomUpgrade( } // 5. Send the tombstone event to the old room - if pErr = r.sendHeaderedEvent(ctx, tombstoneEvent); pErr != nil { + if pErr = r.sendHeaderedEvent(ctx, tombstoneEvent, string(r.Cfg.Matrix.ServerName)); pErr != nil { return "", pErr } @@ -182,7 +182,7 @@ func (r *Upgrader) restrictOldRoomPowerLevels(ctx context.Context, evTime time.T return resErr } } else { - if resErr = r.sendHeaderedEvent(ctx, restrictedPowerLevelsHeadered); resErr != nil { + if resErr = r.sendHeaderedEvent(ctx, restrictedPowerLevelsHeadered, api.DoNotSendToOtherServers); resErr != nil { return resErr } } @@ -253,7 +253,7 @@ func (r *Upgrader) clearOldCanonicalAliasEvent(ctx context.Context, oldRoom *api return resErr } } else { - if resErr = r.sendHeaderedEvent(ctx, emptyCanonicalAliasEvent); resErr != nil { + if resErr = r.sendHeaderedEvent(ctx, emptyCanonicalAliasEvent, api.DoNotSendToOtherServers); resErr != nil { return resErr } } @@ -666,13 +666,14 @@ func createTemporaryPowerLevels(powerLevelContent *gomatrixserverlib.PowerLevelC func (r *Upgrader) sendHeaderedEvent( ctx context.Context, headeredEvent *gomatrixserverlib.HeaderedEvent, + sendAsServer string, ) *api.PerformError { var inputs []api.InputRoomEvent inputs = append(inputs, api.InputRoomEvent{ Kind: api.KindNew, Event: headeredEvent, Origin: r.Cfg.Matrix.ServerName, - SendAsServer: api.DoNotSendToOtherServers, + SendAsServer: sendAsServer, }) if err := api.SendInputRoomEvents(ctx, r.URSAPI, inputs, false); err != nil { return &api.PerformError{ From aafb7bf120d30c37219686a5bb528794b0ab44a2 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 7 Jun 2022 10:46:21 +0100 Subject: [PATCH 04/25] Avoid panicking unnecessarily when shutting down the `gobind` P2P demos (#2520) --- build/gobind-pinecone/monolith.go | 18 +++++++++++++++--- build/gobind-yggdrasil/monolith.go | 4 ++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 664ca85d9..b44e110ad 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -261,7 +261,7 @@ func (m *DendriteMonolith) Start() { cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} cfg.ClientAPI.RegistrationDisabled = false cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true - if err := cfg.Derive(); err != nil { + if err = cfg.Derive(); err != nil { panic(err) } @@ -342,11 +342,23 @@ func (m *DendriteMonolith) Start() { go func() { m.logger.Info("Listening on ", cfg.Global.ServerName) - m.logger.Fatal(m.httpServer.Serve(m.PineconeQUIC.Protocol("matrix"))) + + switch m.httpServer.Serve(m.PineconeQUIC.Protocol("matrix")) { + case net.ErrClosed, http.ErrServerClosed: + m.logger.Info("Stopped listening on ", cfg.Global.ServerName) + default: + m.logger.Fatal(err) + } }() go func() { logrus.Info("Listening on ", m.listener.Addr()) - logrus.Fatal(http.Serve(m.listener, httpRouter)) + + switch http.Serve(m.listener, httpRouter) { + case net.ErrClosed, http.ErrServerClosed: + m.logger.Info("Stopped listening on ", cfg.Global.ServerName) + default: + m.logger.Fatal(err) + } }() } diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 991bc462f..99b180c81 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -170,11 +170,11 @@ func (m *DendriteMonolith) Start() { go func() { m.logger.Info("Listening on ", ygg.DerivedServerName()) - m.logger.Fatal(m.httpServer.Serve(ygg)) + m.logger.Error(m.httpServer.Serve(ygg)) }() go func() { logrus.Info("Listening on ", m.listener.Addr()) - logrus.Fatal(http.Serve(m.listener, httpRouter)) + logrus.Error(http.Serve(m.listener, httpRouter)) }() go func() { logrus.Info("Sending wake-up message to known nodes") From 27948fb30468315ce613402dc8cc1fa7dba01679 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 7 Jun 2022 14:23:26 +0100 Subject: [PATCH 05/25] Optimise `loadAuthEvents`, add roomserver tracing --- roomserver/internal/input/input_events.go | 12 + .../internal/input/input_latest_events.go | 16 +- roomserver/internal/input/input_membership.go | 4 + roomserver/internal/input/input_missing.go | 30 +++ roomserver/state/state.go | 217 ++++++++++++++---- 5 files changed, 228 insertions(+), 51 deletions(-) diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index d437d7768..deb88ea82 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -33,6 +33,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" ) @@ -75,6 +76,11 @@ func (r *Inputer) processRoomEvent( default: } + span, ctx := opentracing.StartSpanFromContext(ctx, "processRoomEvent") + span.SetTag("room_id", input.Event.RoomID()) + span.SetTag("event_id", input.Event.EventID()) + defer span.Finish() + // Measure how long it takes to process this event. started := time.Now() defer func() { @@ -411,6 +417,9 @@ func (r *Inputer) fetchAuthEvents( known map[string]*types.Event, servers []gomatrixserverlib.ServerName, ) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "fetchAuthEvents") + defer span.Finish() + unknown := map[string]struct{}{} authEventIDs := event.AuthEventIDs() if len(authEventIDs) == 0 { @@ -526,6 +535,9 @@ func (r *Inputer) calculateAndSetState( event *gomatrixserverlib.Event, isRejected bool, ) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "calculateAndSetState") + defer span.Finish() + var succeeded bool updater, err := r.DB.GetRoomUpdater(ctx, roomInfo) if err != nil { diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index f772299ab..9738ed4e6 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -27,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/opentracing/opentracing-go" "github.com/sirupsen/logrus" ) @@ -56,6 +57,9 @@ func (r *Inputer) updateLatestEvents( transactionID *api.TransactionID, rewritesState bool, ) (err error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "updateLatestEvents") + defer span.Finish() + var succeeded bool updater, err := r.DB.GetRoomUpdater(ctx, roomInfo) if err != nil { @@ -200,6 +204,9 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { } func (u *latestEventsUpdater) latestState() error { + span, ctx := opentracing.StartSpanFromContext(u.ctx, "processEventWithMissingState") + defer span.Finish() + var err error roomState := state.NewStateResolution(u.updater, u.roomInfo) @@ -246,7 +253,7 @@ func (u *latestEventsUpdater) latestState() error { // of the state after the events. The snapshot state will be resolved // using the correct state resolution algorithm for the room. u.newStateNID, err = roomState.CalculateAndStoreStateAfterEvents( - u.ctx, latestStateAtEvents, + ctx, latestStateAtEvents, ) if err != nil { return fmt.Errorf("roomState.CalculateAndStoreStateAfterEvents: %w", err) @@ -258,7 +265,7 @@ func (u *latestEventsUpdater) latestState() error { // another list of added ones. Replacing a value for a state-key tuple // will result one removed (the old event) and one added (the new event). u.removed, u.added, err = roomState.DifferenceBetweeenStateSnapshots( - u.ctx, u.oldStateNID, u.newStateNID, + ctx, u.oldStateNID, u.newStateNID, ) if err != nil { return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err) @@ -278,7 +285,7 @@ func (u *latestEventsUpdater) latestState() error { // Also work out the state before the event removes and the event // adds. u.stateBeforeEventRemoves, u.stateBeforeEventAdds, err = roomState.DifferenceBetweeenStateSnapshots( - u.ctx, u.newStateNID, u.stateAtEvent.BeforeStateSnapshotNID, + ctx, u.newStateNID, u.stateAtEvent.BeforeStateSnapshotNID, ) if err != nil { return fmt.Errorf("roomState.DifferenceBetweeenStateSnapshots: %w", err) @@ -294,6 +301,9 @@ func (u *latestEventsUpdater) calculateLatest( newEvent *gomatrixserverlib.Event, newStateAndRef types.StateAtEventAndReference, ) (bool, error) { + span, _ := opentracing.StartSpanFromContext(u.ctx, "calculateLatest") + defer span.Finish() + // First of all, get a list of all of the events in our current // set of forward extremities. existingRefs := make(map[string]*types.StateAtEventAndReference) diff --git a/roomserver/internal/input/input_membership.go b/roomserver/internal/input/input_membership.go index 3953586b2..3ce8791a3 100644 --- a/roomserver/internal/input/input_membership.go +++ b/roomserver/internal/input/input_membership.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/opentracing/opentracing-go" ) // updateMembership updates the current membership and the invites for each @@ -34,6 +35,9 @@ func (r *Inputer) updateMemberships( updater *shared.RoomUpdater, removed, added []types.StateEntry, ) ([]api.OutputEvent, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "updateMemberships") + defer span.Finish() + changes := membershipChanges(removed, added) var eventNIDs []types.EventNID for _, change := range changes { diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go index 9c70076c2..edc153b7f 100644 --- a/roomserver/internal/input/input_missing.go +++ b/roomserver/internal/input/input_missing.go @@ -15,6 +15,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/opentracing/opentracing-go" "github.com/sirupsen/logrus" ) @@ -59,6 +60,9 @@ type missingStateReq struct { func (t *missingStateReq) processEventWithMissingState( ctx context.Context, e *gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion, ) (*parsedRespState, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "processEventWithMissingState") + defer span.Finish() + // We are missing the previous events for this events. // This means that there is a gap in our view of the history of the // room. There two ways that we can handle such a gap: @@ -235,6 +239,9 @@ func (t *missingStateReq) processEventWithMissingState( } func (t *missingStateReq) lookupResolvedStateBeforeEvent(ctx context.Context, e *gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) (*parsedRespState, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "lookupResolvedStateBeforeEvent") + defer span.Finish() + type respState struct { // A snapshot is considered trustworthy if it came from our own roomserver. // That's because the state will have been through state resolution once @@ -310,6 +317,9 @@ func (t *missingStateReq) lookupResolvedStateBeforeEvent(ctx context.Context, e // lookupStateAfterEvent returns the room state after `eventID`, which is the state before eventID with the state of `eventID` (if it's a state event) // added into the mix. func (t *missingStateReq) lookupStateAfterEvent(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (*parsedRespState, bool, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "lookupStateAfterEvent") + defer span.Finish() + // try doing all this locally before we resort to querying federation respState := t.lookupStateAfterEventLocally(ctx, roomID, eventID) if respState != nil { @@ -361,6 +371,9 @@ func (t *missingStateReq) cacheAndReturn(ev *gomatrixserverlib.Event) *gomatrixs } func (t *missingStateReq) lookupStateAfterEventLocally(ctx context.Context, roomID, eventID string) *parsedRespState { + span, ctx := opentracing.StartSpanFromContext(ctx, "lookupStateAfterEventLocally") + defer span.Finish() + var res parsedRespState roomInfo, err := t.db.RoomInfo(ctx, roomID) if err != nil { @@ -435,12 +448,17 @@ func (t *missingStateReq) lookupStateAfterEventLocally(ctx context.Context, room // the server supports. func (t *missingStateReq) lookupStateBeforeEvent(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) ( *parsedRespState, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "lookupStateBeforeEvent") + defer span.Finish() // Attempt to fetch the missing state using /state_ids and /events return t.lookupMissingStateViaStateIDs(ctx, roomID, eventID, roomVersion) } func (t *missingStateReq) resolveStatesAndCheck(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, states []*parsedRespState, backwardsExtremity *gomatrixserverlib.Event) (*parsedRespState, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "resolveStatesAndCheck") + defer span.Finish() + var authEventList []*gomatrixserverlib.Event var stateEventList []*gomatrixserverlib.Event for _, state := range states { @@ -484,6 +502,9 @@ retryAllowedState: // get missing events for `e`. If `isGapFilled`=true then `newEvents` contains all the events to inject, // without `e`. If `isGapFilled=false` then `newEvents` contains the response to /get_missing_events func (t *missingStateReq) getMissingEvents(ctx context.Context, e *gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) (newEvents []*gomatrixserverlib.Event, isGapFilled, prevStateKnown bool, err error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "getMissingEvents") + defer span.Finish() + logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID()) latest, _, _, err := t.db.LatestEventIDs(ctx, t.roomInfo.RoomNID) if err != nil { @@ -608,6 +629,9 @@ func (t *missingStateReq) isPrevStateKnown(ctx context.Context, e *gomatrixserve func (t *missingStateReq) lookupMissingStateViaState( ctx context.Context, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion, ) (respState *parsedRespState, err error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "lookupMissingStateViaState") + defer span.Finish() + state, err := t.federation.LookupState(ctx, t.origin, roomID, eventID, roomVersion) if err != nil { return nil, err @@ -637,6 +661,9 @@ func (t *missingStateReq) lookupMissingStateViaState( func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion) ( *parsedRespState, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "lookupMissingStateViaStateIDs") + defer span.Finish() + util.GetLogger(ctx).WithField("room_id", roomID).Infof("lookupMissingStateViaStateIDs %s", eventID) // fetch the state event IDs at the time of the event stateIDs, err := t.federation.LookupStateIDs(ctx, t.origin, roomID, eventID) @@ -799,6 +826,9 @@ func (t *missingStateReq) createRespStateFromStateIDs( } func (t *missingStateReq) lookupEvent(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, _, missingEventID string, localFirst bool) (*gomatrixserverlib.Event, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "lookupEvent") + defer span.Finish() + if localFirst { // fetch from the roomserver events, err := t.db.EventsFromIDs(ctx, []string{missingEventID}) diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 95abdcb36..6c4e4b860 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -20,9 +20,11 @@ import ( "context" "fmt" "sort" + "sync" "time" "github.com/matrix-org/util" + "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" "github.com/matrix-org/dendrite/roomserver/types" @@ -62,6 +64,9 @@ func NewStateResolution(db StateResolutionStorage, roomInfo *types.RoomInfo) Sta func (v *StateResolution) LoadStateAtSnapshot( ctx context.Context, stateNID types.StateSnapshotNID, ) ([]types.StateEntry, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.LoadStateAtSnapshot") + defer span.Finish() + stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) if err != nil { return nil, err @@ -100,6 +105,9 @@ func (v *StateResolution) LoadStateAtSnapshot( func (v *StateResolution) LoadStateAtEvent( ctx context.Context, eventID string, ) ([]types.StateEntry, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.LoadStateAtEvent") + defer span.Finish() + snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID) if err != nil { return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID failed for event %s : %s", eventID, err) @@ -122,6 +130,9 @@ func (v *StateResolution) LoadStateAtEvent( func (v *StateResolution) LoadCombinedStateAfterEvents( ctx context.Context, prevStates []types.StateAtEvent, ) ([]types.StateEntry, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.LoadCombinedStateAfterEvents") + defer span.Finish() + stateNIDs := make([]types.StateSnapshotNID, len(prevStates)) for i, state := range prevStates { stateNIDs[i] = state.BeforeStateSnapshotNID @@ -194,6 +205,9 @@ func (v *StateResolution) LoadCombinedStateAfterEvents( func (v *StateResolution) DifferenceBetweeenStateSnapshots( ctx context.Context, oldStateNID, newStateNID types.StateSnapshotNID, ) (removed, added []types.StateEntry, err error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.DifferenceBetweeenStateSnapshots") + defer span.Finish() + if oldStateNID == newStateNID { // If the snapshot NIDs are the same then nothing has changed return nil, nil, nil @@ -255,6 +269,9 @@ func (v *StateResolution) LoadStateAtSnapshotForStringTuples( stateNID types.StateSnapshotNID, stateKeyTuples []gomatrixserverlib.StateKeyTuple, ) ([]types.StateEntry, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.LoadStateAtSnapshotForStringTuples") + defer span.Finish() + numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples) if err != nil { return nil, err @@ -269,6 +286,9 @@ func (v *StateResolution) stringTuplesToNumericTuples( ctx context.Context, stringTuples []gomatrixserverlib.StateKeyTuple, ) ([]types.StateKeyTuple, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.stringTuplesToNumericTuples") + defer span.Finish() + eventTypes := make([]string, len(stringTuples)) stateKeys := make([]string, len(stringTuples)) for i := range stringTuples { @@ -311,6 +331,9 @@ func (v *StateResolution) loadStateAtSnapshotForNumericTuples( stateNID types.StateSnapshotNID, stateKeyTuples []types.StateKeyTuple, ) ([]types.StateEntry, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.loadStateAtSnapshotForNumericTuples") + defer span.Finish() + stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) if err != nil { return nil, err @@ -359,6 +382,9 @@ func (v *StateResolution) LoadStateAfterEventsForStringTuples( prevStates []types.StateAtEvent, stateKeyTuples []gomatrixserverlib.StateKeyTuple, ) ([]types.StateEntry, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.LoadStateAfterEventsForStringTuples") + defer span.Finish() + numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples) if err != nil { return nil, err @@ -371,6 +397,9 @@ func (v *StateResolution) loadStateAfterEventsForNumericTuples( prevStates []types.StateAtEvent, stateKeyTuples []types.StateKeyTuple, ) ([]types.StateEntry, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.loadStateAfterEventsForNumericTuples") + defer span.Finish() + if len(prevStates) == 1 { // Fast path for a single event. prevState := prevStates[0] @@ -543,6 +572,9 @@ func (v *StateResolution) CalculateAndStoreStateBeforeEvent( event *gomatrixserverlib.Event, isRejected bool, ) (types.StateSnapshotNID, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.CalculateAndStoreStateBeforeEvent") + defer span.Finish() + // Load the state at the prev events. prevStates, err := v.db.StateAtEventIDs(ctx, event.PrevEventIDs()) if err != nil { @@ -559,6 +591,9 @@ func (v *StateResolution) CalculateAndStoreStateAfterEvents( ctx context.Context, prevStates []types.StateAtEvent, ) (types.StateSnapshotNID, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.CalculateAndStoreStateAfterEvents") + defer span.Finish() + metrics := calculateStateMetrics{startTime: time.Now(), prevEventLength: len(prevStates)} if len(prevStates) == 0 { @@ -631,6 +666,9 @@ func (v *StateResolution) calculateAndStoreStateAfterManyEvents( prevStates []types.StateAtEvent, metrics calculateStateMetrics, ) (types.StateSnapshotNID, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.calculateAndStoreStateAfterManyEvents") + defer span.Finish() + state, algorithm, conflictLength, err := v.calculateStateAfterManyEvents(ctx, v.roomInfo.RoomVersion, prevStates) metrics.algorithm = algorithm @@ -649,6 +687,9 @@ func (v *StateResolution) calculateStateAfterManyEvents( ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, prevStates []types.StateAtEvent, ) (state []types.StateEntry, algorithm string, conflictLength int, err error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.calculateStateAfterManyEvents") + defer span.Finish() + var combined []types.StateEntry // Conflict resolution. // First stage: load the state after each of the prev events. @@ -701,6 +742,9 @@ func (v *StateResolution) resolveConflicts( ctx context.Context, version gomatrixserverlib.RoomVersion, notConflicted, conflicted []types.StateEntry, ) ([]types.StateEntry, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.resolveConflicts") + defer span.Finish() + stateResAlgo, err := version.StateResAlgorithm() if err != nil { return nil, err @@ -725,6 +769,8 @@ func (v *StateResolution) resolveConflictsV1( ctx context.Context, notConflicted, conflicted []types.StateEntry, ) ([]types.StateEntry, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.resolveConflictsV1") + defer span.Finish() // Load the conflicted events conflictedEvents, eventIDMap, err := v.loadStateEvents(ctx, conflicted) @@ -788,6 +834,9 @@ func (v *StateResolution) resolveConflictsV2( ctx context.Context, notConflicted, conflicted []types.StateEntry, ) ([]types.StateEntry, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.resolveConflictsV2") + defer span.Finish() + estimate := len(conflicted) + len(notConflicted) eventIDMap := make(map[string]types.StateEntry, estimate) @@ -815,31 +864,47 @@ func (v *StateResolution) resolveConflictsV2( authEvents := make([]*gomatrixserverlib.Event, 0, estimate*3) gotAuthEvents := make(map[string]struct{}, estimate*3) authDifference := make([]*gomatrixserverlib.Event, 0, estimate) + knownAuthEvents := make(map[string]types.Event, estimate*3) // For each conflicted event, let's try and get the needed auth events. - for _, conflictedEvent := range conflictedEvents { - // Work out which auth events we need to load. - key := conflictedEvent.EventID() + if err = func() error { + span, sctx := opentracing.StartSpanFromContext(ctx, "StateResolution.loadAuthEvents") + defer span.Finish() - // Store the newly found auth events in the auth set for this event. - var authEventMap map[string]types.StateEntry - authSets[key], authEventMap, err = v.loadAuthEvents(ctx, conflictedEvent) - if err != nil { - return nil, err - } - for k, v := range authEventMap { - eventIDMap[k] = v + loader := authEventLoader{ + v: v, + lookupFromDB: make([]string, 0, len(conflictedEvents)*3), + lookupFromMem: make([]string, 0, len(conflictedEvents)*3), + lookedUpEvents: make([]types.Event, 0, len(conflictedEvents)*3), + eventMap: map[string]types.Event{}, } + for _, conflictedEvent := range conflictedEvents { + // Work out which auth events we need to load. + key := conflictedEvent.EventID() - // Only add auth events into the authEvents slice once, otherwise the - // check for the auth difference can become expensive and produce - // duplicate entries, which just waste memory and CPU time. - for _, event := range authSets[key] { - if _, ok := gotAuthEvents[event.EventID()]; !ok { - authEvents = append(authEvents, event) - gotAuthEvents[event.EventID()] = struct{}{} + // Store the newly found auth events in the auth set for this event. + var authEventMap map[string]types.StateEntry + authSets[key], authEventMap, err = loader.loadAuthEvents(sctx, conflictedEvent, knownAuthEvents) + if err != nil { + return err + } + for k, v := range authEventMap { + eventIDMap[k] = v + } + + // Only add auth events into the authEvents slice once, otherwise the + // check for the auth difference can become expensive and produce + // duplicate entries, which just waste memory and CPU time. + for _, event := range authSets[key] { + if _, ok := gotAuthEvents[event.EventID()]; !ok { + authEvents = append(authEvents, event) + gotAuthEvents[event.EventID()] = struct{}{} + } } } + return nil + }(); err != nil { + return nil, err } // Kill the reference to this so that the GC may pick it up, since we no @@ -870,19 +935,29 @@ func (v *StateResolution) resolveConflictsV2( // Look through all of the auth events that we've been given and work out if // there are any events which don't appear in all of the auth sets. If they // don't then we add them to the auth difference. - for _, event := range authEvents { - if !isInAllAuthLists(event) { - authDifference = append(authDifference, event) + func() { + span, _ := opentracing.StartSpanFromContext(ctx, "isInAllAuthLists") + defer span.Finish() + + for _, event := range authEvents { + if !isInAllAuthLists(event) { + authDifference = append(authDifference, event) + } } - } + }() // Resolve the conflicts. - resolvedEvents := gomatrixserverlib.ResolveStateConflictsV2( - conflictedEvents, - nonConflictedEvents, - authEvents, - authDifference, - ) + resolvedEvents := func() []*gomatrixserverlib.Event { + span, _ := opentracing.StartSpanFromContext(ctx, "gomatrixserverlib.ResolveStateConflictsV2") + defer span.Finish() + + return gomatrixserverlib.ResolveStateConflictsV2( + conflictedEvents, + nonConflictedEvents, + authEvents, + authDifference, + ) + }() // Map from the full events back to numeric state entries. for _, resolvedEvent := range resolvedEvents { @@ -947,6 +1022,9 @@ func (v *StateResolution) stateKeyTuplesNeeded(stateKeyNIDMap map[string]types.E func (v *StateResolution) loadStateEvents( ctx context.Context, entries []types.StateEntry, ) ([]*gomatrixserverlib.Event, map[string]types.StateEntry, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.loadStateEvents") + defer span.Finish() + result := make([]*gomatrixserverlib.Event, 0, len(entries)) eventEntries := make([]types.StateEntry, 0, len(entries)) eventNIDs := make([]types.EventNID, 0, len(entries)) @@ -975,43 +1053,86 @@ func (v *StateResolution) loadStateEvents( return result, eventIDMap, nil } +type authEventLoader struct { + sync.Mutex + v *StateResolution + lookupFromDB []string // scratch space + lookupFromMem []string // scratch space + lookedUpEvents []types.Event // scratch space + eventMap map[string]types.Event +} + // loadAuthEvents loads all of the auth events for a given event recursively, // along with a map that contains state entries for all of the auth events. -func (v *StateResolution) loadAuthEvents( - ctx context.Context, event *gomatrixserverlib.Event, +func (l *authEventLoader) loadAuthEvents( + ctx context.Context, event *gomatrixserverlib.Event, eventMap map[string]types.Event, ) ([]*gomatrixserverlib.Event, map[string]types.StateEntry, error) { - eventMap := map[string]struct{}{} - var lookup []string - var authEvents []types.Event + l.Lock() + defer l.Unlock() + authEvents := []types.Event{} // our returned list + included := map[string]struct{}{} // dedupes authEvents above queue := event.AuthEventIDs() for i := 0; i < len(queue); i++ { - lookup = lookup[:0] + // Reuse the same underlying memory, since it reduces the + // amount of allocations we make the more times we call + // loadAuthEvents. + l.lookupFromDB = l.lookupFromDB[:0] + l.lookupFromMem = l.lookupFromMem[:0] + l.lookedUpEvents = l.lookedUpEvents[:0] + + // Separate out the list of events in the queue based on if + // we think we already know the event in memory or not. for _, authEventID := range queue { - if _, ok := eventMap[authEventID]; ok { + if _, ok := included[authEventID]; ok { continue } - lookup = append(lookup, authEventID) + if _, ok := eventMap[authEventID]; ok { + l.lookupFromMem = append(l.lookupFromMem, authEventID) + } else { + l.lookupFromDB = append(l.lookupFromDB, authEventID) + } } - if len(lookup) == 0 { + // If there's nothing to do, stop here. + if len(l.lookupFromDB) == 0 && len(l.lookupFromMem) == 0 { break } - events, err := v.db.EventsFromIDs(ctx, lookup) - if err != nil { - return nil, nil, fmt.Errorf("v.db.EventsFromIDs: %w", err) + + // If we need to get events from the database, go and fetch + // those now. + if len(l.lookupFromDB) > 0 { + eventsFromDB, err := l.v.db.EventsFromIDs(ctx, l.lookupFromDB) + if err != nil { + return nil, nil, fmt.Errorf("v.db.EventsFromIDs: %w", err) + } + l.lookedUpEvents = append(l.lookedUpEvents, eventsFromDB...) + for _, event := range eventsFromDB { + eventMap[event.EventID()] = event + } } + + // Fill in the gaps with events that we already have in memory. + if len(l.lookupFromMem) > 0 { + for _, eventID := range l.lookupFromMem { + l.lookedUpEvents = append(l.lookedUpEvents, eventMap[eventID]) + } + } + + // From the events that we've retrieved, work out which auth + // events to look up on the next iteration. add := map[string]struct{}{} - for _, event := range events { - eventMap[event.EventID()] = struct{}{} + for _, event := range l.lookedUpEvents { authEvents = append(authEvents, event) + included[event.EventID()] = struct{}{} + for _, authEventID := range event.AuthEventIDs() { - if _, ok := eventMap[authEventID]; ok { + if _, ok := included[authEventID]; ok { continue } add[authEventID] = struct{}{} } - for authEventID := range add { - queue = append(queue, authEventID) - } + } + for authEventID := range add { + queue = append(queue, authEventID) } } authEventTypes := map[string]struct{}{} @@ -1028,11 +1149,11 @@ func (v *StateResolution) loadAuthEvents( for eventStateKey := range authEventStateKeys { lookupAuthEventStateKeys = append(lookupAuthEventStateKeys, eventStateKey) } - eventTypes, err := v.db.EventTypeNIDs(ctx, lookupAuthEventTypes) + eventTypes, err := l.v.db.EventTypeNIDs(ctx, lookupAuthEventTypes) if err != nil { return nil, nil, fmt.Errorf("v.db.EventTypeNIDs: %w", err) } - eventStateKeys, err := v.db.EventStateKeyNIDs(ctx, lookupAuthEventStateKeys) + eventStateKeys, err := l.v.db.EventStateKeyNIDs(ctx, lookupAuthEventStateKeys) if err != nil { return nil, nil, fmt.Errorf("v.db.EventStateKeyNIDs: %w", err) } From 6d4bd5d890eeab47bddfad5a48d37766f954171f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 7 Jun 2022 14:24:04 +0100 Subject: [PATCH 06/25] Rate limiting changes (#2519) * Rate limiting changes This makes the following changes: * For logged in users, the rate limiting now applies to the device session rather than the remote IP address; * For non-logged in users, the rate limiting continues to apply to remote address as it does today; * It is now possible to add user IDs to the `exempt_user_ids` option under `rate_limiting` to exclude bots from rate limiting; * Admin and appservice users are now exempt from rate limiting by default. * Fix build with media API --- clientapi/routing/routing.go | 50 +++++++++++++++--------------- dendrite-sample.monolith.yaml | 5 ++- dendrite-sample.polylith.yaml | 5 ++- internal/httputil/rate_limiting.go | 31 +++++++++++++++--- mediaapi/routing/routing.go | 6 ++-- setup/config/config_clientapi.go | 4 +++ 6 files changed, 67 insertions(+), 34 deletions(-) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index fab45fdf6..aa4b5a235 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -140,7 +140,7 @@ func Setup( synapseAdminRouter.Handle("/admin/v1/send_server_notice/{txnID}", httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { // not specced, but ensure we're rate limiting requests to this endpoint - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -160,7 +160,7 @@ func Setup( synapseAdminRouter.Handle("/admin/v1/send_server_notice", httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { // not specced, but ensure we're rate limiting requests to this endpoint - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } return SendServerNotice( @@ -190,7 +190,7 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/join/{roomIDOrAlias}", httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -206,7 +206,7 @@ func Setup( if mscCfg.Enabled("msc2753") { v3mux.Handle("/peek/{roomIDOrAlias}", httputil.MakeAuthAPI(gomatrixserverlib.Peek, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -226,7 +226,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/join", httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -240,7 +240,7 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/leave", httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -274,7 +274,7 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/invite", httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -392,14 +392,14 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, nil); r != nil { return *r } return Register(req, userAPI, cfg) })).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, nil); r != nil { return *r } return RegisterAvailable(req, cfg, userAPI) @@ -473,7 +473,7 @@ func Setup( v3mux.Handle("/rooms/{roomID}/typing/{userID}", httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -530,7 +530,7 @@ func Setup( v3mux.Handle("/account/whoami", httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } return Whoami(req, device) @@ -539,7 +539,7 @@ func Setup( v3mux.Handle("/account/password", httputil.MakeAuthAPI("password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } return Password(req, userAPI, device, cfg) @@ -548,7 +548,7 @@ func Setup( v3mux.Handle("/account/deactivate", httputil.MakeAuthAPI("deactivate", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } return Deactivate(req, userInteractiveAuth, userAPI, device) @@ -559,7 +559,7 @@ func Setup( v3mux.Handle("/login", httputil.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, nil); r != nil { return *r } return Login(req, userAPI, cfg) @@ -667,7 +667,7 @@ func Setup( v3mux.Handle("/pushrules/{scope}/{kind}/{ruleID}", httputil.MakeAuthAPI("push_rules", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -733,7 +733,7 @@ func Setup( v3mux.Handle("/profile/{userID}/avatar_url", httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -758,7 +758,7 @@ func Setup( v3mux.Handle("/profile/{userID}/displayname", httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -797,7 +797,7 @@ func Setup( v3mux.Handle("/voip/turnServer", httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } return RequestTurnServer(req, device, cfg) @@ -876,7 +876,7 @@ func Setup( v3mux.Handle("/user/{userID}/openid/request_token", httputil.MakeAuthAPI("openid_request_token", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -889,7 +889,7 @@ func Setup( v3mux.Handle("/user_directory/search", httputil.MakeAuthAPI("userdirectory_search", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } postContent := struct { @@ -935,7 +935,7 @@ func Setup( v3mux.Handle("/rooms/{roomID}/read_markers", httputil.MakeAuthAPI("rooms_read_markers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -948,7 +948,7 @@ func Setup( v3mux.Handle("/rooms/{roomID}/forget", httputil.MakeAuthAPI("rooms_forget", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -1025,7 +1025,7 @@ func Setup( v3mux.Handle("/pushers/set", httputil.MakeAuthAPI("set_pushers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } return SetPusher(req, device, userAPI) @@ -1083,7 +1083,7 @@ func Setup( v3mux.Handle("/capabilities", httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } return GetCapabilities(req, rsAPI) @@ -1299,7 +1299,7 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}", httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml index e974dbcba..ce5da2788 100644 --- a/dendrite-sample.monolith.yaml +++ b/dendrite-sample.monolith.yaml @@ -160,11 +160,14 @@ client_api: # Settings for rate-limited endpoints. Rate limiting kicks in after the threshold # number of "slots" have been taken by requests from a specific host. Each "slot" - # will be released after the cooloff time in milliseconds. + # will be released after the cooloff time in milliseconds. Server administrators + # and appservice users are exempt from rate limiting by default. rate_limiting: enabled: true threshold: 5 cooloff_ms: 500 + exempt_user_ids: + # - @user:domain.com # Configuration for the Federation API. federation_api: diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml index 4b67aaa94..439f09b0a 100644 --- a/dendrite-sample.polylith.yaml +++ b/dendrite-sample.polylith.yaml @@ -163,11 +163,14 @@ client_api: # Settings for rate-limited endpoints. Rate limiting kicks in after the threshold # number of "slots" have been taken by requests from a specific host. Each "slot" - # will be released after the cooloff time in milliseconds. + # will be released after the cooloff time in milliseconds. Server administrators + # and appservice users are exempt from rate limiting by default. rate_limiting: enabled: true threshold: 5 cooloff_ms: 500 + exempt_user_ids: + # - @user:domain.com # Configuration for the Federation API. federation_api: diff --git a/internal/httputil/rate_limiting.go b/internal/httputil/rate_limiting.go index c4f47c7b5..dab36481e 100644 --- a/internal/httputil/rate_limiting.go +++ b/internal/httputil/rate_limiting.go @@ -7,6 +7,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/setup/config" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" ) @@ -17,6 +18,7 @@ type RateLimits struct { enabled bool requestThreshold int64 cooloffDuration time.Duration + exemptUserIDs map[string]struct{} } func NewRateLimits(cfg *config.RateLimiting) *RateLimits { @@ -25,6 +27,10 @@ func NewRateLimits(cfg *config.RateLimiting) *RateLimits { enabled: cfg.Enabled, requestThreshold: cfg.Threshold, cooloffDuration: time.Duration(cfg.CooloffMS) * time.Millisecond, + exemptUserIDs: map[string]struct{}{}, + } + for _, userID := range cfg.ExemptUserIDs { + l.exemptUserIDs[userID] = struct{}{} } if l.enabled { go l.clean() @@ -52,7 +58,7 @@ func (l *RateLimits) clean() { } } -func (l *RateLimits) Limit(req *http.Request) *util.JSONResponse { +func (l *RateLimits) Limit(req *http.Request, device *userapi.Device) *util.JSONResponse { // If rate limiting is disabled then do nothing. if !l.enabled { return nil @@ -67,9 +73,26 @@ func (l *RateLimits) Limit(req *http.Request) *util.JSONResponse { // First of all, work out if X-Forwarded-For was sent to us. If not // then we'll just use the IP address of the caller. - caller := req.RemoteAddr - if forwardedFor := req.Header.Get("X-Forwarded-For"); forwardedFor != "" { - caller = forwardedFor + var caller string + if device != nil { + switch device.AccountType { + case userapi.AccountTypeAdmin: + return nil // don't rate-limit server administrators + case userapi.AccountTypeAppService: + return nil // don't rate-limit appservice users + default: + if _, ok := l.exemptUserIDs[device.UserID]; ok { + // If the user is exempt from rate limiting then do nothing. + return nil + } + caller = device.UserID + device.ID + } + } else { + if forwardedFor := req.Header.Get("X-Forwarded-For"); forwardedFor != "" { + caller = forwardedFor + } else { + caller = req.RemoteAddr + } } // Look up the caller's channel, if they have one. diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 76f07415b..196908184 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -62,7 +62,7 @@ func Setup( uploadHandler := httputil.MakeAuthAPI( "upload", userAPI, func(req *http.Request, dev *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, dev); r != nil { return *r } return Upload(req, cfg, dev, db, activeThumbnailGeneration) @@ -70,7 +70,7 @@ func Setup( ) configHandler := httputil.MakeAuthAPI("config", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, device); r != nil { return *r } respondSize := &cfg.MaxFileSizeBytes @@ -126,7 +126,7 @@ func makeDownloadAPI( // Ratelimit requests // NOTSPEC: The spec says everything at /media/ should be rate limited, but this causes issues with thumbnails (#2243) if name != "thumbnail" { - if r := rateLimits.Limit(req); r != nil { + if r := rateLimits.Limit(req, nil); r != nil { if err := json.NewEncoder(w).Encode(r); err != nil { w.WriteHeader(http.StatusInternalServerError) return diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go index bb786a145..ecf8f6bd5 100644 --- a/setup/config/config_clientapi.go +++ b/setup/config/config_clientapi.go @@ -134,6 +134,10 @@ type RateLimiting struct { // The cooloff period in milliseconds after a request before the "slot" // is freed again CooloffMS int64 `yaml:"cooloff_ms"` + + // A list of users that are exempt from rate limiting, i.e. if you want + // to run Mjolnir or other bots. + ExemptUserIDs []string `yaml:"exempt_user_ids"` } func (r *RateLimiting) Verify(configErrs *ConfigErrors) { From b21a2223ef337c0f53cddfcba0b2e9ea923cd296 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 7 Jun 2022 15:04:00 +0100 Subject: [PATCH 07/25] Optimise state res v2 by parsing power level content less often (update to matrix-org/gomatrixserverlib@e938c07) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c9a7e09b0..6a91b565a 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220531163017-35e1cabf12ee + github.com/matrix-org/gomatrixserverlib v0.0.0-20220607140329-e938c0797d5e github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.13 diff --git a/go.sum b/go.sum index 4460b3905..1953f9dec 100644 --- a/go.sum +++ b/go.sum @@ -418,8 +418,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220531163017-35e1cabf12ee h1:56sxEWrwB3eOmwjP2S4JsrQf29uBUaf+8WrbQJmjaGE= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220531163017-35e1cabf12ee/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220607140329-e938c0797d5e h1:fOMXFu4H0PpFtjocRx/EHxdTyQwQ2kZPA8lDTWYN6kU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220607140329-e938c0797d5e/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE= github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From c4df6d77231358f161e5243e62af6510258943a5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 7 Jun 2022 15:34:58 +0100 Subject: [PATCH 08/25] Optimise state res v2 by only updating the allower when necessary (update to matrix-org/gomatrixserverlib@e55d796) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6a91b565a..ea6e8caeb 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220607140329-e938c0797d5e + github.com/matrix-org/gomatrixserverlib v0.0.0-20220607143425-e55d796fd0b3 github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.13 diff --git a/go.sum b/go.sum index 1953f9dec..e21794f41 100644 --- a/go.sum +++ b/go.sum @@ -418,8 +418,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220607140329-e938c0797d5e h1:fOMXFu4H0PpFtjocRx/EHxdTyQwQ2kZPA8lDTWYN6kU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220607140329-e938c0797d5e/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220607143425-e55d796fd0b3 h1:2eYcBt8Kg+nW/xIJY5x8Uo2dQLjUF+oxLap00uFC5l8= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220607143425-e55d796fd0b3/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE= github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From 3cdefcf765f0373209e10b4a4085a7b41667868d Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 8 Jun 2022 09:13:25 +0100 Subject: [PATCH 09/25] Update sample configs for `exempt_user_ids` --- dendrite-sample.monolith.yaml | 2 +- dendrite-sample.polylith.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml index ce5da2788..c0134c542 100644 --- a/dendrite-sample.monolith.yaml +++ b/dendrite-sample.monolith.yaml @@ -167,7 +167,7 @@ client_api: threshold: 5 cooloff_ms: 500 exempt_user_ids: - # - @user:domain.com + # - "@user:domain.com" # Configuration for the Federation API. federation_api: diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml index 439f09b0a..b9e3e5e56 100644 --- a/dendrite-sample.polylith.yaml +++ b/dendrite-sample.polylith.yaml @@ -170,7 +170,7 @@ client_api: threshold: 5 cooloff_ms: 500 exempt_user_ids: - # - @user:domain.com + # - "@user:domain.com" # Configuration for the Federation API. federation_api: From 289b3c56082e243b2d9674c650d2460226013aa1 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 9 Jun 2022 12:26:48 +0200 Subject: [PATCH 10/25] Allow "registration is idempotent, with username specified" to pass (#2488) Co-authored-by: Neil Alexander --- clientapi/routing/register.go | 57 +++++++++++++++++++++++------------ sytest-whitelist | 3 +- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index eba4920c6..c4ac0f2e7 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -29,9 +29,10 @@ import ( "sync" "time" + "github.com/tidwall/gjson" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/setup/config" - "github.com/tidwall/gjson" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/tokens" @@ -68,9 +69,10 @@ const ( // It shouldn't be passed by value because it contains a mutex. type sessionsDict struct { sync.RWMutex - sessions map[string][]authtypes.LoginType - params map[string]registerRequest - timer map[string]*time.Timer + sessions map[string][]authtypes.LoginType + sessionCompletedResult map[string]registerResponse + params map[string]registerRequest + timer map[string]*time.Timer // deleteSessionToDeviceID protects requests to DELETE /devices/{deviceID} from being abused. // If a UIA session is started by trying to delete device1, and then UIA is completed by deleting device2, // the delete request will fail for device2 since the UIA was initiated by trying to delete device1. @@ -115,6 +117,7 @@ func (d *sessionsDict) deleteSession(sessionID string) { delete(d.params, sessionID) delete(d.sessions, sessionID) delete(d.deleteSessionToDeviceID, sessionID) + delete(d.sessionCompletedResult, sessionID) // stop the timer, e.g. because the registration was completed if t, ok := d.timer[sessionID]; ok { if !t.Stop() { @@ -130,6 +133,7 @@ func (d *sessionsDict) deleteSession(sessionID string) { func newSessionsDict() *sessionsDict { return &sessionsDict{ sessions: make(map[string][]authtypes.LoginType), + sessionCompletedResult: make(map[string]registerResponse), params: make(map[string]registerRequest), timer: make(map[string]*time.Timer), deleteSessionToDeviceID: make(map[string]string), @@ -173,6 +177,19 @@ func (d *sessionsDict) addDeviceToDelete(sessionID, deviceID string) { d.deleteSessionToDeviceID[sessionID] = deviceID } +func (d *sessionsDict) addCompletedRegistration(sessionID string, response registerResponse) { + d.Lock() + defer d.Unlock() + d.sessionCompletedResult[sessionID] = response +} + +func (d *sessionsDict) getCompletedRegistration(sessionID string) (registerResponse, bool) { + d.RLock() + defer d.RUnlock() + result, ok := d.sessionCompletedResult[sessionID] + return result, ok +} + func (d *sessionsDict) getDeviceToDelete(sessionID string) (string, bool) { d.RLock() defer d.RUnlock() @@ -544,6 +561,14 @@ func Register( r.DeviceID = data.DeviceID r.InitialDisplayName = data.InitialDisplayName r.InhibitLogin = data.InhibitLogin + // Check if the user already registered using this session, if so, return that result + if response, ok := sessions.getCompletedRegistration(sessionID); ok { + return util.JSONResponse{ + Code: http.StatusOK, + JSON: response, + } + } + } if resErr := httputil.UnmarshalJSON(reqBody, &r); resErr != nil { return *resErr @@ -839,13 +864,6 @@ func completeRegistration( displayName, deviceID *string, accType userapi.AccountType, ) util.JSONResponse { - var registrationOK bool - defer func() { - if registrationOK { - sessions.deleteSession(sessionID) - } - }() - if username == "" { return util.JSONResponse{ Code: http.StatusBadRequest, @@ -886,7 +904,6 @@ func completeRegistration( // Check whether inhibit_login option is set. If so, don't create an access // token or a device for this user if inhibitLogin { - registrationOK = true return util.JSONResponse{ Code: http.StatusOK, JSON: registerResponse{ @@ -920,15 +937,17 @@ func completeRegistration( } } - registrationOK = true + result := registerResponse{ + UserID: devRes.Device.UserID, + AccessToken: devRes.Device.AccessToken, + HomeServer: accRes.Account.ServerName, + DeviceID: devRes.Device.ID, + } + sessions.addCompletedRegistration(sessionID, result) + return util.JSONResponse{ Code: http.StatusOK, - JSON: registerResponse{ - UserID: devRes.Device.UserID, - AccessToken: devRes.Device.AccessToken, - HomeServer: accRes.Account.ServerName, - DeviceID: devRes.Device.ID, - }, + JSON: result, } } diff --git a/sytest-whitelist b/sytest-whitelist index 5f6797a3e..21bbc396b 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -716,6 +716,7 @@ PUT /rooms/:room_id/redact/:event_id/:txn_id is idempotent Unnamed room comes with a name summary Named room comes with just joined member count summary Room summary only has 5 heroes +registration is idempotent, with username specified Setting state twice is idempotent Joining room twice is idempotent -Inbound federation can return missing events for shared visibility \ No newline at end of file +Inbound federation can return missing events for shared visibility From 83797573be87616bea0644918b82e2b20e8b78ca Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 9 Jun 2022 12:18:50 +0100 Subject: [PATCH 11/25] Version 0.8.8 (#2525) --- CHANGES.md | 17 +++++++++++++++++ internal/version.go | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index a09a80148..0db25f05a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,22 @@ # Changelog +## Dendrite 0.8.8 (2022-06-09) + +### Features + +* The performance of state resolution has been increased significantly for larger rooms +* A number of changes have been made to rate limiting: + * Logged in users will now be rate-limited on a per-session basis rather than by remote IP + * Rate limiting no longer applies to admin or appservice users + * It is now possible to configure additional users that are exempt from rate limiting using the `exempt_user_ids` option in the `rate_limiting` section of the Dendrite config +* Setting state is now idempotent via the client API state endpoints + +### Fixes + +* Room upgrades now properly propagate tombstone events to remote servers +* Room upgrades will no longer send tombstone events if creating the upgraded room fails +* A crash has been fixed when evaluating restricted room joins + ## Dendrite 0.8.7 (2022-06-01) ### Features diff --git a/internal/version.go b/internal/version.go index 2543ec90c..e29996f36 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 8 - VersionPatch = 7 + VersionPatch = 8 VersionTag = "" // example: "rc1" ) From 660f7839f52f319a205dc61b96e43c730e5cb91a Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 9 Jun 2022 18:38:07 +0200 Subject: [PATCH 12/25] Correctly redact events over federation (#2526) * Ensure we check powerlevel/origin before redacting an event * Add passing test * Use pl.UserLevel * Make check more readable, also check for the sender --- roomserver/storage/shared/storage.go | 30 ++++++++++++++++++++++++++-- sytest-whitelist | 1 + 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index cc4a9fff5..67dcfdf38 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -823,13 +823,39 @@ func (d *Database) handleRedactions( return nil, "", nil } + // Get the power level from the database, so we can verify the user is allowed to redact the event + powerLevels, err := d.GetStateEvent(ctx, event.RoomID(), gomatrixserverlib.MRoomPowerLevels, "") + if err != nil { + return nil, "", fmt.Errorf("d.GetStateEvent: %w", err) + } + pl, err := powerLevels.PowerLevels() + if err != nil { + return nil, "", fmt.Errorf("unable to get powerlevels for room: %w", err) + } + + redactUser := pl.UserLevel(redactionEvent.Sender()) + switch { + case redactUser >= pl.Redact: + // The power level of the redaction event’s sender is greater than or equal to the redact level. + case redactedEvent.Origin() == redactionEvent.Origin() && redactedEvent.Sender() == redactionEvent.Sender(): + // The domain of the redaction event’s sender matches that of the original event’s sender. + default: + return nil, "", nil + } + // mark the event as redacted + if redactionsArePermanent { + redactedEvent.Event = redactedEvent.Redact() + } + err = redactedEvent.SetUnsignedField("redacted_because", redactionEvent) if err != nil { return nil, "", fmt.Errorf("redactedEvent.SetUnsignedField: %w", err) } - if redactionsArePermanent { - redactedEvent.Event = redactedEvent.Redact() + // NOTSPEC: sytest relies on this unspecced field existing :( + err = redactedEvent.SetUnsignedField("redacted_by", redactionEvent.EventID()) + if err != nil { + return nil, "", fmt.Errorf("redactedEvent.SetUnsignedField: %w", err) } // overwrite the eventJSON table err = d.EventJSONTable.InsertEventJSON(ctx, txn, redactedEvent.EventNID, redactedEvent.JSON()) diff --git a/sytest-whitelist b/sytest-whitelist index 21bbc396b..60a3b73f6 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -720,3 +720,4 @@ registration is idempotent, with username specified Setting state twice is idempotent Joining room twice is idempotent Inbound federation can return missing events for shared visibility +Inbound federation ignores redactions from invalid servers room > v3 \ No newline at end of file From a37b64c9f12598fdcc3134b90cfc86702fd881c3 Mon Sep 17 00:00:00 2001 From: Tak Wai Wong <64229756+tak-hntlabs@users.noreply.github.com> Date: Thu, 9 Jun 2022 13:03:04 -0400 Subject: [PATCH 13/25] Implement EIP-4361 sign in with Ethereum (#5) * Blacklist some sytest tests that are failing in our environment * Commenting out test that isn't reliably passing or failing, probably a race * refresh latest dendrite main * refresh latest dendrite main * dendrite implementation of eip-4361 * simplify nonce generation Co-authored-by: Brian Meek Co-authored-by: Tak Wai Wong --- clientapi/auth/login_publickey_ethereum.go | 149 +++------------------ clientapi/auth/login_test.go | 18 ++- clientapi/auth/user_interactive.go | 55 +++++--- clientapi/auth/user_interactive_test.go | 13 +- clientapi/routing/deactivate.go | 2 +- clientapi/routing/device.go | 2 +- clientapi/routing/register_publickey.go | 7 + dendrite-sample.monolith.yaml | 10 ++ go.mod | 44 +----- go.sum | 109 ++++++++------- internal/mapsutil/maps.go | 15 +++ setup/config/config.go | 2 +- setup/config/config_clientapi.go | 43 ------ setup/config/config_publickey.go | 81 +++++++++++ 14 files changed, 245 insertions(+), 305 deletions(-) create mode 100644 setup/config/config_publickey.go diff --git a/clientapi/auth/login_publickey_ethereum.go b/clientapi/auth/login_publickey_ethereum.go index 2a8cd78cb..938a9f816 100644 --- a/clientapi/auth/login_publickey_ethereum.go +++ b/clientapi/auth/login_publickey_ethereum.go @@ -16,51 +16,28 @@ package auth import ( "context" - "encoding/base64" "encoding/json" - "errors" - "regexp" "strings" - "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/crypto" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/tidwall/gjson" + "github.com/spruceid/siwe-go" ) type LoginPublicKeyEthereum struct { // https://github.com/tak-hntlabs/matrix-spec-proposals/blob/main/proposals/3782-matrix-publickey-login-spec.md#client-sends-login-request-with-authentication-data - Type string `json:"type"` - Address string `json:"address"` - Session string `json:"session"` - Message string `json:"message"` - Signature string `json:"signature"` - HashFields publicKeyEthereumHashFields `json:"hashFields"` - HashFieldsRaw string // Raw base64 encoded string of MessageFields for hash verification + Type string `json:"type"` + Address string `json:"address"` + Session string `json:"session"` + Message string `json:"message"` + Signature string `json:"signature"` userAPI userapi.ClientUserAPI config *config.ClientAPI } -type publicKeyEthereumHashFields struct { - // Todo: See https://... - Domain string `json:"domain"` // home server domain - Address string `json:"address"` // Ethereum address. 0x... - Nonce string `json:"nonce"` // session ID - Version string `json:"version"` // version of the Matrix public key spec that the client is complying with - ChainId string `json:"chainId"` // blockchain network ID. -} - -type publicKeyEthereumRequiredFields struct { - From string // Sender - To string // Recipient - Hash string // Hash of JSON representation of the message fields -} - func CreatePublicKeyEthereumHandler( reqBytes []byte, userAPI userapi.ClientUserAPI, @@ -71,15 +48,8 @@ func CreatePublicKeyEthereumHandler( return nil, jsonerror.BadJSON("auth") } - hashFields := gjson.GetBytes(reqBytes, "hashFields") - if !hashFields.Exists() { - return nil, jsonerror.BadJSON("auth.hashFields") - } - pk.config = config pk.userAPI = userAPI - // Save raw bytes for hash verification later. - pk.HashFieldsRaw = hashFields.Raw // Case-insensitive pk.Address = strings.ToLower(pk.Address) @@ -116,41 +86,20 @@ func (pk LoginPublicKeyEthereum) AccountExists(ctx context.Context) (string, *js } func (pk LoginPublicKeyEthereum) ValidateLoginResponse() (bool, *jsonerror.MatrixError) { - // Check signature to verify message was not tempered - isVerified := verifySignature(pk.Address, []byte(pk.Message), pk.Signature) - if !isVerified { - return false, jsonerror.InvalidSignature("") - } - - // Extract the required message fields for validation - requiredFields, err := extractRequiredMessageFields(pk.Message) + // Parse the message to extract all the fields. + message, err := siwe.ParseMessage(pk.Message) if err != nil { - return false, jsonerror.MissingParam("message does not contain domain, address, or hash") + return false, jsonerror.InvalidParam("auth.message") } - // Verify that the hash is valid for the message fields. - if !verifyHash(pk.HashFieldsRaw, requiredFields.Hash) { - return false, jsonerror.Forbidden("error verifying message hash") - } - - // Unmarshal the hashFields for further validation - var authData publicKeyEthereumHashFields - if err := json.Unmarshal([]byte(pk.HashFieldsRaw), &authData); err != nil { - return false, jsonerror.BadJSON("auth.hashFields") - } - - // Error if the message is not from the expected public address - if pk.Address != requiredFields.From || requiredFields.From != pk.HashFields.Address { - return false, jsonerror.Forbidden("address") - } - - // Error if the message is not for the home server - if requiredFields.To != pk.HashFields.Domain { - return false, jsonerror.Forbidden("domain") + // Check signature to verify message was not tempered + _, err = message.Verify(pk.Signature, (*string)(&pk.config.Matrix.ServerName), nil, nil) + if err != nil { + return false, jsonerror.InvalidSignature(err.Error()) } // Error if the chainId is not supported by the server. - if !contains(pk.config.PublicKeyAuthentication.Ethereum.ChainIDs, authData.ChainId) { + if !contains(pk.config.PublicKeyAuthentication.Ethereum.ChainIDs, message.GetChainID()) { return false, jsonerror.Forbidden("chainId") } @@ -169,75 +118,7 @@ func (pk LoginPublicKeyEthereum) CreateLogin() *Login { return &login } -// The required fields in the signed message are: -// 1. Domain -- home server. First non-whitespace characters in the first line. -// 2. Address -- public address of the user. Starts with 0x... in the second line on its own. -// 3. Hash -- Base64-encoded hash string of the metadata that represents the message. -// The rest of the fields are informational, and will be used in the future. -var regexpAuthority = regexp.MustCompile(`^\S+`) -var regexpAddress = regexp.MustCompile(`\n(?P
0x\w+)\n`) -var regexpHash = regexp.MustCompile(`\nHash: (?P.*)\n`) - -func extractRequiredMessageFields(message string) (*publicKeyEthereumRequiredFields, error) { - var requiredFields publicKeyEthereumRequiredFields - /* - service.org wants you to sign in with your account: - 0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2 - - I accept the ServiceOrg Terms of Service: https://service.org/tos - - Hash: yfSIwarByPfKFxeYSCWN3XoIgNgeEFJffbwFA+JxYbA= - */ - - requiredFields.To = regexpAuthority.FindString(message) - - from := regexpAddress.FindStringSubmatch(message) - if len(from) == 2 { - requiredFields.From = from[1] - } - - hash := regexpHash.FindStringSubmatch(message) - if len(hash) == 2 { - requiredFields.Hash = hash[1] - } - - if len(requiredFields.To) == 0 || len(requiredFields.From) == 0 || len(requiredFields.Hash) == 0 { - return nil, errors.New("required message fields are missing") - } - - // Make these fields case-insensitive - requiredFields.From = strings.ToLower(requiredFields.From) - requiredFields.To = strings.ToLower(requiredFields.To) - - return &requiredFields, nil -} - -func verifySignature(from string, message []byte, signature string) bool { - decodedSig := hexutil.MustDecode(signature) - - message = accounts.TextHash(message) - // Issue: https://stackoverflow.com/questions/49085737/geth-ecrecover-invalid-signature-recovery-id - // Fix: https://gist.github.com/dcb9/385631846097e1f59e3cba3b1d42f3ed#file-eth_sign_verify-go - decodedSig[crypto.RecoveryIDOffset] -= 27 // Transform yellow paper V from 27/28 to 0/1 - - recovered, err := crypto.SigToPub(message, decodedSig) - if err != nil { - return false - } - - recoveredAddr := crypto.PubkeyToAddress(*recovered) - - addressStr := strings.ToLower(recoveredAddr.Hex()) - return from == addressStr -} - -func verifyHash(rawStr string, expectedHash string) bool { - hash := crypto.Keccak256([]byte(rawStr)) - hashStr := base64.StdEncoding.EncodeToString(hash) - return expectedHash == hashStr -} - -func contains(list []string, element string) bool { +func contains(list []int, element int) bool { for _, i := range list { if i == element { return true diff --git a/clientapi/auth/login_test.go b/clientapi/auth/login_test.go index 655455515..04e51323d 100644 --- a/clientapi/auth/login_test.go +++ b/clientapi/auth/login_test.go @@ -62,11 +62,10 @@ func TestLoginFromJSONReader(t *testing.T) { }, } userInteractive := UserInteractive{ - Completed: []string{}, - Flows: []userInteractiveFlow{}, - Types: make(map[string]Type), - Sessions: make(map[string][]string), - Params: make(map[string]interface{}), + Flows: []userInteractiveFlow{}, + Types: make(map[string]Type), + Sessions: make(map[string][]string), + Params: make(map[string]interface{}), } for _, tst := range tsts { @@ -148,11 +147,10 @@ func TestBadLoginFromJSONReader(t *testing.T) { }, } userInteractive := UserInteractive{ - Completed: []string{}, - Flows: []userInteractiveFlow{}, - Types: make(map[string]Type), - Sessions: make(map[string][]string), - Params: make(map[string]interface{}), + Flows: []userInteractiveFlow{}, + Types: make(map[string]Type), + Sessions: make(map[string][]string), + Params: make(map[string]interface{}), } for _, tst := range tsts { diff --git a/clientapi/auth/user_interactive.go b/clientapi/auth/user_interactive.go index b047fc299..ff7d2ed90 100644 --- a/clientapi/auth/user_interactive.go +++ b/clientapi/auth/user_interactive.go @@ -20,6 +20,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/internal/mapsutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" @@ -102,8 +103,6 @@ type userInteractiveFlow struct { // the user already has a valid access token, but we want to double-check // that it isn't stolen by re-authenticating them. type UserInteractive struct { - Completed []string - Flows []userInteractiveFlow // Map of login type to implementation Types map[string]Type @@ -118,11 +117,10 @@ func NewUserInteractive( cfg *config.ClientAPI, ) *UserInteractive { userInteractive := UserInteractive{ - Completed: []string{}, - Flows: []userInteractiveFlow{}, - Types: make(map[string]Type), - Sessions: make(map[string][]string), - Params: make(map[string]interface{}), + Flows: []userInteractiveFlow{}, + Types: make(map[string]Type), + Sessions: make(map[string][]string), + Params: make(map[string]interface{}), } if !cfg.PasswordAuthenticationDisabled { @@ -130,12 +128,6 @@ func NewUserInteractive( GetAccountByPassword: userAccountAPI.QueryAccountByPassword, Config: cfg, } - - userInteractive.Flows = append(userInteractive.Flows, userInteractiveFlow{ - Stages: []string{typePassword.Name()}, - }, - ) - userInteractive.Types[typePassword.Name()] = typePassword typePassword.AddFLows(&userInteractive) } @@ -179,13 +171,29 @@ type Challenge struct { // Challenge returns an HTTP 401 with the supported flows for authenticating func (u *UserInteractive) Challenge(sessionID string) *util.JSONResponse { + paramsCopy := mapsutil.MapCopy(u.Params) + for key, element := range paramsCopy { + p := getAuthParams(element) + if p != nil { + // If an auth flow has params, + // send it as part of the challenge. + paramsCopy[key] = p + + // If an auth flow generated a nonce, track it as well. + nonce := getAuthParamNonce(p) + if nonce != "" { + u.Sessions[sessionID] = append(u.Sessions[sessionID], nonce) + } + } + } + return &util.JSONResponse{ Code: 401, JSON: Challenge{ Completed: u.Sessions[sessionID], Flows: u.Flows, Session: sessionID, - Params: u.Params, + Params: paramsCopy, }, } } @@ -229,7 +237,7 @@ func (u *UserInteractive) ResponseWithChallenge(sessionID string, response inter // Verify returns an error/challenge response to send to the client, or nil if the user is authenticated. // `bodyBytes` is the HTTP request body which must contain an `auth` key. // Returns the login that was verified for additional checks if required. -func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte) (*Login, *util.JSONResponse) { +func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *api.Device) (*Login, *util.JSONResponse) { // TODO: rate limit // "A client should first make a request with no auth parameter. The homeserver returns an HTTP 401 response, with a JSON body" @@ -271,3 +279,20 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte) (*Login, // TODO: Check if there's more stages to go and return an error return login, nil } + +func getAuthParams(params interface{}) interface{} { + v, ok := params.(config.AuthParams) + if ok { + p := v.GetParams() + return p + } + return nil +} + +func getAuthParamNonce(p interface{}) string { + v, ok := p.(config.AuthParams) + if ok { + return v.GetNonce() + } + return "" +} diff --git a/clientapi/auth/user_interactive_test.go b/clientapi/auth/user_interactive_test.go index c5521d3ff..3dbb9dabc 100644 --- a/clientapi/auth/user_interactive_test.go +++ b/clientapi/auth/user_interactive_test.go @@ -17,6 +17,11 @@ var ( serverName = gomatrixserverlib.ServerName("example.com") // space separated localpart+password -> account lookup = make(map[string]*api.Account) + device = &api.Device{ + AccessToken: "flibble", + DisplayName: "My Device", + ID: "device_id_goes_here", + } ) type fakeAccountDatabase struct { @@ -55,7 +60,7 @@ func setup() *UserInteractive { func TestUserInteractiveChallenge(t *testing.T) { uia := setup() // no auth key results in a challenge - _, errRes := uia.Verify(ctx, []byte(`{}`)) + _, errRes := uia.Verify(ctx, []byte(`{}`), device) if errRes == nil { t.Fatalf("Verify succeeded with {} but expected failure") } @@ -95,7 +100,7 @@ func TestUserInteractivePasswordLogin(t *testing.T) { }`), } for _, tc := range testCases { - _, errRes := uia.Verify(ctx, tc) + _, errRes := uia.Verify(ctx, tc, device) if errRes != nil { t.Errorf("Verify failed but expected success for request: %s - got %+v", string(tc), errRes) } @@ -176,7 +181,7 @@ func TestUserInteractivePasswordBadLogin(t *testing.T) { }, } for _, tc := range testCases { - _, errRes := uia.Verify(ctx, tc.body) + _, errRes := uia.Verify(ctx, tc.body, device) if errRes == nil { t.Errorf("Verify succeeded but expected failure for request: %s", string(tc.body)) continue @@ -209,7 +214,7 @@ func TestUserInteractive_AddCompletedStage(t *testing.T) { ctx := context.Background() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, resp := u.Verify(ctx, []byte("{}")) + _, resp := u.Verify(ctx, []byte("{}"), nil) challenge, ok := resp.JSON.(Challenge) if !ok { t.Fatalf("expected a Challenge, got %T", resp.JSON) diff --git a/clientapi/routing/deactivate.go b/clientapi/routing/deactivate.go index 34e439bab..c8aa6a3bc 100644 --- a/clientapi/routing/deactivate.go +++ b/clientapi/routing/deactivate.go @@ -28,7 +28,7 @@ func Deactivate( } } - login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes) + login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, deviceAPI) if errRes != nil { return *errRes } diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go index f465392aa..bb1cf47bd 100644 --- a/clientapi/routing/device.go +++ b/clientapi/routing/device.go @@ -198,7 +198,7 @@ func DeleteDeviceById( sessionID = s } - login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes) + login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, device) if errRes != nil { switch data := errRes.JSON.(type) { case auth.Challenge: diff --git a/clientapi/routing/register_publickey.go b/clientapi/routing/register_publickey.go index 46807f41e..c6cd5e30a 100644 --- a/clientapi/routing/register_publickey.go +++ b/clientapi/routing/register_publickey.go @@ -68,6 +68,13 @@ func handlePublicKeyRegistration( return false, "", nil } + if _, ok := sessions.sessions[authHandler.GetSession()]; !ok { + return false, "", &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: jsonerror.Unknown("the session ID is missing or unknown."), + } + } + isCompleted, jerr := authHandler.ValidateLoginResponse() if jerr != nil { return false, "", &util.JSONResponse{ diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml index e974dbcba..3f5222167 100644 --- a/dendrite-sample.monolith.yaml +++ b/dendrite-sample.monolith.yaml @@ -137,6 +137,16 @@ client_api: # of whether registration is otherwise disabled. registration_shared_secret: "" + # Disable password authentication. + password_authentication_disabled: false + + # public key authentication + public_key_authentication: + ethereum: + enabled: false + version: 1 + chain_ids: [] + # Whether to require reCAPTCHA for registration. If you have enabled registration # then this is HIGHLY RECOMMENDED to reduce the risk of your homeserver being used # for coordinated spam attacks. diff --git a/go.mod b/go.mod index d10dfaa6f..bf9b0c846 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/DATA-DOG/go-sqlmock v1.5.0 + github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/MFAshby/stdemuxerhook v1.0.0 github.com/Masterminds/semver/v3 v3.1.1 github.com/Microsoft/go-winio v0.5.1 // indirect @@ -55,6 +56,7 @@ require ( github.com/pressly/goose v2.7.0+incompatible github.com/prometheus/client_golang v1.12.2 github.com/sirupsen/logrus v1.8.1 + github.com/spruceid/siwe-go v0.2.0 github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.7.1 github.com/tidwall/gjson v1.14.1 @@ -77,44 +79,4 @@ require ( nhooyr.io/websocket v1.8.7 ) -require github.com/ethereum/go-ethereum v1.10.18 - -require ( - github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/cheekybits/genny v1.0.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect - github.com/fsnotify/fsnotify v1.4.9 // indirect - github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect - github.com/klauspost/compress v1.14.4 // indirect - github.com/lucas-clemente/quic-go v0.26.0 // indirect - github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect - github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect - github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect - github.com/minio/highwayhash v1.0.2 // indirect - github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a // indirect - github.com/nats-io/nkeys v0.3.0 // indirect - github.com/nats-io/nuid v1.0.1 // indirect - github.com/nxadm/tail v1.4.8 // indirect - github.com/onsi/ginkgo v1.16.4 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.32.1 // indirect - github.com/prometheus/procfs v0.7.3 // indirect - github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect - golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect - golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect - golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect - golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - gopkg.in/macaroon.v2 v2.1.0 // indirect - gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect -) - -go 1.17 +go 1.16 diff --git a/go.sum b/go.sum index dab1f0090..e998760d4 100644 --- a/go.sum +++ b/go.sum @@ -48,12 +48,22 @@ github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98 h1:PsaZb47k7WB1V github.com/Arceliar/ironwood v0.0.0-20220306165321-319147a02d98/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk= github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3ugUxx2EnnWmgba1kCqPkd4Gk1yQ= github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI= -github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= -github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= +github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= +github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= +github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= @@ -75,12 +85,11 @@ github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJ github.com/RyanCarrier/dijkstra v1.0.0/go.mod h1:5agGUBNEtUAGIANmbw09fuO3a2htPEkc1jNH01qxCWA= github.com/RyanCarrier/dijkstra-1 v0.0.0-20170512020943-0e5801a26345/go.mod h1:OK4EvWJ441LQqGzed5NGB6vKBAE34n3z7iayPcEwr30= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/albertorestifo/dijkstra v0.0.0-20160910063646-aba76f725f72/go.mod h1:o+JdB7VetTHjLhU0N57x18B9voDBQe0paApdEAEoEfw= @@ -126,10 +135,15 @@ github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2w github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= -github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= -github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= -github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= -github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -163,14 +177,13 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= -github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= -github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= +github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5 h1:RAV05c0xOkJ3dZGS0JFybxFKZ2WMLabgx3uXnd7rpGs= +github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= +github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= @@ -179,11 +192,9 @@ github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMa github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v1.6.2/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.16+incompatible h1:2Db6ZR/+FUR3hqPMwnogOPHFn405crbpxvWzKovETOQ= github.com/docker/docker v20.10.16+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= @@ -191,7 +202,7 @@ github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5Xh github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= +github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -203,13 +214,12 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/ethereum/go-ethereum v1.10.18 h1:hLEd5M+UD0GJWPaROiYMRgZXl6bi5YwoTJSthsx5CZw= -github.com/ethereum/go-ethereum v1.10.18/go.mod h1:RD3NhcSBjZpj3k+SnQq24wBrmnmie78P5R/P62iNBD8= +github.com/ethereum/go-ethereum v1.10.15 h1:E9o0kMbD8HXhp7g6UwIwntY05WTDheCGziMhegcBsQw= +github.com/ethereum/go-ethereum v1.10.15/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= @@ -221,7 +231,6 @@ github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUork github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILDlzrGEckF6HKjXe48EgsY/l7K7vhY4MW8= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= @@ -252,7 +261,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= -github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= @@ -266,7 +274,6 @@ github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7a github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -282,7 +289,6 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -318,7 +324,6 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U= @@ -355,7 +360,7 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= @@ -370,7 +375,7 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= @@ -391,7 +396,7 @@ github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= -github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= +github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= @@ -412,12 +417,14 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/ github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= -github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -468,7 +475,7 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= -github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kardianos/minwinsvc v1.0.0 h1:+JfAi8IBJna0jY2dJGZqi7o15z13JelFIklJCAENALA= github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloLKhfxanPFFRc= github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= @@ -479,6 +486,7 @@ github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubc github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -503,7 +511,6 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= @@ -556,6 +563,8 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.0-20160806122752-66b8e73f3f5c/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -567,7 +576,6 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.13 h1:1tj15ngiFfcZzii7yd82foL+ks+ouQcj8j/TPq3fk1I= @@ -599,7 +607,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= @@ -633,15 +640,15 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= @@ -703,11 +710,11 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/relvacode/iso8601 v1.1.0 h1:2nV8sp0eOjpoKQ2vD3xSDygsjAx37NHG2UlZiCkDH4I= +github.com/relvacode/iso8601 v1.1.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -722,7 +729,6 @@ github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtm github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= @@ -766,6 +772,8 @@ github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tL github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spruceid/siwe-go v0.2.0 h1:MkBZ/TpPlh1mBhul3h/XLSNZJAbbaHF587Q/VQbhPI0= +github.com/spruceid/siwe-go v0.2.0/go.mod h1:rvV+8/z/ryBKqdw9RcexFgtcsrDlESOGR38sPdVWbSI= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -780,7 +788,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -793,9 +800,7 @@ github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= -github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= @@ -837,7 +842,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -852,6 +856,7 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -890,7 +895,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -919,10 +923,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 h1:LQmS1nU0twXLA96Kt7U9qtHJEbBk3z6Q0V4UXjZkpr4= -golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -963,7 +965,6 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -971,13 +972,11 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210927181540-4e4d966f7476/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211011170408-caeb26a5c8c0/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211101193420-4a448f8816b3/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220524220425-1d687d428aca h1:xTaFYiPROfpPhqrfTIDXj0ri1SpfueYT951s4bAuDO8= @@ -1075,9 +1074,9 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211102192858-4dd72447c267/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1138,7 +1137,6 @@ golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191126055441-b0650ceb63d9/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1168,9 +1166,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098 h1:YuekqPskqwCCPM79F1X5Dhv4ezTCj+Ki1oNwiafxkA0= golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 h1:0c3L82FDQ5rt1bjTBlchS8t6RQ6299/+5bWMnRLh+uI= -golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1303,6 +1300,7 @@ gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4/go.mod h1:yeKp02qBN3iKW1OzL3M gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk= @@ -1320,9 +1318,10 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/internal/mapsutil/maps.go b/internal/mapsutil/maps.go index 038ef53a8..b7eaba0dd 100644 --- a/internal/mapsutil/maps.go +++ b/internal/mapsutil/maps.go @@ -27,3 +27,18 @@ func MapsUnion(a map[string]interface{}, b map[string]interface{}) map[string]in return c } + +// Make a copy of the map +func MapCopy(m map[string]interface{}) map[string]interface{} { + cp := make(map[string]interface{}) + for k, v := range m { + vm, ok := v.(map[string]interface{}) + if ok { + cp[k] = MapCopy(vm) + } else { + cp[k] = v + } + } + + return cp +} diff --git a/setup/config/config.go b/setup/config/config.go index 7c8733c78..f8f18c595 100644 --- a/setup/config/config.go +++ b/setup/config/config.go @@ -280,7 +280,7 @@ func (config *Dendrite) Derive() error { config.Derived.Registration.Params[authtypes.LoginTypeRecaptcha] = map[string]string{"public_key": config.ClientAPI.RecaptchaPublicKey} config.Derived.Registration.Flows = append(config.Derived.Registration.Flows, authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeRecaptcha}}) - } else { + } else if !config.ClientAPI.PasswordAuthenticationDisabled { config.Derived.Registration.Flows = append(config.Derived.Registration.Flows, authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeDummy}}) } diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go index f056738c1..17ead4ed5 100644 --- a/setup/config/config_clientapi.go +++ b/setup/config/config_clientapi.go @@ -3,8 +3,6 @@ package config import ( "fmt" "time" - - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" ) type ClientAPI struct { @@ -156,44 +154,3 @@ func (r *RateLimiting) Defaults() { r.Threshold = 5 r.CooloffMS = 500 } - -type ethereumAuthParams struct { - Version uint32 `json:"version"` - ChainIDs []string `json:"chain_ids"` -} - -type ethereumAuthConfig struct { - Enabled bool `yaml:"enabled"` - Version uint32 `yaml:"version"` - ChainIDs []string `yaml:"chain_ids"` -} - -type publicKeyAuthentication struct { - Ethereum ethereumAuthConfig `yaml:"ethereum"` -} - -func (pk *publicKeyAuthentication) Enabled() bool { - return pk.Ethereum.Enabled -} - -func (pk *publicKeyAuthentication) GetPublicKeyRegistrationFlows() []authtypes.Flow { - var flows []authtypes.Flow - if pk.Ethereum.Enabled { - flows = append(flows, authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypePublicKeyEthereum}}) - } - - return flows -} - -func (pk *publicKeyAuthentication) GetPublicKeyRegistrationParams() map[string]interface{} { - params := make(map[string]interface{}) - if pk.Ethereum.Enabled { - p := ethereumAuthParams{ - Version: pk.Ethereum.Version, - ChainIDs: pk.Ethereum.ChainIDs, - } - params[authtypes.LoginTypePublicKeyEthereum] = p - } - - return params -} diff --git a/setup/config/config_publickey.go b/setup/config/config_publickey.go new file mode 100644 index 000000000..5d67afd9c --- /dev/null +++ b/setup/config/config_publickey.go @@ -0,0 +1,81 @@ +package config + +import ( + "math/rand" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" +) + +var nonceLength = 32 + +type AuthParams interface { + GetParams() interface{} + GetNonce() string +} + +type EthereumAuthParams struct { + Version uint `json:"version"` + ChainIDs []int `json:"chain_ids"` + Nonce string `json:"nonce"` +} + +func (p EthereumAuthParams) GetParams() interface{} { + copyP := p + copyP.ChainIDs = make([]int, len(p.ChainIDs)) + copy(copyP.ChainIDs, p.ChainIDs) + copyP.Nonce = newNonce(nonceLength) + return copyP +} + +func (p EthereumAuthParams) GetNonce() string { + return p.Nonce +} + +type ethereumAuthConfig struct { + Enabled bool `yaml:"enabled"` + Version uint `yaml:"version"` + ChainIDs []int `yaml:"chain_ids"` +} + +type publicKeyAuthentication struct { + Ethereum ethereumAuthConfig `yaml:"ethereum"` +} + +func (pk *publicKeyAuthentication) Enabled() bool { + return pk.Ethereum.Enabled +} + +func (pk *publicKeyAuthentication) GetPublicKeyRegistrationFlows() []authtypes.Flow { + var flows []authtypes.Flow + if pk.Ethereum.Enabled { + flows = append(flows, authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypePublicKeyEthereum}}) + } + + return flows +} + +func (pk *publicKeyAuthentication) GetPublicKeyRegistrationParams() map[string]interface{} { + params := make(map[string]interface{}) + if pk.Ethereum.Enabled { + p := EthereumAuthParams{ + Version: pk.Ethereum.Version, + ChainIDs: pk.Ethereum.ChainIDs, + Nonce: "", + } + params[authtypes.LoginTypePublicKeyEthereum] = p + } + + return params +} + +const lettersAndNumbers = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +func newNonce(n int) string { + nonce := make([]byte, n) + + for i := range nonce { + nonce[i] = lettersAndNumbers[rand.Int63()%int64(len(lettersAndNumbers))] + } + + return string(nonce) +} From f510b6c718e6b84b2589793c00b0ff40e65fbf68 Mon Sep 17 00:00:00 2001 From: Tak Wai Wong <64229756+tak-hntlabs@users.noreply.github.com> Date: Thu, 9 Jun 2022 16:25:12 -0400 Subject: [PATCH 14/25] Use rand.Seed to seed the random function generator (#6) * Blacklist some sytest tests that are failing in our environment * Commenting out test that isn't reliably passing or failing, probably a race * refresh latest dendrite main * use rand.Seed to seed the random function Co-authored-by: Brian Meek Co-authored-by: Tak Wai Wong --- setup/config/config_publickey.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/config/config_publickey.go b/setup/config/config_publickey.go index 5d67afd9c..9820a5969 100644 --- a/setup/config/config_publickey.go +++ b/setup/config/config_publickey.go @@ -2,6 +2,7 @@ package config import ( "math/rand" + "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" ) @@ -72,6 +73,7 @@ const lettersAndNumbers = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0 func newNonce(n int) string { nonce := make([]byte, n) + rand.Seed(time.Now().UnixNano()) for i := range nonce { nonce[i] = lettersAndNumbers[rand.Int63()%int64(len(lettersAndNumbers))] From e2a64773cef3ed22544990e7e04daf91a80a473e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 10 Jun 2022 10:14:15 +0100 Subject: [PATCH 15/25] Add new next steps page to the documentation --- docs/installation/10_next_steps.md | 71 ++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 docs/installation/10_next_steps.md diff --git a/docs/installation/10_next_steps.md b/docs/installation/10_next_steps.md new file mode 100644 index 000000000..3110b5562 --- /dev/null +++ b/docs/installation/10_next_steps.md @@ -0,0 +1,71 @@ +--- +title: Next steps +parent: Installation +has_toc: true +nav_order: 10 +permalink: /installation/start/nextsteps +--- + +# Next steps + +Now that you have Dendrite running, the following tweaks will improve the reliability +of your installation. + +## File descriptor limit + +Most platforms have a limit on how many file descriptors a single process can open. All +connections made by Dendrite consume file descriptors — this includes database connections +and network requests to remote homeservers. When participating in large federated rooms +where Dendrite must talk to many remote servers, it is often very easy to exhaust default +limits which are quite low. + +We currently recommend setting the file descriptor limit to 65535 to avoid such +issues. Dendrite will log immediately after startup if the file descriptor limit is too low: + +``` +level=warning msg="IMPORTANT: Process file descriptor limit is currently 1024, it is recommended to raise the limit for Dendrite to at least 65535 to avoid issues" +``` + +UNIX systems have two limits: a hard limit and a soft limit. You can view the soft limit +by running `ulimit -Sn` and the hard limit with `ulimit -Hn`: + +``` +$ ulimit -Hn +1048576 + +$ ulimit -Sn +1024 +``` + +Increase the soft limit before starting Dendrite: + +``` +ulimit -Sn 65535 +``` + +The log line at startup should no longer appear if the limit is sufficient. + +If you are running under a systemd service, you can instead add `LimitNOFILE=65535` option +to the `[Service]` section of your service unit file. + +## DNS caching + +Dendrite has a built-in DNS cache which significantly reduces the load that Dendrite will +place on your DNS resolver. This may also speed up outbound federation. + +Consider enabling the DNS cache by modifying the `global` section of your configuration file: + +``` + dns_cache: + enabled: true + cache_size: 4096 + cache_lifetime: 600s +``` + +## Time synchronisation + +Matrix relies heavily on TLS which requires the system time to be correct. If the clock +drifts then you may find that federation no works reliably (or at all) and clients may +struggle to connect to your Dendrite server. + +Ensure that the time is synchronised on your system by enabling NTP sync. From 16ed1633b6f50da7a865ae7a09b03168fd496c4f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 10 Jun 2022 10:15:14 +0100 Subject: [PATCH 16/25] Highlighting in docs --- docs/installation/10_next_steps.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/installation/10_next_steps.md b/docs/installation/10_next_steps.md index 3110b5562..02542b082 100644 --- a/docs/installation/10_next_steps.md +++ b/docs/installation/10_next_steps.md @@ -29,7 +29,7 @@ level=warning msg="IMPORTANT: Process file descriptor limit is currently 1024, i UNIX systems have two limits: a hard limit and a soft limit. You can view the soft limit by running `ulimit -Sn` and the hard limit with `ulimit -Hn`: -``` +```bash $ ulimit -Hn 1048576 @@ -39,7 +39,7 @@ $ ulimit -Sn Increase the soft limit before starting Dendrite: -``` +```bash ulimit -Sn 65535 ``` @@ -55,7 +55,7 @@ place on your DNS resolver. This may also speed up outbound federation. Consider enabling the DNS cache by modifying the `global` section of your configuration file: -``` +```yaml dns_cache: enabled: true cache_size: 4096 From 10300722859ba345b8b21980e39a8a7022e6d35b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 10 Jun 2022 10:18:32 +0100 Subject: [PATCH 17/25] Rename the page to "Optimise your installation" --- .../installation/{10_next_steps.md => 10_optimisation.md} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename docs/installation/{10_next_steps.md => 10_optimisation.md} (93%) diff --git a/docs/installation/10_next_steps.md b/docs/installation/10_optimisation.md similarity index 93% rename from docs/installation/10_next_steps.md rename to docs/installation/10_optimisation.md index 02542b082..c19b7a75e 100644 --- a/docs/installation/10_next_steps.md +++ b/docs/installation/10_optimisation.md @@ -1,15 +1,15 @@ --- -title: Next steps +title: Optimise your installation parent: Installation has_toc: true nav_order: 10 -permalink: /installation/start/nextsteps +permalink: /installation/start/optimisation --- -# Next steps +# Optimise your installation Now that you have Dendrite running, the following tweaks will improve the reliability -of your installation. +and performance of your installation. ## File descriptor limit From 89d2adadbdef4ce62338d55d56b58fa58055d495 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 10 Jun 2022 10:58:04 +0100 Subject: [PATCH 18/25] Attempt to raise the file descriptor limit at startup (#2527) --- setup/base/sanity_unix.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/setup/base/sanity_unix.go b/setup/base/sanity_unix.go index 0c1543e0b..c630d3f19 100644 --- a/setup/base/sanity_unix.go +++ b/setup/base/sanity_unix.go @@ -15,8 +15,21 @@ func platformSanityChecks() { // If we run out of file descriptors, we might run into problems accessing // PostgreSQL amongst other things. Complain at startup if we think the // number of file descriptors is too low. - var rLimit syscall.Rlimit - if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err == nil && rLimit.Cur < 65535 { + warn := func(rLimit *syscall.Rlimit) { logrus.Warnf("IMPORTANT: Process file descriptor limit is currently %d, it is recommended to raise the limit for Dendrite to at least 65535 to avoid issues", rLimit.Cur) } + var rLimit syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err == nil && rLimit.Cur < 65535 { + // The file descriptor count is too low. Let's try to raise it. + rLimit.Cur = 65535 + if err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil { + // We failed to raise it, so log an error. + logrus.WithError(err).Warn("IMPORTANT: Failed to raise the file descriptor limit") + warn(&rLimit) + } else if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err == nil && rLimit.Cur < 65535 { + // We think we successfully raised the limit, but a second call to + // get the limit told us that we didn't succeed. Log an error. + warn(&rLimit) + } + } } From 0a7f7dc716a25b1071bacc16cb91b31441174453 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 13 Jun 2022 10:16:30 +0100 Subject: [PATCH 19/25] Add `--difference` to `resolve-state` tool --- cmd/resolve-state/main.go | 54 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go index 6ed6ebdb8..da0f70f01 100644 --- a/cmd/resolve-state/main.go +++ b/cmd/resolve-state/main.go @@ -28,7 +28,9 @@ import ( var roomVersion = flag.String("roomversion", "5", "the room version to parse events as") var filterType = flag.String("filtertype", "", "the event types to filter on") +var difference = flag.Bool("difference", false, "whether to calculate the difference between snapshots") +// nolint:gocyclo func main() { ctx := context.Background() cfg := setup.ParseFlags(true) @@ -64,6 +66,58 @@ func main() { RoomVersion: gomatrixserverlib.RoomVersion(*roomVersion), }) + if *difference { + if len(snapshotNIDs) != 2 { + panic("need exactly two state snapshot NIDs to calculate difference") + } + + removed, added, err := stateres.DifferenceBetweeenStateSnapshots(ctx, snapshotNIDs[0], snapshotNIDs[1]) + if err != nil { + panic(err) + } + + var eventNIDs []types.EventNID + for _, entry := range append(removed, added...) { + eventNIDs = append(eventNIDs, entry.EventNID) + } + + eventEntries, err := roomserverDB.Events(ctx, eventNIDs) + if err != nil { + panic(err) + } + + events := make(map[types.EventNID]*gomatrixserverlib.Event, len(eventEntries)) + for _, entry := range eventEntries { + events[entry.EventNID] = entry.Event + } + + if len(removed) > 0 { + fmt.Println("Removed:") + for _, r := range removed { + event := events[r.EventNID] + fmt.Println() + fmt.Printf("* %s %s %q\n", event.EventID(), event.Type(), *event.StateKey()) + fmt.Printf(" %s\n", string(event.Content())) + } + } + + if len(removed) > 0 && len(added) > 0 { + fmt.Println() + } + + if len(added) > 0 { + fmt.Println("Added:") + for _, a := range added { + event := events[a.EventNID] + fmt.Println() + fmt.Printf("* %s %s %q\n", event.EventID(), event.Type(), *event.StateKey()) + fmt.Printf(" %s\n", string(event.Content())) + } + } + + return + } + var stateEntries []types.StateEntry for _, snapshotNID := range snapshotNIDs { var entries []types.StateEntry From e1136f4d3e372f9c0b8a309a387193621a53ae6d Mon Sep 17 00:00:00 2001 From: Till Faelligen Date: Mon, 13 Jun 2022 11:46:59 +0200 Subject: [PATCH 20/25] Make the linter happy again --- cmd/resolve-state/main.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go index da0f70f01..c02140003 100644 --- a/cmd/resolve-state/main.go +++ b/cmd/resolve-state/main.go @@ -38,6 +38,7 @@ func main() { Type: "std", Level: "error", }) + cfg.ClientAPI.RegistrationDisabled = true base := base.NewBaseDendrite(cfg, "ResolveState", base.DisableMetrics) args := flag.Args() @@ -70,8 +71,8 @@ func main() { if len(snapshotNIDs) != 2 { panic("need exactly two state snapshot NIDs to calculate difference") } - - removed, added, err := stateres.DifferenceBetweeenStateSnapshots(ctx, snapshotNIDs[0], snapshotNIDs[1]) + var removed, added []types.StateEntry + removed, added, err = stateres.DifferenceBetweeenStateSnapshots(ctx, snapshotNIDs[0], snapshotNIDs[1]) if err != nil { panic(err) } @@ -81,7 +82,8 @@ func main() { eventNIDs = append(eventNIDs, entry.EventNID) } - eventEntries, err := roomserverDB.Events(ctx, eventNIDs) + var eventEntries []types.Event + eventEntries, err = roomserverDB.Events(ctx, eventNIDs) if err != nil { panic(err) } From c50095858341cc051e2db97fb85a1bb985f90c66 Mon Sep 17 00:00:00 2001 From: Emanuele Aliberti Date: Mon, 13 Jun 2022 13:08:46 +0200 Subject: [PATCH 21/25] generic CaddyFile in front of Dendrite (monolith) (#2531) for Caddy 2.5.x Co-authored-by: emanuele.aliberti --- docs/caddy/monolith/CaddyFile | 68 +++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 docs/caddy/monolith/CaddyFile diff --git a/docs/caddy/monolith/CaddyFile b/docs/caddy/monolith/CaddyFile new file mode 100644 index 000000000..cd93f9e10 --- /dev/null +++ b/docs/caddy/monolith/CaddyFile @@ -0,0 +1,68 @@ +{ + # debug + admin off + email example@example.com + default_sni example.com + # Debug endpoint + # acme_ca https://acme-staging-v02.api.letsencrypt.org/directory +} + +####################################################################### +# Snippets +#______________________________________________________________________ + +(handle_errors_maintenance) { + handle_errors { + @maintenance expression {http.error.status_code} == 502 + rewrite @maintenance maintenance.html + root * "/path/to/service/pages" + file_server + } +} + +(matrix-well-known-header) { + # Headers + header Access-Control-Allow-Origin "*" + header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" + header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization" + header Content-Type "application/json" +} + +####################################################################### + +example.com { + + # ... + + handle /.well-known/matrix/server { + import matrix-well-known-header + respond `{ "m.server": "matrix.example.com:443" }` 200 + } + + handle /.well-known/matrix/client { + import matrix-well-known-header + respond `{ "m.homeserver": { "base_url": "https://matrix.example.com" } }` 200 + } + + import handle_errors_maintenance +} + +example.com:8448 { + # server<->server HTTPS traffic + reverse_proxy http://dendrite-host:8008 +} + +matrix.example.com { + + handle /_matrix/* { + # client<->server HTTPS traffic + reverse_proxy http://dendrite-host:8008 + } + + handle_path /* { + # Client webapp (Element SPA or ...) + file_server { + root /path/to/www/example.com/matrix-web-client/ + } + } +} From 4c2a10f1a61a79ed8bbe17af1b28532c3d24c261 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 13 Jun 2022 15:11:10 +0100 Subject: [PATCH 22/25] Handle state before, send history visibility in output (#2532) * Check state before event * Tweaks * Refactor a bit, include in output events * Don't waste time if soft failed either * Tweak control flow, comments, use GMSL history visibility type --- clientapi/routing/aliases.go | 2 +- go.mod | 2 +- go.sum | 4 +- roomserver/api/output.go | 5 +- roomserver/internal/input/input_events.go | 114 +++++++++++++++++- .../internal/input/input_latest_events.go | 35 +++--- syncapi/routing/context.go | 2 +- 7 files changed, 142 insertions(+), 22 deletions(-) diff --git a/clientapi/routing/aliases.go b/clientapi/routing/aliases.go index 504d60265..68d0f4195 100644 --- a/clientapi/routing/aliases.go +++ b/clientapi/routing/aliases.go @@ -44,7 +44,7 @@ func GetAliases( return util.ErrorResponse(fmt.Errorf("rsAPI.QueryCurrentState: %w", err)) } - visibility := "invite" + visibility := gomatrixserverlib.HistoryVisibilityInvited if historyVisEvent, ok := stateRes.StateEvents[stateTuple]; ok { var err error visibility, err = historyVisEvent.HistoryVisibility() diff --git a/go.mod b/go.mod index ea6e8caeb..b2a096751 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220607143425-e55d796fd0b3 + github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.13 diff --git a/go.sum b/go.sum index e21794f41..3a35c47da 100644 --- a/go.sum +++ b/go.sum @@ -418,8 +418,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220607143425-e55d796fd0b3 h1:2eYcBt8Kg+nW/xIJY5x8Uo2dQLjUF+oxLap00uFC5l8= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220607143425-e55d796fd0b3/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a h1:jOkrb6twViAGTHHadA51sQwdloHT0Vx1MCptk9InTHo= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE= github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= diff --git a/roomserver/api/output.go b/roomserver/api/output.go index a82bf8701..36d0625c7 100644 --- a/roomserver/api/output.go +++ b/roomserver/api/output.go @@ -161,6 +161,8 @@ type OutputNewRoomEvent struct { // The transaction ID of the send request if sent by a local user and one // was specified TransactionID *TransactionID `json:"transaction_id,omitempty"` + // The history visibility of the event. + HistoryVisibility gomatrixserverlib.HistoryVisibility `json:"history_visibility"` } func (o *OutputNewRoomEvent) NeededStateEventIDs() ([]*gomatrixserverlib.HeaderedEvent, []string) { @@ -187,7 +189,8 @@ func (o *OutputNewRoomEvent) NeededStateEventIDs() ([]*gomatrixserverlib.Headere // should build their current room state up from OutputNewRoomEvents only. type OutputOldRoomEvent struct { // The Event. - Event *gomatrixserverlib.HeaderedEvent `json:"event"` + Event *gomatrixserverlib.HeaderedEvent `json:"event"` + HistoryVisibility gomatrixserverlib.HistoryVisibility `json:"history_visibility"` } // An OutputNewInviteEvent is written whenever an invite becomes active. diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index deb88ea82..ff05f798c 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -295,6 +295,22 @@ func (r *Inputer) processRoomEvent( } } + // Get the state before the event so that we can work out if the event was + // allowed at the time, and also to get the history visibility. We won't + // bother doing this if the event was already rejected as it just ends up + // burning CPU time. + historyVisibility := gomatrixserverlib.HistoryVisibilityJoined // Default to restrictive. + if rejectionErr == nil && !isRejected && !softfail { + var err error + historyVisibility, rejectionErr, err = r.processStateBefore(ctx, input, missingPrev) + if err != nil { + return fmt.Errorf("r.processStateBefore: %w", err) + } + if rejectionErr != nil { + isRejected = true + } + } + // Store the event. _, _, stateAtEvent, redactionEvent, redactedEventID, err := r.DB.StoreEvent(ctx, event, authEventNIDs, isRejected || softfail) if err != nil { @@ -360,6 +376,7 @@ func (r *Inputer) processRoomEvent( input.SendAsServer, // send as server input.TransactionID, // transaction ID input.HasState, // rewrites state? + historyVisibility, // the history visibility before the event ); err != nil { return fmt.Errorf("r.updateLatestEvents: %w", err) } @@ -368,7 +385,8 @@ func (r *Inputer) processRoomEvent( { Type: api.OutputTypeOldRoomEvent, OldRoomEvent: &api.OutputOldRoomEvent{ - Event: headered, + Event: headered, + HistoryVisibility: historyVisibility, }, }, }) @@ -402,6 +420,100 @@ func (r *Inputer) processRoomEvent( return nil } +// processStateBefore works out what the state is before the event and +// then checks the event auths against the state at the time. It also +// tries to determine what the history visibility was of the event at +// the time, so that it can be sent in the output event to downstream +// components. +// nolint:nakedret +func (r *Inputer) processStateBefore( + ctx context.Context, + input *api.InputRoomEvent, + missingPrev bool, +) (historyVisibility gomatrixserverlib.HistoryVisibility, rejectionErr error, err error) { + historyVisibility = gomatrixserverlib.HistoryVisibilityJoined // Default to restrictive. + event := input.Event.Unwrap() + isCreateEvent := event.Type() == gomatrixserverlib.MRoomCreate && event.StateKeyEquals("") + var stateBeforeEvent []*gomatrixserverlib.Event + switch { + case isCreateEvent: + // There's no state before a create event so there is nothing + // else to do. + return + case input.HasState: + // If we're overriding the state then we need to go and retrieve + // them from the database. It's a hard error if they are missing. + stateEvents, err := r.DB.EventsFromIDs(ctx, input.StateEventIDs) + if err != nil { + return "", nil, fmt.Errorf("r.DB.EventsFromIDs: %w", err) + } + stateBeforeEvent = make([]*gomatrixserverlib.Event, 0, len(stateEvents)) + for _, entry := range stateEvents { + stateBeforeEvent = append(stateBeforeEvent, entry.Event) + } + case missingPrev: + // We don't know all of the prev events, so we can't work out + // the state before the event. Reject it in that case. + rejectionErr = fmt.Errorf("event %q has missing prev events", event.EventID()) + return + case len(event.PrevEventIDs()) == 0: + // There should be prev events since it's not a create event. + // A non-create event that claims to have no prev events is + // invalid, so reject it. + rejectionErr = fmt.Errorf("event %q must have prev events", event.EventID()) + return + default: + // For all non-create events, there must be prev events, so we'll + // ask the query API for the relevant tuples needed for auth. We + // will include the history visibility here even though we don't + // actually need it for auth, because we want to send it in the + // output events. + tuplesNeeded := gomatrixserverlib.StateNeededForAuth([]*gomatrixserverlib.Event{event}).Tuples() + tuplesNeeded = append(tuplesNeeded, gomatrixserverlib.StateKeyTuple{ + EventType: gomatrixserverlib.MRoomHistoryVisibility, + StateKey: "", + }) + stateBeforeReq := &api.QueryStateAfterEventsRequest{ + RoomID: event.RoomID(), + PrevEventIDs: event.PrevEventIDs(), + StateToFetch: tuplesNeeded, + } + stateBeforeRes := &api.QueryStateAfterEventsResponse{} + if err := r.Queryer.QueryStateAfterEvents(ctx, stateBeforeReq, stateBeforeRes); err != nil { + return "", nil, fmt.Errorf("r.Queryer.QueryStateAfterEvents: %w", err) + } + switch { + case !stateBeforeRes.RoomExists: + rejectionErr = fmt.Errorf("room %q does not exist", event.RoomID()) + return + case !stateBeforeRes.PrevEventsExist: + rejectionErr = fmt.Errorf("prev events of %q are not known", event.EventID()) + return + default: + stateBeforeEvent = gomatrixserverlib.UnwrapEventHeaders(stateBeforeRes.StateEvents) + } + } + // At this point, stateBeforeEvent should be populated either by + // the supplied state in the input request, or from the prev events. + // Check whether the event is allowed or not. + stateBeforeAuth := gomatrixserverlib.NewAuthEvents(stateBeforeEvent) + if rejectionErr = gomatrixserverlib.Allowed(event, &stateBeforeAuth); rejectionErr != nil { + return + } + // Work out what the history visibility was at the time of the + // event. + for _, event := range stateBeforeEvent { + if event.Type() != gomatrixserverlib.MRoomHistoryVisibility || !event.StateKeyEquals("") { + continue + } + if hisVis, err := event.HistoryVisibility(); err == nil { + historyVisibility = hisVis + break + } + } + return +} + // fetchAuthEvents will check to see if any of the // auth events specified by the given event are unknown. If they are // then we will go off and request them from the federation and then diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index 9738ed4e6..e76f4ba8d 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -56,6 +56,7 @@ func (r *Inputer) updateLatestEvents( sendAsServer string, transactionID *api.TransactionID, rewritesState bool, + historyVisibility gomatrixserverlib.HistoryVisibility, ) (err error) { span, ctx := opentracing.StartSpanFromContext(ctx, "updateLatestEvents") defer span.Finish() @@ -69,15 +70,16 @@ func (r *Inputer) updateLatestEvents( defer sqlutil.EndTransactionWithCheck(updater, &succeeded, &err) u := latestEventsUpdater{ - ctx: ctx, - api: r, - updater: updater, - roomInfo: roomInfo, - stateAtEvent: stateAtEvent, - event: event, - sendAsServer: sendAsServer, - transactionID: transactionID, - rewritesState: rewritesState, + ctx: ctx, + api: r, + updater: updater, + roomInfo: roomInfo, + stateAtEvent: stateAtEvent, + event: event, + sendAsServer: sendAsServer, + transactionID: transactionID, + rewritesState: rewritesState, + historyVisibility: historyVisibility, } if err = u.doUpdateLatestEvents(); err != nil { @@ -119,6 +121,8 @@ type latestEventsUpdater struct { // The snapshots of current state before and after processing this event oldStateNID types.StateSnapshotNID newStateNID types.StateSnapshotNID + // The history visibility of the event itself (from the state before the event). + historyVisibility gomatrixserverlib.HistoryVisibility } func (u *latestEventsUpdater) doUpdateLatestEvents() error { @@ -365,12 +369,13 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) } ore := api.OutputNewRoomEvent{ - Event: u.event.Headered(u.roomInfo.RoomVersion), - RewritesState: u.rewritesState, - LastSentEventID: u.lastEventIDSent, - LatestEventIDs: latestEventIDs, - TransactionID: u.transactionID, - SendAsServer: u.sendAsServer, + Event: u.event.Headered(u.roomInfo.RoomVersion), + RewritesState: u.rewritesState, + LastSentEventID: u.lastEventIDSent, + LatestEventIDs: latestEventIDs, + TransactionID: u.transactionID, + SendAsServer: u.sendAsServer, + HistoryVisibility: u.historyVisibility, } eventIDMap, err := u.stateEventMap() diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go index 96438e184..d021d365d 100644 --- a/syncapi/routing/context.go +++ b/syncapi/routing/context.go @@ -97,7 +97,7 @@ func Context( state, _ := syncDB.CurrentState(ctx, roomID, &stateFilter, nil) // verify the user is allowed to see the context for this room/event for _, x := range state { - var hisVis string + var hisVis gomatrixserverlib.HistoryVisibility hisVis, err = x.HistoryVisibility() if err != nil { continue From fef29593c4c6bb5c6ec497460476fb2e7a57448b Mon Sep 17 00:00:00 2001 From: Tak Wai Wong <64229756+tak-hntlabs@users.noreply.github.com> Date: Tue, 14 Jun 2022 19:11:02 -0400 Subject: [PATCH 23/25] Create session ID during registration (#8) * Blacklist some sytest tests that are failing in our environment * Commenting out test that isn't reliably passing or failing, probably a race * refresh latest dendrite main * pull latest from dendrite-fork subtree * refresh latest dendrite main * Create session ID during registration Co-authored-by: Brian Meek Co-authored-by: Tak Wai Wong --- clientapi/auth/login_publickey_ethereum.go | 8 ++++---- clientapi/routing/register.go | 2 +- clientapi/routing/register_publickey.go | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/clientapi/auth/login_publickey_ethereum.go b/clientapi/auth/login_publickey_ethereum.go index 938a9f816..592f02383 100644 --- a/clientapi/auth/login_publickey_ethereum.go +++ b/clientapi/auth/login_publickey_ethereum.go @@ -29,7 +29,7 @@ import ( type LoginPublicKeyEthereum struct { // https://github.com/tak-hntlabs/matrix-spec-proposals/blob/main/proposals/3782-matrix-publickey-login-spec.md#client-sends-login-request-with-authentication-data Type string `json:"type"` - Address string `json:"address"` + UserId string `json:"user_id"` Session string `json:"session"` Message string `json:"message"` Signature string `json:"signature"` @@ -51,7 +51,7 @@ func CreatePublicKeyEthereumHandler( pk.config = config pk.userAPI = userAPI // Case-insensitive - pk.Address = strings.ToLower(pk.Address) + pk.UserId = strings.ToLower(pk.UserId) return &pk, nil } @@ -65,7 +65,7 @@ func (pk LoginPublicKeyEthereum) GetType() string { } func (pk LoginPublicKeyEthereum) AccountExists(ctx context.Context) (string, *jsonerror.MatrixError) { - localPart, err := userutil.ParseUsernameParam(pk.Address, &pk.config.Matrix.ServerName) + localPart, err := userutil.ParseUsernameParam(pk.UserId, &pk.config.Matrix.ServerName) if err != nil { // userId does not exist return "", jsonerror.Forbidden("the address is incorrect, or the account does not exist.") @@ -110,7 +110,7 @@ func (pk LoginPublicKeyEthereum) ValidateLoginResponse() (bool, *jsonerror.Matri func (pk LoginPublicKeyEthereum) CreateLogin() *Login { identifier := LoginIdentifier{ Type: "m.id.publickey", - User: pk.Address, + User: pk.UserId, } login := Login{ Identifier: identifier, diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 0e3a7554e..83c89356a 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -775,7 +775,7 @@ func handleRegistrationFlow( if isCompleted { sessions.addCompletedSessionStage(sessionID, authType) } else { - newPublicKeyAuthSession(&r) + newPublicKeyAuthSession(&r, sessions, sessionID) } case "": diff --git a/clientapi/routing/register_publickey.go b/clientapi/routing/register_publickey.go index c6cd5e30a..aa0fea656 100644 --- a/clientapi/routing/register_publickey.go +++ b/clientapi/routing/register_publickey.go @@ -26,7 +26,8 @@ import ( "github.com/tidwall/gjson" ) -func newPublicKeyAuthSession(request *registerRequest) { +func newPublicKeyAuthSession(request *registerRequest, sessions *sessionsDict, sessionID string) { + sessions.sessions[sessionID] = append(sessions.sessions[sessionID], authtypes.LoginTypePublicKey) // Public key auth does not use password. But the registration flow // requires setting a password in order to create the account. // Create a random password to satisfy the requirement. From 1b90cc95367947fa00616b4426d0c894b33c9862 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 15 Jun 2022 12:50:02 +0200 Subject: [PATCH 24/25] Fix rare panic when returning user devices over federation (#2534) --- federationapi/routing/devices.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go index 1a092645f..2f9da1f25 100644 --- a/federationapi/routing/devices.go +++ b/federationapi/routing/devices.go @@ -85,6 +85,9 @@ func GetUserDevices( if targetKey, ok := targetUser[gomatrixserverlib.KeyID(dev.DeviceID)]; ok { for sourceUserID, forSourceUser := range targetKey { for sourceKeyID, sourceKey := range forSourceUser { + if device.Keys.Signatures == nil { + device.Keys.Signatures = map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{} + } if _, ok := device.Keys.Signatures[sourceUserID]; !ok { device.Keys.Signatures[sourceUserID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{} } From 7120eb6bc943af6f725b0c61cfd110330f04064a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 15 Jun 2022 14:27:07 +0100 Subject: [PATCH 25/25] Add `InputDeviceListUpdate` to the keyserver, remove old input API (#2536) * Add `InputDeviceListUpdate` to the keyserver, remove old input API * Fix copyright * Log more information when a device list update fails --- federationapi/federationapi.go | 1 + federationapi/producers/syncapi.go | 17 +++++ federationapi/routing/send.go | 8 +-- keyserver/api/api.go | 10 --- keyserver/consumers/devicelistupdate.go | 82 +++++++++++++++++++++++++ keyserver/internal/internal.go | 11 ---- keyserver/inthttp/client.go | 14 ----- keyserver/inthttp/server.go | 11 ---- keyserver/keyserver.go | 10 ++- setup/jetstream/streams.go | 6 ++ syncapi/internal/keychange_test.go | 3 - 11 files changed, 117 insertions(+), 56 deletions(-) create mode 100644 keyserver/consumers/devicelistupdate.go diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index ff159beea..97bcc12a5 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -63,6 +63,7 @@ func AddPublicRoutes( TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), TopicTypingEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), TopicPresenceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent), + TopicDeviceListUpdate: cfg.Matrix.JetStream.Prefixed(jetstream.InputDeviceListUpdate), ServerName: cfg.Matrix.ServerName, UserAPI: userAPI, } diff --git a/federationapi/producers/syncapi.go b/federationapi/producers/syncapi.go index 494150036..6453d026c 100644 --- a/federationapi/producers/syncapi.go +++ b/federationapi/producers/syncapi.go @@ -17,6 +17,7 @@ package producers import ( "context" "encoding/json" + "fmt" "strconv" "time" @@ -34,6 +35,7 @@ type SyncAPIProducer struct { TopicSendToDeviceEvent string TopicTypingEvent string TopicPresenceEvent string + TopicDeviceListUpdate string JetStream nats.JetStreamContext ServerName gomatrixserverlib.ServerName UserAPI userapi.UserInternalAPI @@ -161,3 +163,18 @@ func (p *SyncAPIProducer) SendPresence( _, err := p.JetStream.PublishMsg(m, nats.Context(ctx)) return err } + +func (p *SyncAPIProducer) SendDeviceListUpdate( + ctx context.Context, deviceListUpdate *gomatrixserverlib.DeviceListUpdateEvent, +) (err error) { + m := nats.NewMsg(p.TopicDeviceListUpdate) + m.Header.Set(jetstream.UserID, deviceListUpdate.UserID) + m.Data, err = json.Marshal(deviceListUpdate) + if err != nil { + return fmt.Errorf("json.Marshal: %w", err) + } + + log.Debugf("Sending device list update: %+v", m.Header) + _, err = p.JetStream.PublishMsg(m, nats.Context(ctx)) + return err +} diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index c25dabce9..43003be38 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -501,11 +501,7 @@ func (t *txnReq) processDeviceListUpdate(ctx context.Context, e gomatrixserverli } else if serverName != t.Origin { return } - var inputRes keyapi.InputDeviceListUpdateResponse - t.keyAPI.InputDeviceListUpdate(context.Background(), &keyapi.InputDeviceListUpdateRequest{ - Event: payload, - }, &inputRes) - if inputRes.Error != nil { - util.GetLogger(ctx).WithError(inputRes.Error).WithField("user_id", payload.UserID).Error("failed to InputDeviceListUpdate") + if err := t.producer.SendDeviceListUpdate(ctx, &payload); err != nil { + util.GetLogger(ctx).WithError(err).WithField("user_id", payload.UserID).Error("failed to InputDeviceListUpdate") } } diff --git a/keyserver/api/api.go b/keyserver/api/api.go index 140f03569..c0a1eedbb 100644 --- a/keyserver/api/api.go +++ b/keyserver/api/api.go @@ -62,8 +62,6 @@ type FederationKeyAPI interface { QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse) QuerySignatures(ctx context.Context, req *QuerySignaturesRequest, res *QuerySignaturesResponse) QueryDeviceMessages(ctx context.Context, req *QueryDeviceMessagesRequest, res *QueryDeviceMessagesResponse) - // InputDeviceListUpdate from a federated server EDU - InputDeviceListUpdate(ctx context.Context, req *InputDeviceListUpdateRequest, res *InputDeviceListUpdateResponse) PerformUploadDeviceKeys(ctx context.Context, req *PerformUploadDeviceKeysRequest, res *PerformUploadDeviceKeysResponse) PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse) } @@ -337,11 +335,3 @@ type QuerySignaturesResponse struct { // The request error, if any Error *KeyError } - -type InputDeviceListUpdateRequest struct { - Event gomatrixserverlib.DeviceListUpdateEvent -} - -type InputDeviceListUpdateResponse struct { - Error *KeyError -} diff --git a/keyserver/consumers/devicelistupdate.go b/keyserver/consumers/devicelistupdate.go new file mode 100644 index 000000000..f4f246280 --- /dev/null +++ b/keyserver/consumers/devicelistupdate.go @@ -0,0 +1,82 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "context" + "encoding/json" + + "github.com/matrix-org/dendrite/keyserver/internal" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + "github.com/sirupsen/logrus" +) + +// DeviceListUpdateConsumer consumes device list updates that came in over federation. +type DeviceListUpdateConsumer struct { + ctx context.Context + jetstream nats.JetStreamContext + durable string + topic string + updater *internal.DeviceListUpdater +} + +// NewDeviceListUpdateConsumer creates a new DeviceListConsumer. Call Start() to begin consuming from key servers. +func NewDeviceListUpdateConsumer( + process *process.ProcessContext, + cfg *config.KeyServer, + js nats.JetStreamContext, + updater *internal.DeviceListUpdater, +) *DeviceListUpdateConsumer { + return &DeviceListUpdateConsumer{ + ctx: process.Context(), + jetstream: js, + durable: cfg.Matrix.JetStream.Prefixed("KeyServerInputDeviceListConsumer"), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.InputDeviceListUpdate), + updater: updater, + } +} + +// Start consuming from key servers +func (t *DeviceListUpdateConsumer) Start() error { + return jetstream.JetStreamConsumer( + t.ctx, t.jetstream, t.topic, t.durable, t.onMessage, + nats.DeliverAll(), nats.ManualAck(), + ) +} + +// onMessage is called in response to a message received on the +// key change events topic from the key server. +func (t *DeviceListUpdateConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool { + var m gomatrixserverlib.DeviceListUpdateEvent + if err := json.Unmarshal(msg.Data, &m); err != nil { + logrus.WithError(err).Errorf("Failed to read from device list update input topic") + return true + } + err := t.updater.Update(ctx, m) + if err != nil { + logrus.WithFields(logrus.Fields{ + "user_id": m.UserID, + "device_id": m.DeviceID, + "stream_id": m.StreamID, + "prev_id": m.PrevID, + }).WithError(err).Errorf("Failed to update device list") + return false + } + return true +} diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index f8d0d69c3..c146b2aa0 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -47,17 +47,6 @@ func (a *KeyInternalAPI) SetUserAPI(i userapi.KeyserverUserAPI) { a.UserAPI = i } -func (a *KeyInternalAPI) InputDeviceListUpdate( - ctx context.Context, req *api.InputDeviceListUpdateRequest, res *api.InputDeviceListUpdateResponse, -) { - err := a.Updater.Update(ctx, req.Event) - if err != nil { - res.Error = &api.KeyError{ - Err: fmt.Sprintf("failed to update device list: %s", err), - } - } -} - func (a *KeyInternalAPI) QueryKeyChanges(ctx context.Context, req *api.QueryKeyChangesRequest, res *api.QueryKeyChangesResponse) { userIDs, latest, err := a.DB.KeyChanges(ctx, req.Offset, req.ToOffset) if err != nil { diff --git a/keyserver/inthttp/client.go b/keyserver/inthttp/client.go index abce81582..dac61d1ea 100644 --- a/keyserver/inthttp/client.go +++ b/keyserver/inthttp/client.go @@ -63,20 +63,6 @@ type httpKeyInternalAPI struct { func (h *httpKeyInternalAPI) SetUserAPI(i userapi.KeyserverUserAPI) { // no-op: doesn't need it } -func (h *httpKeyInternalAPI) InputDeviceListUpdate( - ctx context.Context, req *api.InputDeviceListUpdateRequest, res *api.InputDeviceListUpdateResponse, -) { - span, ctx := opentracing.StartSpanFromContext(ctx, "InputDeviceListUpdate") - defer span.Finish() - - apiURL := h.apiURL + InputDeviceListUpdatePath - err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) - if err != nil { - res.Error = &api.KeyError{ - Err: err.Error(), - } - } -} func (h *httpKeyInternalAPI) PerformClaimKeys( ctx context.Context, diff --git a/keyserver/inthttp/server.go b/keyserver/inthttp/server.go index 8d557a768..5bf5976a8 100644 --- a/keyserver/inthttp/server.go +++ b/keyserver/inthttp/server.go @@ -25,17 +25,6 @@ import ( ) func AddRoutes(internalAPIMux *mux.Router, s api.KeyInternalAPI) { - internalAPIMux.Handle(InputDeviceListUpdatePath, - httputil.MakeInternalAPI("inputDeviceListUpdate", func(req *http.Request) util.JSONResponse { - request := api.InputDeviceListUpdateRequest{} - response := api.InputDeviceListUpdateResponse{} - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - s.InputDeviceListUpdate(req.Context(), &request, &response) - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) internalAPIMux.Handle(PerformClaimKeysPath, httputil.MakeInternalAPI("performClaimKeys", func(req *http.Request) util.JSONResponse { request := api.PerformClaimKeysRequest{} diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index 3ffd3ba1e..cd506f981 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -18,6 +18,7 @@ import ( "github.com/gorilla/mux" fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/keyserver/consumers" "github.com/matrix-org/dendrite/keyserver/internal" "github.com/matrix-org/dendrite/keyserver/inthttp" "github.com/matrix-org/dendrite/keyserver/producers" @@ -59,10 +60,17 @@ func NewInternalAPI( updater := internal.NewDeviceListUpdater(db, ap, keyChangeProducer, fedClient, 8) // 8 workers TODO: configurable ap.Updater = updater go func() { - if err := updater.Start(); err != nil { + if err = updater.Start(); err != nil { logrus.WithError(err).Panicf("failed to start device list updater") } }() + dlConsumer := consumers.NewDeviceListUpdateConsumer( + base.ProcessContext, cfg, js, updater, + ) + if err = dlConsumer.Start(); err != nil { + logrus.WithError(err).Panic("failed to start device list consumer") + } + return ap } diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go index 6594e6941..110808b1b 100644 --- a/setup/jetstream/streams.go +++ b/setup/jetstream/streams.go @@ -16,6 +16,7 @@ const ( var ( InputRoomEvent = "InputRoomEvent" + InputDeviceListUpdate = "InputDeviceListUpdate" OutputRoomEvent = "OutputRoomEvent" OutputSendToDeviceEvent = "OutputSendToDeviceEvent" OutputKeyChangeEvent = "OutputKeyChangeEvent" @@ -45,6 +46,11 @@ var streams = []*nats.StreamConfig{ Retention: nats.InterestPolicy, Storage: nats.FileStorage, }, + { + Name: InputDeviceListUpdate, + Retention: nats.InterestPolicy, + Storage: nats.FileStorage, + }, { Name: OutputRoomEvent, Retention: nats.InterestPolicy, diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go index d9fb9cf82..219b35e2c 100644 --- a/syncapi/internal/keychange_test.go +++ b/syncapi/internal/keychange_test.go @@ -43,9 +43,6 @@ func (k *mockKeyAPI) QueryOneTimeKeys(ctx context.Context, req *keyapi.QueryOneT } func (k *mockKeyAPI) QueryDeviceMessages(ctx context.Context, req *keyapi.QueryDeviceMessagesRequest, res *keyapi.QueryDeviceMessagesResponse) { -} -func (k *mockKeyAPI) InputDeviceListUpdate(ctx context.Context, req *keyapi.InputDeviceListUpdateRequest, res *keyapi.InputDeviceListUpdateResponse) { - } func (k *mockKeyAPI) QuerySignatures(ctx context.Context, req *keyapi.QuerySignaturesRequest, res *keyapi.QuerySignaturesResponse) { }