From 5bbe73747c8fea387c97c79de3917e61b487c646 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 27 Sep 2020 22:23:42 +0100 Subject: [PATCH 001/104] unbreak CORS on /capabilities. fixes https://github.com/vector-im/element-web/issues/15297 --- clientapi/routing/routing.go | 2 +- go.mod | 2 ++ syncapi/routing/messages.go | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 999946a65..ab56c5f4d 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -793,7 +793,7 @@ func Setup( } return GetCapabilities(req, rsAPI) }), - ).Methods(http.MethodGet) + ).Methods(http.MethodGet, http.MethodOptions) // Supplying a device ID is deprecated. r0mux.Handle("/keys/upload/{deviceID}", diff --git a/go.mod b/go.mod index ca0c2710c..d60fe2354 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,7 @@ module github.com/matrix-org/dendrite +replace github.com/matrix-org/gomatrixserverlib => ../gomatrixserverlib + require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/Shopify/sarama v1.27.0 diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 0999d3e8c..6447e5d59 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -179,7 +179,7 @@ func OnIncomingMessagesRequest( } } -// retrieveEvents retrieve events from the local database for a request on +// retrieveEvents retrieves events from the local database for a request on // /messages. If there's not enough events to retrieve, it asks another // homeserver in the room for older events. // Returns an error if there was an issue talking to the database or with the From 15fe61ed582b9f0f9bac9d9491b0e621ef4b2a43 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 27 Sep 2020 22:24:17 +0100 Subject: [PATCH 002/104] oops --- go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.mod b/go.mod index d60fe2354..ca0c2710c 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,5 @@ module github.com/matrix-org/dendrite -replace github.com/matrix-org/gomatrixserverlib => ../gomatrixserverlib - require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/Shopify/sarama v1.27.0 From 40dd16a6e64f61e00f54bafc2ab3b2b5b831810b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 28 Sep 2020 10:03:18 +0100 Subject: [PATCH 003/104] Don't fall back to /state on incoming /send (#1446) --- federationapi/routing/send.go | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index cb7bea6c4..5d5ea310d 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -557,18 +557,12 @@ func (t *txnReq) lookupStateAfterEventLocally(ctx context.Context, roomID, event // lookuptStateBeforeEvent returns the room state before the event e, which is just /state_ids and/or /state depending on what // the server supports. func (t *txnReq) lookupStateBeforeEvent(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) ( - respState *gomatrixserverlib.RespState, err error) { + *gomatrixserverlib.RespState, error) { util.GetLogger(ctx).Infof("lookupStateBeforeEvent %s", eventID) // Attempt to fetch the missing state using /state_ids and /events - respState, err = t.lookupMissingStateViaStateIDs(ctx, roomID, eventID, roomVersion) - if err != nil { - // Fallback to /state - util.GetLogger(ctx).WithError(err).Warn("lookupStateBeforeEvent failed to /state_ids, falling back to /state") - respState, err = t.lookupMissingStateViaState(ctx, roomID, eventID, roomVersion) - } - return + return t.lookupMissingStateViaStateIDs(ctx, roomID, eventID, roomVersion) } func (t *txnReq) resolveStatesAndCheck(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, states []*gomatrixserverlib.RespState, backwardsExtremity *gomatrixserverlib.Event) (*gomatrixserverlib.RespState, error) { @@ -711,19 +705,6 @@ Event: return nil, nil } -func (t *txnReq) lookupMissingStateViaState(ctx context.Context, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion) ( - respState *gomatrixserverlib.RespState, err error) { - state, err := t.federation.LookupState(ctx, t.Origin, roomID, eventID, roomVersion) - if err != nil { - return nil, err - } - // Check that the returned state is valid. - if err := state.Check(ctx, t.keys, nil); err != nil { - return nil, err - } - return &state, nil -} - func (t *txnReq) lookupMissingStateViaStateIDs(ctx context.Context, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion) ( *gomatrixserverlib.RespState, error) { util.GetLogger(ctx).Infof("lookupMissingStateViaStateIDs %s", eventID) From ce318f53bc99d5fb4014d4cca7408c113a89d2ef Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 28 Sep 2020 11:32:59 +0100 Subject: [PATCH 004/104] Use workers when fetching events from /state_ids, use /state only if significant portion of events missing (#1447) * Don't fall back to /state on incoming /send * Event workers for /state_ids, use /state only if significant percentage of events are missing --- federationapi/routing/send.go | 94 ++++++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 8 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 5d5ea310d..5f20b2d8e 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" "net/http" + "sync" "time" "github.com/matrix-org/dendrite/clientapi/jsonerror" @@ -705,6 +706,20 @@ Event: return nil, nil } +func (t *txnReq) lookupMissingStateViaState(ctx context.Context, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion) ( + respState *gomatrixserverlib.RespState, err error) { + state, err := t.federation.LookupState(ctx, t.Origin, roomID, eventID, roomVersion) + if err != nil { + return nil, err + } + // Check that the returned state is valid. + if err := state.Check(ctx, t.keys, nil); err != nil { + return nil, err + } + return &state, nil +} + +// nolint:gocyclo func (t *txnReq) lookupMissingStateViaStateIDs(ctx context.Context, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion) ( *gomatrixserverlib.RespState, error) { util.GetLogger(ctx).Infof("lookupMissingStateViaStateIDs %s", eventID) @@ -742,27 +757,90 @@ func (t *txnReq) lookupMissingStateViaStateIDs(ctx context.Context, roomID, even } } + concurrentRequests := 8 + missingCount := len(missing) + + // If over 50% of the auth/state events from /state_ids are missing + // then we'll just call /state instead, otherwise we'll just end up + // hammering the remote side with /event requests unnecessarily. + if missingCount > concurrentRequests && missingCount > len(wantIDs)/2 { + util.GetLogger(ctx).WithFields(logrus.Fields{ + "missing": missingCount, + "event_id": eventID, + "room_id": roomID, + "total_state": len(stateIDs.StateEventIDs), + "total_auth_events": len(stateIDs.AuthEventIDs), + }).Info("Fetching all state at event") + return t.lookupMissingStateViaState(ctx, roomID, eventID, roomVersion) + } + util.GetLogger(ctx).WithFields(logrus.Fields{ - "missing": len(missing), - "event_id": eventID, - "room_id": roomID, - "total_state": len(stateIDs.StateEventIDs), - "total_auth_events": len(stateIDs.AuthEventIDs), + "missing": missingCount, + "event_id": eventID, + "room_id": roomID, + "total_state": len(stateIDs.StateEventIDs), + "total_auth_events": len(stateIDs.AuthEventIDs), + "concurrent_requests": concurrentRequests, }).Info("Fetching missing state at event") + // Create a queue containing all of the missing event IDs that we want + // to retrieve. + pending := make(chan string, missingCount) for missingEventID := range missing { + pending <- missingEventID + } + close(pending) + + // Define how many workers we should start to do this. + if missingCount < concurrentRequests { + concurrentRequests = missingCount + } + + // Create the wait group. + var fetchgroup sync.WaitGroup + fetchgroup.Add(concurrentRequests) + + // This is the only place where we'll write to t.haveEvents from + // multiple goroutines, and everywhere else is blocked on this + // synchronous function anyway. + var haveEventsMutex sync.Mutex + + // Define what we'll do in order to fetch the missing event ID. + fetch := func(missingEventID string) { var h *gomatrixserverlib.HeaderedEvent h, err = t.lookupEvent(ctx, roomVersion, missingEventID, false) switch err.(type) { case verifySigError: - continue + break case nil: - // do nothing + break default: - return nil, err + util.GetLogger(ctx).WithFields(logrus.Fields{ + "event_id": missingEventID, + "room_id": roomID, + }).Info("Failed to fetch missing event") + return } + haveEventsMutex.Lock() t.haveEvents[h.EventID()] = h + haveEventsMutex.Unlock() } + + // Create the worker. + worker := func(ch <-chan string) { + defer fetchgroup.Done() + for missingEventID := range ch { + fetch(missingEventID) + } + } + + // Start the workers. + for i := 0; i < concurrentRequests; i++ { + go worker(pending) + } + + // Wait for the workers to finish. + fetchgroup.Wait() resp, err := t.createRespStateFromStateIDs(stateIDs) return resp, err } From da89f2bd1fbc7738d5f56ac1e92d34fc797d6092 Mon Sep 17 00:00:00 2001 From: bn4t <17193640+bn4t@users.noreply.github.com> Date: Tue, 29 Sep 2020 09:06:59 +0000 Subject: [PATCH 005/104] Finish implementing the capabilities endpoint (#1449) Closes #1310 Signed-off-by: Benjamin Nater --- clientapi/routing/capabilities.go | 7 +++++-- sytest-whitelist | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/clientapi/routing/capabilities.go b/clientapi/routing/capabilities.go index 199b15240..72668fa5a 100644 --- a/clientapi/routing/capabilities.go +++ b/clientapi/routing/capabilities.go @@ -23,8 +23,8 @@ import ( "github.com/matrix-org/util" ) -// SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite) -// by building a m.room.member event then sending it to the room server +// GetCapabilities returns information about the server's supported feature set +// and other relevant capabilities to an authenticated user. func GetCapabilities( req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { @@ -41,6 +41,9 @@ func GetCapabilities( response := map[string]interface{}{ "capabilities": map[string]interface{}{ + "m.change_password": map[string]bool{ + "enabled": true, + }, "m.room_versions": roomVersionsQueryRes, }, } diff --git a/sytest-whitelist b/sytest-whitelist index e049f8e7c..29e9166ae 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -480,3 +480,4 @@ Federation key API can act as a notary server via a GET request Inbound /make_join rejects attempts to join rooms where all users have left Inbound federation rejects invites which include invalid JSON for room version 6 Inbound federation rejects invite rejections which include invalid JSON for room version 6 +GET /capabilities is present and well formed for registered user \ No newline at end of file From 4ff7ac7b6574d6adec775057dc2c798a1fe10248 Mon Sep 17 00:00:00 2001 From: bn4t <17193640+bn4t@users.noreply.github.com> Date: Tue, 29 Sep 2020 09:07:23 +0000 Subject: [PATCH 006/104] Fix a small typo (#1448) Signed-off-by: Benjamin Nater --- docs/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 30ef65ea9..1ab885ebc 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -72,7 +72,7 @@ issue](https://github.com/matrix-org/dendrite/labels/good%20first%20issue). These should be well-contained, small pieces of work that can be picked up to help you get familiar with the code base. -Once you're comfortable with hacking on Dendrite there are issues lablled as +Once you're comfortable with hacking on Dendrite there are issues labelled as [help wanted](https://github.com/matrix-org/dendrite/labels/help-wanted), these are often slightly larger or more complicated pieces of work but are hopefully nonetheless fairly well-contained. From 738b829a23d4e50e68f98acb72f7d10a16009f8b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 29 Sep 2020 13:40:29 +0100 Subject: [PATCH 007/104] Fetch missing auth events, implement QueryMissingAuthPrevEvents, try other servers in room for /event and /get_missing_events (#1450) * Try to ask other servers in the room for missing events if the origin won't provide them * Logging * More logging * Implement QueryMissingAuthPrevEvents * Try to get missing auth events badly * Use processEvent * Logging * Update QueryMissingAuthPrevEvents * Try to find missing auth events * Patchy fix for test * Logging tweaks * Send auth events as outliers * Update check in QueryMissingAuthPrevEvents * Error responses * More return codes * Don't return error on reject/soft-fail since it was ultimately handled * More tweaks * More error tweaks --- federationapi/routing/send.go | 130 +++++++++++++----- federationapi/routing/send_test.go | 98 +++++++++---- roomserver/api/api.go | 7 + roomserver/api/api_trace.go | 10 ++ roomserver/api/query.go | 23 ++++ roomserver/internal/helpers/auth.go | 6 +- roomserver/internal/input/input_events.go | 2 +- roomserver/internal/perform/perform_invite.go | 19 +-- roomserver/internal/perform/perform_join.go | 10 +- roomserver/internal/query/query.go | 39 +++++- roomserver/inthttp/client.go | 14 ++ roomserver/inthttp/server.go | 14 ++ 12 files changed, 291 insertions(+), 81 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 5f20b2d8e..4a30f8d7f 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -206,10 +206,10 @@ func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.Res return nil, &jsonErr } else { // Auth errors mean the event is 'rejected' which have to be silent to appease sytest + errMsg := "" _, rejected := err.(*gomatrixserverlib.NotAllowed) - errMsg := err.Error() - if rejected { - errMsg = "" + if !rejected { + errMsg = err.Error() } util.GetLogger(ctx).WithError(err).WithField("event_id", e.EventID()).WithField("rejected", rejected).Warn( "Failed to process incoming federation event, skipping", @@ -345,17 +345,17 @@ func (t *txnReq) processDeviceListUpdate(ctx context.Context, e gomatrixserverli } func (t *txnReq) processEvent(ctx context.Context, e gomatrixserverlib.Event, isInboundTxn bool) error { - prevEventIDs := e.PrevEventIDs() + logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID()) - // Fetch the state needed to authenticate the event. - needed := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{e}) - stateReq := api.QueryStateAfterEventsRequest{ + // Work out if the roomserver knows everything it needs to know to auth + // the event. + stateReq := api.QueryMissingAuthPrevEventsRequest{ RoomID: e.RoomID(), - PrevEventIDs: prevEventIDs, - StateToFetch: needed.Tuples(), + AuthEventIDs: e.AuthEventIDs(), + PrevEventIDs: e.PrevEventIDs(), } - var stateResp api.QueryStateAfterEventsResponse - if err := t.rsAPI.QueryStateAfterEvents(ctx, &stateReq, &stateResp); err != nil { + var stateResp api.QueryMissingAuthPrevEventsResponse + if err := t.rsAPI.QueryMissingAuthPrevEvents(ctx, &stateReq, &stateResp); err != nil { return err } @@ -369,7 +369,53 @@ func (t *txnReq) processEvent(ctx context.Context, e gomatrixserverlib.Event, is return roomNotFoundError{e.RoomID()} } - if !stateResp.PrevEventsExist { + if len(stateResp.MissingAuthEventIDs) > 0 { + logger.Infof("Event refers to %d unknown auth_events", len(stateResp.MissingAuthEventIDs)) + + servers := []gomatrixserverlib.ServerName{t.Origin} + serverReq := &api.QueryServerJoinedToRoomRequest{ + RoomID: e.RoomID(), + } + serverRes := &api.QueryServerJoinedToRoomResponse{} + if err := t.rsAPI.QueryServerJoinedToRoom(ctx, serverReq, serverRes); err == nil { + servers = append(servers, serverRes.ServerNames...) + logger.Infof("Found %d server(s) to query for missing events", len(servers)) + } + + getAuthEvent: + for _, missingAuthEventID := range stateResp.MissingAuthEventIDs { + for _, server := range servers { + logger.Infof("Retrieving missing auth event %q from %q", missingAuthEventID, server) + tx, err := t.federation.GetEvent(ctx, server, missingAuthEventID) + if err != nil { + continue // try the next server + } + ev, err := gomatrixserverlib.NewEventFromUntrustedJSON(tx.PDUs[0], stateResp.RoomVersion) + if err != nil { + logger.WithError(err).Errorf("Failed to unmarshal auth event %q", missingAuthEventID) + continue // try the next server + } + if err = api.SendInputRoomEvents( + context.Background(), + t.rsAPI, + []api.InputRoomEvent{ + { + Kind: api.KindOutlier, + Event: ev.Headered(stateResp.RoomVersion), + AuthEventIDs: ev.AuthEventIDs(), + SendAsServer: api.DoNotSendToOtherServers, + }, + }, + ); err != nil { + logger.WithError(err).Errorf("Failed to send auth event %q to roomserver", missingAuthEventID) + continue getAuthEvent // move onto the next event + } + } + } + } + + if len(stateResp.MissingPrevEventIDs) > 0 { + logger.Infof("Event refers to %d unknown prev_events", len(stateResp.MissingPrevEventIDs)) return t.processEventWithMissingState(ctx, e, stateResp.RoomVersion, isInboundTxn) } @@ -611,6 +657,7 @@ retryAllowedState: // begin from. Returns an error only if we should terminate the transaction which initiated /get_missing_events // This function recursively calls txnReq.processEvent with the missing events, which will be processed before this function returns. // This means that we may recursively call this function, as we spider back up prev_events to the min depth. +// nolint:gocyclo func (t *txnReq) getMissingEvents(ctx context.Context, e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion, isInboundTxn bool) (backwardsExtremity *gomatrixserverlib.Event, err error) { if !isInboundTxn { // we've recursed here, so just take a state snapshot please! @@ -637,15 +684,46 @@ func (t *txnReq) getMissingEvents(ctx context.Context, e gomatrixserverlib.Event if minDepth < 0 { minDepth = 0 } - missingResp, err := t.federation.LookupMissingEvents(ctx, t.Origin, e.RoomID(), gomatrixserverlib.MissingEvents{ - Limit: 20, - // synapse uses the min depth they've ever seen in that room - MinDepth: minDepth, - // The latest event IDs that the sender already has. These are skipped when retrieving the previous events of latest_events. - EarliestEvents: latestEvents, - // The event IDs to retrieve the previous events for. - LatestEvents: []string{e.EventID()}, - }, roomVersion) + + servers := []gomatrixserverlib.ServerName{t.Origin} + serverReq := &api.QueryServerJoinedToRoomRequest{ + RoomID: e.RoomID(), + } + serverRes := &api.QueryServerJoinedToRoomResponse{} + if err = t.rsAPI.QueryServerJoinedToRoom(ctx, serverReq, serverRes); err == nil { + servers = append(servers, serverRes.ServerNames...) + logger.Infof("Found %d server(s) to query for missing events", len(servers)) + } + + var missingResp *gomatrixserverlib.RespMissingEvents + for _, server := range servers { + var m gomatrixserverlib.RespMissingEvents + if m, err = t.federation.LookupMissingEvents(ctx, server, e.RoomID(), gomatrixserverlib.MissingEvents{ + Limit: 20, + // synapse uses the min depth they've ever seen in that room + MinDepth: minDepth, + // The latest event IDs that the sender already has. These are skipped when retrieving the previous events of latest_events. + EarliestEvents: latestEvents, + // The event IDs to retrieve the previous events for. + LatestEvents: []string{e.EventID()}, + }, roomVersion); err == nil { + missingResp = &m + break + } else { + logger.WithError(err).Errorf("%s pushed us an event but %q did not respond to /get_missing_events", t.Origin, server) + } + } + + if missingResp == nil { + logger.WithError(err).Errorf( + "%s pushed us an event but %d server(s) couldn't give us details about prev_events via /get_missing_events - dropping this event until it can", + t.Origin, len(servers), + ) + return nil, missingPrevEventsError{ + eventID: e.EventID(), + err: err, + } + } // security: how we handle failures depends on whether or not this event will become the new forward extremity for the room. // There's 2 scenarios to consider: @@ -658,16 +736,6 @@ func (t *txnReq) getMissingEvents(ctx context.Context, e gomatrixserverlib.Event // https://github.com/matrix-org/synapse/pull/3456 // https://github.com/matrix-org/synapse/blob/229eb81498b0fe1da81e9b5b333a0285acde9446/synapse/handlers/federation.py#L335 // For now, we do not allow Case B, so reject the event. - if err != nil { - logger.WithError(err).Errorf( - "%s pushed us an event but couldn't give us details about prev_events via /get_missing_events - dropping this event until it can", - t.Origin, - ) - return nil, missingPrevEventsError{ - eventID: e.EventID(), - err: err, - } - } logger.Infof("get_missing_events returned %d events", len(missingResp.Events)) // topologically sort and sanity check that we are making forward progress diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index e1211ffe9..ba653c1e8 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -77,10 +77,11 @@ func (p *testEDUProducer) InputSendToDeviceEvent( } type testRoomserverAPI struct { - inputRoomEvents []api.InputRoomEvent - queryStateAfterEvents func(*api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse - queryEventsByID func(req *api.QueryEventsByIDRequest) api.QueryEventsByIDResponse - queryLatestEventsAndState func(*api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse + inputRoomEvents []api.InputRoomEvent + queryMissingAuthPrevEvents func(*api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse + queryStateAfterEvents func(*api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse + queryEventsByID func(req *api.QueryEventsByIDRequest) api.QueryEventsByIDResponse + queryLatestEventsAndState func(*api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse } func (t *testRoomserverAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) {} @@ -162,6 +163,20 @@ func (t *testRoomserverAPI) QueryStateAfterEvents( return nil } +// Query the state after a list of events in a room from the room server. +func (t *testRoomserverAPI) QueryMissingAuthPrevEvents( + ctx context.Context, + request *api.QueryMissingAuthPrevEventsRequest, + response *api.QueryMissingAuthPrevEventsResponse, +) error { + response.RoomVersion = testRoomVersion + res := t.queryMissingAuthPrevEvents(request) + response.RoomExists = res.RoomExists + response.MissingAuthEventIDs = res.MissingAuthEventIDs + response.MissingPrevEventIDs = res.MissingPrevEventIDs + return nil +} + // Query a list of events by event ID. func (t *testRoomserverAPI) QueryEventsByID( ctx context.Context, @@ -453,11 +468,11 @@ func assertInputRoomEvents(t *testing.T, got []api.InputRoomEvent, want []gomatr // to the roomserver. It's the most basic test possible. func TestBasicTransaction(t *testing.T) { rsAPI := &testRoomserverAPI{ - queryStateAfterEvents: func(req *api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse { - return api.QueryStateAfterEventsResponse{ - PrevEventsExist: true, - RoomExists: true, - StateEvents: fromStateTuples(req.StateToFetch, nil), + queryMissingAuthPrevEvents: func(req *api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse { + return api.QueryMissingAuthPrevEventsResponse{ + RoomExists: true, + MissingAuthEventIDs: []string{}, + MissingPrevEventIDs: []string{}, } }, } @@ -473,14 +488,11 @@ func TestBasicTransaction(t *testing.T) { // as it does the auth check. func TestTransactionFailAuthChecks(t *testing.T) { rsAPI := &testRoomserverAPI{ - queryStateAfterEvents: func(req *api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse { - return api.QueryStateAfterEventsResponse{ - PrevEventsExist: true, - RoomExists: true, - // omit the create event so auth checks fail - StateEvents: fromStateTuples(req.StateToFetch, []gomatrixserverlib.StateKeyTuple{ - {EventType: gomatrixserverlib.MRoomCreate, StateKey: ""}, - }), + queryMissingAuthPrevEvents: func(req *api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse { + return api.QueryMissingAuthPrevEventsResponse{ + RoomExists: true, + MissingAuthEventIDs: []string{"create_event"}, + MissingPrevEventIDs: []string{}, } }, } @@ -504,28 +516,24 @@ func TestTransactionFetchMissingPrevEvents(t *testing.T) { var rsAPI *testRoomserverAPI // ref here so we can refer to inputRoomEvents inside these functions rsAPI = &testRoomserverAPI{ - queryStateAfterEvents: func(req *api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse { - // we expect this to be called three times: - // - first with input event to realise there's a gap - // - second with the prevEvent to realise there is no gap - // - third with the input event to realise there is no longer a gap - prevEventsExist := false + queryMissingAuthPrevEvents: func(req *api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse { + missingPrevEvent := []string{"missing_prev_event"} if len(req.PrevEventIDs) == 1 { switch req.PrevEventIDs[0] { case haveEvent.EventID(): - prevEventsExist = true + missingPrevEvent = []string{} case prevEvent.EventID(): // we only have this event if we've been send prevEvent if len(rsAPI.inputRoomEvents) == 1 && rsAPI.inputRoomEvents[0].Event.EventID() == prevEvent.EventID() { - prevEventsExist = true + missingPrevEvent = []string{} } } } - return api.QueryStateAfterEventsResponse{ - PrevEventsExist: prevEventsExist, - RoomExists: true, - StateEvents: fromStateTuples(req.StateToFetch, nil), + return api.QueryMissingAuthPrevEventsResponse{ + RoomExists: true, + MissingAuthEventIDs: []string{}, + MissingPrevEventIDs: missingPrevEvent, } }, queryLatestEventsAndState: func(req *api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse { @@ -626,6 +634,38 @@ func TestTransactionFetchMissingStateByStateIDs(t *testing.T) { StateEvents: stateEvents, } }, + + queryMissingAuthPrevEvents: func(req *api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse { + askingForEvent := req.PrevEventIDs[0] + haveEventB := false + haveEventC := false + for _, ev := range rsAPI.inputRoomEvents { + switch ev.Event.EventID() { + case eventB.EventID(): + haveEventB = true + case eventC.EventID(): + haveEventC = true + } + } + prevEventExists := false + if askingForEvent == eventC.EventID() { + prevEventExists = haveEventC + } else if askingForEvent == eventB.EventID() { + prevEventExists = haveEventB + } + + var missingPrevEvent []string + if !prevEventExists { + missingPrevEvent = []string{"test"} + } + + return api.QueryMissingAuthPrevEventsResponse{ + RoomExists: true, + MissingAuthEventIDs: []string{}, + MissingPrevEventIDs: missingPrevEvent, + } + }, + queryLatestEventsAndState: func(req *api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse { omitTuples := []gomatrixserverlib.StateKeyTuple{ {EventType: gomatrixserverlib.MRoomPowerLevels, StateKey: ""}, diff --git a/roomserver/api/api.go b/roomserver/api/api.go index 159c18299..043f72221 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -68,6 +68,13 @@ type RoomserverInternalAPI interface { response *QueryStateAfterEventsResponse, ) error + // Query whether the roomserver is missing any auth or prev events. + QueryMissingAuthPrevEvents( + ctx context.Context, + request *QueryMissingAuthPrevEventsRequest, + response *QueryMissingAuthPrevEventsResponse, + ) error + // Query a list of events by event ID. QueryEventsByID( ctx context.Context, diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go index 5fabbc21d..f4eaddc1e 100644 --- a/roomserver/api/api_trace.go +++ b/roomserver/api/api_trace.go @@ -104,6 +104,16 @@ func (t *RoomserverInternalAPITrace) QueryStateAfterEvents( return err } +func (t *RoomserverInternalAPITrace) QueryMissingAuthPrevEvents( + ctx context.Context, + req *QueryMissingAuthPrevEventsRequest, + res *QueryMissingAuthPrevEventsResponse, +) error { + err := t.Impl.QueryMissingAuthPrevEvents(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryMissingAuthPrevEvents req=%+v res=%+v", js(req), js(res)) + return err +} + func (t *RoomserverInternalAPITrace) QueryEventsByID( ctx context.Context, req *QueryEventsByIDRequest, diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 5d61e862c..aff6ee07a 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -82,6 +82,27 @@ type QueryStateAfterEventsResponse struct { StateEvents []gomatrixserverlib.HeaderedEvent `json:"state_events"` } +type QueryMissingAuthPrevEventsRequest struct { + // The room ID to query the state in. + RoomID string `json:"room_id"` + // The list of auth events to check the existence of. + AuthEventIDs []string `json:"auth_event_ids"` + // The list of previous events to check the existence of. + PrevEventIDs []string `json:"prev_event_ids"` +} + +type QueryMissingAuthPrevEventsResponse struct { + // Does the room exist on this roomserver? + // If the room doesn't exist all other fields will be empty. + RoomExists bool `json:"room_exists"` + // The room version of the room. + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` + // The event IDs of the auth events that we don't know locally. + MissingAuthEventIDs []string `json:"missing_auth_event_ids"` + // The event IDs of the previous events that we don't know locally. + MissingPrevEventIDs []string `json:"missing_prev_event_ids"` +} + // QueryEventsByIDRequest is a request to QueryEventsByID type QueryEventsByIDRequest struct { // The event IDs to look up. @@ -154,6 +175,8 @@ type QueryServerJoinedToRoomResponse struct { RoomExists bool `json:"room_exists"` // True if we still believe that we are participating in the room IsInRoom bool `json:"is_in_room"` + // List of servers that are also in the room + ServerNames []gomatrixserverlib.ServerName `json:"server_names"` } // QueryServerAllowedToSeeEventRequest is a request to QueryServerAllowedToSeeEvent diff --git a/roomserver/internal/helpers/auth.go b/roomserver/internal/helpers/auth.go index 834bc0c6e..0fa89d9c4 100644 --- a/roomserver/internal/helpers/auth.go +++ b/roomserver/internal/helpers/auth.go @@ -83,7 +83,7 @@ func CheckForSoftFail( // Check if the event is allowed. if err = gomatrixserverlib.Allowed(event.Event, &authEvents); err != nil { // return true, nil - return true, fmt.Errorf("gomatrixserverlib.Allowed: %w", err) + return true, err } return false, nil } @@ -99,7 +99,7 @@ func CheckAuthEvents( // Grab the numeric IDs for the supplied auth state events from the database. authStateEntries, err := db.StateEntriesForEventIDs(ctx, authEventIDs) if err != nil { - return nil, err + return nil, fmt.Errorf("db.StateEntriesForEventIDs: %w", err) } authStateEntries = types.DeduplicateStateEntries(authStateEntries) @@ -109,7 +109,7 @@ func CheckAuthEvents( // Load the actual auth events from the database. authEvents, err := loadAuthEvents(ctx, db, stateNeeded, authStateEntries) if err != nil { - return nil, err + return nil, fmt.Errorf("loadAuthEvents: %w", err) } // Check if the event is allowed. diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index f953a9259..3d44f0486 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -49,7 +49,7 @@ func (r *Inputer) processRoomEvent( isRejected := false authEventNIDs, rejectionErr := helpers.CheckAuthEvents(ctx, r.DB, headered, input.AuthEventIDs) if rejectionErr != nil { - logrus.WithError(rejectionErr).WithField("event_id", event.EventID()).WithField("auth_event_ids", input.AuthEventIDs).Error("processRoomEvent.checkAuthEvents failed for event, rejecting event") + logrus.WithError(rejectionErr).WithField("event_id", event.EventID()).WithField("auth_event_ids", input.AuthEventIDs).Error("helpers.CheckAuthEvents failed for event, rejecting event") isRejected = true } diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go index d6a64e7e8..734e73d43 100644 --- a/roomserver/internal/perform/perform_invite.go +++ b/roomserver/internal/perform/perform_invite.go @@ -136,14 +136,10 @@ func (r *Inviter) PerformInvite( log.WithError(err).WithField("event_id", event.EventID()).WithField("auth_event_ids", event.AuthEventIDs()).Error( "processInviteEvent.checkAuthEvents failed for event", ) - if _, ok := err.(*gomatrixserverlib.NotAllowed); ok { - res.Error = &api.PerformError{ - Msg: err.Error(), - Code: api.PerformErrorNotAllowed, - } - return nil, nil + res.Error = &api.PerformError{ + Msg: err.Error(), + Code: api.PerformErrorNotAllowed, } - return nil, fmt.Errorf("checkAuthEvents: %w", err) } // If the invite originated from us and the target isn't local then we @@ -160,7 +156,7 @@ func (r *Inviter) PerformInvite( if err = r.FSAPI.PerformInvite(ctx, fsReq, fsRes); err != nil { res.Error = &api.PerformError{ Msg: err.Error(), - Code: api.PerformErrorNoOperation, + Code: api.PerformErrorNotAllowed, } log.WithError(err).WithField("event_id", event.EventID()).Error("r.FSAPI.PerformInvite failed") return nil, nil @@ -185,7 +181,12 @@ func (r *Inviter) PerformInvite( inputRes := &api.InputRoomEventsResponse{} r.Inputer.InputRoomEvents(context.Background(), inputReq, inputRes) if err = inputRes.Err(); err != nil { - return nil, fmt.Errorf("r.InputRoomEvents: %w", err) + res.Error = &api.PerformError{ + Msg: fmt.Sprintf("r.InputRoomEvents: %s", err.Error()), + Code: api.PerformErrorNotAllowed, + } + log.WithError(err).WithField("event_id", event.EventID()).Error("r.InputRoomEvents failed") + return nil, nil } } else { // The invite originated over federation. Process the membership diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go index e9aebb839..56ae6d0b1 100644 --- a/roomserver/internal/perform/perform_join.go +++ b/roomserver/internal/perform/perform_join.go @@ -249,14 +249,10 @@ func (r *Joiner) performJoinRoomByID( inputRes := api.InputRoomEventsResponse{} r.Inputer.InputRoomEvents(ctx, &inputReq, &inputRes) if err = inputRes.Err(); err != nil { - var notAllowed *gomatrixserverlib.NotAllowed - if errors.As(err, ¬Allowed) { - return "", &api.PerformError{ - Code: api.PerformErrorNotAllowed, - Msg: fmt.Sprintf("InputRoomEvents auth failed: %s", err), - } + return "", &api.PerformError{ + Code: api.PerformErrorNotAllowed, + Msg: fmt.Sprintf("InputRoomEvents auth failed: %s", err), } - return "", fmt.Errorf("r.InputRoomEvents: %w", err) } } diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 58cb44933..736604217 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -98,6 +98,38 @@ func (r *Queryer) QueryStateAfterEvents( return nil } +// QueryMissingAuthPrevEvents implements api.RoomserverInternalAPI +func (r *Queryer) QueryMissingAuthPrevEvents( + ctx context.Context, + request *api.QueryMissingAuthPrevEventsRequest, + response *api.QueryMissingAuthPrevEventsResponse, +) error { + info, err := r.DB.RoomInfo(ctx, request.RoomID) + if err != nil { + return err + } + if info == nil { + return errors.New("room doesn't exist") + } + + response.RoomExists = !info.IsStub + response.RoomVersion = info.RoomVersion + + for _, authEventID := range request.AuthEventIDs { + if nids, err := r.DB.EventNIDs(ctx, []string{authEventID}); err != nil || len(nids) == 0 { + response.MissingAuthEventIDs = append(response.MissingAuthEventIDs, authEventID) + } + } + + for _, prevEventID := range request.PrevEventIDs { + if nids, err := r.DB.EventNIDs(ctx, []string{prevEventID}); err != nil || len(nids) == 0 { + response.MissingPrevEventIDs = append(response.MissingPrevEventIDs, prevEventID) + } + } + + return nil +} + // QueryEventsByID implements api.RoomserverInternalAPI func (r *Queryer) QueryEventsByID( ctx context.Context, @@ -255,19 +287,24 @@ func (r *Queryer) QueryServerJoinedToRoom( return fmt.Errorf("r.DB.Events: %w", err) } + servers := map[gomatrixserverlib.ServerName]struct{}{} for _, e := range events { if e.Type() == gomatrixserverlib.MRoomMember && e.StateKey() != nil { _, serverName, err := gomatrixserverlib.SplitID('@', *e.StateKey()) if err != nil { continue } + servers[serverName] = struct{}{} if serverName == request.ServerName { response.IsInRoom = true - break } } } + for server := range servers { + response.ServerNames = append(response.ServerNames, server) + } + return nil } diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index 3dd3edaff..24a82adf8 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -35,6 +35,7 @@ const ( // Query operations RoomserverQueryLatestEventsAndStatePath = "/roomserver/queryLatestEventsAndState" RoomserverQueryStateAfterEventsPath = "/roomserver/queryStateAfterEvents" + RoomserverQueryMissingAuthPrevEventsPath = "/roomserver/queryMissingAuthPrevEvents" RoomserverQueryEventsByIDPath = "/roomserver/queryEventsByID" RoomserverQueryMembershipForUserPath = "/roomserver/queryMembershipForUser" RoomserverQueryMembershipsForRoomPath = "/roomserver/queryMembershipsForRoom" @@ -262,6 +263,19 @@ func (h *httpRoomserverInternalAPI) QueryStateAfterEvents( return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } +// QueryStateAfterEvents implements RoomserverQueryAPI +func (h *httpRoomserverInternalAPI) QueryMissingAuthPrevEvents( + ctx context.Context, + request *api.QueryMissingAuthPrevEventsRequest, + response *api.QueryMissingAuthPrevEventsResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryMissingAuthPrevEvents") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryMissingAuthPrevEventsPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + // QueryEventsByID implements RoomserverQueryAPI func (h *httpRoomserverInternalAPI) QueryEventsByID( ctx context.Context, diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go index c7e541dd6..9c9d4d4ae 100644 --- a/roomserver/inthttp/server.go +++ b/roomserver/inthttp/server.go @@ -125,6 +125,20 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle( + RoomserverQueryMissingAuthPrevEventsPath, + httputil.MakeInternalAPI("queryMissingAuthPrevEvents", func(req *http.Request) util.JSONResponse { + var request api.QueryMissingAuthPrevEventsRequest + var response api.QueryMissingAuthPrevEventsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryMissingAuthPrevEvents(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) internalAPIMux.Handle( RoomserverQueryEventsByIDPath, httputil.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { From b0d5d1cc9f0de61ee0738581db11fd218be65dc4 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 29 Sep 2020 14:00:12 +0100 Subject: [PATCH 008/104] Fix old verify keys --- federationapi/routing/keys.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go index 17762b03e..cbec88c13 100644 --- a/federationapi/routing/keys.go +++ b/federationapi/routing/keys.go @@ -151,7 +151,7 @@ func localKeys(cfg *config.FederationAPI, validUntil time.Time) (*gomatrixserver for _, oldVerifyKey := range cfg.Matrix.OldVerifyKeys { keys.OldVerifyKeys[oldVerifyKey.KeyID] = gomatrixserverlib.OldVerifyKey{ VerifyKey: gomatrixserverlib.VerifyKey{ - Key: gomatrixserverlib.Base64Bytes(oldVerifyKey.PrivateKey), + Key: gomatrixserverlib.Base64Bytes(oldVerifyKey.PrivateKey.Public().(ed25519.PublicKey)), }, ExpiredTS: oldVerifyKey.ExpiredAt, } From 43cdba9a69674899a5900aee976ebc7add286914 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 29 Sep 2020 14:07:59 +0100 Subject: [PATCH 009/104] Ignore depth in federation API (#1451) --- federationapi/routing/missingevents.go | 10 +++++----- federationapi/routing/send.go | 9 +-------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go index f93e0eb41..5118b34e5 100644 --- a/federationapi/routing/missingevents.go +++ b/federationapi/routing/missingevents.go @@ -26,7 +26,7 @@ type getMissingEventRequest struct { EarliestEvents []string `json:"earliest_events"` LatestEvents []string `json:"latest_events"` Limit int `json:"limit"` - MinDepth int64 `json:"min_depth"` + MinDepth int64 `json:"min_depth"` // not used } // GetMissingEvents returns missing events between earliest_events & latest_events. @@ -59,7 +59,7 @@ func GetMissingEvents( return jsonerror.InternalServerError() } - eventsResponse.Events = filterEvents(eventsResponse.Events, gme.MinDepth, roomID) + eventsResponse.Events = filterEvents(eventsResponse.Events, roomID) resp := gomatrixserverlib.RespMissingEvents{ Events: gomatrixserverlib.UnwrapEventHeaders(eventsResponse.Events), @@ -71,13 +71,13 @@ func GetMissingEvents( } } -// filterEvents returns only those events with matching roomID and having depth greater than minDepth +// filterEvents returns only those events with matching roomID func filterEvents( - events []gomatrixserverlib.HeaderedEvent, minDepth int64, roomID string, + events []gomatrixserverlib.HeaderedEvent, roomID string, ) []gomatrixserverlib.HeaderedEvent { ref := events[:0] for _, ev := range events { - if ev.Depth() >= minDepth && ev.RoomID() == roomID { + if ev.RoomID() == roomID { ref = append(ref, ev) } } diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 4a30f8d7f..f8b8bcd6f 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -656,7 +656,7 @@ retryAllowedState: // getMissingEvents returns a nil backwardsExtremity if missing events were fetched and handled, else returns the new backwards extremity which we should // begin from. Returns an error only if we should terminate the transaction which initiated /get_missing_events // This function recursively calls txnReq.processEvent with the missing events, which will be processed before this function returns. -// This means that we may recursively call this function, as we spider back up prev_events to the min depth. +// This means that we may recursively call this function, as we spider back up prev_events. // nolint:gocyclo func (t *txnReq) getMissingEvents(ctx context.Context, e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion, isInboundTxn bool) (backwardsExtremity *gomatrixserverlib.Event, err error) { if !isInboundTxn { @@ -679,11 +679,6 @@ func (t *txnReq) getMissingEvents(ctx context.Context, e gomatrixserverlib.Event for i := range res.LatestEvents { latestEvents[i] = res.LatestEvents[i].EventID } - // this server just sent us an event for which we do not know its prev_events - ask that server for those prev_events. - minDepth := int(res.Depth) - 20 - if minDepth < 0 { - minDepth = 0 - } servers := []gomatrixserverlib.ServerName{t.Origin} serverReq := &api.QueryServerJoinedToRoomRequest{ @@ -700,8 +695,6 @@ func (t *txnReq) getMissingEvents(ctx context.Context, e gomatrixserverlib.Event var m gomatrixserverlib.RespMissingEvents if m, err = t.federation.LookupMissingEvents(ctx, server, e.RoomID(), gomatrixserverlib.MissingEvents{ Limit: 20, - // synapse uses the min depth they've ever seen in that room - MinDepth: minDepth, // The latest event IDs that the sender already has. These are skipped when retrieving the previous events of latest_events. EarliestEvents: latestEvents, // The event IDs to retrieve the previous events for. From f290e92a34c67e4d9673629810ac8f0f85b28b7c Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 29 Sep 2020 17:08:18 +0100 Subject: [PATCH 010/104] Remove TLS fingerprints, improve perspective unmarshal handling (#1452) * Add prefer_direct_fetch option * Update gomatrixserverlib * Update gomatrixserverlib * Update gomatrixserverlib * Don't deal in TLS fingerprints anymore --- dendrite-config.yaml | 5 +++++ federationapi/routing/keys.go | 1 - go.mod | 2 +- go.sum | 10 +++++++-- internal/config/config.go | 29 ------------------------- internal/config/config_federationapi.go | 6 ----- internal/config/config_serverkey.go | 3 +++ internal/config/config_test.go | 13 ----------- serverkeyapi/internal/api.go | 2 +- serverkeyapi/serverkeyapi.go | 21 +++++++++++++----- 10 files changed, 34 insertions(+), 58 deletions(-) diff --git a/dendrite-config.yaml b/dendrite-config.yaml index b71fb5091..74fa9b3e1 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -274,6 +274,11 @@ server_key_api: - key_id: ed25519:a_RXGa public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ + # This option will control whether Dendrite will prefer to look up keys directly + # or whether it should try perspective servers first, using direct fetches as a + # last resort. + prefer_direct_fetch: false + # Configuration for the Sync API. sync_api: internal_api: diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go index cbec88c13..4779bcb2b 100644 --- a/federationapi/routing/keys.go +++ b/federationapi/routing/keys.go @@ -136,7 +136,6 @@ func localKeys(cfg *config.FederationAPI, validUntil time.Time) (*gomatrixserver var keys gomatrixserverlib.ServerKeys keys.ServerName = cfg.Matrix.ServerName - keys.TLSFingerprints = cfg.TLSFingerPrints keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(validUntil) publicKey := cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey) diff --git a/go.mod b/go.mod index ca0c2710c..c0dda07cd 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20200925165243-b9780a852681 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200929155210-32fc5888d26a github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index 829977785..648807285 100644 --- a/go.sum +++ b/go.sum @@ -569,8 +569,14 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200925165243-b9780a852681 h1:75fM7vPHiFGt+XxktT17LJD972XMtJ1n7FU1MpC08Zc= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200925165243-b9780a852681/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200929152221-6fe6457127ad h1:n0P/Oy8ZqqTPzum6FEayAjamsmvJTuIcA10WQ8GcK70= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200929152221-6fe6457127ad/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154026-a52e7a5f0553 h1:tiel2c3I9xr0SRS05g3UvOjj6Sgg1I3Yn2/oGA1GgLk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154026-a52e7a5f0553/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154241-9414c4d0b5f2 h1:a07U5eFT521mFiUtA/A8NwiZp5vfRU59/QKs+pa3Fkc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154241-9414c4d0b5f2/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200929155210-32fc5888d26a h1:kIwbS7eY7P/MX0oN4wRHGkoc4eTTnwOcdCawBZ3SrJI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200929155210-32fc5888d26a/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/internal/config/config.go b/internal/config/config.go index 7528aa237..74d3f4fa5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -16,7 +16,6 @@ package config import ( "bytes" - "crypto/sha256" "encoding/pem" "fmt" "io" @@ -252,20 +251,6 @@ func loadConfig( c.Global.OldVerifyKeys[i].KeyID, c.Global.OldVerifyKeys[i].PrivateKey = keyID, privateKey } - for _, certPath := range c.FederationAPI.FederationCertificatePaths { - absCertPath := absPath(basePath, certPath) - var pemData []byte - pemData, err = readFile(absCertPath) - if err != nil { - return nil, err - } - fingerprint := fingerprintPEM(pemData) - if fingerprint == nil { - return nil, fmt.Errorf("no certificate PEM data in %q", absCertPath) - } - c.FederationAPI.TLSFingerPrints = append(c.FederationAPI.TLSFingerPrints, *fingerprint) - } - c.MediaAPI.AbsBasePath = Path(absPath(basePath, c.MediaAPI.BasePath)) // Generate data from config options @@ -494,20 +479,6 @@ func readKeyPEM(path string, data []byte, enforceKeyIDFormat bool) (gomatrixserv } } -func fingerprintPEM(data []byte) *gomatrixserverlib.TLSFingerprint { - for { - var certDERBlock *pem.Block - certDERBlock, data = pem.Decode(data) - if data == nil { - return nil - } - if certDERBlock.Type == "CERTIFICATE" { - digest := sha256.Sum256(certDERBlock.Bytes) - return &gomatrixserverlib.TLSFingerprint{SHA256: digest[:]} - } - } -} - // AppServiceURL returns a HTTP URL for where the appservice component is listening. func (config *Dendrite) AppServiceURL() string { // Hard code the appservice server to talk HTTP for now. diff --git a/internal/config/config_federationapi.go b/internal/config/config_federationapi.go index 727bfce2f..64803d95e 100644 --- a/internal/config/config_federationapi.go +++ b/internal/config/config_federationapi.go @@ -1,7 +1,5 @@ package config -import "github.com/matrix-org/gomatrixserverlib" - type FederationAPI struct { Matrix *Global `yaml:"-"` @@ -14,10 +12,6 @@ type FederationAPI struct { // to match one of these certificates. // The certificates should be in PEM format. FederationCertificatePaths []Path `yaml:"federation_certificates"` - - // A list of SHA256 TLS fingerprints for the X509 certificates used by the - // federation listener for this server. - TLSFingerPrints []gomatrixserverlib.TLSFingerprint `yaml:"-"` } func (c *FederationAPI) Defaults() { diff --git a/internal/config/config_serverkey.go b/internal/config/config_serverkey.go index 40506d233..788a2fa05 100644 --- a/internal/config/config_serverkey.go +++ b/internal/config/config_serverkey.go @@ -14,6 +14,9 @@ type ServerKeyAPI struct { // Perspective keyservers, to use as a backup when direct key fetch // requests don't succeed KeyPerspectives KeyPerspectives `yaml:"key_perspectives"` + + // Should we prefer direct key fetches over perspective ones? + PreferDirectFetch bool `yaml:"prefer_direct_fetch"` } func (c *ServerKeyAPI) Defaults() { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7549fa024..4107b6845 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -253,19 +253,6 @@ Key-ID: ` + testKeyID + ` -----END MATRIX PRIVATE KEY----- ` -func TestFingerprintPEM(t *testing.T) { - got := fingerprintPEM([]byte(testCert)) - if got == nil { - t.Error("failed to calculate fingerprint") - } - if string(got.SHA256) != testCertFingerprint { - t.Errorf("bad fingerprint: wanted %q got %q", got, testCertFingerprint) - } - -} - -const testCertFingerprint = "56.\\SPQxE\xd4\x95\xfb\xf6\xd5\x04\x91\xcb/\x07\xb1^\x88\x08\xe3\xc1p\xdfY\x04\x19w\xcb" - const testCert = ` -----BEGIN CERTIFICATE----- MIIE0zCCArugAwIBAgIJAPype3u24LJeMA0GCSqGSIb3DQEBCwUAMAAwHhcNMTcw diff --git a/serverkeyapi/internal/api.go b/serverkeyapi/internal/api.go index bc02ac2df..415093fd4 100644 --- a/serverkeyapi/internal/api.go +++ b/serverkeyapi/internal/api.go @@ -191,7 +191,7 @@ func (s *ServerKeyAPI) handleFetcherKeys( // Try to fetch the keys. fetcherResults, err := fetcher.FetchKeys(fetcherCtx, requests) if err != nil { - return err + return fmt.Errorf("fetcher.FetchKeys: %w", err) } // Build a map of the results that we want to commit to the diff --git a/serverkeyapi/serverkeyapi.go b/serverkeyapi/serverkeyapi.go index 783402b25..92781f464 100644 --- a/serverkeyapi/serverkeyapi.go +++ b/serverkeyapi/serverkeyapi.go @@ -51,15 +51,26 @@ func NewInternalAPI( ServerKeyValidity: cfg.Matrix.KeyValidityPeriod, FedClient: fedClient, OurKeyRing: gomatrixserverlib.KeyRing{ - KeyFetchers: []gomatrixserverlib.KeyFetcher{ - &gomatrixserverlib.DirectKeyFetcher{ - Client: fedClient, - }, - }, + KeyFetchers: []gomatrixserverlib.KeyFetcher{}, KeyDatabase: serverKeyDB, }, } + addDirectFetcher := func() { + internalAPI.OurKeyRing.KeyFetchers = append( + internalAPI.OurKeyRing.KeyFetchers, + &gomatrixserverlib.DirectKeyFetcher{ + Client: fedClient, + }, + ) + } + + if cfg.PreferDirectFetch { + addDirectFetcher() + } else { + defer addDirectFetcher() + } + var b64e = base64.StdEncoding.WithPadding(base64.NoPadding) for _, ps := range cfg.KeyPerspectives { perspective := &gomatrixserverlib.PerspectiveKeyFetcher{ From d63d7c564022c4e273adaafc0dec2fb182694bc9 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 29 Sep 2020 17:08:47 +0100 Subject: [PATCH 011/104] Tweak log level of a fairly common log line --- federationsender/queue/destinationqueue.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index 12a04d4b9..e87f00634 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -224,7 +224,7 @@ func (oq *destinationQueue) backgroundSend() { // The worker is idle so stop the goroutine. It'll get // restarted automatically the next time we have an event to // send. - log.Debugf("Queue %q has been idle for %s, going to sleep", oq.destination, queueIdleTimeout) + log.Tracef("Queue %q has been idle for %s, going to sleep", oq.destination, queueIdleTimeout) return } From beaf4cc00f6f198202742693cef40d71ed0b255e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 29 Sep 2020 17:34:48 +0100 Subject: [PATCH 012/104] Fix perspective fetching --- serverkeyapi/internal/api.go | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/serverkeyapi/internal/api.go b/serverkeyapi/internal/api.go index 415093fd4..09707ae5b 100644 --- a/serverkeyapi/internal/api.go +++ b/serverkeyapi/internal/api.go @@ -201,6 +201,10 @@ func (s *ServerKeyAPI) handleFetcherKeys( // Now let's look at the results that we got from this fetcher. for req, res := range fetcherResults { + if req.ServerName == s.ServerName { + continue + } + if prev, ok := results[req]; ok { // We've already got a previous entry for this request // so let's see if the newly retrieved one contains a more @@ -208,31 +212,24 @@ func (s *ServerKeyAPI) handleFetcherKeys( if res.ValidUntilTS > prev.ValidUntilTS { // This key is newer than the one we had so let's store // it in the database. - if req.ServerName != s.ServerName { - storeResults[req] = res - } + storeResults[req] = res } } else { // We didn't already have a previous entry for this request // so store it in the database anyway for now. - if req.ServerName != s.ServerName { - storeResults[req] = res - } + storeResults[req] = res } // Update the results map with this new result. If nothing // else, we can try verifying against this key. results[req] = res - // If the key is valid right now then we can remove it from the - // request list as we won't need to re-fetch it. - if res.WasValidAt(now, true) { - delete(requests, req) - } + // Remove it from the request list so we won't re-fetch it. + delete(requests, req) } // Store the keys from our store map. - if err = s.OurKeyRing.KeyDatabase.StoreKeys(ctx, storeResults); err != nil { + if err = s.OurKeyRing.KeyDatabase.StoreKeys(context.Background(), storeResults); err != nil { logrus.WithError(err).WithFields(logrus.Fields{ "fetcher_name": fetcher.FetcherName(), "database_name": s.OurKeyRing.KeyDatabase.FetcherName(), @@ -243,7 +240,7 @@ func (s *ServerKeyAPI) handleFetcherKeys( if len(storeResults) > 0 { logrus.WithFields(logrus.Fields{ "fetcher_name": fetcher.FetcherName(), - }).Infof("Updated %d of %d key(s) in database", len(storeResults), len(results)) + }).Infof("Updated %d of %d key(s) in database (%d keys remaining)", len(storeResults), len(results), len(requests)) } return nil From 135b5e264f1e183994ca0ddca21574d2b2081bba Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 30 Sep 2020 13:51:54 +0100 Subject: [PATCH 013/104] Fix panic on verifySigError in fetching missing events --- federationapi/routing/send.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index f8b8bcd6f..d4bbe31c6 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -872,7 +872,7 @@ func (t *txnReq) lookupMissingStateViaStateIDs(ctx context.Context, roomID, even h, err = t.lookupEvent(ctx, roomVersion, missingEventID, false) switch err.(type) { case verifySigError: - break + return case nil: break default: From 05e5386fb09c4661d16bc75640fc195d94b14ed7 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 30 Sep 2020 15:57:31 +0100 Subject: [PATCH 014/104] Update gobind database paths --- build/gobind/monolith.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/build/gobind/monolith.go b/build/gobind/monolith.go index 27b116487..996a6b68d 100644 --- a/build/gobind/monolith.go +++ b/build/gobind/monolith.go @@ -88,16 +88,16 @@ func (m *DendriteMonolith) Start() { cfg.Global.PrivateKey = ygg.SigningPrivateKey() cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) cfg.Global.Kafka.UseNaffka = true - cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-naffka.db", m.StorageDirectory)) - cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-account.db", m.StorageDirectory)) - cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-device.db", m.StorageDirectory)) - cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-mediaapi.db", m.StorageDirectory)) - cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-syncapi.db", m.StorageDirectory)) - cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-roomserver.db", m.StorageDirectory)) - cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-serverkey.db", m.StorageDirectory)) - cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-keyserver.db", m.StorageDirectory)) - cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-federationsender.db", m.StorageDirectory)) - cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-appservice.db", m.StorageDirectory)) + cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-naffka.db", m.StorageDirectory)) + cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-account.db", m.StorageDirectory)) + cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-device.db", m.StorageDirectory)) + cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-mediaapi.db", m.StorageDirectory)) + cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-syncapi.db", m.StorageDirectory)) + cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-roomserver.db", m.StorageDirectory)) + cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-serverkey.db", m.StorageDirectory)) + cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-keyserver.db", m.StorageDirectory)) + cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-federationsender.db", m.StorageDirectory)) + cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-appservice.db", m.StorageDirectory)) cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory)) cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory)) cfg.FederationSender.FederationMaxRetries = 8 From 0caad67abcc55b6d0812bede8bdfa853aafb5316 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 30 Sep 2020 19:52:36 +0100 Subject: [PATCH 015/104] Consider old keys in handleLocalKeys (#1454) --- serverkeyapi/internal/api.go | 33 ++++++++++++++++++++++++++++++--- serverkeyapi/serverkeyapi.go | 1 + 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/serverkeyapi/internal/api.go b/serverkeyapi/internal/api.go index 09707ae5b..b8a362259 100644 --- a/serverkeyapi/internal/api.go +++ b/serverkeyapi/internal/api.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -18,6 +19,7 @@ type ServerKeyAPI struct { ServerPublicKey ed25519.PublicKey ServerKeyID gomatrixserverlib.KeyID ServerKeyValidity time.Duration + OldServerKeys []config.OldVerifyKeys OurKeyRing gomatrixserverlib.KeyRing FedClient gomatrixserverlib.KeyClient @@ -112,14 +114,17 @@ func (s *ServerKeyAPI) FetcherName() string { } // handleLocalKeys handles cases where the key request contains -// a request for our own server keys. +// a request for our own server keys, either current or old. func (s *ServerKeyAPI) handleLocalKeys( _ context.Context, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, ) { for req := range requests { - if req.ServerName == s.ServerName { + if req.ServerName != s.ServerName { + continue + } + if req.KeyID == s.ServerKeyID { // We found a key request that is supposed to be for our own // keys. Remove it from the request list so we don't hit the // database or the fetchers for it. @@ -133,6 +138,28 @@ func (s *ServerKeyAPI) handleLocalKeys( ExpiredTS: gomatrixserverlib.PublicKeyNotExpired, ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(s.ServerKeyValidity)), } + } else { + // The key request doesn't match our current key. Let's see + // if it matches any of our old verify keys. + for _, oldVerifyKey := range s.OldServerKeys { + if req.KeyID == oldVerifyKey.KeyID { + // We found a key request that is supposed to be an expired + // key. + delete(requests, req) + + // Insert our own key into the response. + results[req] = gomatrixserverlib.PublicKeyLookupResult{ + VerifyKey: gomatrixserverlib.VerifyKey{ + Key: gomatrixserverlib.Base64Bytes(oldVerifyKey.PrivateKey.Public().(ed25519.PublicKey)), + }, + ExpiredTS: oldVerifyKey.ExpiredAt, + ValidUntilTS: gomatrixserverlib.PublicKeyNotValid, + } + + // No need to look at the other keys. + break + } + } } } } @@ -175,7 +202,7 @@ func (s *ServerKeyAPI) handleDatabaseKeys( // the remaining requests. func (s *ServerKeyAPI) handleFetcherKeys( ctx context.Context, - now gomatrixserverlib.Timestamp, + _ gomatrixserverlib.Timestamp, fetcher gomatrixserverlib.KeyFetcher, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, diff --git a/serverkeyapi/serverkeyapi.go b/serverkeyapi/serverkeyapi.go index 92781f464..da239eb05 100644 --- a/serverkeyapi/serverkeyapi.go +++ b/serverkeyapi/serverkeyapi.go @@ -49,6 +49,7 @@ func NewInternalAPI( ServerPublicKey: cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), ServerKeyID: cfg.Matrix.KeyID, ServerKeyValidity: cfg.Matrix.KeyValidityPeriod, + OldServerKeys: cfg.Matrix.OldVerifyKeys, FedClient: fedClient, OurKeyRing: gomatrixserverlib.KeyRing{ KeyFetchers: []gomatrixserverlib.KeyFetcher{}, From fed3ebd2f14c70364f45128b387efe1ff9a5ee2f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 30 Sep 2020 21:18:35 +0100 Subject: [PATCH 016/104] CodeQL analysis --- .github/workflows/codeql-analysis.yml | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..a4ef8b395 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,34 @@ +name: "CodeQL" + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: ['go'] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + fetch-depth: 2 + + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 91fc1f1c92d06d875b2e6293ac8d352716bdfb20 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 10:55:00 +0100 Subject: [PATCH 017/104] Fix bug in error handling in SQLite InsertPreviousEvent (#1456) --- roomserver/storage/sqlite3/previous_events_table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roomserver/storage/sqlite3/previous_events_table.go b/roomserver/storage/sqlite3/previous_events_table.go index 222b53b93..aaee62733 100644 --- a/roomserver/storage/sqlite3/previous_events_table.go +++ b/roomserver/storage/sqlite3/previous_events_table.go @@ -98,7 +98,7 @@ func (s *previousEventStatements) InsertPreviousEvent( eventNIDAsString := fmt.Sprintf("%d", eventNID) selectStmt := sqlutil.TxStmt(txn, s.selectPreviousEventExistsStmt) err := selectStmt.QueryRowContext(ctx, previousEventID, previousEventReferenceSHA256).Scan(&eventNIDs) - if err != sql.ErrNoRows { + if err != nil && err != sql.ErrNoRows { return fmt.Errorf("selectStmt.QueryRowContext.Scan: %w", err) } var nids []string From b1d5360335ef9b26f62504282669403f6f3e6df5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 11:55:17 +0100 Subject: [PATCH 018/104] Update HTTP clients (#1457) * Update gomatrixserverlib * Use separate HTTP client for API calls, set User-Agent for outbound HTTP requests --- cmd/dendrite-media-api-server/main.go | 3 +- cmd/dendrite-monolith-server/main.go | 3 +- go.mod | 2 +- go.sum | 10 ++----- internal/setup/base.go | 41 ++++++++++++++++++--------- 5 files changed, 33 insertions(+), 26 deletions(-) diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go index f442abfa7..2c2fe3b36 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-media-api-server/main.go @@ -17,7 +17,6 @@ package main import ( "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/mediaapi" - "github.com/matrix-org/gomatrixserverlib" ) func main() { @@ -26,7 +25,7 @@ func main() { defer base.Close() // nolint: errcheck userAPI := base.UserAPIClient() - client := gomatrixserverlib.NewClient(cfg.FederationSender.DisableTLSValidation) + client := base.CreateClient() mediaapi.AddPublicRoutes(base.PublicMediaAPIMux, &base.Cfg.MediaAPI, userAPI, client) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 759f1c9ff..28a349a76 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -29,7 +29,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/serverkeyapi" "github.com/matrix-org/dendrite/userapi" - "github.com/matrix-org/gomatrixserverlib" ) var ( @@ -125,7 +124,7 @@ func main() { monolith := setup.Monolith{ Config: base.Cfg, AccountDB: accountDB, - Client: gomatrixserverlib.NewClient(cfg.FederationSender.DisableTLSValidation), + Client: base.CreateClient(), FedClient: federation, KeyRing: keyRing, KafkaConsumer: base.KafkaConsumer, diff --git a/go.mod b/go.mod index c0dda07cd..b4e16c0c4 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20200929155210-32fc5888d26a + github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index 648807285..461b3d3c0 100644 --- a/go.sum +++ b/go.sum @@ -569,14 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929152221-6fe6457127ad h1:n0P/Oy8ZqqTPzum6FEayAjamsmvJTuIcA10WQ8GcK70= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929152221-6fe6457127ad/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154026-a52e7a5f0553 h1:tiel2c3I9xr0SRS05g3UvOjj6Sgg1I3Yn2/oGA1GgLk= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154026-a52e7a5f0553/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154241-9414c4d0b5f2 h1:a07U5eFT521mFiUtA/A8NwiZp5vfRU59/QKs+pa3Fkc= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929154241-9414c4d0b5f2/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929155210-32fc5888d26a h1:kIwbS7eY7P/MX0oN4wRHGkoc4eTTnwOcdCawBZ3SrJI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200929155210-32fc5888d26a/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae h1:9vxYG3+pXD3G74lZ8s67/afs2RkvyrJy6HEcqfHEVgw= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/internal/setup/base.go b/internal/setup/base.go index ef956dd2a..f0009d19d 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -69,6 +69,7 @@ type BaseDendrite struct { PublicMediaAPIMux *mux.Router InternalAPIMux *mux.Router UseHTTPAPIs bool + apiHttpClient *http.Client httpClient *http.Client Cfg *config.Dendrite Caches *caching.Caches @@ -118,6 +119,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo logrus.WithError(err).Warnf("Failed to create cache") } + apiClient := http.Client{} client := http.Client{Timeout: HTTPClientTimeout} if cfg.FederationSender.Proxy.Enabled { client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{ @@ -148,6 +150,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo PublicKeyAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicKeyPathPrefix).Subrouter().UseEncodedPath(), PublicMediaAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicMediaPathPrefix).Subrouter().UseEncodedPath(), InternalAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(), + apiHttpClient: &apiClient, httpClient: &client, KafkaConsumer: kafkaConsumer, KafkaProducer: kafkaProducer, @@ -161,7 +164,7 @@ func (b *BaseDendrite) Close() error { // AppserviceHTTPClient returns the AppServiceQueryAPI for hitting the appservice component over HTTP. func (b *BaseDendrite) AppserviceHTTPClient() appserviceAPI.AppServiceQueryAPI { - a, err := asinthttp.NewAppserviceClient(b.Cfg.AppServiceURL(), b.httpClient) + a, err := asinthttp.NewAppserviceClient(b.Cfg.AppServiceURL(), b.apiHttpClient) if err != nil { logrus.WithError(err).Panic("CreateHTTPAppServiceAPIs failed") } @@ -170,27 +173,27 @@ func (b *BaseDendrite) AppserviceHTTPClient() appserviceAPI.AppServiceQueryAPI { // RoomserverHTTPClient returns RoomserverInternalAPI for hitting the roomserver over HTTP. func (b *BaseDendrite) RoomserverHTTPClient() roomserverAPI.RoomserverInternalAPI { - rsAPI, err := rsinthttp.NewRoomserverClient(b.Cfg.RoomServerURL(), b.httpClient, b.Caches) + rsAPI, err := rsinthttp.NewRoomserverClient(b.Cfg.RoomServerURL(), b.apiHttpClient, b.Caches) if err != nil { - logrus.WithError(err).Panic("RoomserverHTTPClient failed", b.httpClient) + logrus.WithError(err).Panic("RoomserverHTTPClient failed", b.apiHttpClient) } return rsAPI } // UserAPIClient returns UserInternalAPI for hitting the userapi over HTTP. func (b *BaseDendrite) UserAPIClient() userapi.UserInternalAPI { - userAPI, err := userapiinthttp.NewUserAPIClient(b.Cfg.UserAPIURL(), b.httpClient) + userAPI, err := userapiinthttp.NewUserAPIClient(b.Cfg.UserAPIURL(), b.apiHttpClient) if err != nil { - logrus.WithError(err).Panic("UserAPIClient failed", b.httpClient) + logrus.WithError(err).Panic("UserAPIClient failed", b.apiHttpClient) } return userAPI } // EDUServerClient returns EDUServerInputAPI for hitting the EDU server over HTTP func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI { - e, err := eduinthttp.NewEDUServerClient(b.Cfg.EDUServerURL(), b.httpClient) + e, err := eduinthttp.NewEDUServerClient(b.Cfg.EDUServerURL(), b.apiHttpClient) if err != nil { - logrus.WithError(err).Panic("EDUServerClient failed", b.httpClient) + logrus.WithError(err).Panic("EDUServerClient failed", b.apiHttpClient) } return e } @@ -198,9 +201,9 @@ func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI { // FederationSenderHTTPClient returns FederationSenderInternalAPI for hitting // the federation sender over HTTP func (b *BaseDendrite) FederationSenderHTTPClient() federationSenderAPI.FederationSenderInternalAPI { - f, err := fsinthttp.NewFederationSenderClient(b.Cfg.FederationSenderURL(), b.httpClient) + f, err := fsinthttp.NewFederationSenderClient(b.Cfg.FederationSenderURL(), b.apiHttpClient) if err != nil { - logrus.WithError(err).Panic("FederationSenderHTTPClient failed", b.httpClient) + logrus.WithError(err).Panic("FederationSenderHTTPClient failed", b.apiHttpClient) } return f } @@ -209,7 +212,7 @@ func (b *BaseDendrite) FederationSenderHTTPClient() federationSenderAPI.Federati func (b *BaseDendrite) ServerKeyAPIClient() serverKeyAPI.ServerKeyInternalAPI { f, err := skinthttp.NewServerKeyClient( b.Cfg.ServerKeyAPIURL(), - b.httpClient, + b.apiHttpClient, b.Caches, ) if err != nil { @@ -220,9 +223,9 @@ func (b *BaseDendrite) ServerKeyAPIClient() serverKeyAPI.ServerKeyInternalAPI { // KeyServerHTTPClient returns KeyInternalAPI for hitting the key server over HTTP func (b *BaseDendrite) KeyServerHTTPClient() keyserverAPI.KeyInternalAPI { - f, err := keyinthttp.NewKeyServerClient(b.Cfg.KeyServerURL(), b.httpClient) + f, err := keyinthttp.NewKeyServerClient(b.Cfg.KeyServerURL(), b.apiHttpClient) if err != nil { - logrus.WithError(err).Panic("KeyServerHTTPClient failed", b.httpClient) + logrus.WithError(err).Panic("KeyServerHTTPClient failed", b.apiHttpClient) } return f } @@ -238,13 +241,25 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database { return db } +// CreateClient creates a new client (normally used for media fetch requests). +// Should only be called once per component. +func (b *BaseDendrite) CreateClient() *gomatrixserverlib.Client { + client := gomatrixserverlib.NewClient( + b.Cfg.FederationSender.DisableTLSValidation, + ) + client.SetUserAgent(fmt.Sprintf("Dendrite/%s", internal.VersionString())) + return client +} + // CreateFederationClient creates a new federation client. Should only be called // once per component. func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationClient { - return gomatrixserverlib.NewFederationClient( + client := gomatrixserverlib.NewFederationClient( b.Cfg.Global.ServerName, b.Cfg.Global.KeyID, b.Cfg.Global.PrivateKey, b.Cfg.FederationSender.DisableTLSValidation, ) + client.SetUserAgent(fmt.Sprintf("Dendrite/%s", internal.VersionString())) + return client } // SetupAndServeHTTP sets up the HTTP server to serve endpoints registered on From cb4b93b16ca60099699bfcde0e2dcec30b4153d8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 14:33:48 +0100 Subject: [PATCH 019/104] Add nginx sample reverse proxy configs (#1458) * Add nginx sample reverse proxy configs * Add line breaks at end of files --- docs/nginx/monolith-sample.conf | 24 ++++++++++++++++++++++ docs/nginx/polylith-sample.conf | 36 +++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 docs/nginx/monolith-sample.conf create mode 100644 docs/nginx/polylith-sample.conf diff --git a/docs/nginx/monolith-sample.conf b/docs/nginx/monolith-sample.conf new file mode 100644 index 000000000..4129729f4 --- /dev/null +++ b/docs/nginx/monolith-sample.conf @@ -0,0 +1,24 @@ +server { + listen 443 ssl; + server_name my.hostname.com; + + ssl_certificate /path/to/fullchain.pem; + ssl_certificate_key /path/to/privkey.pem; + ssl_dhparam /path/to/ssl-dhparams.pem; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_read_timeout 600; + + location = /.well-known/matrix/server { + return 200 '{ "m.server": "my.hostname.com:443" }'; + } + + location = /.well-known/matrix/client { + return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }'; + } + + location = /_matrix { + proxy_pass http://monolith:8008; + } +} diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf new file mode 100644 index 000000000..b2a91b0c5 --- /dev/null +++ b/docs/nginx/polylith-sample.conf @@ -0,0 +1,36 @@ +server { + listen 443 ssl; + server_name my.hostname.com; + + ssl_certificate /path/to/fullchain.pem; + ssl_certificate_key /path/to/privkey.pem; + ssl_dhparam /path/to/ssl-dhparams.pem; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_read_timeout 600; + + location = /.well-known/matrix/server { + return 200 '{ "m.server": "my.hostname.com:443" }'; + } + + location = /.well-known/matrix/client { + return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }'; + } + + location = /_matrix/client { + proxy_pass http://client_api:8071; + } + + location = /_matrix/federation { + proxy_pass http://federation_api:8072; + } + + location = /_matrix/key { + proxy_pass http://federation_api:8072; + } + + location = /_matrix/media { + proxy_pass http://media_api:8074; + } +} From 43b3c4a2fc493fa6cacf0174f8c9f31f6b9c88c1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 14:41:08 +0100 Subject: [PATCH 020/104] Fix bugs in nginx sample configs --- docs/nginx/monolith-sample.conf | 6 +++--- docs/nginx/polylith-sample.conf | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/nginx/monolith-sample.conf b/docs/nginx/monolith-sample.conf index 4129729f4..9ee5e1ac1 100644 --- a/docs/nginx/monolith-sample.conf +++ b/docs/nginx/monolith-sample.conf @@ -10,15 +10,15 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_read_timeout 600; - location = /.well-known/matrix/server { + location /.well-known/matrix/server { return 200 '{ "m.server": "my.hostname.com:443" }'; } - location = /.well-known/matrix/client { + location /.well-known/matrix/client { return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }'; } - location = /_matrix { + location /_matrix { proxy_pass http://monolith:8008; } } diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf index b2a91b0c5..658e0e4a4 100644 --- a/docs/nginx/polylith-sample.conf +++ b/docs/nginx/polylith-sample.conf @@ -10,27 +10,27 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_read_timeout 600; - location = /.well-known/matrix/server { + location /.well-known/matrix/server { return 200 '{ "m.server": "my.hostname.com:443" }'; } - location = /.well-known/matrix/client { + location /.well-known/matrix/client { return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }'; } - location = /_matrix/client { + location /_matrix/client { proxy_pass http://client_api:8071; } - location = /_matrix/federation { + location /_matrix/federation { proxy_pass http://federation_api:8072; } - location = /_matrix/key { + location /_matrix/key { proxy_pass http://federation_api:8072; } - location = /_matrix/media { + location /_matrix/media { proxy_pass http://media_api:8074; } } From 378a0520be8a03ee4c0bab6de8a1fff24bf4dcdf Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 1 Oct 2020 15:00:16 +0100 Subject: [PATCH 021/104] Add custom issue templates; fixes #1421 (#1460) --- .github/ISSUE_TEMPLATE/BUG_REPORT.md | 49 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/FEATURE_REQUEST.md | 14 +++++++ .github/PULL_REQUEST_TEMPLATE.md | 2 +- 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 .github/ISSUE_TEMPLATE/BUG_REPORT.md create mode 100644 .github/ISSUE_TEMPLATE/FEATURE_REQUEST.md diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md new file mode 100644 index 000000000..68ae922a3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -0,0 +1,49 @@ +--- +name: Bug report +about: Create a report to help us improve + +--- + + + +### Background information + +- **Dendrite version or git SHA**: +- **Monolith or Polylith?**: +- **SQLite3 or Postgres?**: +- **Running in Docker?**: +- **`go version`**: + + + +### Description + + + +### Steps to reproduce + +- list the steps +- that reproduce the bug +- using hyphens as bullet points + + diff --git a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md new file mode 100644 index 000000000..084f683bd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md @@ -0,0 +1,14 @@ +--- +name: Feature request +about: Suggest an idea for this project + +--- + + + +**Description:** + + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 8d22f7f72..c1cb8d7cc 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,5 +2,5 @@ -* [ ] I have added any new tests that need to pass to `testfile` as specified in [docs/sytest.md](https://github.com/matrix-org/dendrite/blob/master/docs/sytest.md) +* [ ] I have added any new tests that need to pass to `sytest-whitelist` as specified in [docs/sytest.md](https://github.com/matrix-org/dendrite/blob/master/docs/sytest.md) * [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/master/docs/CONTRIBUTING.md#sign-off) From b471d4d77ac884bc232a6a09d01f3fd4c3aa3a38 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 15:00:35 +0100 Subject: [PATCH 022/104] Set internal HTTP API timeout to 10 minutes (#1459) --- internal/setup/base.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/setup/base.go b/internal/setup/base.go index f0009d19d..f9ddfdf7d 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -119,7 +119,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo logrus.WithError(err).Warnf("Failed to create cache") } - apiClient := http.Client{} + apiClient := http.Client{Timeout: time.Minute * 10} client := http.Client{Timeout: HTTPClientTimeout} if cfg.FederationSender.Proxy.Enabled { client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{ From 4f87df198acfcd804753df37a81d6b5a0fa735ba Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 15:27:16 +0100 Subject: [PATCH 023/104] Update README.md (#1461) * Update README.md * Maybe bullet points * Tweaks * Put the shields back, use text instead, less OCD problems * One more tweak --- README.md | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index ecb3cdb9b..15e0d6b53 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,14 @@ -# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) +# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite Alerts](https://img.shields.io/matrix/dendrite-alerts:matrix.org.svg?label=%23dendrite-alerts%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-alerts:matrix.org) -Dendrite is a second-generation Matrix homeserver written in Go. It is not recommended to use Dendrite as -a production homeserver at this time as there is no stable release. +Dendrite is a second-generation Matrix homeserver written in Go! -Dendrite will start to receive versioned releases stable enough to run [once we enter beta](https://github.com/matrix-org/dendrite/milestone/8). +Join us in: -# Quick start +- **[#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org)** - General chat about the Dendrite project, for users and server admins alike +- **[#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org)** - The place for developers, where all Dendrite development discussion happens +- **[#dendrite-alerts:matrix.org](https://matrix.to/#/#dendrite-alerts:matrix.org)** - Release notifications and important info, highly recommended for all Dendrite server admins + +## Quick start Requires Go 1.13+ and SQLite3 (Postgres is also supported): @@ -30,7 +33,7 @@ $ ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config Then point your favourite Matrix client at `http://localhost:8008`. For full installation information, see [INSTALL.md](docs/INSTALL.md). For running in Docker, see [build/docker](build/docker). -# Progress +## Progress We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it @@ -59,7 +62,7 @@ This means Dendrite supports amongst others: - E2E keys and device lists -# Contributing +## Contributing We would be grateful for any help on issues marked as [Are We Synapse Yet](https://github.com/matrix-org/dendrite/labels/are-we-synapse-yet). These issues @@ -101,7 +104,7 @@ look for [Good First Issues](https://github.com/matrix-org/dendrite/labels/good% familiar with the project, look for [Help Wanted](https://github.com/matrix-org/dendrite/labels/help-wanted) issues. -# Hardware requirements +## Hardware requirements Dendrite in Monolith + SQLite works in a range of environments including iOS and in-browser via WASM. @@ -112,12 +115,3 @@ encrypted rooms: - CPU: Brief spikes when processing events, typically idles at 1% CPU. This means Dendrite should comfortably work on things like Raspberry Pis. - -# Discussion - -For questions about Dendrite we have a dedicated room on Matrix -[#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org). Development -discussion should happen in -[#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org). - - From 9b2d8f69aa2bab5dfc1ae9e52a3cc20c7201a1d7 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 15:28:01 +0100 Subject: [PATCH 024/104] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 15e0d6b53..72c0df07d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite Alerts](https://img.shields.io/matrix/dendrite-alerts:matrix.org.svg?label=%23dendrite-alerts%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-alerts:matrix.org) +# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) Dendrite is a second-generation Matrix homeserver written in Go! From dbae85283f3927444aa89221523405ada07a0f4a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 15:37:45 +0100 Subject: [PATCH 025/104] Bring docker sample config into sync with normal one --- build/docker/config/dendrite-config.yaml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/build/docker/config/dendrite-config.yaml b/build/docker/config/dendrite-config.yaml index 7ebeeb6e7..2bf8dd85f 100644 --- a/build/docker/config/dendrite-config.yaml +++ b/build/docker/config/dendrite-config.yaml @@ -38,8 +38,13 @@ global: # The path to the signing private key file, used to sign requests and events. private_key: matrix_key.pem - # A unique identifier for this private key. Must start with the prefix "ed25519:". - key_id: ed25519:auto + # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) + # to old signing private keys that were formerly in use on this domain. These + # keys will not be used for federation request or event signing, but will be + # provided to any other homeserver that asks when trying to verify old events. + # old_private_keys: + # - private_key: old_matrix_key.pem + # expired_at: 1601024554498 # How long a remote server can cache our server signing key before requesting it # again. Increasing this number will reduce the number of requests made by other @@ -133,6 +138,14 @@ client_api: turn_username: "" turn_password: "" + # Settings for rate-limited endpoints. Rate limiting will kick in after the + # threshold number of "slots" have been taken by requests from a specific + # host. Each "slot" will be released after the cooloff time in milliseconds. + rate_limiting: + enabled: true + threshold: 5 + cooloff_ms: 500 + # Configuration for the EDU server. edu_server: internal_api: @@ -260,6 +273,11 @@ server_key_api: public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw - key_id: ed25519:a_RXGa public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ + + # This option will control whether Dendrite will prefer to look up keys directly + # or whether it should try perspective servers first, using direct fetches as a + # last resort. + prefer_direct_fetch: false # Configuration for the Sync API. sync_api: @@ -291,6 +309,8 @@ user_api: conn_max_lifetime: -1 # Configuration for Opentracing. +# See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on +# how this works and how to set it up. tracing: enabled: false jaeger: From 7048532bc4adcc017739fae358fd6b6c44b63ebd Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 1 Oct 2020 16:09:11 +0100 Subject: [PATCH 026/104] Update version imprinting (#1462) * Add version tag constant * Update build imprinting --- build.sh | 12 +++++++++--- internal/version.go | 3 ++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index 34e4b1153..f790077e3 100755 --- a/build.sh +++ b/build.sh @@ -3,10 +3,16 @@ # Put installed packages into ./bin export GOBIN=$PWD/`dirname $0`/bin -export BRANCH=`(git symbolic-ref --short HEAD | cut -d'/' -f 3 )|| ""` -export BUILD=`git rev-parse --short HEAD || ""` +if [ -d ".git" ] +then + export BUILD=`git rev-parse --short HEAD || ""` + export BRANCH=`(git symbolic-ref --short HEAD | tr -d \/ ) || ""` + [[ $BRANCH == "master" ]] && export BRANCH="" -export FLAGS="-X github.com/matrix-org/dendrite/internal.branch=$BRANCH -X github.com/matrix-org/dendrite/internal.build=$BUILD" + export FLAGS="-X github.com/matrix-org/dendrite/internal.branch=$BRANCH -X github.com/matrix-org/dendrite/internal.build=$BUILD" +else + export FLAGS="" +fi go install -trimpath -ldflags "$FLAGS" -v $PWD/`dirname $0`/cmd/... diff --git a/internal/version.go b/internal/version.go index 851a09384..718273e72 100644 --- a/internal/version.go +++ b/internal/version.go @@ -12,10 +12,11 @@ const ( VersionMajor = 0 VersionMinor = 0 VersionPatch = 0 + VersionTag = "" // example: "rc1" ) func VersionString() string { - version := fmt.Sprintf("%d.%d.%d", VersionMajor, VersionMinor, VersionPatch) + version := fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionTag) if branch != "" { version += fmt.Sprintf("-%s", branch) } From 3e01db0049a839eb163523990b383156b69527c2 Mon Sep 17 00:00:00 2001 From: S7evinK Date: Thu, 1 Oct 2020 21:00:56 +0200 Subject: [PATCH 027/104] Fix golangci-lint issues (#1464) * Fix S1039: unnecessary use of fmt.Sprintf * Fix S1036: unnecessary guard around map access Signed-off-by: Till Faelligen --- federationapi/routing/join.go | 4 +--- roomserver/state/state.go | 6 +----- syncapi/storage/storage_test.go | 16 ++++++++-------- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 9fa0794ef..c637116f7 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -192,9 +192,7 @@ func SendJoin( if event.StateKey() == nil || event.StateKeyEquals("") { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON( - fmt.Sprintf("No state key was provided in the join event."), - ), + JSON: jsonerror.BadJSON("No state key was provided in the join event."), } } diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 9ee6f40d4..0663499e7 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -717,11 +717,7 @@ func ResolveConflictsAdhoc( // Append the events if there is already a conflicted list for // this tuple key, create it if not. tuple := stateKeyTuple{event.Type(), *event.StateKey()} - if _, ok := eventMap[tuple]; ok { - eventMap[tuple] = append(eventMap[tuple], event) - } else { - eventMap[tuple] = []gomatrixserverlib.Event{event} - } + eventMap[tuple] = append(eventMap[tuple], event) } // Split out the events in the map into conflicted and unconflicted diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 0e827c95d..8f16642f0 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -81,7 +81,7 @@ func SimpleRoom(t *testing.T, roomID, userA, userB string) (msgs []gomatrixserve })) state = append(state, events[len(events)-1]) events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Content: []byte(`{"membership":"join"}`), Type: "m.room.member", StateKey: &userA, Sender: userA, @@ -97,7 +97,7 @@ func SimpleRoom(t *testing.T, roomID, userA, userB string) (msgs []gomatrixserve })) } events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Content: []byte(`{"membership":"join"}`), Type: "m.room.member", StateKey: &userB, Sender: userB, @@ -348,7 +348,7 @@ func TestGetEventsInRangeWithEventsSameDepth(t *testing.T) { Depth: int64(len(events) + 1), })) events = append(events, MustCreateEvent(t, testRoomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Content: []byte(`{"membership":"join"}`), Type: "m.room.member", StateKey: &testUserIDA, Sender: testUserIDA, @@ -367,7 +367,7 @@ func TestGetEventsInRangeWithEventsSameDepth(t *testing.T) { } // merge the fork, prev_events are all 3 messages, depth is increased by 1. events = append(events, MustCreateEvent(t, testRoomID, events[len(events)-3:], &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"body":"Message merge"}`)), + Content: []byte(`{"body":"Message merge"}`), Type: "m.room.message", Sender: testUserIDA, Depth: depth + 1, @@ -438,7 +438,7 @@ func TestGetEventsInTopologicalRangeMultiRoom(t *testing.T) { Depth: int64(len(events) + 1), })) events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Content: []byte(`{"membership":"join"}`), Type: "m.room.member", StateKey: &testUserIDA, Sender: testUserIDA, @@ -484,7 +484,7 @@ func TestGetEventsInRangeWithEventsInsertedLikeBackfill(t *testing.T) { // "federation" join userC := fmt.Sprintf("@radiance:%s", testOrigin) joinEvent := MustCreateEvent(t, testRoomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Content: []byte(`{"membership":"join"}`), Type: "m.room.member", StateKey: &userC, Sender: userC, @@ -615,14 +615,14 @@ func TestInviteBehaviour(t *testing.T) { db := MustCreateDatabase(t) inviteRoom1 := "!inviteRoom1:somewhere" inviteEvent1 := MustCreateEvent(t, inviteRoom1, nil, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"membership":"invite"}`)), + Content: []byte(`{"membership":"invite"}`), Type: "m.room.member", StateKey: &testUserIDA, Sender: "@inviteUser1:somewhere", }) inviteRoom2 := "!inviteRoom2:somewhere" inviteEvent2 := MustCreateEvent(t, inviteRoom2, nil, &gomatrixserverlib.EventBuilder{ - Content: []byte(fmt.Sprintf(`{"membership":"invite"}`)), + Content: []byte(`{"membership":"invite"}`), Type: "m.room.member", StateKey: &testUserIDA, Sender: "@inviteUser2:somewhere", From 92ceb46b4996330326aaf0cb99c3fdd523b9afba Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 2 Oct 2020 09:43:02 +0100 Subject: [PATCH 028/104] Update to matrix-org/gomatrixserverlib#227 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b4e16c0c4..b1119f608 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae + github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index 461b3d3c0..c657cc457 100644 --- a/go.sum +++ b/go.sum @@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae h1:9vxYG3+pXD3G74lZ8s67/afs2RkvyrJy6HEcqfHEVgw= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290 h1:ilT9QNIh2KXfvzIALtAe31IvLVZH7mVjVtOOTxdd0tY= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= From 28454d6fb712d69e299660bef098be0be45c9475 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 2 Oct 2020 11:38:35 +0100 Subject: [PATCH 029/104] Log origin in /send --- federationapi/routing/send.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index d4bbe31c6..e2ab9b334 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -81,7 +81,7 @@ func Send( t.TransactionID = txnID t.Destination = cfg.Matrix.ServerName - util.GetLogger(httpReq.Context()).Infof("Received transaction %q containing %d PDUs, %d EDUs", txnID, len(t.PDUs), len(t.EDUs)) + util.GetLogger(httpReq.Context()).Infof("Received transaction %q from %q containing %d PDUs, %d EDUs", txnID, request.Origin(), len(t.PDUs), len(t.EDUs)) resp, jsonErr := t.processTransaction(httpReq.Context()) if jsonErr != nil { From 1b29e5771feb335afe0e39e7f33f1d1896dbdb2f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 2 Oct 2020 11:49:13 +0100 Subject: [PATCH 030/104] Fix build.sh --- build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.sh b/build.sh index f790077e3..31e0519f5 100755 --- a/build.sh +++ b/build.sh @@ -7,7 +7,10 @@ if [ -d ".git" ] then export BUILD=`git rev-parse --short HEAD || ""` export BRANCH=`(git symbolic-ref --short HEAD | tr -d \/ ) || ""` - [[ $BRANCH == "master" ]] && export BRANCH="" + if [[ $BRANCH == "master" ]] + then + export BRANCH="" + fi export FLAGS="-X github.com/matrix-org/dendrite/internal.branch=$BRANCH -X github.com/matrix-org/dendrite/internal.build=$BUILD" else From fb9a8f215b0af25c02d62c6f44ed91adc3349a6e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 2 Oct 2020 12:50:58 +0100 Subject: [PATCH 031/104] Fix initial sync (#1465) * Fix complete sync check * Remove unnecessary 'since' copy * Fix failing test * Un-whitelist a couple of tests Co-authored-by: Kegan Dougal --- syncapi/storage/shared/syncserver.go | 17 +++++++++-------- syncapi/sync/requestpool.go | 13 ++++--------- sytest-whitelist | 2 -- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 05a8768e8..edb51b347 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -670,7 +670,7 @@ func (d *Database) RedactEvent(ctx context.Context, redactedEventID string, reda // nolint:nakedret func (d *Database) getResponseWithPDUsForCompleteSync( ctx context.Context, res *types.Response, - userID string, deviceID string, + userID string, device userapi.Device, numRecentEventsPerRoom int, ) ( toPos types.StreamingToken, @@ -712,7 +712,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync( for _, roomID := range joinedRoomIDs { var jr *types.JoinResponse jr, err = d.getJoinResponseForCompleteSync( - ctx, txn, roomID, r, &stateFilter, numRecentEventsPerRoom, + ctx, txn, roomID, r, &stateFilter, numRecentEventsPerRoom, device, ) if err != nil { return @@ -721,7 +721,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync( } // Add peeked rooms. - peeks, err := d.Peeks.SelectPeeksInRange(ctx, txn, userID, deviceID, r) + peeks, err := d.Peeks.SelectPeeksInRange(ctx, txn, userID, device.ID, r) if err != nil { return } @@ -729,7 +729,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync( if !peek.Deleted { var jr *types.JoinResponse jr, err = d.getJoinResponseForCompleteSync( - ctx, txn, peek.RoomID, r, &stateFilter, numRecentEventsPerRoom, + ctx, txn, peek.RoomID, r, &stateFilter, numRecentEventsPerRoom, device, ) if err != nil { return @@ -751,7 +751,7 @@ func (d *Database) getJoinResponseForCompleteSync( roomID string, r types.Range, stateFilter *gomatrixserverlib.StateFilter, - numRecentEventsPerRoom int, + numRecentEventsPerRoom int, device userapi.Device, ) (jr *types.JoinResponse, err error) { var stateEvents []gomatrixserverlib.HeaderedEvent stateEvents, err = d.CurrentRoomState.SelectCurrentState(ctx, txn, roomID, stateFilter) @@ -784,8 +784,9 @@ func (d *Database) getJoinResponseForCompleteSync( } // We don't include a device here as we don't need to send down - // transaction IDs for complete syncs - recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents) + // transaction IDs for complete syncs, but we do it anyway because Sytest demands it for: + // "Can sync a room with a message with a transaction id" - which does a complete sync to check. + recentEvents := d.StreamEventsToEvents(&device, recentStreamEvents) stateEvents = removeDuplicates(stateEvents, recentEvents) jr = types.NewJoinResponse() jr.Timeline.PrevBatch = prevBatchStr @@ -800,7 +801,7 @@ func (d *Database) CompleteSync( device userapi.Device, numRecentEventsPerRoom int, ) (*types.Response, error) { toPos, joinedRoomIDs, err := d.getResponseWithPDUsForCompleteSync( - ctx, res, device.UserID, device.ID, numRecentEventsPerRoom, + ctx, res, device.UserID, device, numRecentEventsPerRoom, ) if err != nil { return nil, fmt.Errorf("d.getResponseWithPDUsForCompleteSync: %w", err) diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index aaaf94917..8a79737aa 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -197,19 +197,14 @@ func (rp *RequestPool) OnIncomingKeyChangeRequest(req *http.Request, device *use func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.StreamingToken) (*types.Response, error) { res := types.NewResponse() - since := types.NewStreamToken(0, 0, nil) - if req.since != nil { - since = *req.since - } - // See if we have any new tasks to do for the send-to-device messaging. - events, updates, deletions, err := rp.db.SendToDeviceUpdatesForSync(req.ctx, req.device.UserID, req.device.ID, since) + events, updates, deletions, err := rp.db.SendToDeviceUpdatesForSync(req.ctx, req.device.UserID, req.device.ID, *req.since) if err != nil { return nil, fmt.Errorf("rp.db.SendToDeviceUpdatesForSync: %w", err) } // TODO: handle ignored users - if req.since == nil { + if req.since.PDUPosition() == 0 && req.since.EDUPosition() == 0 { res, err = rp.db.CompleteSync(req.ctx, res, req.device, req.limit) if err != nil { return res, fmt.Errorf("rp.db.CompleteSync: %w", err) @@ -226,7 +221,7 @@ func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.Strea if err != nil { return res, fmt.Errorf("rp.appendAccountData: %w", err) } - res, err = rp.appendDeviceLists(res, req.device.UserID, since, latestPos) + res, err = rp.appendDeviceLists(res, req.device.UserID, *req.since, latestPos) if err != nil { return res, fmt.Errorf("rp.appendDeviceLists: %w", err) } @@ -240,7 +235,7 @@ func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.Strea // Then add the updates into the sync response. if len(updates) > 0 || len(deletions) > 0 { // Handle the updates and deletions in the database. - err = rp.db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, since) + err = rp.db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, *req.since) if err != nil { return res, fmt.Errorf("rp.db.CleanSendToDeviceUpdates: %w", err) } diff --git a/sytest-whitelist b/sytest-whitelist index 29e9166ae..df71e275f 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -396,8 +396,6 @@ GET /rooms/:room_id/state fetches entire room state Setting room topic reports m.room.topic to myself setting 'm.room.name' respects room powerlevel Syncing a new room with a large timeline limit isn't limited -Left rooms appear in the leave section of sync -Banned rooms appear in the leave section of sync Getting state checks the events requested belong to the room Getting state IDs checks the events requested belong to the room Can invite users to invite-only rooms From 439bc235d7855d7efe7aed864eef7c6ee9eef88c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFck=20Bonniot?= Date: Fri, 2 Oct 2020 16:05:23 +0200 Subject: [PATCH 032/104] Add 'completed' field in Interactive Auth API (#1469) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This field must be present even after authentication failure, as tested by sytest. This is needed by #1455. Signed-off-by: Loïck Bonniot --- clientapi/auth/user_interactive.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/clientapi/auth/user_interactive.go b/clientapi/auth/user_interactive.go index c67eba150..b7414ebe9 100644 --- a/clientapi/auth/user_interactive.go +++ b/clientapi/auth/user_interactive.go @@ -103,7 +103,8 @@ type userInteractiveFlow struct { // the user already has a valid access token, but we want to double-check // that it isn't stolen by re-authenticating them. type UserInteractive struct { - Flows []userInteractiveFlow + Completed []string + Flows []userInteractiveFlow // Map of login type to implementation Types map[string]Type // Map of session ID to completed login types, will need to be extended in future @@ -117,6 +118,7 @@ func NewUserInteractive(getAccByPass GetAccountByPassword, cfg *config.ClientAPI } // TODO: Add SSO login return &UserInteractive{ + Completed: []string{}, Flows: []userInteractiveFlow{ { Stages: []string{typePassword.Name()}, @@ -140,6 +142,7 @@ func (u *UserInteractive) IsSingleStageFlow(authType string) bool { func (u *UserInteractive) AddCompletedStage(sessionID, authType string) { // TODO: Handle multi-stage flows + u.Completed = append(u.Completed, authType) delete(u.Sessions, sessionID) } @@ -148,11 +151,13 @@ func (u *UserInteractive) Challenge(sessionID string) *util.JSONResponse { return &util.JSONResponse{ Code: 401, JSON: struct { - Flows []userInteractiveFlow `json:"flows"` - Session string `json:"session"` + Completed []string `json:"completed"` + Flows []userInteractiveFlow `json:"flows"` + Session string `json:"session"` // TODO: Return any additional `params` Params map[string]interface{} `json:"params"` }{ + u.Completed, u.Flows, sessionID, make(map[string]interface{}), From 85c43e1befa9255c541a17abdab4a32be8b80695 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 2 Oct 2020 15:14:01 +0100 Subject: [PATCH 033/104] Don't drop federation retries for Yggdrasil demos --- build/gobind/monolith.go | 1 - cmd/dendrite-demo-yggdrasil/main.go | 1 - 2 files changed, 2 deletions(-) diff --git a/build/gobind/monolith.go b/build/gobind/monolith.go index 996a6b68d..b4740ed42 100644 --- a/build/gobind/monolith.go +++ b/build/gobind/monolith.go @@ -100,7 +100,6 @@ func (m *DendriteMonolith) Start() { cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-appservice.db", m.StorageDirectory)) cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory)) cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory)) - cfg.FederationSender.FederationMaxRetries = 8 if err = cfg.Derive(); err != nil { panic(err) } diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 7a370bda5..257ddb58a 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -73,7 +73,6 @@ func main() { cfg.Global.PrivateKey = ygg.SigningPrivateKey() cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) cfg.Global.Kafka.UseNaffka = true - cfg.FederationSender.FederationMaxRetries = 8 cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName)) cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) From c870435c17676d2b0272420242c1b0761c38fe35 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 2 Oct 2020 16:56:13 +0100 Subject: [PATCH 034/104] Update PULL_REQUEST_TEMPLATE.md --- .github/PULL_REQUEST_TEMPLATE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c1cb8d7cc..92253214a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -4,3 +4,5 @@ * [ ] I have added any new tests that need to pass to `sytest-whitelist` as specified in [docs/sytest.md](https://github.com/matrix-org/dendrite/blob/master/docs/sytest.md) * [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/master/docs/CONTRIBUTING.md#sign-off) + +Signed-off-by: `Your Name ` From 279044cd90722ba98b018b0aa277285113454822 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 2 Oct 2020 17:08:13 +0100 Subject: [PATCH 035/104] Add history visibility guards (#1470) * Add history visibility guards Default to 'joined' visibility to avoid leaking events, until we get around to implementing history visibility completely. Related #617 * Don't apply his vis checks on shared rooms * Fix order of checks * Linting and remove another misleading check * Update whitelist --- syncapi/routing/messages.go | 93 +++++++++++++++++++++++++++- syncapi/routing/routing.go | 2 +- syncapi/storage/shared/syncserver.go | 27 ++++++++ syncapi/storage/storage_test.go | 1 + sytest-whitelist | 10 +-- 5 files changed, 124 insertions(+), 9 deletions(-) diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 6447e5d59..9c6c6a80d 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/sirupsen/logrus" @@ -41,6 +42,7 @@ type messagesReq struct { from *types.TopologyToken to *types.TopologyToken fromStream *types.StreamingToken + device *userapi.Device wasToProvided bool limit int backwardOrdering bool @@ -58,7 +60,7 @@ const defaultMessagesLimit = 10 // client-server API. // See: https://matrix.org/docs/spec/client_server/latest.html#get-matrix-client-r0-rooms-roomid-messages func OnIncomingMessagesRequest( - req *http.Request, db storage.Database, roomID string, + req *http.Request, db storage.Database, roomID string, device *userapi.Device, federation *gomatrixserverlib.FederationClient, rsAPI api.RoomserverInternalAPI, cfg *config.SyncAPI, @@ -151,6 +153,7 @@ func OnIncomingMessagesRequest( wasToProvided: wasToProvided, limit: limit, backwardOrdering: backwardOrdering, + device: device, } clientEvents, start, end, err := mReq.retrieveEvents() @@ -238,6 +241,10 @@ func (r *messagesReq) retrieveEvents() ( } events = reversed(events) } + events = r.filterHistoryVisible(events) + if len(events) == 0 { + return []gomatrixserverlib.ClientEvent{}, *r.from, *r.to, nil + } // Convert all of the events into client events. clientEvents = gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatAll) @@ -252,6 +259,90 @@ func (r *messagesReq) retrieveEvents() ( return clientEvents, start, end, err } +// nolint:gocyclo +func (r *messagesReq) filterHistoryVisible(events []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { + // TODO FIXME: We don't fully implement history visibility yet. To avoid leaking events which the + // user shouldn't see, we check the recent events and remove any prior to the join event of the user + // which is equiv to history_visibility: joined + joinEventIndex := -1 + for i, ev := range events { + if ev.Type() == gomatrixserverlib.MRoomMember && ev.StateKeyEquals(r.device.UserID) { + membership, _ := ev.Membership() + if membership == "join" { + joinEventIndex = i + break + } + } + } + + var result []gomatrixserverlib.HeaderedEvent + var eventsToCheck []gomatrixserverlib.HeaderedEvent + if joinEventIndex != -1 { + if r.backwardOrdering { + result = events[:joinEventIndex+1] + eventsToCheck = append(eventsToCheck, result[0]) + } else { + result = events[joinEventIndex:] + eventsToCheck = append(eventsToCheck, result[len(result)-1]) + } + } else { + eventsToCheck = []gomatrixserverlib.HeaderedEvent{events[0], events[len(events)-1]} + result = events + } + // make sure the user was in the room for both the earliest and latest events, we need this because + // some backpagination results will not have the join event (e.g if they hit /messages at the join event itself) + wasJoined := true + for _, ev := range eventsToCheck { + var queryRes api.QueryStateAfterEventsResponse + err := r.rsAPI.QueryStateAfterEvents(r.ctx, &api.QueryStateAfterEventsRequest{ + RoomID: ev.RoomID(), + PrevEventIDs: ev.PrevEventIDs(), + StateToFetch: []gomatrixserverlib.StateKeyTuple{ + {EventType: gomatrixserverlib.MRoomMember, StateKey: r.device.UserID}, + {EventType: gomatrixserverlib.MRoomHistoryVisibility, StateKey: ""}, + }, + }, &queryRes) + if err != nil { + wasJoined = false + break + } + var hisVisEvent, membershipEvent *gomatrixserverlib.HeaderedEvent + for i := range queryRes.StateEvents { + switch queryRes.StateEvents[i].Type() { + case gomatrixserverlib.MRoomMember: + membershipEvent = &queryRes.StateEvents[i] + case gomatrixserverlib.MRoomHistoryVisibility: + hisVisEvent = &queryRes.StateEvents[i] + } + } + if hisVisEvent == nil { + return events // apply no filtering as it defaults to Shared. + } + hisVis, _ := hisVisEvent.HistoryVisibility() + if hisVis == "shared" { + return events // apply no filtering + } + if membershipEvent == nil { + wasJoined = false + break + } + membership, err := membershipEvent.Membership() + if err != nil { + wasJoined = false + break + } + if membership != "join" { + wasJoined = false + break + } + } + if !wasJoined { + util.GetLogger(r.ctx).WithField("num_events", len(events)).Warnf("%s was not joined to room during these events, omitting them", r.device.UserID) + return []gomatrixserverlib.HeaderedEvent{} + } + return result +} + func (r *messagesReq) getStartEnd(events []gomatrixserverlib.HeaderedEvent) (start, end types.TopologyToken, err error) { start, err = r.db.EventPositionInTopology( r.ctx, events[0].EventID(), diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index f42679c67..141eec799 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -51,7 +51,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], federation, rsAPI, cfg) + return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], device, federation, rsAPI, cfg) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/filter", diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index edb51b347..b9f21913e 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -769,6 +769,33 @@ func (d *Database) getJoinResponseForCompleteSync( return } + // TODO FIXME: We don't fully implement history visibility yet. To avoid leaking events which the + // user shouldn't see, we check the recent events and remove any prior to the join event of the user + // which is equiv to history_visibility: joined + joinEventIndex := -1 + for i := len(recentStreamEvents) - 1; i >= 0; i-- { + ev := recentStreamEvents[i] + if ev.Type() == gomatrixserverlib.MRoomMember && ev.StateKeyEquals(device.UserID) { + membership, _ := ev.Membership() + if membership == "join" { + joinEventIndex = i + if i > 0 { + // the create event happens before the first join, so we should cut it at that point instead + if recentStreamEvents[i-1].Type() == gomatrixserverlib.MRoomCreate && recentStreamEvents[i-1].StateKeyEquals("") { + joinEventIndex = i - 1 + break + } + } + break + } + } + } + if joinEventIndex != -1 { + // cut all events earlier than the join (but not the join itself) + recentStreamEvents = recentStreamEvents[joinEventIndex:] + limited = false // so clients know not to try to backpaginate + } + // Retrieve the backward topology position, i.e. the position of the // oldest event in the room's topology. var prevBatchStr string diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 8f16642f0..2869ac5d2 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -689,6 +689,7 @@ func assertInvitedToRooms(t *testing.T, res *types.Response, roomIDs []string) { } func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatrixserverlib.ClientEvent, wants []gomatrixserverlib.HeaderedEvent) { + t.Helper() if len(gots) != len(wants) { t.Fatalf("%s response returned %d events, want %d", msg, len(gots), len(wants)) } diff --git a/sytest-whitelist b/sytest-whitelist index df71e275f..eaaeea00f 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -90,8 +90,6 @@ Real non-joined users can get state for world_readable rooms Real non-joined users can get individual state for world_readable rooms #Real non-joined users can get individual state for world_readable rooms after leaving Real non-joined users cannot send messages to guest_access rooms if not joined -Real users can sync from world_readable guest_access rooms if joined -Real users can sync from default guest_access rooms if joined Can't forget room you're still in Can get rooms/{roomId}/members Can create filter @@ -236,13 +234,11 @@ Outbound federation can query v2 /send_join Inbound federation can receive v2 /send_join Message history can be paginated Getting messages going forward is limited for a departed room (SPEC-216) -m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users Backfill works correctly with history visibility set to joined Guest user cannot call /events globally Guest users can join guest_access rooms Guest user can set display names Guest user cannot upgrade other users -m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users Guest non-joined user cannot call /events on shared room Guest non-joined user cannot call /events on invited room Guest non-joined user cannot call /events on joined room @@ -252,8 +248,6 @@ Guest non-joined users can get individual state for world_readable rooms Guest non-joined users cannot room initalSync for non-world_readable rooms Guest non-joined users can get individual state for world_readable rooms after leaving Guest non-joined users cannot send messages to guest_access rooms if not joined -Guest users can sync from world_readable guest_access rooms if joined -Guest users can sync from default guest_access rooms if joined Real non-joined users cannot room initalSync for non-world_readable rooms Push rules come down in an initial /sync Regular users can add and delete aliases in the default room configuration @@ -478,4 +472,6 @@ Federation key API can act as a notary server via a GET request Inbound /make_join rejects attempts to join rooms where all users have left Inbound federation rejects invites which include invalid JSON for room version 6 Inbound federation rejects invite rejections which include invalid JSON for room version 6 -GET /capabilities is present and well formed for registered user \ No newline at end of file +GET /capabilities is present and well formed for registered user +m.room.history_visibility == "joined" allows/forbids appropriately for Guest users +m.room.history_visibility == "joined" allows/forbids appropriately for Real users \ No newline at end of file From 4e8c484618f5bfd5b7349cc2afd102cd5feb3861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFck=20Bonniot?= Date: Fri, 2 Oct 2020 18:18:20 +0200 Subject: [PATCH 036/104] Implement account deactivation (#1455) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Implement account deactivation See #610 Signed-off-by: Loïck Bonniot * Rename 'is_active' to 'is_deactivated' Signed-off-by: Loïck Bonniot Co-authored-by: Kegsay --- clientapi/routing/deactivate.go | 55 +++++++++++++++++++ clientapi/routing/routing.go | 9 +++ sytest-whitelist | 3 + userapi/api/api.go | 11 ++++ userapi/internal/api.go | 7 +++ userapi/inthttp/client.go | 19 +++++-- userapi/inthttp/server.go | 13 +++++ userapi/storage/accounts/interface.go | 1 + .../accounts/postgres/accounts_table.go | 20 ++++++- .../deltas/20200929203058_is_active.sql | 9 +++ userapi/storage/accounts/postgres/storage.go | 5 ++ .../accounts/sqlite3/accounts_table.go | 20 ++++++- .../deltas/20200929203058_is_active.sql | 38 +++++++++++++ userapi/storage/accounts/sqlite3/storage.go | 5 ++ 14 files changed, 206 insertions(+), 9 deletions(-) create mode 100644 clientapi/routing/deactivate.go create mode 100644 userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql create mode 100644 userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql diff --git a/clientapi/routing/deactivate.go b/clientapi/routing/deactivate.go new file mode 100644 index 000000000..effe3769d --- /dev/null +++ b/clientapi/routing/deactivate.go @@ -0,0 +1,55 @@ +package routing + +import ( + "io/ioutil" + "net/http" + + "github.com/matrix-org/dendrite/clientapi/auth" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +// Deactivate handles POST requests to /account/deactivate +func Deactivate( + req *http.Request, + userInteractiveAuth *auth.UserInteractive, + userAPI api.UserInternalAPI, + deviceAPI *api.Device, +) util.JSONResponse { + ctx := req.Context() + defer req.Body.Close() // nolint:errcheck + bodyBytes, err := ioutil.ReadAll(req.Body) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()), + } + } + + login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, deviceAPI) + if errRes != nil { + return *errRes + } + + localpart, _, err := gomatrixserverlib.SplitID('@', login.User) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() + } + + var res api.PerformAccountDeactivationResponse + err = userAPI.PerformAccountDeactivation(ctx, &api.PerformAccountDeactivationRequest{ + Localpart: localpart, + }, &res) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("userAPI.PerformAccountDeactivation failed") + return jsonerror.InternalServerError() + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index ab56c5f4d..8606f69c3 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -435,6 +435,15 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) + r0mux.Handle("/account/deactivate", + httputil.MakeAuthAPI("deactivate", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + if r := rateLimits.rateLimit(req); r != nil { + return *r + } + return Deactivate(req, userInteractiveAuth, userAPI, device) + }), + ).Methods(http.MethodPost, http.MethodOptions) + // Stub endpoints required by Riot r0mux.Handle("/login", diff --git a/sytest-whitelist b/sytest-whitelist index eaaeea00f..a811259fb 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -219,6 +219,9 @@ Regular users cannot create room aliases within the AS namespace Deleting a non-existent alias should return a 404 Users can't delete other's aliases Outbound federation can query room alias directory +Can deactivate account +Can't deactivate account with wrong password +After deactivating account, can't log in with password After deactivating account, can't log in with an email Remote room alias queries can handle Unicode Newly joined room is included in an incremental sync after invite diff --git a/userapi/api/api.go b/userapi/api/api.go index 3baaa1002..d384c5b18 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -30,6 +30,7 @@ type UserInternalAPI interface { PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error PerformDeviceDeletion(ctx context.Context, req *PerformDeviceDeletionRequest, res *PerformDeviceDeletionResponse) error PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error + PerformAccountDeactivation(ctx context.Context, req *PerformAccountDeactivationRequest, res *PerformAccountDeactivationResponse) error QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error @@ -199,6 +200,16 @@ type PerformDeviceCreationResponse struct { Device *Device } +// PerformAccountDeactivationRequest is the request for PerformAccountDeactivation +type PerformAccountDeactivationRequest struct { + Localpart string +} + +// PerformAccountDeactivationResponse is the response for PerformAccountDeactivation +type PerformAccountDeactivationResponse struct { + AccountDeactivated bool +} + // Device represents a client's device (mobile, web, etc) type Device struct { ID string diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 461c548cc..ec8284397 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -388,3 +388,10 @@ func (a *UserInternalAPI) queryAppServiceToken(ctx context.Context, token, appSe dev.UserID = appService.SenderLocalpart return &dev, nil } + +// PerformAccountDeactivation deactivates the user's account, removing all ability for the user to login again. +func (a *UserInternalAPI) PerformAccountDeactivation(ctx context.Context, req *api.PerformAccountDeactivationRequest, res *api.PerformAccountDeactivationResponse) error { + err := a.AccountDB.DeactivateAccount(ctx, req.Localpart) + res.AccountDeactivated = err == nil + return err +} diff --git a/userapi/inthttp/client.go b/userapi/inthttp/client.go index 6dcaf7568..4d9dcc416 100644 --- a/userapi/inthttp/client.go +++ b/userapi/inthttp/client.go @@ -28,11 +28,12 @@ import ( const ( InputAccountDataPath = "/userapi/inputAccountData" - PerformDeviceCreationPath = "/userapi/performDeviceCreation" - PerformAccountCreationPath = "/userapi/performAccountCreation" - PerformPasswordUpdatePath = "/userapi/performPasswordUpdate" - PerformDeviceDeletionPath = "/userapi/performDeviceDeletion" - PerformDeviceUpdatePath = "/userapi/performDeviceUpdate" + PerformDeviceCreationPath = "/userapi/performDeviceCreation" + PerformAccountCreationPath = "/userapi/performAccountCreation" + PerformPasswordUpdatePath = "/userapi/performPasswordUpdate" + PerformDeviceDeletionPath = "/userapi/performDeviceDeletion" + PerformDeviceUpdatePath = "/userapi/performDeviceUpdate" + PerformAccountDeactivationPath = "/userapi/performAccountDeactivation" QueryProfilePath = "/userapi/queryProfile" QueryAccessTokenPath = "/userapi/queryAccessToken" @@ -126,6 +127,14 @@ func (h *httpUserInternalAPI) PerformDeviceUpdate(ctx context.Context, req *api. return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) } +func (h *httpUserInternalAPI) PerformAccountDeactivation(ctx context.Context, req *api.PerformAccountDeactivationRequest, res *api.PerformAccountDeactivationResponse) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformAccountDeactivation") + defer span.Finish() + + apiURL := h.apiURL + PerformAccountDeactivationPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) +} + func (h *httpUserInternalAPI) QueryProfile( ctx context.Context, request *api.QueryProfileRequest, diff --git a/userapi/inthttp/server.go b/userapi/inthttp/server.go index d26746788..e24aad3a9 100644 --- a/userapi/inthttp/server.go +++ b/userapi/inthttp/server.go @@ -91,6 +91,19 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(PerformAccountDeactivationPath, + httputil.MakeInternalAPI("performAccountDeactivation", func(req *http.Request) util.JSONResponse { + request := api.PerformAccountDeactivationRequest{} + response := api.PerformAccountDeactivationResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.PerformAccountDeactivation(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) internalAPIMux.Handle(QueryProfilePath, httputil.MakeInternalAPI("queryProfile", func(req *http.Request) util.JSONResponse { request := api.QueryProfileRequest{} diff --git a/userapi/storage/accounts/interface.go b/userapi/storage/accounts/interface.go index 49446f11f..c86b2c391 100644 --- a/userapi/storage/accounts/interface.go +++ b/userapi/storage/accounts/interface.go @@ -51,6 +51,7 @@ type Database interface { CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) GetAccountByLocalpart(ctx context.Context, localpart string) (*api.Account, error) SearchProfiles(ctx context.Context, searchString string, limit int) ([]authtypes.Profile, error) + DeactivateAccount(ctx context.Context, localpart string) (err error) } // Err3PIDInUse is the error returned when trying to save an association involving diff --git a/userapi/storage/accounts/postgres/accounts_table.go b/userapi/storage/accounts/postgres/accounts_table.go index 7500e1e82..254da84c3 100644 --- a/userapi/storage/accounts/postgres/accounts_table.go +++ b/userapi/storage/accounts/postgres/accounts_table.go @@ -37,7 +37,9 @@ CREATE TABLE IF NOT EXISTS account_accounts ( -- The password hash for this account. Can be NULL if this is a passwordless account. password_hash TEXT, -- Identifies which application service this account belongs to, if any. - appservice_id TEXT + appservice_id TEXT, + -- If the account is currently active + is_deactivated BOOLEAN DEFAULT FALSE -- TODO: -- is_guest, is_admin, upgraded_ts, devices, any email reset stuff? ); @@ -51,11 +53,14 @@ const insertAccountSQL = "" + const updatePasswordSQL = "" + "UPDATE account_accounts SET password_hash = $1 WHERE localpart = $2" +const deactivateAccountSQL = "" + + "UPDATE account_accounts SET is_deactivated = TRUE WHERE localpart = $1" + const selectAccountByLocalpartSQL = "" + "SELECT localpart, appservice_id FROM account_accounts WHERE localpart = $1" const selectPasswordHashSQL = "" + - "SELECT password_hash FROM account_accounts WHERE localpart = $1" + "SELECT password_hash FROM account_accounts WHERE localpart = $1 AND is_deactivated = FALSE" const selectNewNumericLocalpartSQL = "" + "SELECT nextval('numeric_username_seq')" @@ -63,6 +68,7 @@ const selectNewNumericLocalpartSQL = "" + type accountsStatements struct { insertAccountStmt *sql.Stmt updatePasswordStmt *sql.Stmt + deactivateAccountStmt *sql.Stmt selectAccountByLocalpartStmt *sql.Stmt selectPasswordHashStmt *sql.Stmt selectNewNumericLocalpartStmt *sql.Stmt @@ -80,6 +86,9 @@ func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.Server if s.updatePasswordStmt, err = db.Prepare(updatePasswordSQL); err != nil { return } + if s.deactivateAccountStmt, err = db.Prepare(deactivateAccountSQL); err != nil { + return + } if s.selectAccountByLocalpartStmt, err = db.Prepare(selectAccountByLocalpartSQL); err != nil { return } @@ -127,6 +136,13 @@ func (s *accountsStatements) updatePassword( return } +func (s *accountsStatements) deactivateAccount( + ctx context.Context, localpart string, +) (err error) { + _, err = s.deactivateAccountStmt.ExecContext(ctx, localpart) + return +} + func (s *accountsStatements) selectPasswordHash( ctx context.Context, localpart string, ) (hash string, err error) { diff --git a/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql b/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql new file mode 100644 index 000000000..32e6e1664 --- /dev/null +++ b/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql @@ -0,0 +1,9 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS is_deactivated BOOLEAN DEFAULT FALSE; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE account_accounts DROP COLUMN is_deactivated; +-- +goose StatementEnd diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go index 8b9ebef80..2230f7e79 100644 --- a/userapi/storage/accounts/postgres/storage.go +++ b/userapi/storage/accounts/postgres/storage.go @@ -317,3 +317,8 @@ func (d *Database) SearchProfiles(ctx context.Context, searchString string, limi ) ([]authtypes.Profile, error) { return d.profiles.selectProfilesBySearch(ctx, searchString, limit) } + +// DeactivateAccount deactivates the user's account, removing all ability for the user to login again. +func (d *Database) DeactivateAccount(ctx context.Context, localpart string) (err error) { + return d.accounts.deactivateAccount(ctx, localpart) +} diff --git a/userapi/storage/accounts/sqlite3/accounts_table.go b/userapi/storage/accounts/sqlite3/accounts_table.go index 2d935fb63..d0ea8a8bc 100644 --- a/userapi/storage/accounts/sqlite3/accounts_table.go +++ b/userapi/storage/accounts/sqlite3/accounts_table.go @@ -37,7 +37,9 @@ CREATE TABLE IF NOT EXISTS account_accounts ( -- The password hash for this account. Can be NULL if this is a passwordless account. password_hash TEXT, -- Identifies which application service this account belongs to, if any. - appservice_id TEXT + appservice_id TEXT, + -- If the account is currently active + is_deactivated BOOLEAN DEFAULT 0 -- TODO: -- is_guest, is_admin, upgraded_ts, devices, any email reset stuff? ); @@ -49,11 +51,14 @@ const insertAccountSQL = "" + const updatePasswordSQL = "" + "UPDATE account_accounts SET password_hash = $1 WHERE localpart = $2" +const deactivateAccountSQL = "" + + "UPDATE account_accounts SET is_deactivated = 1 WHERE localpart = $1" + const selectAccountByLocalpartSQL = "" + "SELECT localpart, appservice_id FROM account_accounts WHERE localpart = $1" const selectPasswordHashSQL = "" + - "SELECT password_hash FROM account_accounts WHERE localpart = $1" + "SELECT password_hash FROM account_accounts WHERE localpart = $1 AND is_deactivated = 0" const selectNewNumericLocalpartSQL = "" + "SELECT COUNT(localpart) FROM account_accounts" @@ -62,6 +67,7 @@ type accountsStatements struct { db *sql.DB insertAccountStmt *sql.Stmt updatePasswordStmt *sql.Stmt + deactivateAccountStmt *sql.Stmt selectAccountByLocalpartStmt *sql.Stmt selectPasswordHashStmt *sql.Stmt selectNewNumericLocalpartStmt *sql.Stmt @@ -81,6 +87,9 @@ func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.Server if s.updatePasswordStmt, err = db.Prepare(updatePasswordSQL); err != nil { return } + if s.deactivateAccountStmt, err = db.Prepare(deactivateAccountSQL); err != nil { + return + } if s.selectAccountByLocalpartStmt, err = db.Prepare(selectAccountByLocalpartSQL); err != nil { return } @@ -128,6 +137,13 @@ func (s *accountsStatements) updatePassword( return } +func (s *accountsStatements) deactivateAccount( + ctx context.Context, localpart string, +) (err error) { + _, err = s.deactivateAccountStmt.ExecContext(ctx, localpart) + return +} + func (s *accountsStatements) selectPasswordHash( ctx context.Context, localpart string, ) (hash string, err error) { diff --git a/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql b/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql new file mode 100644 index 000000000..51e9bae3c --- /dev/null +++ b/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql @@ -0,0 +1,38 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE account_accounts RENAME TO account_accounts_tmp; +CREATE TABLE account_accounts ( + localpart TEXT NOT NULL PRIMARY KEY, + created_ts BIGINT NOT NULL, + password_hash TEXT, + appservice_id TEXT, + is_deactivated BOOLEAN DEFAULT 0 +); +INSERT + INTO account_accounts ( + localpart, created_ts, password_hash, appservice_id + ) SELECT + localpart, created_ts, password_hash, appservice_id + FROM account_accounts_tmp +; +DROP TABLE account_accounts_tmp; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE account_accounts RENAME TO account_accounts_tmp; +CREATE TABLE account_accounts ( + localpart TEXT NOT NULL PRIMARY KEY, + created_ts BIGINT NOT NULL, + password_hash TEXT, + appservice_id TEXT +); +INSERT + INTO account_accounts ( + localpart, created_ts, password_hash, appservice_id + ) SELECT + localpart, created_ts, password_hash, appservice_id + FROM account_accounts_tmp +; +DROP TABLE account_accounts_tmp; +-- +goose StatementEnd diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go index 4b66304c2..7a2830a93 100644 --- a/userapi/storage/accounts/sqlite3/storage.go +++ b/userapi/storage/accounts/sqlite3/storage.go @@ -359,3 +359,8 @@ func (d *Database) SearchProfiles(ctx context.Context, searchString string, limi ) ([]authtypes.Profile, error) { return d.profiles.selectProfilesBySearch(ctx, searchString, limit) } + +// DeactivateAccount deactivates the user's account, removing all ability for the user to login again. +func (d *Database) DeactivateAccount(ctx context.Context, localpart string) (err error) { + return d.accounts.deactivateAccount(ctx, localpart) +} From 3bd66ff196a84b0c96763850228c379b2a124fb6 Mon Sep 17 00:00:00 2001 From: S7evinK Date: Fri, 2 Oct 2020 23:35:54 +0200 Subject: [PATCH 037/104] Route several paths to sync_api (#1473) --- docs/nginx/polylith-sample.conf | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf index 658e0e4a4..ab3461848 100644 --- a/docs/nginx/polylith-sample.conf +++ b/docs/nginx/polylith-sample.conf @@ -18,6 +18,17 @@ server { return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }'; } + # route requests to: + # /_matrix/client/.*/sync + # /_matrix/client/.*/user/{userId}/filter + # /_matrix/client/.*/user/{userId}/filter/{filterID} + # /_matrix/client/.*/keys/changes + # /_matrix/client/.*/rooms/{roomId}/messages + # to sync_api + location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$ { + proxy_pass http://sync_api:8073; + } + location /_matrix/client { proxy_pass http://client_api:8071; } From c4756eee0e59ba730fa7855171e18eda9dba8b5f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Oct 2020 11:06:31 +0100 Subject: [PATCH 038/104] Don't store backfilled events using request context (#1478) --- syncapi/routing/messages.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 9c6c6a80d..e5299f200 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -503,7 +503,7 @@ func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][] // up in responses to sync requests. for i := range res.Events { _, err = r.db.WriteEvent( - r.ctx, + context.Background(), &res.Events[i], []gomatrixserverlib.HeaderedEvent{}, []string{}, From 2bfab5f58b77002b1838b52f6edc369ea280ced7 Mon Sep 17 00:00:00 2001 From: S7evinK Date: Mon, 5 Oct 2020 12:28:10 +0200 Subject: [PATCH 039/104] Add example goose sqlite3 migration (#1474) Signed-off-by: Till Faelligen --- cmd/goose/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/goose/README.md b/cmd/goose/README.md index c7f085d80..725c6a586 100644 --- a/cmd/goose/README.md +++ b/cmd/goose/README.md @@ -104,4 +104,6 @@ You __must__ import the package in `/cmd/goose/main.go` so `func init()` gets ca #### Database limitations - SQLite3 does NOT support `ALTER TABLE table_name DROP COLUMN` - you would have to rename the column or drop the table - entirely and recreate it. + entirely and recreate it. ([example](https://github.com/matrix-org/dendrite/blob/master/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql)) + + More information: [sqlite.org](https://www.sqlite.org/lang_altertable.html) From 4e6b7f726d6741e0faa77ad0c8ba373ea58ef3f8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Oct 2020 12:42:56 +0100 Subject: [PATCH 040/104] Update to matrix-org/gomatrixserverlib#230 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b1119f608..5808c55ae 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290 + github.com/matrix-org/gomatrixserverlib v0.0.0-20201005114154-18abed1923fd github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index c657cc457..7dae1ef90 100644 --- a/go.sum +++ b/go.sum @@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290 h1:ilT9QNIh2KXfvzIALtAe31IvLVZH7mVjVtOOTxdd0tY= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201005114154-18abed1923fd h1:30Mlf0wdEj4dtVQdpSRT1aVkFu2qXh5fK9qrExPbPIM= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201005114154-18abed1923fd/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= From 2e71d2708f7c922dcb6361c8a49022470b8b6e33 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Oct 2020 17:47:08 +0100 Subject: [PATCH 041/104] Resolve state after event against current room state when determining latest state changes (#1479) * Resolve state after event against current room state when determining latest state changes * Update sytest-whitelist * Update sytest-whitelist, blacklist --- .../internal/input/input_latest_events.go | 23 ++++++++++++++++--- roomserver/state/state.go | 8 ++++--- sytest-blacklist | 7 +++++- sytest-whitelist | 4 ---- 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index 5c2a1de6a..2e9f3b4e4 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -215,10 +215,27 @@ func (u *latestEventsUpdater) latestState() error { var err error roomState := state.NewStateResolution(u.api.DB, *u.roomInfo) - // Get a list of the current latest events. - latestStateAtEvents := make([]types.StateAtEvent, len(u.latest)) + // Get a list of the current room state events if available. + var currentState []types.StateEntry + if u.roomInfo.StateSnapshotNID != 0 { + currentState, _ = roomState.LoadStateAtSnapshot(u.ctx, u.roomInfo.StateSnapshotNID) + } + + // Get a list of the current latest events. This will include both + // the current room state and the latest events after the input event. + // The idea is that we will perform state resolution on this set and + // any conflicting events will be resolved properly. + latestStateAtEvents := make([]types.StateAtEvent, len(u.latest)+len(currentState)) + offset := 0 + for i := range currentState { + latestStateAtEvents[i] = types.StateAtEvent{ + BeforeStateSnapshotNID: u.roomInfo.StateSnapshotNID, + StateEntry: currentState[i], + } + offset++ + } for i := range u.latest { - latestStateAtEvents[i] = u.latest[i].StateAtEvent + latestStateAtEvents[offset+i] = u.latest[i].StateAtEvent } // Takes the NIDs of the latest events and creates a state snapshot diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 0663499e7..2944f71c1 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -118,7 +118,7 @@ func (v StateResolution) LoadCombinedStateAfterEvents( // the snapshot of the room state before them was the same. stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, uniqueStateSnapshotNIDs(stateNIDs)) if err != nil { - return nil, err + return nil, fmt.Errorf("v.db.StateBlockNIDs: %w", err) } var stateBlockNIDs []types.StateBlockNID @@ -131,7 +131,7 @@ func (v StateResolution) LoadCombinedStateAfterEvents( // multiple snapshots. stateEntryLists, err := v.db.StateEntries(ctx, uniqueStateBlockNIDs(stateBlockNIDs)) if err != nil { - return nil, err + return nil, fmt.Errorf("v.db.StateEntries: %w", err) } stateBlockNIDsMap := stateBlockNIDListMap(stateBlockNIDLists) stateEntriesMap := stateEntryListMap(stateEntryLists) @@ -623,7 +623,7 @@ func (v StateResolution) calculateAndStoreStateAfterManyEvents( v.calculateStateAfterManyEvents(ctx, v.roomInfo.RoomVersion, prevStates) metrics.algorithm = algorithm if err != nil { - return metrics.stop(0, err) + return metrics.stop(0, fmt.Errorf("v.calculateStateAfterManyEvents: %w", err)) } // TODO: Check if we can encode the new state as a delta against the @@ -642,6 +642,7 @@ func (v StateResolution) calculateStateAfterManyEvents( // First stage: load the state after each of the prev events. combined, err = v.LoadCombinedStateAfterEvents(ctx, prevStates) if err != nil { + err = fmt.Errorf("v.LoadCombinedStateAfterEvents: %w", err) algorithm = "_load_combined_state" return } @@ -672,6 +673,7 @@ func (v StateResolution) calculateStateAfterManyEvents( var resolved []types.StateEntry resolved, err = v.resolveConflicts(ctx, roomVersion, notConflicted, conflicts) if err != nil { + err = fmt.Errorf("v.resolveConflits: %w", err) algorithm = "_resolve_conflicts" return } diff --git a/sytest-blacklist b/sytest-blacklist index 2f80fc789..ff7fdf7e0 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -52,4 +52,9 @@ Inbound federation accepts a second soft-failed event Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state # We don't implement lazy membership loading yet. -The only membership state included in a gapped incremental sync is for senders in the timeline \ No newline at end of file +The only membership state included in a gapped incremental sync is for senders in the timeline + +# Blacklisted out of flakiness after #1479 +Invited user can reject local invite after originator leaves +Invited user can reject invite for empty room +If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes \ No newline at end of file diff --git a/sytest-whitelist b/sytest-whitelist index a811259fb..9a013cbf3 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -400,8 +400,6 @@ Uninvited users cannot join the room Users cannot invite themselves to a room Users cannot invite a user that is already in the room Invited user can reject invite -Invited user can reject invite for empty room -Invited user can reject local invite after originator leaves PUT /rooms/:room_id/typing/:user_id sets typing notification Typing notification sent to local room members Typing notifications also sent to remote room members @@ -431,7 +429,6 @@ A prev_batch token can be used in the v1 messages API We don't send redundant membership state across incremental syncs by default Typing notifications don't leak Users cannot kick users from a room they are not in -Users cannot kick users who have already left a room User appears in user directory User directory correctly update on display name change User in shared private room does appear in user directory @@ -451,7 +448,6 @@ Banned servers cannot backfill Inbound /v1/send_leave rejects leaves from other servers Guest users can accept invites to private rooms over federation AS user (not ghost) can join room without registering -If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes Can search public room list Can get remote public room list Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list From 52ddded72d05924434f7284bd686629e1cca06c6 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 5 Oct 2020 17:56:18 +0100 Subject: [PATCH 042/104] Update to matrix-org/gomatrixserverlib#232 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5808c55ae..5793897af 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20201005114154-18abed1923fd + github.com/matrix-org/gomatrixserverlib v0.0.0-20201005165532-90f2e192f625 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index 7dae1ef90..20479015d 100644 --- a/go.sum +++ b/go.sum @@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201005114154-18abed1923fd h1:30Mlf0wdEj4dtVQdpSRT1aVkFu2qXh5fK9qrExPbPIM= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201005114154-18abed1923fd/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201005165532-90f2e192f625 h1:9ShgY0ZkfLzqe3gv18V5WxDAZ4dgUvJwnGORycox680= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201005165532-90f2e192f625/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= From 8fb74fe99a968aa438d436bd251baf5a790e8156 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 6 Oct 2020 10:37:52 +0100 Subject: [PATCH 043/104] Yggdrasil demo tweaks --- cmd/dendrite-demo-yggdrasil/yggconn/node.go | 10 ++++------ go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go index 9b123aa64..c036c0d87 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go @@ -205,13 +205,11 @@ func (n *Node) SessionCount() int { func (n *Node) KnownNodes() []gomatrixserverlib.ServerName { nodemap := map[string]struct{}{ - "b5ae50589e50991dd9dd7d59c5c5f7a4521e8da5b603b7f57076272abc58b374": {}, + //"b5ae50589e50991dd9dd7d59c5c5f7a4521e8da5b603b7f57076272abc58b374": {}, + } + for _, peer := range n.core.GetSwitchPeers() { + nodemap[hex.EncodeToString(peer.SigPublicKey[:])] = struct{}{} } - /* - for _, peer := range n.core.GetSwitchPeers() { - nodemap[hex.EncodeToString(peer.PublicKey[:])] = struct{}{} - } - */ n.sessions.Range(func(_, v interface{}) bool { session, ok := v.(quic.Session) if !ok { diff --git a/go.mod b/go.mod index 5793897af..4a382d6d8 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/tidwall/sjson v1.1.1 github.com/uber/jaeger-client-go v2.25.0+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible - github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200806125501-cd4685a3b4de + github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20201006093556-760d9a7fd5ee go.uber.org/atomic v1.6.0 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a gopkg.in/h2non/bimg.v1 v1.1.4 diff --git a/go.sum b/go.sum index 20479015d..adb0f0e88 100644 --- a/go.sum +++ b/go.sum @@ -851,8 +851,8 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhe github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yggdrasil-network/yggdrasil-extras v0.0.0-20200525205615-6c8a4a2e8855/go.mod h1:xQdsh08Io6nV4WRnOVTe6gI8/2iTvfLDQ0CYa5aMt+I= -github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200806125501-cd4685a3b4de h1:p91aw0Mvol825U+5bvV9BBPl+HQxIczj7wxIOxZs70M= -github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200806125501-cd4685a3b4de/go.mod h1:d+Nz6SPeG6kmeSPFL0cvfWfgwEql75fUnZiAONgvyBE= +github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20201006093556-760d9a7fd5ee h1:Kot820OfxWfYrk5di5f4S5s0jXXrQj8w8BG5826HAv4= +github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20201006093556-760d9a7fd5ee/go.mod h1:d+Nz6SPeG6kmeSPFL0cvfWfgwEql75fUnZiAONgvyBE= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= From bf90db5b60d694d8af0e8ec1d90d2501604ab219 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 6 Oct 2020 11:05:00 +0100 Subject: [PATCH 044/104] Remove KindRewrite (#1481) * Don't send rewrite events * Remove final traces of rewrite events * Remove test that is no longer needed * Revert "Remove test that is no longer needed" This reverts commit 9a45babff690480acd656a52f2c2950a5f7e9ada. * Update test to use KindOutlier --- federationsender/internal/perform.go | 2 +- roomserver/api/input.go | 4 - roomserver/api/wrapper.go | 93 ----------------------- roomserver/internal/input/input_events.go | 9 --- roomserver/roomserver_test.go | 4 +- 5 files changed, 3 insertions(+), 109 deletions(-) diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index 6aea296bd..0c9dd2572 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -232,7 +232,7 @@ func (r *FederationSenderInternalAPI) performJoinUsingServer( // If we successfully performed a send_join above then the other // server now thinks we're a part of the room. Send the newly // returned state to the roomserver to update our local view. - if err = roomserverAPI.SendEventWithRewrite( + if err = roomserverAPI.SendEventWithState( ctx, r.rsAPI, respState, event.Headered(respMakeJoin.RoomVersion), diff --git a/roomserver/api/input.go b/roomserver/api/input.go index 862a6fa1f..a72e2d9a2 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -35,10 +35,6 @@ const ( // KindBackfill event extend the contiguous graph going backwards. // They always have state. KindBackfill = 3 - // KindRewrite events are used when rewriting the head of the room - // graph with entirely new state. The output events generated will - // be state events rather than timeline events. - KindRewrite = 4 ) // DoNotSendToOtherServers tells us not to send the event to other matrix diff --git a/roomserver/api/wrapper.go b/roomserver/api/wrapper.go index 24949fc63..a38c00df7 100644 --- a/roomserver/api/wrapper.go +++ b/roomserver/api/wrapper.go @@ -80,99 +80,6 @@ func SendEventWithState( return SendInputRoomEvents(ctx, rsAPI, ires) } -// SendEventWithRewrite writes an event with KindNew to the roomserver along -// with a number of rewrite and outlier events for state and auth events -// respectively. -func SendEventWithRewrite( - ctx context.Context, rsAPI RoomserverInternalAPI, state *gomatrixserverlib.RespState, - event gomatrixserverlib.HeaderedEvent, haveEventIDs map[string]bool, -) error { - isCurrentState := map[string]struct{}{} - for _, se := range state.StateEvents { - isCurrentState[se.EventID()] = struct{}{} - } - - authAndStateEvents, err := state.Events() - if err != nil { - return err - } - - var ires []InputRoomEvent - var stateIDs []string - - // This function generates three things: - // A - A set of "rewrite" events, which will form the newly rewritten - // state before the event, which includes every rewrite event that - // came before it in its state - // B - A set of "outlier" events, which are auth events but not part - // of the rewritten state - // C - A "new" event, which include all of the rewrite events in its - // state - for _, authOrStateEvent := range authAndStateEvents { - if authOrStateEvent.StateKey() == nil { - continue - } - if haveEventIDs[authOrStateEvent.EventID()] { - continue - } - if event.StateKey() == nil { - continue - } - - // We will handle an event as if it's an outlier if one of the - // following conditions is true: - storeAsOutlier := false - if _, ok := isCurrentState[authOrStateEvent.EventID()]; !ok { - // The event is an auth event and isn't a part of the state set. - // We'll send it as an outlier because we need it to be stored - // in case something is referring to it as an auth event. - storeAsOutlier = true - } - - if storeAsOutlier { - ires = append(ires, InputRoomEvent{ - Kind: KindOutlier, - Event: authOrStateEvent.Headered(event.RoomVersion), - AuthEventIDs: authOrStateEvent.AuthEventIDs(), - }) - continue - } - - // If the event isn't an outlier then we'll instead send it as a - // rewrite event, so that it'll form part of the rewritten state. - // These events will go through the membership and latest event - // updaters and we will generate output events, but they will be - // flagged as non-current (i.e. didn't just happen) events. - // Each of these rewrite events includes all of the rewrite events - // that came before in their StateEventIDs. - ires = append(ires, InputRoomEvent{ - Kind: KindRewrite, - Event: authOrStateEvent.Headered(event.RoomVersion), - AuthEventIDs: authOrStateEvent.AuthEventIDs(), - HasState: true, - StateEventIDs: stateIDs, - }) - - // Add the event ID into the StateEventIDs of all subsequent - // rewrite events, and the new event. - stateIDs = append(stateIDs, authOrStateEvent.EventID()) - } - - // Send the final event as a new event, which will generate - // a timeline output event for it. All of the rewrite events - // that came before will be sent as StateEventIDs, forming a - // new clean state before the event. - ires = append(ires, InputRoomEvent{ - Kind: KindNew, - Event: event, - AuthEventIDs: event.AuthEventIDs(), - HasState: true, - StateEventIDs: stateIDs, - }) - - return SendInputRoomEvents(ctx, rsAPI, ires) -} - // SendInputRoomEvents to the roomserver. func SendInputRoomEvents( ctx context.Context, rsAPI RoomserverInternalAPI, ires []InputRoomEvent, diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 3d44f0486..810d8cdaf 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -136,15 +136,6 @@ func (r *Inputer) processRoomEvent( return event.EventID(), rejectionErr } - if input.Kind == api.KindRewrite { - logrus.WithFields(logrus.Fields{ - "event_id": event.EventID(), - "type": event.Type(), - "room": event.RoomID(), - }).Debug("Stored rewrite") - return event.EventID(), nil - } - if err = r.updateLatestEvents( ctx, // context roomInfo, // room info for the room being updated diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index 912c5852f..2a03195c9 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -238,7 +238,7 @@ func TestOutputRedactedEvent(t *testing.T) { } } -// This tests that rewriting state via KindRewrite works correctly. +// This tests that rewriting state works correctly. // This creates a small room with a create/join/name state, then replays it // with a new room name. We expect the output events to contain the original events, // followed by a single OutputNewRoomEvent with RewritesState set to true with the @@ -344,7 +344,7 @@ func TestOutputRewritesState(t *testing.T) { for i := 0; i < len(rewriteEvents)-1; i++ { ev := rewriteEvents[i] inputEvents = append(inputEvents, api.InputRoomEvent{ - Kind: api.KindRewrite, + Kind: api.KindOutlier, Event: ev, AuthEventIDs: ev.AuthEventIDs(), HasState: true, From 1eaf7aa27e5e4592cd5f8d8c3d9c42cece798748 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 6 Oct 2020 11:05:15 +0100 Subject: [PATCH 045/104] Use [] not null when there are no devices (#1480) --- federationapi/routing/devices.go | 1 + 1 file changed, 1 insertion(+) diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go index 00631b9b6..07862451f 100644 --- a/federationapi/routing/devices.go +++ b/federationapi/routing/devices.go @@ -40,6 +40,7 @@ func GetUserDevices( response := gomatrixserverlib.RespUserDevices{ UserID: userID, StreamID: res.StreamID, + Devices: []gomatrixserverlib.RespUserDevice{}, } for _, dev := range res.Devices { From d69eba10e5fb4cbe847776f1fbc6b96cbced3d66 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 6 Oct 2020 14:51:32 +0100 Subject: [PATCH 046/104] Add furl (#1482) * Add furl * Add POST support --- cmd/furl/main.go | 123 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 cmd/furl/main.go diff --git a/cmd/furl/main.go b/cmd/furl/main.go new file mode 100644 index 000000000..efaaa4b86 --- /dev/null +++ b/cmd/furl/main.go @@ -0,0 +1,123 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "crypto/ed25519" + "encoding/json" + "encoding/pem" + "flag" + "fmt" + "io/ioutil" + "net/url" + "os" + + "github.com/matrix-org/gomatrixserverlib" +) + +var requestFrom = flag.String("from", "", "the server name that the request should originate from") +var requestKey = flag.String("key", "matrix_key.pem", "the private key to use when signing the request") +var requestPost = flag.Bool("post", false, "send a POST request instead of GET (pipe input into stdin or type followed by Ctrl-D)") + +func main() { + flag.Parse() + + if requestFrom == nil || *requestFrom == "" { + fmt.Println("expecting: furl -from origin.com [-key matrix_key.pem] https://path/to/url") + fmt.Println("supported flags:") + flag.PrintDefaults() + os.Exit(1) + } + + data, err := ioutil.ReadFile(*requestKey) + if err != nil { + panic(err) + } + + var privateKey ed25519.PrivateKey + keyBlock, _ := pem.Decode(data) + if keyBlock == nil { + panic("keyBlock is nil") + } + if keyBlock.Type == "MATRIX PRIVATE KEY" { + _, privateKey, err = ed25519.GenerateKey(bytes.NewReader(keyBlock.Bytes)) + if err != nil { + panic(err) + } + } else { + panic("unexpected key block") + } + + client := gomatrixserverlib.NewFederationClient( + gomatrixserverlib.ServerName(*requestFrom), + gomatrixserverlib.KeyID(keyBlock.Headers["Key-ID"]), + privateKey, + false, + ) + + u, err := url.Parse(flag.Arg(0)) + if err != nil { + panic(err) + } + + var bodyObj interface{} + var bodyBytes []byte + method := "GET" + if *requestPost { + method = "POST" + fmt.Println("Waiting for JSON input. Press Enter followed by Ctrl-D when done...") + + scan := bufio.NewScanner(os.Stdin) + for scan.Scan() { + bytes := scan.Bytes() + bodyBytes = append(bodyBytes, bytes...) + } + fmt.Println("Done!") + if err = json.Unmarshal(bodyBytes, &bodyObj); err != nil { + panic(err) + } + } + + req := gomatrixserverlib.NewFederationRequest( + method, + gomatrixserverlib.ServerName(u.Host), + u.RequestURI(), + ) + + if *requestPost { + if err = req.SetContent(bodyObj); err != nil { + panic(err) + } + } + + if err = req.Sign( + gomatrixserverlib.ServerName(*requestFrom), + gomatrixserverlib.KeyID(keyBlock.Headers["Key-ID"]), + privateKey, + ); err != nil { + panic(err) + } + + httpReq, err := req.HTTPRequest() + if err != nil { + panic(err) + } + + var res interface{} + err = client.DoRequestAndParseResponse( + context.TODO(), + httpReq, + &res, + ) + if err != nil { + panic(err) + } + + j, err := json.MarshalIndent(res, "", " ") + if err != nil { + panic(err) + } + + fmt.Println(string(j)) +} From ee79d662e764576e02e16a74c62349e81326c64a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 6 Oct 2020 15:37:31 +0100 Subject: [PATCH 047/104] Update to matrix-org/gomatrixserverlib#233 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4a382d6d8..c98aa61e9 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20201005165532-90f2e192f625 + github.com/matrix-org/gomatrixserverlib v0.0.0-20201006143701-222e7423a5e3 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index adb0f0e88..c92774bd0 100644 --- a/go.sum +++ b/go.sum @@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201005165532-90f2e192f625 h1:9ShgY0ZkfLzqe3gv18V5WxDAZ4dgUvJwnGORycox680= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201005165532-90f2e192f625/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201006143701-222e7423a5e3 h1:lWR/w6rXKZJJU1yGHb2zem/EK7+aYhUcRgAOiouZAxk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201006143701-222e7423a5e3/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= From 4feff8e8d9efd36b5d202ba219af997a8313866a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 6 Oct 2020 17:59:08 +0100 Subject: [PATCH 048/104] Don't give up if we fail to fetch a key (#1483) * Don't give up if we fail to fetch a key * Fix logging line * furl nolint --- cmd/furl/main.go | 1 + roomserver/internal/perform/perform_backfill.go | 2 +- serverkeyapi/internal/api.go | 4 ---- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cmd/furl/main.go b/cmd/furl/main.go index efaaa4b86..3955ef0cd 100644 --- a/cmd/furl/main.go +++ b/cmd/furl/main.go @@ -20,6 +20,7 @@ var requestFrom = flag.String("from", "", "the server name that the request shou var requestKey = flag.String("key", "matrix_key.pem", "the private key to use when signing the request") var requestPost = flag.Bool("post", false, "send a POST request instead of GET (pipe input into stdin or type followed by Ctrl-D)") +// nolint:gocyclo func main() { flag.Parse() diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go index eb1aa99b8..f60919948 100644 --- a/roomserver/internal/perform/perform_backfill.go +++ b/roomserver/internal/perform/perform_backfill.go @@ -195,7 +195,7 @@ func (r *Backfiller) fetchAndStoreMissingEvents(ctx context.Context, roomVer gom logger.Infof("returned %d PDUs which made events %+v", len(res.PDUs), result) for _, res := range result { if res.Error != nil { - logger.WithError(err).Warn("event failed PDU checks") + logger.WithError(res.Error).Warn("event failed PDU checks") continue } missingMap[id] = res.Event diff --git a/serverkeyapi/internal/api.go b/serverkeyapi/internal/api.go index b8a362259..335bfe4ce 100644 --- a/serverkeyapi/internal/api.go +++ b/serverkeyapi/internal/api.go @@ -98,10 +98,6 @@ func (s *ServerKeyAPI) FetchKeys( // we've failed to satisfy it from local keys, database keys or from // all of the fetchers. Report an error. logrus.Warnf("Failed to retrieve key %q for server %q", req.KeyID, req.ServerName) - return results, fmt.Errorf( - "server key API failed to satisfy key request for server %q key ID %q", - req.ServerName, req.KeyID, - ) } } From 0f7e707f399e7f633c58f4e1a5aedc0e45f90241 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 6 Oct 2020 18:09:02 +0100 Subject: [PATCH 049/104] Optimise servers to backfill from (#1485) - Prefer perspective servers if they are in the room. - Limit the number of backfill servers to 5 to avoid taking too long. --- roomserver/internal/api.go | 34 +++++++++------- .../internal/perform/perform_backfill.go | 39 +++++++++++++++---- roomserver/roomserver.go | 7 +++- 3 files changed, 58 insertions(+), 22 deletions(-) diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index 8dc1a170b..ee4e4ec96 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -26,28 +26,30 @@ type RoomserverInternalAPI struct { *perform.Leaver *perform.Publisher *perform.Backfiller - DB storage.Database - Cfg *config.RoomServer - Producer sarama.SyncProducer - Cache caching.RoomServerCaches - ServerName gomatrixserverlib.ServerName - KeyRing gomatrixserverlib.JSONVerifier - fsAPI fsAPI.FederationSenderInternalAPI - OutputRoomEventTopic string // Kafka topic for new output room events + DB storage.Database + Cfg *config.RoomServer + Producer sarama.SyncProducer + Cache caching.RoomServerCaches + ServerName gomatrixserverlib.ServerName + KeyRing gomatrixserverlib.JSONVerifier + fsAPI fsAPI.FederationSenderInternalAPI + OutputRoomEventTopic string // Kafka topic for new output room events + PerspectiveServerNames []gomatrixserverlib.ServerName } func NewRoomserverAPI( cfg *config.RoomServer, roomserverDB storage.Database, producer sarama.SyncProducer, outputRoomEventTopic string, caches caching.RoomServerCaches, - keyRing gomatrixserverlib.JSONVerifier, + keyRing gomatrixserverlib.JSONVerifier, perspectiveServerNames []gomatrixserverlib.ServerName, ) *RoomserverInternalAPI { serverACLs := acls.NewServerACLs(roomserverDB) a := &RoomserverInternalAPI{ - DB: roomserverDB, - Cfg: cfg, - Cache: caches, - ServerName: cfg.Matrix.ServerName, - KeyRing: keyRing, + DB: roomserverDB, + Cfg: cfg, + Cache: caches, + ServerName: cfg.Matrix.ServerName, + PerspectiveServerNames: perspectiveServerNames, + KeyRing: keyRing, Queryer: &query.Queryer{ DB: roomserverDB, Cache: caches, @@ -105,6 +107,10 @@ func (r *RoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSen DB: r.DB, FSAPI: r.fsAPI, KeyRing: r.KeyRing, + // Perspective servers are trusted to not lie about server keys, so we will also + // prefer these servers when backfilling (assuming they are in the room) rather + // than trying random servers + PreferServers: r.PerspectiveServerNames, } } diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go index f60919948..d90ac8fcc 100644 --- a/roomserver/internal/perform/perform_backfill.go +++ b/roomserver/internal/perform/perform_backfill.go @@ -30,11 +30,19 @@ import ( "github.com/sirupsen/logrus" ) +// the max number of servers to backfill from per request. If this is too low we may fail to backfill when +// we could've from another server. If this is too high we may take far too long to successfully backfill +// as we try dead servers. +const maxBackfillServers = 5 + type Backfiller struct { ServerName gomatrixserverlib.ServerName DB storage.Database FSAPI federationSenderAPI.FederationSenderInternalAPI KeyRing gomatrixserverlib.JSONVerifier + + // The servers which should be preferred above other servers when backfilling + PreferServers []gomatrixserverlib.ServerName } // PerformBackfill implements api.RoomServerQueryAPI @@ -96,7 +104,7 @@ func (r *Backfiller) backfillViaFederation(ctx context.Context, req *api.Perform if info == nil || info.IsStub { return fmt.Errorf("backfillViaFederation: missing room info for room %s", req.RoomID) } - requester := newBackfillRequester(r.DB, r.FSAPI, r.ServerName, req.BackwardsExtremities) + requester := newBackfillRequester(r.DB, r.FSAPI, r.ServerName, req.BackwardsExtremities, r.PreferServers) // Request 100 items regardless of what the query asks for. // We don't want to go much higher than this. // We can't honour exactly the limit as some sytests rely on requesting more for tests to pass @@ -215,10 +223,11 @@ func (r *Backfiller) fetchAndStoreMissingEvents(ctx context.Context, roomVer gom // backfillRequester implements gomatrixserverlib.BackfillRequester type backfillRequester struct { - db storage.Database - fsAPI federationSenderAPI.FederationSenderInternalAPI - thisServer gomatrixserverlib.ServerName - bwExtrems map[string][]string + db storage.Database + fsAPI federationSenderAPI.FederationSenderInternalAPI + thisServer gomatrixserverlib.ServerName + preferServer map[gomatrixserverlib.ServerName]bool + bwExtrems map[string][]string // per-request state servers []gomatrixserverlib.ServerName @@ -226,7 +235,14 @@ type backfillRequester struct { eventIDMap map[string]gomatrixserverlib.Event } -func newBackfillRequester(db storage.Database, fsAPI federationSenderAPI.FederationSenderInternalAPI, thisServer gomatrixserverlib.ServerName, bwExtrems map[string][]string) *backfillRequester { +func newBackfillRequester( + db storage.Database, fsAPI federationSenderAPI.FederationSenderInternalAPI, thisServer gomatrixserverlib.ServerName, + bwExtrems map[string][]string, preferServers []gomatrixserverlib.ServerName, +) *backfillRequester { + preferServer := make(map[gomatrixserverlib.ServerName]bool) + for _, p := range preferServers { + preferServer[p] = true + } return &backfillRequester{ db: db, fsAPI: fsAPI, @@ -234,6 +250,7 @@ func newBackfillRequester(db storage.Database, fsAPI federationSenderAPI.Federat eventIDToBeforeStateIDs: make(map[string][]string), eventIDMap: make(map[string]gomatrixserverlib.Event), bwExtrems: bwExtrems, + preferServer: preferServer, } } @@ -436,8 +453,16 @@ FindSuccessor: if server == b.thisServer { continue } - servers = append(servers, server) + if b.preferServer[server] { // insert at the front + servers = append([]gomatrixserverlib.ServerName{server}, servers...) + } else { // insert at the back + servers = append(servers, server) + } } + if len(servers) > maxBackfillServers { + servers = servers[:maxBackfillServers] + } + b.servers = servers return servers } diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 2eabf4504..98a86e5bb 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -41,6 +41,11 @@ func NewInternalAPI( ) api.RoomserverInternalAPI { cfg := &base.Cfg.RoomServer + var perspectiveServerNames []gomatrixserverlib.ServerName + for _, kp := range base.Cfg.ServerKeyAPI.KeyPerspectives { + perspectiveServerNames = append(perspectiveServerNames, kp.ServerName) + } + roomserverDB, err := storage.Open(&cfg.Database, base.Caches) if err != nil { logrus.WithError(err).Panicf("failed to connect to room server db") @@ -48,6 +53,6 @@ func NewInternalAPI( return internal.NewRoomserverAPI( cfg, roomserverDB, base.KafkaProducer, string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputRoomEvent)), - base.Caches, keyRing, + base.Caches, keyRing, perspectiveServerNames, ) } From f7c15071decd9a33fabece54b86e92e10009a034 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 7 Oct 2020 10:30:27 +0100 Subject: [PATCH 050/104] Don't return 500s on checking to see if a remote server is allowed to see an event we don't know about (#1490) --- roomserver/internal/helpers/helpers.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go index b7e6ce86c..a2fbd287b 100644 --- a/roomserver/internal/helpers/helpers.go +++ b/roomserver/internal/helpers/helpers.go @@ -2,6 +2,8 @@ package helpers import ( "context" + "database/sql" + "errors" "fmt" "github.com/matrix-org/dendrite/roomserver/api" @@ -217,6 +219,9 @@ func CheckServerAllowedToSeeEvent( roomState := state.NewStateResolution(db, info) stateEntries, err := roomState.LoadStateAtEvent(ctx, eventID) if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } return false, err } From d821f9d3c92adde5b0576de03d0d44ffce5f0182 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 7 Oct 2020 14:05:33 +0100 Subject: [PATCH 051/104] Deep checking of forward extremities (#1491) * Deep forward extremity calculation * Use updater txn * Update error * Update error * Create previous event references in StoreEvent * Use latest events updater to row-lock prev events * Fix unexpected fallthrough * Fix deadlock * Don't roll back * Update comments in calculateLatest * Don't include events that we can't find references for in the forward extremities * Add another passing test --- roomserver/api/input.go | 3 - roomserver/internal/input/input_events.go | 2 +- .../internal/input/input_latest_events.go | 118 ++++++++---------- roomserver/storage/shared/storage.go | 28 ++++- sytest-whitelist | 4 +- 5 files changed, 81 insertions(+), 74 deletions(-) diff --git a/roomserver/api/input.go b/roomserver/api/input.go index a72e2d9a2..dd693203b 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -32,9 +32,6 @@ const ( // there was a new event that references an event that we don't // have a copy of. KindNew = 2 - // KindBackfill event extend the contiguous graph going backwards. - // They always have state. - KindBackfill = 3 ) // DoNotSendToOtherServers tells us not to send the event to other matrix diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 810d8cdaf..113341591 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -54,7 +54,7 @@ func (r *Inputer) processRoomEvent( } var softfail bool - if input.Kind == api.KindBackfill || input.Kind == api.KindNew { + if input.Kind == api.KindNew { // Check that the event passes authentication checks based on the // current room state. softfail, err = helpers.CheckForSoftFail(ctx, r.DB, headered, input.StateEventIDs) diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index 2e9f3b4e4..7be6372b2 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -28,6 +28,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/sirupsen/logrus" ) // updateLatestEvents updates the list of latest events for this room in the database and writes the @@ -116,7 +117,6 @@ type latestEventsUpdater struct { } func (u *latestEventsUpdater) doUpdateLatestEvents() error { - prevEvents := u.event.PrevEvents() u.lastEventIDSent = u.updater.LastEventIDSent() u.oldStateNID = u.updater.CurrentStateSnapshotNID() @@ -140,30 +140,12 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { return nil } - // Update the roomserver_previous_events table with references. This - // is effectively tracking the structure of the DAG. - if err = u.updater.StorePreviousEvents(u.stateAtEvent.EventNID, prevEvents); err != nil { - return fmt.Errorf("u.updater.StorePreviousEvents: %w", err) - } - - // Get the event reference for our new event. This will be used when - // determining if the event is referenced by an existing event. - eventReference := u.event.EventReference() - - // Check if our new event is already referenced by an existing event - // in the room. If it is then it isn't a latest event. - alreadyReferenced, err := u.updater.IsReferenced(eventReference) - if err != nil { - return fmt.Errorf("u.updater.IsReferenced: %w", err) - } - - // Work out what the latest events are. - u.latest = calculateLatest( + // Work out what the latest events are. This will include the new + // event if it is not already referenced. + u.calculateLatest( oldLatest, - alreadyReferenced, - prevEvents, types.StateAtEventAndReference{ - EventReference: eventReference, + EventReference: u.event.EventReference(), StateAtEvent: u.stateAtEvent, }, ) @@ -215,27 +197,12 @@ func (u *latestEventsUpdater) latestState() error { var err error roomState := state.NewStateResolution(u.api.DB, *u.roomInfo) - // Get a list of the current room state events if available. - var currentState []types.StateEntry - if u.roomInfo.StateSnapshotNID != 0 { - currentState, _ = roomState.LoadStateAtSnapshot(u.ctx, u.roomInfo.StateSnapshotNID) - } - - // Get a list of the current latest events. This will include both - // the current room state and the latest events after the input event. - // The idea is that we will perform state resolution on this set and - // any conflicting events will be resolved properly. - latestStateAtEvents := make([]types.StateAtEvent, len(u.latest)+len(currentState)) - offset := 0 - for i := range currentState { - latestStateAtEvents[i] = types.StateAtEvent{ - BeforeStateSnapshotNID: u.roomInfo.StateSnapshotNID, - StateEntry: currentState[i], - } - offset++ - } + // Get a list of the current latest events. This may or may not + // include the new event from the input path, depending on whether + // it is a forward extremity or not. + latestStateAtEvents := make([]types.StateAtEvent, len(u.latest)) for i := range u.latest { - latestStateAtEvents[offset+i] = u.latest[i].StateAtEvent + latestStateAtEvents[i] = u.latest[i].StateAtEvent } // Takes the NIDs of the latest events and creates a state snapshot @@ -266,6 +233,14 @@ func (u *latestEventsUpdater) latestState() error { if err != nil { return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err) } + if len(u.removed) > len(u.added) { + // This really shouldn't happen. + // TODO: What is ultimately the best way to handle this situation? + return fmt.Errorf( + "invalid state delta wants to remove %d state but only add %d state (between state snapshots %d and %d)", + len(u.removed), len(u.added), u.oldStateNID, u.newStateNID, + ) + } // Also work out the state before the event removes and the event // adds. @@ -279,42 +254,49 @@ func (u *latestEventsUpdater) latestState() error { return nil } -func calculateLatest( +func (u *latestEventsUpdater) calculateLatest( oldLatest []types.StateAtEventAndReference, - alreadyReferenced bool, - prevEvents []gomatrixserverlib.EventReference, newEvent types.StateAtEventAndReference, -) []types.StateAtEventAndReference { - var alreadyInLatest bool +) { var newLatest []types.StateAtEventAndReference + + // First of all, let's see if any of the existing forward extremities + // now have entries in the previous events table. If they do then we + // will no longer include them as forward extremities. for _, l := range oldLatest { - keep := true - for _, prevEvent := range prevEvents { - if l.EventID == prevEvent.EventID && bytes.Equal(l.EventSHA256, prevEvent.EventSHA256) { - // This event can be removed from the latest events cause we've found an event that references it. - // (If an event is referenced by another event then it can't be one of the latest events in the room - // because we have an event that comes after it) - keep = false - break - } - } - if l.EventNID == newEvent.EventNID { - alreadyInLatest = true - } - if keep { - // Keep the event in the latest events. + referenced, err := u.updater.IsReferenced(l.EventReference) + if err != nil { + logrus.WithError(err).Errorf("Failed to retrieve event reference for %q", l.EventID) + } else if !referenced { newLatest = append(newLatest, l) } } - if !alreadyReferenced && !alreadyInLatest { - // This event is not referenced by any of the events in the room - // and the event is not already in the latest events. - // Add it to the latest events + // Then check and see if our new event is already included in that set. + // This ordinarily won't happen but it covers the edge-case that we've + // already seen this event before and it's a forward extremity, so rather + // than adding a duplicate, we'll just return the set as complete. + for _, l := range newLatest { + if l.EventReference.EventID == newEvent.EventReference.EventID && bytes.Equal(l.EventReference.EventSHA256, newEvent.EventReference.EventSHA256) { + // We've already referenced this new event so we can just return + // the newly completed extremities at this point. + u.latest = newLatest + return + } + } + + // At this point we've processed the old extremities, and we've checked + // that our new event isn't already in that set. Therefore now we can + // check if our *new* event is a forward extremity, and if it is, add + // it in. + referenced, err := u.updater.IsReferenced(newEvent.EventReference) + if err != nil { + logrus.WithError(err).Errorf("Failed to retrieve event reference for %q", newEvent.EventReference.EventID) + } else if !referenced { newLatest = append(newLatest, newEvent) } - return newLatest + u.latest = newLatest } func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) { diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index f8e733ab7..e96eab71b 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -474,6 +474,32 @@ func (d *Database) StoreEvent( return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("d.Writer.Do: %w", err) } + // We should attempt to update the previous events table with any + // references that this new event makes. We do this using a latest + // events updater because it somewhat works as a mutex, ensuring + // that there's a row-level lock on the latest room events (well, + // on Postgres at least). + var roomInfo *types.RoomInfo + var updater *LatestEventsUpdater + if prevEvents := event.PrevEvents(); len(prevEvents) > 0 { + roomInfo, err = d.RoomInfo(ctx, event.RoomID()) + if err != nil { + return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("d.RoomInfo: %w", err) + } + if roomInfo == nil && len(prevEvents) > 0 { + return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("expected room %q to exist", event.RoomID()) + } + updater, err = d.GetLatestEventsForUpdate(ctx, *roomInfo) + if err != nil { + return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("NewLatestEventsUpdater: %w", err) + } + if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil { + return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("updater.StorePreviousEvents: %w", err) + } + succeeded := true + err = sqlutil.EndTransaction(updater, &succeeded) + } + return roomNID, types.StateAtEvent{ BeforeStateSnapshotNID: stateNID, StateEntry: types.StateEntry{ @@ -483,7 +509,7 @@ func (d *Database) StoreEvent( }, EventNID: eventNID, }, - }, redactionEvent, redactedEventID, nil + }, redactionEvent, redactedEventID, err } func (d *Database) PublishRoom(ctx context.Context, roomID string, publish bool) error { diff --git a/sytest-whitelist b/sytest-whitelist index 9a013cbf3..e0f1f311e 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -473,4 +473,6 @@ Inbound federation rejects invites which include invalid JSON for room version 6 Inbound federation rejects invite rejections which include invalid JSON for room version 6 GET /capabilities is present and well formed for registered user m.room.history_visibility == "joined" allows/forbids appropriately for Guest users -m.room.history_visibility == "joined" allows/forbids appropriately for Real users \ No newline at end of file +m.room.history_visibility == "joined" allows/forbids appropriately for Real users +Users cannot kick users who have already left a room +A prev_batch token from incremental sync can be used in the v1 messages API From 533006141ecbe18fc82d63a400cea57def8791d8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 7 Oct 2020 15:29:14 +0100 Subject: [PATCH 052/104] Return 200 on join before time out (#1493) * Return 200 on join afer 15 seconds if nothing better has happened by that point * Return 202 instead, 20 second timeout --- clientapi/routing/joinroom.go | 38 ++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index c10113574..578aaec56 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -16,9 +16,11 @@ package routing import ( "net/http" + "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/storage/accounts" @@ -74,16 +76,32 @@ func JoinRoomByIDOrAlias( } // Ask the roomserver to perform the join. - rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes) - if joinRes.Error != nil { - return joinRes.Error.JSONResponse() - } + done := make(chan util.JSONResponse, 1) + go func() { + defer close(done) + rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes) + if joinRes.Error != nil { + done <- joinRes.Error.JSONResponse() + } else { + done <- util.JSONResponse{ + Code: http.StatusOK, + // TODO: Put the response struct somewhere internal. + JSON: struct { + RoomID string `json:"room_id"` + }{joinRes.RoomID}, + } + } + }() - return util.JSONResponse{ - Code: http.StatusOK, - // TODO: Put the response struct somewhere internal. - JSON: struct { - RoomID string `json:"room_id"` - }{joinRes.RoomID}, + // Wait either for the join to finish, or for us to hit a reasonable + // timeout, at which point we'll just return a 200 to placate clients. + select { + case <-time.After(time.Second * 20): + return util.JSONResponse{ + Code: http.StatusAccepted, + JSON: jsonerror.Unknown("The room join will continue in the background."), + } + case result := <-done: + return result } } From bf7e85848bce3ec9ef89e485699d1c5fc6b34e6b Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 7 Oct 2020 16:23:18 +0100 Subject: [PATCH 053/104] Rename serverkeyapi to signingkeyserver (#1492) * Rename serverkeyapi to signingkeyserver We use "api" for public facing stuff and "server" for internal stuff. As the server key API is internal only, we call it 'signing key server', which also clarifies the type of key (as opposed to TLS keys, E2E keys, etc) * Convert docker/scripts to use signing-key-server * Rename missed bits --- build/docker/config/dendrite-config.yaml | 6 +++--- build/docker/docker-compose.polylith.yml | 6 +++--- build/docker/images-build.sh | 2 +- build/docker/images-pull.sh | 1 + build/docker/images-push.sh | 2 +- build/docker/postgres/create_db.sh | 2 +- build/gobind/monolith.go | 2 +- cmd/dendrite-demo-libp2p/main.go | 8 ++++---- cmd/dendrite-demo-yggdrasil/main.go | 2 +- cmd/dendrite-federation-api-server/main.go | 2 +- cmd/dendrite-federation-sender-server/main.go | 2 +- cmd/dendrite-monolith-server/main.go | 16 ++++++++-------- cmd/dendrite-room-server/main.go | 2 +- .../main.go | 10 +++++----- cmd/dendritejs/main.go | 2 +- cmd/generate-config/main.go | 2 +- dendrite-config.yaml | 6 +++--- docs/INSTALL.md | 2 +- internal/config/config.go | 16 ++++++++-------- ...serverkey.go => config_signingkeyserver.go} | 16 ++++++++-------- internal/setup/base.go | 14 +++++++------- internal/setup/monolith.go | 4 ++-- internal/test/config.go | 6 +++--- roomserver/roomserver.go | 2 +- {serverkeyapi => signingkeyserver}/api/api.go | 2 +- .../internal/api.go | 4 ++-- .../inthttp/client.go | 14 +++++++------- .../inthttp/server.go | 4 ++-- .../serverkeyapi_test.go | 12 ++++++------ .../signingkeyserver.go | 18 +++++++++--------- .../storage/cache/keydb.go | 0 .../storage/interface.go | 0 .../storage/keydb.go | 4 ++-- .../storage/keydb_wasm.go | 2 +- .../storage/postgres/keydb.go | 0 .../storage/postgres/server_key_table.go | 0 .../storage/sqlite3/keydb.go | 0 .../storage/sqlite3/server_key_table.go | 0 38 files changed, 97 insertions(+), 96 deletions(-) rename cmd/{dendrite-server-key-api-server => dendrite-signing-key-server}/main.go (72%) rename internal/config/{config_serverkey.go => config_signingkeyserver.go} (69%) rename {serverkeyapi => signingkeyserver}/api/api.go (95%) rename {serverkeyapi => signingkeyserver}/internal/api.go (99%) rename {serverkeyapi => signingkeyserver}/inthttp/client.go (90%) rename {serverkeyapi => signingkeyserver}/inthttp/server.go (89%) rename {serverkeyapi => signingkeyserver}/serverkeyapi_test.go (96%) rename serverkeyapi/serverkeyapi.go => signingkeyserver/signingkeyserver.go (85%) rename {serverkeyapi => signingkeyserver}/storage/cache/keydb.go (100%) rename {serverkeyapi => signingkeyserver}/storage/interface.go (100%) rename {serverkeyapi => signingkeyserver}/storage/keydb.go (90%) rename {serverkeyapi => signingkeyserver}/storage/keydb_wasm.go (95%) rename {serverkeyapi => signingkeyserver}/storage/postgres/keydb.go (100%) rename {serverkeyapi => signingkeyserver}/storage/postgres/server_key_table.go (100%) rename {serverkeyapi => signingkeyserver}/storage/sqlite3/keydb.go (100%) rename {serverkeyapi => signingkeyserver}/storage/sqlite3/server_key_table.go (100%) diff --git a/build/docker/config/dendrite-config.yaml b/build/docker/config/dendrite-config.yaml index 2bf8dd85f..2dc2f3b7d 100644 --- a/build/docker/config/dendrite-config.yaml +++ b/build/docker/config/dendrite-config.yaml @@ -253,12 +253,12 @@ room_server: conn_max_lifetime: -1 # Configuration for the Server Key API (for server signing keys). -server_key_api: +signing_key_server: internal_api: listen: http://0.0.0.0:7780 - connect: http://server_key_api:7780 + connect: http://signing_key_server:7780 database: - connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_serverkey?sslmode=disable + connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_signingkeyserver?sslmode=disable max_open_conns: 100 max_idle_conns: 2 conn_max_lifetime: -1 diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index 6dd743141..8a4c50e06 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -128,9 +128,9 @@ services: networks: - internal - server_key_api: - hostname: server_key_api - image: matrixdotorg/dendrite:serverkeyapi + signing_key_server: + hostname: signing_key_server + image: matrixdotorg/dendrite:signingkeyserver command: [ "--config=dendrite.yaml" ] diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index fdff51320..d72bac214 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -17,5 +17,5 @@ docker build -t matrixdotorg/dendrite:keyserver --build-arg component=de docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:serverkeyapi --build-arg component=dendrite-server-key-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:signingkeyserver --build-arg component=dendrite-signing-key-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:userapi --build-arg component=dendrite-user-api-server -f build/docker/Dockerfile.component . diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh index c6b09b6a4..be9185464 100755 --- a/build/docker/images-pull.sh +++ b/build/docker/images-pull.sh @@ -13,4 +13,5 @@ docker pull matrixdotorg/dendrite:keyserver docker pull matrixdotorg/dendrite:mediaapi docker pull matrixdotorg/dendrite:roomserver docker pull matrixdotorg/dendrite:syncapi +docker pull matrixdotorg/dendrite:signingkeyserver docker pull matrixdotorg/dendrite:userapi diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index 4838c76f6..64920171b 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -13,5 +13,5 @@ docker push matrixdotorg/dendrite:keyserver docker push matrixdotorg/dendrite:mediaapi docker push matrixdotorg/dendrite:roomserver docker push matrixdotorg/dendrite:syncapi -docker push matrixdotorg/dendrite:serverkeyapi +docker push matrixdotorg/dendrite:signingkeyserver docker push matrixdotorg/dendrite:userapi diff --git a/build/docker/postgres/create_db.sh b/build/docker/postgres/create_db.sh index 70d6743e4..f8ee715a9 100644 --- a/build/docker/postgres/create_db.sh +++ b/build/docker/postgres/create_db.sh @@ -1,5 +1,5 @@ #!/bin/bash -for db in account device mediaapi syncapi roomserver serverkey keyserver federationsender appservice e2ekey naffka; do +for db in account device mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice e2ekey naffka; do createdb -U dendrite -O dendrite dendrite_$db done diff --git a/build/gobind/monolith.go b/build/gobind/monolith.go index b4740ed42..7d10b87e4 100644 --- a/build/gobind/monolith.go +++ b/build/gobind/monolith.go @@ -94,7 +94,7 @@ func (m *DendriteMonolith) Start() { cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-mediaapi.db", m.StorageDirectory)) cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-syncapi.db", m.StorageDirectory)) cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-roomserver.db", m.StorageDirectory)) - cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-serverkey.db", m.StorageDirectory)) + cfg.SigningKeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-signingkeyserver.db", m.StorageDirectory)) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-keyserver.db", m.StorageDirectory)) cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-federationsender.db", m.StorageDirectory)) cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-appservice.db", m.StorageDirectory)) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 1f6748865..0f30e8d30 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -36,7 +36,7 @@ import ( "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" - "github.com/matrix-org/dendrite/serverkeyapi" + "github.com/matrix-org/dendrite/signingkeyserver" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" @@ -125,7 +125,7 @@ func main() { cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) - cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName)) + cfg.SigningKeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-signingkeyserver.db", *instanceName)) cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) @@ -143,8 +143,8 @@ func main() { userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI) keyAPI.SetUserAPI(userAPI) - serverKeyAPI := serverkeyapi.NewInternalAPI( - &base.Base.Cfg.ServerKeyAPI, federation, base.Base.Caches, + serverKeyAPI := signingkeyserver.NewInternalAPI( + &base.Base.Cfg.SigningKeyServer, federation, base.Base.Caches, ) keyRing := serverKeyAPI.KeyRing() createKeyDB( diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 257ddb58a..5e8b92318 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -78,7 +78,7 @@ func main() { cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) - cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName)) + cfg.SigningKeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-signingkeyserver.db", *instanceName)) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName)) cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index cab304e6b..3ebb16f4b 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -26,7 +26,7 @@ func main() { userAPI := base.UserAPIClient() federation := base.CreateFederationClient() - serverKeyAPI := base.ServerKeyAPIClient() + serverKeyAPI := base.SigningKeyServerHTTPClient() keyRing := serverKeyAPI.KeyRing() fsAPI := base.FederationSenderHTTPClient() rsAPI := base.RoomserverHTTPClient() diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 4d918f6b1..07380bb05 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -26,7 +26,7 @@ func main() { federation := base.CreateFederationClient() - serverKeyAPI := base.ServerKeyAPIClient() + serverKeyAPI := base.SigningKeyServerHTTPClient() keyRing := serverKeyAPI.KeyRing() rsAPI := base.RoomserverHTTPClient() diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 28a349a76..c50c0c218 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -27,7 +27,7 @@ import ( "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/serverkeyapi" + "github.com/matrix-org/dendrite/signingkeyserver" "github.com/matrix-org/dendrite/userapi" ) @@ -58,7 +58,7 @@ func main() { cfg.KeyServer.InternalAPI.Connect = httpAddr cfg.MediaAPI.InternalAPI.Connect = httpAddr cfg.RoomServer.InternalAPI.Connect = httpAddr - cfg.ServerKeyAPI.InternalAPI.Connect = httpAddr + cfg.SigningKeyServer.InternalAPI.Connect = httpAddr cfg.SyncAPI.InternalAPI.Connect = httpAddr } @@ -68,14 +68,14 @@ func main() { accountDB := base.CreateAccountsDB() federation := base.CreateFederationClient() - serverKeyAPI := serverkeyapi.NewInternalAPI( - &base.Cfg.ServerKeyAPI, federation, base.Caches, + skAPI := signingkeyserver.NewInternalAPI( + &base.Cfg.SigningKeyServer, federation, base.Caches, ) if base.UseHTTPAPIs { - serverkeyapi.AddInternalRoutes(base.InternalAPIMux, serverKeyAPI, base.Caches) - serverKeyAPI = base.ServerKeyAPIClient() + signingkeyserver.AddInternalRoutes(base.InternalAPIMux, skAPI, base.Caches) + skAPI = base.SigningKeyServerHTTPClient() } - keyRing := serverKeyAPI.KeyRing() + keyRing := skAPI.KeyRing() rsImpl := roomserver.NewInternalAPI( base, keyRing, @@ -134,7 +134,7 @@ func main() { EDUInternalAPI: eduInputAPI, FederationSenderAPI: fsAPI, RoomserverAPI: rsAPI, - ServerKeyAPI: serverKeyAPI, + ServerKeyAPI: skAPI, UserAPI: userAPI, KeyAPI: keyAPI, } diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index 08ad34bfd..c61368bf4 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -24,7 +24,7 @@ func main() { base := setup.NewBaseDendrite(cfg, "RoomServerAPI", true) defer base.Close() // nolint: errcheck - serverKeyAPI := base.ServerKeyAPIClient() + serverKeyAPI := base.SigningKeyServerHTTPClient() keyRing := serverKeyAPI.KeyRing() fsAPI := base.FederationSenderHTTPClient() diff --git a/cmd/dendrite-server-key-api-server/main.go b/cmd/dendrite-signing-key-server/main.go similarity index 72% rename from cmd/dendrite-server-key-api-server/main.go rename to cmd/dendrite-signing-key-server/main.go index 1ad4ede26..003bd755e 100644 --- a/cmd/dendrite-server-key-api-server/main.go +++ b/cmd/dendrite-signing-key-server/main.go @@ -16,21 +16,21 @@ package main import ( "github.com/matrix-org/dendrite/internal/setup" - "github.com/matrix-org/dendrite/serverkeyapi" + "github.com/matrix-org/dendrite/signingkeyserver" ) func main() { cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "ServerKeyAPI", true) + base := setup.NewBaseDendrite(cfg, "SigningKeyServer", true) defer base.Close() // nolint: errcheck federation := base.CreateFederationClient() - intAPI := serverkeyapi.NewInternalAPI(&base.Cfg.ServerKeyAPI, federation, base.Caches) - serverkeyapi.AddInternalRoutes(base.InternalAPIMux, intAPI, base.Caches) + intAPI := signingkeyserver.NewInternalAPI(&base.Cfg.SigningKeyServer, federation, base.Caches) + signingkeyserver.AddInternalRoutes(base.InternalAPIMux, intAPI, base.Caches) base.SetupAndServeHTTP( - base.Cfg.ServerKeyAPI.InternalAPI.Listen, + base.Cfg.SigningKeyServer.InternalAPI.Listen, setup.NoExternalListener, nil, nil, ) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 12dc2d7cc..267259c78 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -168,7 +168,7 @@ func main() { cfg.FederationSender.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db" cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db" cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db" - cfg.ServerKeyAPI.Database.ConnectionString = "file:/idb/dendritejs_serverkey.db" + cfg.SigningKeyServer.Database.ConnectionString = "file:/idb/dendritejs_signingkeyserver.db" cfg.SyncAPI.Database.ConnectionString = "file:/idb/dendritejs_syncapi.db" cfg.KeyServer.Database.ConnectionString = "file:/idb/dendritejs_e2ekey.db" cfg.Global.Kafka.UseNaffka = true diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index 78ed3af6c..e65723e65 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -27,7 +27,7 @@ func main() { }, }, } - cfg.ServerKeyAPI.KeyPerspectives = config.KeyPerspectives{ + cfg.SigningKeyServer.KeyPerspectives = config.KeyPerspectives{ { ServerName: "matrix.org", Keys: []config.KeyPerspectiveTrustKey{ diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 74fa9b3e1..6e87bc709 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -252,13 +252,13 @@ room_server: max_idle_conns: 2 conn_max_lifetime: -1 -# Configuration for the Server Key API (for server signing keys). -server_key_api: +# Configuration for the Signing Key Server (for server signing keys). +signing_key_server: internal_api: listen: http://localhost:7780 connect: http://localhost:7780 database: - connection_string: file:serverkeyapi.db + connection_string: file:signingkeyserver.db max_open_conns: 100 max_idle_conns: 2 conn_max_lifetime: -1 diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 7a7fb03ee..913bc5832 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -109,7 +109,7 @@ Assuming that Postgres 9.5 (or later) is installed: * Create the component databases: ```bash - for i in account device mediaapi syncapi roomserver serverkey federationsender appservice e2ekey naffka; do + for i in account device mediaapi syncapi roomserver signingkeyserver federationsender appservice e2ekey naffka; do sudo -u postgres createdb -O dendrite dendrite_$i done ``` diff --git a/internal/config/config.go b/internal/config/config.go index 74d3f4fa5..9d9e2414f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -62,7 +62,7 @@ type Dendrite struct { KeyServer KeyServer `yaml:"key_server"` MediaAPI MediaAPI `yaml:"media_api"` RoomServer RoomServer `yaml:"room_server"` - ServerKeyAPI ServerKeyAPI `yaml:"server_key_api"` + SigningKeyServer SigningKeyServer `yaml:"signing_key_server"` SyncAPI SyncAPI `yaml:"sync_api"` UserAPI UserAPI `yaml:"user_api"` @@ -302,7 +302,7 @@ func (c *Dendrite) Defaults() { c.KeyServer.Defaults() c.MediaAPI.Defaults() c.RoomServer.Defaults() - c.ServerKeyAPI.Defaults() + c.SigningKeyServer.Defaults() c.SyncAPI.Defaults() c.UserAPI.Defaults() c.AppServiceAPI.Defaults() @@ -318,7 +318,7 @@ func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) { &c.Global, &c.ClientAPI, &c.EDUServer, &c.FederationAPI, &c.FederationSender, &c.KeyServer, &c.MediaAPI, &c.RoomServer, - &c.ServerKeyAPI, &c.SyncAPI, &c.UserAPI, + &c.SigningKeyServer, &c.SyncAPI, &c.UserAPI, &c.AppServiceAPI, } { c.Verify(configErrs, isMonolith) @@ -333,7 +333,7 @@ func (c *Dendrite) Wiring() { c.KeyServer.Matrix = &c.Global c.MediaAPI.Matrix = &c.Global c.RoomServer.Matrix = &c.Global - c.ServerKeyAPI.Matrix = &c.Global + c.SigningKeyServer.Matrix = &c.Global c.SyncAPI.Matrix = &c.Global c.UserAPI.Matrix = &c.Global c.AppServiceAPI.Matrix = &c.Global @@ -524,13 +524,13 @@ func (config *Dendrite) FederationSenderURL() string { return string(config.FederationSender.InternalAPI.Connect) } -// ServerKeyAPIURL returns an HTTP URL for where the server key API is listening. -func (config *Dendrite) ServerKeyAPIURL() string { - // Hard code the server key API server to talk HTTP for now. +// SigningKeyServerURL returns an HTTP URL for where the signing key server is listening. +func (config *Dendrite) SigningKeyServerURL() string { + // Hard code the signing key server to talk HTTP for now. // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return string(config.ServerKeyAPI.InternalAPI.Connect) + return string(config.SigningKeyServer.InternalAPI.Connect) } // KeyServerURL returns an HTTP URL for where the key server is listening. diff --git a/internal/config/config_serverkey.go b/internal/config/config_signingkeyserver.go similarity index 69% rename from internal/config/config_serverkey.go rename to internal/config/config_signingkeyserver.go index 788a2fa05..51aca38bd 100644 --- a/internal/config/config_serverkey.go +++ b/internal/config/config_signingkeyserver.go @@ -2,12 +2,12 @@ package config import "github.com/matrix-org/gomatrixserverlib" -type ServerKeyAPI struct { +type SigningKeyServer struct { Matrix *Global `yaml:"-"` InternalAPI InternalAPIOptions `yaml:"internal_api"` - // The ServerKey database caches the public keys of remote servers. + // The SigningKeyServer database caches the public keys of remote servers. // It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI. Database DatabaseOptions `yaml:"database"` @@ -19,17 +19,17 @@ type ServerKeyAPI struct { PreferDirectFetch bool `yaml:"prefer_direct_fetch"` } -func (c *ServerKeyAPI) Defaults() { +func (c *SigningKeyServer) Defaults() { c.InternalAPI.Listen = "http://localhost:7780" c.InternalAPI.Connect = "http://localhost:7780" c.Database.Defaults() - c.Database.ConnectionString = "file:serverkeyapi.db" + c.Database.ConnectionString = "file:signingkeyserver.db" } -func (c *ServerKeyAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { - checkURL(configErrs, "server_key_api.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "server_key_api.internal_api.bind", string(c.InternalAPI.Connect)) - checkNotEmpty(configErrs, "server_key_api.database.connection_string", string(c.Database.ConnectionString)) +func (c *SigningKeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) { + checkURL(configErrs, "signing_key_server.internal_api.listen", string(c.InternalAPI.Listen)) + checkURL(configErrs, "signing_key_server.internal_api.bind", string(c.InternalAPI.Connect)) + checkNotEmpty(configErrs, "signing_key_server.database.connection_string", string(c.Database.ConnectionString)) } // KeyPerspectives are used to configure perspective key servers for diff --git a/internal/setup/base.go b/internal/setup/base.go index f9ddfdf7d..6a0a8bbd2 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -46,8 +46,8 @@ import ( keyinthttp "github.com/matrix-org/dendrite/keyserver/inthttp" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" rsinthttp "github.com/matrix-org/dendrite/roomserver/inthttp" - serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api" - skinthttp "github.com/matrix-org/dendrite/serverkeyapi/inthttp" + skapi "github.com/matrix-org/dendrite/signingkeyserver/api" + skinthttp "github.com/matrix-org/dendrite/signingkeyserver/inthttp" userapi "github.com/matrix-org/dendrite/userapi/api" userapiinthttp "github.com/matrix-org/dendrite/userapi/inthttp" "github.com/sirupsen/logrus" @@ -208,15 +208,15 @@ func (b *BaseDendrite) FederationSenderHTTPClient() federationSenderAPI.Federati return f } -// ServerKeyAPIClient returns ServerKeyInternalAPI for hitting the server key API over HTTP -func (b *BaseDendrite) ServerKeyAPIClient() serverKeyAPI.ServerKeyInternalAPI { - f, err := skinthttp.NewServerKeyClient( - b.Cfg.ServerKeyAPIURL(), +// SigningKeyServerHTTPClient returns SigningKeyServer for hitting the signing key server over HTTP +func (b *BaseDendrite) SigningKeyServerHTTPClient() skapi.SigningKeyServerAPI { + f, err := skinthttp.NewSigningKeyServerClient( + b.Cfg.SigningKeyServerURL(), b.apiHttpClient, b.Caches, ) if err != nil { - logrus.WithError(err).Panic("NewServerKeyInternalAPIHTTP failed", b.httpClient) + logrus.WithError(err).Panic("SigningKeyServerHTTPClient failed", b.httpClient) } return f } diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index 2274283e6..a0675d61f 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -28,7 +28,7 @@ import ( keyAPI "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/mediaapi" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api" + serverKeyAPI "github.com/matrix-org/dendrite/signingkeyserver/api" "github.com/matrix-org/dendrite/syncapi" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/storage/accounts" @@ -50,7 +50,7 @@ type Monolith struct { EDUInternalAPI eduServerAPI.EDUServerInputAPI FederationSenderAPI federationSenderAPI.FederationSenderInternalAPI RoomserverAPI roomserverAPI.RoomserverInternalAPI - ServerKeyAPI serverKeyAPI.ServerKeyInternalAPI + ServerKeyAPI serverKeyAPI.SigningKeyServerAPI UserAPI userapi.UserInternalAPI KeyAPI keyAPI.KeyInternalAPI diff --git a/internal/test/config.go b/internal/test/config.go index 8080988f3..69fc5a873 100644 --- a/internal/test/config.go +++ b/internal/test/config.go @@ -92,7 +92,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.KeyServer.Database.ConnectionString = config.DataSource(database) cfg.MediaAPI.Database.ConnectionString = config.DataSource(database) cfg.RoomServer.Database.ConnectionString = config.DataSource(database) - cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource(database) + cfg.SigningKeyServer.Database.ConnectionString = config.DataSource(database) cfg.SyncAPI.Database.ConnectionString = config.DataSource(database) cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(database) cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(database) @@ -104,7 +104,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.KeyServer.InternalAPI.Listen = assignAddress() cfg.MediaAPI.InternalAPI.Listen = assignAddress() cfg.RoomServer.InternalAPI.Listen = assignAddress() - cfg.ServerKeyAPI.InternalAPI.Listen = assignAddress() + cfg.SigningKeyServer.InternalAPI.Listen = assignAddress() cfg.SyncAPI.InternalAPI.Listen = assignAddress() cfg.UserAPI.InternalAPI.Listen = assignAddress() @@ -115,7 +115,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.KeyServer.InternalAPI.Connect = cfg.KeyServer.InternalAPI.Listen cfg.MediaAPI.InternalAPI.Connect = cfg.MediaAPI.InternalAPI.Listen cfg.RoomServer.InternalAPI.Connect = cfg.RoomServer.InternalAPI.Listen - cfg.ServerKeyAPI.InternalAPI.Connect = cfg.ServerKeyAPI.InternalAPI.Listen + cfg.SigningKeyServer.InternalAPI.Connect = cfg.SigningKeyServer.InternalAPI.Listen cfg.SyncAPI.InternalAPI.Connect = cfg.SyncAPI.InternalAPI.Listen cfg.UserAPI.InternalAPI.Connect = cfg.UserAPI.InternalAPI.Listen diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 98a86e5bb..4c138116f 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -42,7 +42,7 @@ func NewInternalAPI( cfg := &base.Cfg.RoomServer var perspectiveServerNames []gomatrixserverlib.ServerName - for _, kp := range base.Cfg.ServerKeyAPI.KeyPerspectives { + for _, kp := range base.Cfg.SigningKeyServer.KeyPerspectives { perspectiveServerNames = append(perspectiveServerNames, kp.ServerName) } diff --git a/serverkeyapi/api/api.go b/signingkeyserver/api/api.go similarity index 95% rename from serverkeyapi/api/api.go rename to signingkeyserver/api/api.go index 7af626345..f053d72e2 100644 --- a/serverkeyapi/api/api.go +++ b/signingkeyserver/api/api.go @@ -6,7 +6,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -type ServerKeyInternalAPI interface { +type SigningKeyServerAPI interface { gomatrixserverlib.KeyDatabase KeyRing() *gomatrixserverlib.KeyRing diff --git a/serverkeyapi/internal/api.go b/signingkeyserver/internal/api.go similarity index 99% rename from serverkeyapi/internal/api.go rename to signingkeyserver/internal/api.go index 335bfe4ce..54c41b52f 100644 --- a/serverkeyapi/internal/api.go +++ b/signingkeyserver/internal/api.go @@ -7,13 +7,13 @@ import ( "time" "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/serverkeyapi/api" + "github.com/matrix-org/dendrite/signingkeyserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) type ServerKeyAPI struct { - api.ServerKeyInternalAPI + api.SigningKeyServerAPI ServerName gomatrixserverlib.ServerName ServerPublicKey ed25519.PublicKey diff --git a/serverkeyapi/inthttp/client.go b/signingkeyserver/inthttp/client.go similarity index 90% rename from serverkeyapi/inthttp/client.go rename to signingkeyserver/inthttp/client.go index 39ab8c6c5..71e40b8f0 100644 --- a/serverkeyapi/inthttp/client.go +++ b/signingkeyserver/inthttp/client.go @@ -7,26 +7,26 @@ import ( "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/serverkeyapi/api" + "github.com/matrix-org/dendrite/signingkeyserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/opentracing/opentracing-go" ) // HTTP paths for the internal HTTP APIs const ( - ServerKeyInputPublicKeyPath = "/serverkeyapi/inputPublicKey" - ServerKeyQueryPublicKeyPath = "/serverkeyapi/queryPublicKey" + ServerKeyInputPublicKeyPath = "/signingkeyserver/inputPublicKey" + ServerKeyQueryPublicKeyPath = "/signingkeyserver/queryPublicKey" ) -// NewServerKeyClient creates a ServerKeyInternalAPI implemented by talking to a HTTP POST API. +// NewSigningKeyServerClient creates a SigningKeyServerAPI implemented by talking to a HTTP POST API. // If httpClient is nil an error is returned -func NewServerKeyClient( +func NewSigningKeyServerClient( serverKeyAPIURL string, httpClient *http.Client, cache caching.ServerKeyCache, -) (api.ServerKeyInternalAPI, error) { +) (api.SigningKeyServerAPI, error) { if httpClient == nil { - return nil, errors.New("NewRoomserverInternalAPIHTTP: httpClient is ") + return nil, errors.New("NewSigningKeyServerClient: httpClient is ") } return &httpServerKeyInternalAPI{ serverKeyAPIURL: serverKeyAPIURL, diff --git a/serverkeyapi/inthttp/server.go b/signingkeyserver/inthttp/server.go similarity index 89% rename from serverkeyapi/inthttp/server.go rename to signingkeyserver/inthttp/server.go index cd4748392..d26f73805 100644 --- a/serverkeyapi/inthttp/server.go +++ b/signingkeyserver/inthttp/server.go @@ -7,11 +7,11 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/serverkeyapi/api" + "github.com/matrix-org/dendrite/signingkeyserver/api" "github.com/matrix-org/util" ) -func AddRoutes(s api.ServerKeyInternalAPI, internalAPIMux *mux.Router, cache caching.ServerKeyCache) { +func AddRoutes(s api.SigningKeyServerAPI, internalAPIMux *mux.Router, cache caching.ServerKeyCache) { internalAPIMux.Handle(ServerKeyQueryPublicKeyPath, httputil.MakeInternalAPI("queryPublicKeys", func(req *http.Request) util.JSONResponse { request := api.QueryPublicKeysRequest{} diff --git a/serverkeyapi/serverkeyapi_test.go b/signingkeyserver/serverkeyapi_test.go similarity index 96% rename from serverkeyapi/serverkeyapi_test.go rename to signingkeyserver/serverkeyapi_test.go index 152a853e3..e5578f43c 100644 --- a/serverkeyapi/serverkeyapi_test.go +++ b/signingkeyserver/serverkeyapi_test.go @@ -1,4 +1,4 @@ -package serverkeyapi +package signingkeyserver import ( "bytes" @@ -16,18 +16,18 @@ import ( "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/serverkeyapi/api" + "github.com/matrix-org/dendrite/signingkeyserver/api" "github.com/matrix-org/gomatrixserverlib" ) type server struct { name gomatrixserverlib.ServerName // server name validity time.Duration // key validity duration from now - config *config.ServerKeyAPI // skeleton config, from TestMain + config *config.SigningKeyServer // skeleton config, from TestMain fedconfig *config.FederationAPI // fedclient *gomatrixserverlib.FederationClient // uses MockRoundTripper cache *caching.Caches // server-specific cache - api api.ServerKeyInternalAPI // server-specific server key API + api api.SigningKeyServerAPI // server-specific server key API } func (s *server) renew() { @@ -76,8 +76,8 @@ func TestMain(m *testing.M) { cfg.Global.PrivateKey = testPriv cfg.Global.KeyID = serverKeyID cfg.Global.KeyValidityPeriod = s.validity - cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource("file::memory:") - s.config = &cfg.ServerKeyAPI + cfg.SigningKeyServer.Database.ConnectionString = config.DataSource("file::memory:") + s.config = &cfg.SigningKeyServer s.fedconfig = &cfg.FederationAPI // Create a transport which redirects federation requests to diff --git a/serverkeyapi/serverkeyapi.go b/signingkeyserver/signingkeyserver.go similarity index 85% rename from serverkeyapi/serverkeyapi.go rename to signingkeyserver/signingkeyserver.go index da239eb05..27b4c7035 100644 --- a/serverkeyapi/serverkeyapi.go +++ b/signingkeyserver/signingkeyserver.go @@ -1,4 +1,4 @@ -package serverkeyapi +package signingkeyserver import ( "crypto/ed25519" @@ -7,28 +7,28 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/serverkeyapi/api" - "github.com/matrix-org/dendrite/serverkeyapi/internal" - "github.com/matrix-org/dendrite/serverkeyapi/inthttp" - "github.com/matrix-org/dendrite/serverkeyapi/storage" - "github.com/matrix-org/dendrite/serverkeyapi/storage/cache" + "github.com/matrix-org/dendrite/signingkeyserver/api" + "github.com/matrix-org/dendrite/signingkeyserver/internal" + "github.com/matrix-org/dendrite/signingkeyserver/inthttp" + "github.com/matrix-org/dendrite/signingkeyserver/storage" + "github.com/matrix-org/dendrite/signingkeyserver/storage/cache" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) // AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions // on the given input API. -func AddInternalRoutes(router *mux.Router, intAPI api.ServerKeyInternalAPI, caches *caching.Caches) { +func AddInternalRoutes(router *mux.Router, intAPI api.SigningKeyServerAPI, caches *caching.Caches) { inthttp.AddRoutes(intAPI, router, caches) } // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - cfg *config.ServerKeyAPI, + cfg *config.SigningKeyServer, fedClient gomatrixserverlib.KeyClient, caches *caching.Caches, -) api.ServerKeyInternalAPI { +) api.SigningKeyServerAPI { innerDB, err := storage.NewDatabase( &cfg.Database, cfg.Matrix.ServerName, diff --git a/serverkeyapi/storage/cache/keydb.go b/signingkeyserver/storage/cache/keydb.go similarity index 100% rename from serverkeyapi/storage/cache/keydb.go rename to signingkeyserver/storage/cache/keydb.go diff --git a/serverkeyapi/storage/interface.go b/signingkeyserver/storage/interface.go similarity index 100% rename from serverkeyapi/storage/interface.go rename to signingkeyserver/storage/interface.go diff --git a/serverkeyapi/storage/keydb.go b/signingkeyserver/storage/keydb.go similarity index 90% rename from serverkeyapi/storage/keydb.go rename to signingkeyserver/storage/keydb.go index 3d3a0c303..ef1077fc9 100644 --- a/serverkeyapi/storage/keydb.go +++ b/signingkeyserver/storage/keydb.go @@ -22,8 +22,8 @@ import ( "golang.org/x/crypto/ed25519" "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/serverkeyapi/storage/postgres" - "github.com/matrix-org/dendrite/serverkeyapi/storage/sqlite3" + "github.com/matrix-org/dendrite/signingkeyserver/storage/postgres" + "github.com/matrix-org/dendrite/signingkeyserver/storage/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/serverkeyapi/storage/keydb_wasm.go b/signingkeyserver/storage/keydb_wasm.go similarity index 95% rename from serverkeyapi/storage/keydb_wasm.go rename to signingkeyserver/storage/keydb_wasm.go index de66a1d63..187d9669f 100644 --- a/serverkeyapi/storage/keydb_wasm.go +++ b/signingkeyserver/storage/keydb_wasm.go @@ -23,7 +23,7 @@ import ( "golang.org/x/crypto/ed25519" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/serverkeyapi/storage/sqlite3" + "github.com/matrix-org/dendrite/signingkeyserver/storage/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/serverkeyapi/storage/postgres/keydb.go b/signingkeyserver/storage/postgres/keydb.go similarity index 100% rename from serverkeyapi/storage/postgres/keydb.go rename to signingkeyserver/storage/postgres/keydb.go diff --git a/serverkeyapi/storage/postgres/server_key_table.go b/signingkeyserver/storage/postgres/server_key_table.go similarity index 100% rename from serverkeyapi/storage/postgres/server_key_table.go rename to signingkeyserver/storage/postgres/server_key_table.go diff --git a/serverkeyapi/storage/sqlite3/keydb.go b/signingkeyserver/storage/sqlite3/keydb.go similarity index 100% rename from serverkeyapi/storage/sqlite3/keydb.go rename to signingkeyserver/storage/sqlite3/keydb.go diff --git a/serverkeyapi/storage/sqlite3/server_key_table.go b/signingkeyserver/storage/sqlite3/server_key_table.go similarity index 100% rename from serverkeyapi/storage/sqlite3/server_key_table.go rename to signingkeyserver/storage/sqlite3/server_key_table.go From 8bca7a83a98a310e4adae405d125dda93c8db1a0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 7 Oct 2020 16:59:22 +0100 Subject: [PATCH 054/104] Update monolith -api behaviour (#1484) * Update monolith -api mode listeners * Fix check * Fix another check * Update HTTP API addr behaviour * Redefine NoExternalListener * NoListener --- cmd/dendrite-appservice-server/main.go | 4 +- cmd/dendrite-edu-server/main.go | 4 +- cmd/dendrite-federation-sender-server/main.go | 4 +- cmd/dendrite-key-server/main.go | 4 +- cmd/dendrite-monolith-server/main.go | 37 ++++++----- cmd/dendrite-room-server/main.go | 4 +- cmd/dendrite-signing-key-server/main.go | 2 +- cmd/dendrite-user-api-server/main.go | 4 +- internal/setup/base.go | 61 ++++++++++--------- 9 files changed, 65 insertions(+), 59 deletions(-) diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index 72b243e28..6adbdb17c 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -31,8 +31,8 @@ func main() { appservice.AddInternalRoutes(base.InternalAPIMux, intAPI) base.SetupAndServeHTTP( - base.Cfg.AppServiceAPI.InternalAPI.Listen, - setup.NoExternalListener, + base.Cfg.AppServiceAPI.InternalAPI.Listen, // internal listener + setup.NoListener, // external listener nil, nil, ) } diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-edu-server/main.go index e0956619e..3a34b9a68 100644 --- a/cmd/dendrite-edu-server/main.go +++ b/cmd/dendrite-edu-server/main.go @@ -34,8 +34,8 @@ func main() { eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI) base.SetupAndServeHTTP( - base.Cfg.EDUServer.InternalAPI.Listen, - setup.NoExternalListener, + base.Cfg.EDUServer.InternalAPI.Listen, // internal listener + setup.NoListener, // external listener nil, nil, ) } diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 07380bb05..99b416c45 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -36,8 +36,8 @@ func main() { federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI) base.SetupAndServeHTTP( - base.Cfg.FederationSender.InternalAPI.Listen, - setup.NoExternalListener, + base.Cfg.FederationSender.InternalAPI.Listen, // internal listener + setup.NoListener, // external listener nil, nil, ) } diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-key-server/main.go index 2110b216d..92d18ac38 100644 --- a/cmd/dendrite-key-server/main.go +++ b/cmd/dendrite-key-server/main.go @@ -30,8 +30,8 @@ func main() { keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI) base.SetupAndServeHTTP( - base.Cfg.KeyServer.InternalAPI.Listen, - setup.NoExternalListener, + base.Cfg.KeyServer.InternalAPI.Listen, // internal listener + setup.NoListener, // external listener nil, nil, ) } diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index c50c0c218..0fe70ca8c 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -29,11 +29,13 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/signingkeyserver" "github.com/matrix-org/dendrite/userapi" + "github.com/sirupsen/logrus" ) var ( httpBindAddr = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server") httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server") + apiBindAddr = flag.String("api-bind-address", "localhost:18008", "The HTTP listening port for the internal HTTP APIs (if -api is enabled)") certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS") keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS") enableHTTPAPIs = flag.Bool("api", false, "Use HTTP APIs instead of short-circuiting (warning: exposes API endpoints!)") @@ -44,22 +46,25 @@ func main() { cfg := setup.ParseFlags(true) httpAddr := config.HTTPAddress("http://" + *httpBindAddr) httpsAddr := config.HTTPAddress("https://" + *httpsBindAddr) + httpAPIAddr := httpAddr if *enableHTTPAPIs { + logrus.Warnf("DANGER! The -api option is enabled, exposing internal APIs on %q!", *apiBindAddr) + httpAPIAddr = config.HTTPAddress("http://" + *apiBindAddr) // If the HTTP APIs are enabled then we need to update the Listen // statements in the configuration so that we know where to find // the API endpoints. They'll listen on the same port as the monolith // itself. - cfg.AppServiceAPI.InternalAPI.Connect = httpAddr - cfg.ClientAPI.InternalAPI.Connect = httpAddr - cfg.EDUServer.InternalAPI.Connect = httpAddr - cfg.FederationAPI.InternalAPI.Connect = httpAddr - cfg.FederationSender.InternalAPI.Connect = httpAddr - cfg.KeyServer.InternalAPI.Connect = httpAddr - cfg.MediaAPI.InternalAPI.Connect = httpAddr - cfg.RoomServer.InternalAPI.Connect = httpAddr - cfg.SigningKeyServer.InternalAPI.Connect = httpAddr - cfg.SyncAPI.InternalAPI.Connect = httpAddr + cfg.AppServiceAPI.InternalAPI.Connect = httpAPIAddr + cfg.ClientAPI.InternalAPI.Connect = httpAPIAddr + cfg.EDUServer.InternalAPI.Connect = httpAPIAddr + cfg.FederationAPI.InternalAPI.Connect = httpAPIAddr + cfg.FederationSender.InternalAPI.Connect = httpAPIAddr + cfg.KeyServer.InternalAPI.Connect = httpAPIAddr + cfg.MediaAPI.InternalAPI.Connect = httpAPIAddr + cfg.RoomServer.InternalAPI.Connect = httpAPIAddr + cfg.SigningKeyServer.InternalAPI.Connect = httpAPIAddr + cfg.SyncAPI.InternalAPI.Connect = httpAPIAddr } base := setup.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs) @@ -148,18 +153,18 @@ func main() { // Expose the matrix APIs directly rather than putting them under a /api path. go func() { base.SetupAndServeHTTP( - config.HTTPAddress(httpAddr), // internal API - config.HTTPAddress(httpAddr), // external API - nil, nil, // TLS settings + httpAPIAddr, // internal API + httpAddr, // external API + nil, nil, // TLS settings ) }() // Handle HTTPS if certificate and key are provided if *certFile != "" && *keyFile != "" { go func() { base.SetupAndServeHTTP( - config.HTTPAddress(httpsAddr), // internal API - config.HTTPAddress(httpsAddr), // external API - certFile, keyFile, // TLS settings + setup.NoListener, // internal API + httpsAddr, // external API + certFile, keyFile, // TLS settings ) }() } diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index c61368bf4..d3f145745 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -33,8 +33,8 @@ func main() { roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI) base.SetupAndServeHTTP( - base.Cfg.RoomServer.InternalAPI.Listen, - setup.NoExternalListener, + base.Cfg.RoomServer.InternalAPI.Listen, // internal listener + setup.NoListener, // external listener nil, nil, ) } diff --git a/cmd/dendrite-signing-key-server/main.go b/cmd/dendrite-signing-key-server/main.go index 003bd755e..a4d48d361 100644 --- a/cmd/dendrite-signing-key-server/main.go +++ b/cmd/dendrite-signing-key-server/main.go @@ -31,7 +31,7 @@ func main() { base.SetupAndServeHTTP( base.Cfg.SigningKeyServer.InternalAPI.Listen, - setup.NoExternalListener, + setup.NoListener, nil, nil, ) } diff --git a/cmd/dendrite-user-api-server/main.go b/cmd/dendrite-user-api-server/main.go index c8e2e2a37..fb65fefbc 100644 --- a/cmd/dendrite-user-api-server/main.go +++ b/cmd/dendrite-user-api-server/main.go @@ -31,8 +31,8 @@ func main() { userapi.AddInternalRoutes(base.InternalAPIMux, userAPI) base.SetupAndServeHTTP( - base.Cfg.UserAPI.InternalAPI.Listen, - setup.NoExternalListener, + base.Cfg.UserAPI.InternalAPI.Listen, // internal listener + setup.NoListener, // external listener nil, nil, ) } diff --git a/internal/setup/base.go b/internal/setup/base.go index 6a0a8bbd2..77fdb04a1 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -80,7 +80,7 @@ type BaseDendrite struct { const HTTPServerTimeout = time.Minute * 5 const HTTPClientTimeout = time.Second * 30 -const NoExternalListener = "" +const NoListener = "" // NewBaseDendrite creates a new instance to be used by a component. // The componentName is used for logging purposes, and should be a friendly name @@ -272,22 +272,21 @@ func (b *BaseDendrite) SetupAndServeHTTP( internalAddr, _ := internalHTTPAddr.Address() externalAddr, _ := externalHTTPAddr.Address() - internalRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() - externalRouter := internalRouter + externalRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() + internalRouter := externalRouter - internalServ := &http.Server{ - Addr: string(internalAddr), + externalServ := &http.Server{ + Addr: string(externalAddr), WriteTimeout: HTTPServerTimeout, - Handler: internalRouter, + Handler: externalRouter, } - externalServ := internalServ + internalServ := externalServ - if externalAddr != NoExternalListener && externalAddr != internalAddr { - externalRouter = mux.NewRouter().SkipClean(true).UseEncodedPath() - externalServ = &http.Server{ - Addr: string(externalAddr), - WriteTimeout: HTTPServerTimeout, - Handler: externalRouter, + if internalAddr != NoListener && externalAddr != internalAddr { + internalRouter = mux.NewRouter().SkipClean(true).UseEncodedPath() + internalServ = &http.Server{ + Addr: string(internalAddr), + Handler: internalRouter, } } @@ -301,23 +300,25 @@ func (b *BaseDendrite) SetupAndServeHTTP( externalRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(b.PublicFederationAPIMux) externalRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(b.PublicMediaAPIMux) - go func() { - logrus.Infof("Starting %s listener on %s", b.componentName, internalServ.Addr) - if certFile != nil && keyFile != nil { - if err := internalServ.ListenAndServeTLS(*certFile, *keyFile); err != nil { - logrus.WithError(err).Fatal("failed to serve HTTPS") - } - } else { - if err := internalServ.ListenAndServe(); err != nil { - logrus.WithError(err).Fatal("failed to serve HTTP") - } - } - logrus.Infof("Stopped %s listener on %s", b.componentName, internalServ.Addr) - }() - - if externalAddr != NoExternalListener && internalAddr != externalAddr { + if internalAddr != NoListener && internalAddr != externalAddr { go func() { - logrus.Infof("Starting %s listener on %s", b.componentName, externalServ.Addr) + logrus.Infof("Starting internal %s listener on %s", b.componentName, internalServ.Addr) + if certFile != nil && keyFile != nil { + if err := internalServ.ListenAndServeTLS(*certFile, *keyFile); err != nil { + logrus.WithError(err).Fatal("failed to serve HTTPS") + } + } else { + if err := internalServ.ListenAndServe(); err != nil { + logrus.WithError(err).Fatal("failed to serve HTTP") + } + } + logrus.Infof("Stopped internal %s listener on %s", b.componentName, internalServ.Addr) + }() + } + + if externalAddr != NoListener { + go func() { + logrus.Infof("Starting external %s listener on %s", b.componentName, externalServ.Addr) if certFile != nil && keyFile != nil { if err := externalServ.ListenAndServeTLS(*certFile, *keyFile); err != nil { logrus.WithError(err).Fatal("failed to serve HTTPS") @@ -327,7 +328,7 @@ func (b *BaseDendrite) SetupAndServeHTTP( logrus.WithError(err).Fatal("failed to serve HTTP") } } - logrus.Infof("Stopped %s listener on %s", b.componentName, externalServ.Addr) + logrus.Infof("Stopped external %s listener on %s", b.componentName, externalServ.Addr) }() } From 8b880be57e41c47b11b8f238daed7ae0151959f4 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 8 Oct 2020 10:03:37 +0100 Subject: [PATCH 055/104] Include a stripped version of the invite itself (#1495) --- syncapi/types/types.go | 20 +++++++++++++++++--- syncapi/types/types_test.go | 23 +++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 2499976e5..9be83f5fa 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -466,17 +466,31 @@ func NewJoinResponse() *JoinResponse { // InviteResponse represents a /sync response for a room which is under the 'invite' key. type InviteResponse struct { InviteState struct { - Events json.RawMessage `json:"events"` + Events []json.RawMessage `json:"events"` } `json:"invite_state"` } // NewInviteResponse creates an empty response with initialised arrays. func NewInviteResponse(event gomatrixserverlib.HeaderedEvent) *InviteResponse { res := InviteResponse{} - res.InviteState.Events = json.RawMessage{'[', ']'} + res.InviteState.Events = []json.RawMessage{} + + // First see if there's invite_room_state in the unsigned key of the invite. + // If there is then unmarshal it into the response. This will contain the + // partial room state such as join rules, room name etc. if inviteRoomState := gjson.GetBytes(event.Unsigned(), "invite_room_state"); inviteRoomState.Exists() { - res.InviteState.Events = json.RawMessage(inviteRoomState.Raw) + _ = json.Unmarshal([]byte(inviteRoomState.Raw), &res.InviteState.Events) } + + // Then we'll see if we can create a partial of the invite event itself. + // This is needed for clients to work out *who* sent the invite. + format, _ := event.RoomVersion.EventFormat() + inviteEvent := gomatrixserverlib.ToClientEvent(event.Unwrap(), format) + inviteEvent.Unsigned = nil + if ev, err := json.Marshal(inviteEvent); err == nil { + res.InviteState.Events = append(res.InviteState.Events, ev) + } + return &res } diff --git a/syncapi/types/types_test.go b/syncapi/types/types_test.go index 634f84dc9..34c73dc29 100644 --- a/syncapi/types/types_test.go +++ b/syncapi/types/types_test.go @@ -1,8 +1,11 @@ package types import ( + "encoding/json" "reflect" "testing" + + "github.com/matrix-org/gomatrixserverlib" ) func TestNewSyncTokenWithLogs(t *testing.T) { @@ -87,3 +90,23 @@ func TestNewSyncTokenFromString(t *testing.T) { } } } + +func TestNewInviteResponse(t *testing.T) { + event := `{"auth_events":["$SbSsh09j26UAXnjd3RZqf2lyA3Kw2sY_VZJVZQAV9yA","$EwL53onrLwQ5gL8Dv3VrOOCvHiueXu2ovLdzqkNi3lo","$l2wGmz9iAwevBDGpHT_xXLUA5O8BhORxWIGU1cGi1ZM","$GsWFJLXgdlF5HpZeyWkP72tzXYWW3uQ9X28HBuTztHE"],"content":{"avatar_url":"","displayname":"neilalexander","membership":"invite"},"depth":9,"hashes":{"sha256":"8p+Ur4f8vLFX6mkIXhxI0kegPG7X3tWy56QmvBkExAg"},"origin":"matrix.org","origin_server_ts":1602087113066,"prev_events":["$1v-O6tNwhOZcA8bvCYY-Dnj1V2ZDE58lLPxtlV97S28"],"prev_state":[],"room_id":"!XbeXirGWSPXbEaGokF:matrix.org","sender":"@neilalexander:matrix.org","signatures":{"dendrite.neilalexander.dev":{"ed25519:BMJi":"05KQ5lPw0cSFsE4A0x1z7vi/3cc8bG4WHUsFWYkhxvk/XkXMGIYAYkpNThIvSeLfdcHlbm/k10AsBSKH8Uq4DA"},"matrix.org":{"ed25519:a_RXGa":"jeovuHr9E/x0sHbFkdfxDDYV/EyoeLi98douZYqZ02iYddtKhfB7R3WLay/a+D3V3V7IW0FUmPh/A404x5sYCw"}},"state_key":"@neilalexander:dendrite.neilalexander.dev","type":"m.room.member","unsigned":{"age":2512,"invite_room_state":[{"content":{"join_rule":"invite"},"sender":"@neilalexander:matrix.org","state_key":"","type":"m.room.join_rules"},{"content":{"avatar_url":"mxc://matrix.org/BpDaozLwgLnlNStxDxvLzhPr","displayname":"neilalexander","membership":"join"},"sender":"@neilalexander:matrix.org","state_key":"@neilalexander:matrix.org","type":"m.room.member"},{"content":{"name":"Test room"},"sender":"@neilalexander:matrix.org","state_key":"","type":"m.room.name"}]},"_room_version":"5"}` + expected := `{"invite_state":{"events":[{"content":{"join_rule":"invite"},"sender":"@neilalexander:matrix.org","state_key":"","type":"m.room.join_rules"},{"content":{"avatar_url":"mxc://matrix.org/BpDaozLwgLnlNStxDxvLzhPr","displayname":"neilalexander","membership":"join"},"sender":"@neilalexander:matrix.org","state_key":"@neilalexander:matrix.org","type":"m.room.member"},{"content":{"name":"Test room"},"sender":"@neilalexander:matrix.org","state_key":"","type":"m.room.name"},{"content":{"avatar_url":"","displayname":"neilalexander","membership":"invite"},"event_id":"$GQmw8e8-26CQv1QuFoHBHpKF1hQj61Flg3kvv_v_XWs","origin_server_ts":1602087113066,"sender":"@neilalexander:matrix.org","state_key":"@neilalexander:dendrite.neilalexander.dev","type":"m.room.member"}]}}` + + ev, err := gomatrixserverlib.NewEventFromTrustedJSON([]byte(event), false, gomatrixserverlib.RoomVersionV5) + if err != nil { + t.Fatal(err) + } + + res := NewInviteResponse(ev.Headered(gomatrixserverlib.RoomVersionV5)) + j, err := json.Marshal(res) + if err != nil { + t.Fatal(err) + } + + if string(j) != expected { + t.Fatalf("Invite response didn't contain correct info") + } +} From 3e12f6e9c210824ecd80c5c4dcccabf742eb4183 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 8 Oct 2020 10:27:10 +0100 Subject: [PATCH 056/104] Remove notifs about key changes in syncapi (#1496) The join/leave events themselves will wake up the right people so we needn't do it twice. --- syncapi/consumers/keychange.go | 28 ---------------------------- syncapi/consumers/roomserver.go | 20 -------------------- syncapi/syncapi.go | 2 +- 3 files changed, 1 insertion(+), 49 deletions(-) diff --git a/syncapi/consumers/keychange.go b/syncapi/consumers/keychange.go index 200ac85cc..3fc6120d2 100644 --- a/syncapi/consumers/keychange.go +++ b/syncapi/consumers/keychange.go @@ -125,31 +125,3 @@ func (s *OutputKeyChangeEventConsumer) onMessage(msg *sarama.ConsumerMessage) er } return nil } - -func (s *OutputKeyChangeEventConsumer) OnJoinEvent(ev *gomatrixserverlib.HeaderedEvent) { - // work out who we are now sharing rooms with which we previously were not and notify them about the joining - // users keys: - changed, _, err := syncinternal.TrackChangedUsers(context.Background(), s.rsAPI, *ev.StateKey(), []string{ev.RoomID()}, nil) - if err != nil { - log.WithError(err).Error("OnJoinEvent: failed to work out changed users") - return - } - // TODO: f.e changed, wake up stream - for _, userID := range changed { - log.Infof("OnJoinEvent:Notify %s that %s should have device lists tracked", userID, *ev.StateKey()) - } -} - -func (s *OutputKeyChangeEventConsumer) OnLeaveEvent(ev *gomatrixserverlib.HeaderedEvent) { - // work out who we are no longer sharing any rooms with and notify them about the leaving user - _, left, err := syncinternal.TrackChangedUsers(context.Background(), s.rsAPI, *ev.StateKey(), nil, []string{ev.RoomID()}) - if err != nil { - log.WithError(err).Error("OnLeaveEvent: failed to work out left users") - return - } - // TODO: f.e left, wake up stream - for _, userID := range left { - log.Infof("OnLeaveEvent:Notify %s that %s should no longer track device lists", userID, *ev.StateKey()) - } - -} diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index d8d0a298a..ca48c8300 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -38,7 +38,6 @@ type OutputRoomEventConsumer struct { rsConsumer *internal.ContinualConsumer db storage.Database notifier *sync.Notifier - keyChanges *OutputKeyChangeEventConsumer } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -48,7 +47,6 @@ func NewOutputRoomEventConsumer( n *sync.Notifier, store storage.Database, rsAPI api.RoomserverInternalAPI, - keyChanges *OutputKeyChangeEventConsumer, ) *OutputRoomEventConsumer { consumer := internal.ContinualConsumer{ @@ -63,7 +61,6 @@ func NewOutputRoomEventConsumer( db: store, notifier: n, rsAPI: rsAPI, - keyChanges: keyChanges, } consumer.ProcessMessage = s.onMessage @@ -182,26 +179,9 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( s.notifier.OnNewEvent(&ev, "", nil, types.NewStreamToken(pduPos, 0, nil)) - s.notifyKeyChanges(&ev) - return nil } -func (s *OutputRoomEventConsumer) notifyKeyChanges(ev *gomatrixserverlib.HeaderedEvent) { - membership, err := ev.Membership() - if err != nil { - return - } - switch membership { - case gomatrixserverlib.Join: - s.keyChanges.OnJoinEvent(ev) - case gomatrixserverlib.Ban: - fallthrough - case gomatrixserverlib.Leave: - s.keyChanges.OnLeaveEvent(ev) - } -} - func (s *OutputRoomEventConsumer) notifyJoinedPeeks(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent, sp types.StreamPosition) (types.StreamPosition, error) { if ev.Type() != gomatrixserverlib.MRoomMember { return sp, nil diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index c77c55412..43e2455b6 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -71,7 +71,7 @@ func AddPublicRoutes( } roomConsumer := consumers.NewOutputRoomEventConsumer( - cfg, consumer, notifier, syncDB, rsAPI, keyChangeConsumer, + cfg, consumer, notifier, syncDB, rsAPI, ) if err = roomConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start room server consumer") From 429bd48129b375c853dec5416212d44dd14cc0db Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 8 Oct 2020 12:13:50 +0100 Subject: [PATCH 057/104] Return a non-fatal error to the federation API on a state regression (#1498) * Return a non-fatal error to the federation API on a state regression * Return no error but don't morph state --- roomserver/internal/input/input_latest_events.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index 7be6372b2..9c6e6a25e 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -236,10 +236,13 @@ func (u *latestEventsUpdater) latestState() error { if len(u.removed) > len(u.added) { // This really shouldn't happen. // TODO: What is ultimately the best way to handle this situation? - return fmt.Errorf( - "invalid state delta wants to remove %d state but only add %d state (between state snapshots %d and %d)", - len(u.removed), len(u.added), u.oldStateNID, u.newStateNID, + logrus.Errorf( + "Invalid state delta on event %q wants to remove %d state but only add %d state (between state snapshots %d and %d)", + u.event.EventID(), len(u.removed), len(u.added), u.oldStateNID, u.newStateNID, ) + u.added = u.added[:0] + u.removed = u.removed[:0] + return nil } // Also work out the state before the event removes and the event From a846dad0e1c2cd8d43d57bc72d106574bc350c39 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 8 Oct 2020 12:22:00 +0100 Subject: [PATCH 058/104] Return what we have when we encounter missing events when servicing backfill/gme (#1499) We expect to have missing events as we walk back in the DAG over federation as we didn't always create the room. When checking if the server is allowed to see those events, just give up and stop rather than fail the request. --- roomserver/internal/helpers/helpers.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go index a2fbd287b..4c072e44a 100644 --- a/roomserver/internal/helpers/helpers.go +++ b/roomserver/internal/helpers/helpers.go @@ -309,7 +309,9 @@ BFSLoop: util.GetLogger(ctx).WithField("server", serverName).WithField("event_id", pre).WithError(err).Error( "Error checking if allowed to see event", ) - return resultNIDs, err + // drop the error, as we will often error at the DB level if we don't have the prev_event itself. Let's + // just return what we have. + return resultNIDs, nil } // If the event hasn't been seen before and the HS From 78f6e1a31e9d034d8ffca28d38830d014a82e64f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 8 Oct 2020 13:30:30 +0100 Subject: [PATCH 059/104] Don't set new state NID if state regression --- roomserver/internal/input/input_latest_events.go | 1 + 1 file changed, 1 insertion(+) diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index 9c6e6a25e..b6666cc03 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -242,6 +242,7 @@ func (u *latestEventsUpdater) latestState() error { ) u.added = u.added[:0] u.removed = u.removed[:0] + u.newStateNID = u.oldStateNID return nil } From 8035c50c066209108f9ccefebb9e12fdc1df3cab Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 8 Oct 2020 14:46:25 +0100 Subject: [PATCH 060/104] Don't get into situations where we have no forward extremities --- roomserver/internal/input/input_latest_events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index b6666cc03..229665a0b 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -296,7 +296,7 @@ func (u *latestEventsUpdater) calculateLatest( referenced, err := u.updater.IsReferenced(newEvent.EventReference) if err != nil { logrus.WithError(err).Errorf("Failed to retrieve event reference for %q", newEvent.EventReference.EventID) - } else if !referenced { + } else if !referenced || len(newLatest) == 0 { newLatest = append(newLatest, newEvent) } From b12b7abcc0a61c3b422ca0f337aec7533ba7371f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 8 Oct 2020 14:53:46 +0100 Subject: [PATCH 061/104] v0.1.0 Squashed commit of the following: commit 570528e5f19d76fe2fd7ea646246eea9650b1b60 Author: Kegan Dougal Date: Thu Oct 8 14:52:10 2020 +0100 v0.1.0 commit 02c020bced3be6d1a63e1001ab9c0ef6231ba087 Merge: db840f02 8035c50c Author: Neil Alexander Date: Thu Oct 8 14:46:32 2020 +0100 Merge branch 'master' into v0.1.0 commit db840f025b5d7b7c7e1ba905646571cb03dd7b22 Merge: adc19a3d 78f6e1a3 Author: Neil Alexander Date: Thu Oct 8 13:31:36 2020 +0100 Merge branch 'master' into v0.1.0 commit adc19a3d5f0c9d5a85b3c1dd7d1772236fb8cc9b Merge: c8fc6855 3e12f6e9 Author: Neil Alexander Date: Thu Oct 8 10:31:58 2020 +0100 Merge branch 'master' into v0.1.0 commit c8fc68555c3607b0153d10df91d357d9603ccf90 Author: Neil Alexander Date: Wed Oct 7 18:41:04 2020 +0100 Version 0.1.0rc3 commit 15bf3851415dc21ebcfa98e0f2a5ec725034d6dd Merge: e7d9eea4 8bca7a83 Author: Neil Alexander Date: Wed Oct 7 18:39:25 2020 +0100 Merge branch 'master' into v0.1.0 commit e7d9eea4a09be7b05a87b1df2a9e87d3109e8fcc Author: Neil Alexander Date: Mon Oct 5 17:56:45 2020 +0100 v0.1.0rc2 commit 3fa76370f214e2ba3ec04f4c6f0f63d3baa273e7 Merge: f7cecdd9 52ddded7 Author: Neil Alexander Date: Mon Oct 5 17:56:28 2020 +0100 Merge branch 'master' into v0.1.0 commit f7cecdd9a85fe2806a99e426b806832e7036da1e Author: Kegan Dougal Date: Fri Oct 2 17:25:59 2020 +0100 Bump to 0.1.0rc1 --- internal/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/version.go b/internal/version.go index 718273e72..2ffd7c90e 100644 --- a/internal/version.go +++ b/internal/version.go @@ -10,7 +10,7 @@ var build string const ( VersionMajor = 0 - VersionMinor = 0 + VersionMinor = 1 VersionPatch = 0 VersionTag = "" // example: "rc1" ) From 009401ad4df2e8f5c0c2f08b8c73f807e99b2c13 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 8 Oct 2020 17:45:55 +0100 Subject: [PATCH 062/104] Version 0.1.0 Beta README (#1466) * Beta docs * More tweaks * More docs * Update README.md (#1497) * Call out missing features * Add CHANGES * Call out CHANGES * Update INSTALL.md * Update README.md Co-authored-by: Neil Alexander --- CHANGES.md | 89 +++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 50 +++++++++++++++++++++++---- docs/INSTALL.md | 9 +++-- 3 files changed, 139 insertions(+), 9 deletions(-) create mode 100644 CHANGES.md diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 000000000..17fb75bed --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,89 @@ +# Dendrite 0.1.0 (2020-10-08) + +First versioned release of Dendrite. + +## Client-Server API Features + +### Account registration and management +- Registration: By password only. +- Login: By password only. No fallback. +- Logout: Yes. +- Change password: Yes. +- Link email/msisdn to account: No. +- Deactivate account: Yes. +- Check if username is available: Yes. +- Account data: Yes. +- OpenID: No. + +### Rooms +- Room creation: Yes, including presets. +- Joining rooms: Yes, including by alias or `?server_name=`. +- Event sending: Yes, including transaction IDs. +- Aliases: Yes. +- Published room directory: Yes. +- Kicking users: Yes. +- Banning users: Yes. +- Inviting users: Yes, but not third-party invites. +- Forgetting rooms: No. +- Room versions: All (v1 - v6) +- Tagging: Yes. + +### User management +- User directory: Basic support. +- Ignoring users: No. +- Groups/Communities: No. + +### Device management +- Creating devices: Yes. +- Deleting devices: Yes. +- Send-to-device messaging: Yes. + +### Sync +- Filters: Timeline limit only. Rest unimplemented. +- Deprecated `/events` and `/initialSync`: No. + +### Room events +- Typing: Yes. +- Receipts: No. +- Read Markers: No. +- Presence: No. +- Content repository (attachments): Yes. +- History visibility: No, defaults to `joined`. +- Push notifications: No. +- Event context: No. +- Reporting content: No. + +### End-to-End Encryption +- Uploading device keys: Yes. +- Downloading device keys: Yes. +- Claiming one-time keys: Yes. +- Querying key changes: Yes. +- Cross-Signing: No. + +### Misc +- Server-side search: No. +- Guest access: Partial. +- Room previews: No, partial support for Peeking via MSC2753. +- Third-Party networks: No. +- Server notices: No. +- Policy lists: No. + +## Federation Features +- Querying keys (incl. notary): Yes. +- Server ACLs: Yes. +- Sending transactions: Yes. +- Joining rooms: Yes. +- Inviting to rooms: Yes, but not third-party invites. +- Leaving rooms: Yes. +- Content repository: Yes. +- Backfilling / get_missing_events: Yes. +- Retrieving state of the room (`/state` and `/state_ids`): Yes. +- Public rooms: Yes. +- Querying profile data: Yes. +- Device management: Yes. +- Send-to-Device messaging: Yes. +- Querying/Claiming E2E Keys: Yes. +- Typing: Yes. +- Presence: No. +- Receipts: No. +- OpenID: No. \ No newline at end of file diff --git a/README.md b/README.md index 72c0df07d..f27cb4029 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,28 @@ # Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) -Dendrite is a second-generation Matrix homeserver written in Go! +Dendrite is a second-generation Matrix homeserver written in Go. +It intends to provide an **efficient**, **reliable** and **scalable** alternative to Synapse: + - Efficient: A small memory footprint with better baseline performance than an out-of-the-box Synapse. + - Reliable: Implements the Matrix specification as written, using the + [same test suite](https://github.com/matrix-org/sytest) as Synapse as well as + a [brand new Go test suite](https://github.com/matrix-org/complement). + - Scalable: can run on multiple machines and eventually scale to massive homeserver deployments. + + +As of October 2020, Dendrite has now entered **beta** which means: +- Dendrite is ready for early adopters. We recommend running in Monolith mode with a PostgreSQL database. +- Dendrite has periodic semver releases. We intend to release new versions as we land significant features. +- Dendrite supports database schema upgrades between releases. This means you should never lose your messages when upgrading Dendrite. +- Breaking changes will not occur on minor releases. This means you can safely upgrade Dendrite without modifying your database or config file. + +This does not mean: + - Dendrite is bug-free. It has not yet been battle-tested in the real world and so will be error prone initially. + - All of the CS/Federation APIs are implemented. We are tracking progress via a script called 'Are We Synapse Yet?'. In particular, + read receipts, presence and push notifications are entirely missing from Dendrite. See [CHANGES.md](CHANGES.md) for updates. + - Dendrite is ready for massive homeserver deployments. You cannot shard each microservice, only run each one on a different machine. + +Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices. +In the future, we will be able to scale up to gigantic servers (equivalent to matrix.org) via polylith mode. Join us in: @@ -8,9 +30,26 @@ Join us in: - **[#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org)** - The place for developers, where all Dendrite development discussion happens - **[#dendrite-alerts:matrix.org](https://matrix.to/#/#dendrite-alerts:matrix.org)** - Release notifications and important info, highly recommended for all Dendrite server admins -## Quick start +## Requirements -Requires Go 1.13+ and SQLite3 (Postgres is also supported): +To build Dendrite, you will need Go 1.13 or later. + +For a usable federating Dendrite deployment, you will also need: +- A domain name (or subdomain) +- A valid TLS certificate issued by a trusted authority for that domain +- SRV records or a well-known file pointing to your deployment + +Also recommended are: +- A PostgreSQL database engine, which will perform better than SQLite with many users and/or larger rooms +- A reverse proxy server, such as nginx, configured [like this sample](https://github.com/matrix-org/dendrite/blob/master/docs/nginx/monolith-sample.conf) + +The [Federation Tester](https://federationtester.matrix.org) can be used to verify your deployment. + +## Get started + +If you wish to build a fully-federating Dendrite instance, see [INSTALL.md](docs/INSTALL.md). For running in Docker, see [build/docker](build/docker). + +The following instructions are enough to get Dendrite started as a non-federating test deployment using self-signed certificates and SQLite databases: ```bash $ git clone https://github.com/matrix-org/dendrite @@ -30,14 +69,13 @@ $ go build ./cmd/dendrite-monolith-server $ ./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml ``` -Then point your favourite Matrix client at `http://localhost:8008`. For full installation information, see -[INSTALL.md](docs/INSTALL.md). For running in Docker, see [build/docker](build/docker). +Then point your favourite Matrix client at `http://localhost:8008`. ## Progress We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it -updates with CI. As of August 2020 we're at around 52% CS API coverage and 65% Federation coverage, though check +updates with CI. As of October 2020 we're at around 56% CS API coverage and 77% Federation coverage, though check CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse servers such as matrix.org reasonably well. There's a long list of features that are not implemented, notably: - Receipts diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 913bc5832..be12d7b86 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -120,7 +120,10 @@ Assuming that Postgres 9.5 (or later) is installed: Each Dendrite server requires unique server keys. -Generate the self-signed SSL certificate for federation and the server signing key: +In order for an instance to federate correctly, you should have a valid +certificate issued by a trusted authority, and private key to match. If you +don't and just want to test locally, generate the self-signed SSL certificate +for federation and the server signing key: ```bash ./bin/generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key @@ -267,12 +270,12 @@ This manages end-to-end encryption keys for users. ./bin/dendrite-key-server --config dendrite.yaml ``` -#### Server Key server +#### Signing key server This manages signing keys for servers. ```bash -./bin/dendrite-server-key-api-server --config dendrite.yaml +./bin/dendrite-signing-key-server --config dendrite.yaml ``` #### EDU server From f3e8ae01efb0abd0904509ddaa2ae85017ca4aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFck=20Bonniot?= Date: Fri, 9 Oct 2020 10:15:35 +0200 Subject: [PATCH 063/104] Implement fully read markers (#1475) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See #653 Signed-off-by: Loïck Bonniot Co-authored-by: Kegsay --- clientapi/routing/account_data.go | 75 ++++++++++++++++++++++++++++++- clientapi/routing/routing.go | 9 ++-- sytest-whitelist | 3 ++ 3 files changed, 83 insertions(+), 4 deletions(-) diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index d5fafedb1..48303c97f 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -20,8 +20,10 @@ import ( "io/ioutil" "net/http" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" @@ -91,6 +93,13 @@ func SaveAccountData( } } + if dataType == "m.fully_read" { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("Unable to set read marker"), + } + } + body, err := ioutil.ReadAll(req.Body) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("ioutil.ReadAll failed") @@ -112,7 +121,7 @@ func SaveAccountData( } dataRes := api.InputAccountDataResponse{} if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccountData failed") + util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") return util.ErrorResponse(err) } @@ -127,3 +136,67 @@ func SaveAccountData( JSON: struct{}{}, } } + +type readMarkerJSON struct { + FullyRead string `json:"m.fully_read"` + Read string `json:"m.read"` +} + +type fullyReadEvent struct { + EventID string `json:"event_id"` +} + +// SaveReadMarker implements POST /rooms/{roomId}/read_markers +func SaveReadMarker( + req *http.Request, userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, + syncProducer *producers.SyncAPIProducer, device *api.Device, roomID string, +) util.JSONResponse { + // Verify that the user is a member of this room + resErr := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID) + if resErr != nil { + return *resErr + } + + var r readMarkerJSON + resErr = httputil.UnmarshalJSONRequest(req, &r) + if resErr != nil { + return *resErr + } + + if r.FullyRead == "" { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("Missing m.fully_read mandatory field"), + } + } + + data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead}) + if err != nil { + return jsonerror.InternalServerError() + } + + dataReq := api.InputAccountDataRequest{ + UserID: device.UserID, + DataType: "m.fully_read", + RoomID: roomID, + AccountData: data, + } + dataRes := api.InputAccountDataResponse{} + if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") + return util.ErrorResponse(err) + } + + if err := syncProducer.SendData(device.UserID, roomID, "m.fully_read"); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed") + return jsonerror.InternalServerError() + } + + // TODO handle the read receipt that may be included in the read marker + // See https://matrix.org/docs/spec/client_server/r0.6.0#post-matrix-client-r0-rooms-roomid-read-markers + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 8606f69c3..b547efb4b 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -695,12 +695,15 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/read_markers", - httputil.MakeExternalAPI("rooms_read_markers", func(req *http.Request) util.JSONResponse { + httputil.MakeAuthAPI("rooms_read_markers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { if r := rateLimits.rateLimit(req); r != nil { return *r } - // TODO: return the read_markers. - return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}} + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return SaveReadMarker(req, userAPI, rsAPI, syncProducer, device, vars["roomID"]) }), ).Methods(http.MethodPost, http.MethodOptions) diff --git a/sytest-whitelist b/sytest-whitelist index e0f1f311e..420acb22c 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -456,6 +456,9 @@ After changing password, can log in with new password After changing password, existing session still works After changing password, different sessions can optionally be kept After changing password, a different session no longer works by default +Read markers appear in incremental v2 /sync +Read markers appear in initial v2 /sync +Read markers can be updated Local users can peek into world_readable rooms by room ID We can't peek into rooms with shared history_visibility We can't peek into rooms with invited history_visibility From c4c8bfd0270f3d7009f0fb7c953a26e2cb65442d Mon Sep 17 00:00:00 2001 From: Pestdoktor Date: Fri, 9 Oct 2020 10:15:51 +0200 Subject: [PATCH 064/104] reject invalid UTF-8 (#1472) * reject invalid UTF-8 Signed-off-by: Jonas Fentker * update sytest-whitelist Signed-off-by: Jonas Fentker Co-authored-by: Kegsay --- clientapi/httputil/httputil.go | 20 +++++++++++++++++++- clientapi/routing/device.go | 12 +++++------- clientapi/routing/routing.go | 6 ++++-- sytest-whitelist | 1 + 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/clientapi/httputil/httputil.go b/clientapi/httputil/httputil.go index b0fe6a6cb..29d7b0b37 100644 --- a/clientapi/httputil/httputil.go +++ b/clientapi/httputil/httputil.go @@ -16,7 +16,9 @@ package httputil import ( "encoding/json" + "io/ioutil" "net/http" + "unicode/utf8" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/util" @@ -25,7 +27,23 @@ import ( // UnmarshalJSONRequest into the given interface pointer. Returns an error JSON response if // there was a problem unmarshalling. Calling this function consumes the request body. func UnmarshalJSONRequest(req *http.Request, iface interface{}) *util.JSONResponse { - if err := json.NewDecoder(req.Body).Decode(iface); err != nil { + // encoding/json allows invalid utf-8, matrix does not + // https://matrix.org/docs/spec/client_server/r0.6.1#api-standards + body, err := ioutil.ReadAll(req.Body) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("ioutil.ReadAll failed") + resp := jsonerror.InternalServerError() + return &resp + } + + if !utf8.Valid(body) { + return &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.NotJSON("Body contains invalid UTF-8"), + } + } + + if err := json.Unmarshal(body, iface); err != nil { // TODO: We may want to suppress the Error() return in production? It's useful when // debugging because an error will be produced for both invalid/malformed JSON AND // valid JSON with incorrect types for values. diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go index 56886d57f..d50c73b35 100644 --- a/clientapi/routing/device.go +++ b/clientapi/routing/device.go @@ -15,11 +15,11 @@ package routing import ( - "encoding/json" "io/ioutil" "net/http" "github.com/matrix-org/dendrite/clientapi/auth" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/userapi/api" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -121,9 +121,8 @@ func UpdateDeviceByID( payload := deviceUpdateJSON{} - if err := json.NewDecoder(req.Body).Decode(&payload); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("json.NewDecoder.Decode failed") - return jsonerror.InternalServerError() + if resErr := httputil.UnmarshalJSONRequest(req, &payload); resErr != nil { + return *resErr } var performRes api.PerformDeviceUpdateResponse @@ -211,9 +210,8 @@ func DeleteDevices( ctx := req.Context() payload := devicesDeleteJSON{} - if err := json.NewDecoder(req.Body).Decode(&payload); err != nil { - util.GetLogger(ctx).WithError(err).Error("json.NewDecoder.Decode failed") - return jsonerror.InternalServerError() + if resErr := httputil.UnmarshalJSONRequest(req, &payload); resErr != nil { + return *resErr } defer req.Body.Close() // nolint: errcheck diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index b547efb4b..4f99237f5 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -23,6 +23,7 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/auth" + clientutil "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" @@ -659,8 +660,9 @@ func Setup( SearchString string `json:"search_term"` Limit int `json:"limit"` }{} - if err := json.NewDecoder(req.Body).Decode(&postContent); err != nil { - return util.ErrorResponse(err) + + if resErr := clientutil.UnmarshalJSONRequest(req, &postContent); resErr != nil { + return *resErr } return *SearchUserDirectory( req.Context(), diff --git a/sytest-whitelist b/sytest-whitelist index 420acb22c..099fc6cbd 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -477,5 +477,6 @@ Inbound federation rejects invite rejections which include invalid JSON for room GET /capabilities is present and well formed for registered user m.room.history_visibility == "joined" allows/forbids appropriately for Guest users m.room.history_visibility == "joined" allows/forbids appropriately for Real users +POST rejects invalid utf-8 in JSON Users cannot kick users who have already left a room A prev_batch token from incremental sync can be used in the v1 messages API From 1cd525ef0d7b6209232f93c4d0527a0fccfcdb4f Mon Sep 17 00:00:00 2001 From: S7evinK Date: Fri, 9 Oct 2020 10:17:23 +0200 Subject: [PATCH 065/104] Extend device_devices table (#1471) * Add last_used_ts and IP to database * Add migrations * Rename column Prepare statements * Add interface method and implement it Signed-off-by: Till Faelligen * Rename struct fields * Add user_agent to database * Add userAgent to registration calls * Add missing "IF NOT EXISTS" * use txn writer * Add UserAgent to Device Co-authored-by: Kegsay --- clientapi/routing/login.go | 5 ++- clientapi/routing/register.go | 14 +++--- cmd/create-account/main.go | 2 +- userapi/api/api.go | 7 +++ userapi/internal/api.go | 2 +- userapi/storage/devices/interface.go | 3 +- .../deltas/20201001204705_last_seen_ts_ip.sql | 13 ++++++ .../storage/devices/postgres/devices_table.go | 34 +++++++++++--- userapi/storage/devices/postgres/storage.go | 13 ++++-- .../deltas/20201001204705_last_seen_ts_ip.sql | 44 +++++++++++++++++++ .../storage/devices/sqlite3/devices_table.go | 28 ++++++++++-- userapi/storage/devices/sqlite3/storage.go | 13 ++++-- 12 files changed, 154 insertions(+), 24 deletions(-) create mode 100644 userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql create mode 100644 userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.sql diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index 772775aa0..f84f078dd 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -79,7 +79,7 @@ func Login( return *authErr } // make a device/access token - return completeAuth(req.Context(), cfg.Matrix.ServerName, userAPI, login) + return completeAuth(req.Context(), cfg.Matrix.ServerName, userAPI, login, req.RemoteAddr, req.UserAgent()) } return util.JSONResponse{ Code: http.StatusMethodNotAllowed, @@ -89,6 +89,7 @@ func Login( func completeAuth( ctx context.Context, serverName gomatrixserverlib.ServerName, userAPI userapi.UserInternalAPI, login *auth.Login, + ipAddr, userAgent string, ) util.JSONResponse { token, err := auth.GenerateAccessToken() if err != nil { @@ -108,6 +109,8 @@ func completeAuth( DeviceID: login.DeviceID, AccessToken: token, Localpart: localpart, + IPAddr: ipAddr, + UserAgent: userAgent, }, &performRes) if err != nil { return util.JSONResponse{ diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 937abc83d..756eafe2f 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -543,6 +543,8 @@ func handleGuestRegistration( Localpart: res.Account.Localpart, DeviceDisplayName: r.InitialDisplayName, AccessToken: token, + IPAddr: req.RemoteAddr, + UserAgent: req.UserAgent(), }, &devRes) if err != nil { return util.JSONResponse{ @@ -691,7 +693,7 @@ func handleApplicationServiceRegistration( // Don't need to worry about appending to registration stages as // application service registration is entirely separate. return completeRegistration( - req.Context(), userAPI, r.Username, "", appserviceID, + req.Context(), userAPI, r.Username, "", appserviceID, req.RemoteAddr, req.UserAgent(), r.InhibitLogin, r.InitialDisplayName, r.DeviceID, ) } @@ -710,7 +712,7 @@ func checkAndCompleteFlow( if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) { // This flow was completed, registration can continue return completeRegistration( - req.Context(), userAPI, r.Username, r.Password, "", + req.Context(), userAPI, r.Username, r.Password, "", req.RemoteAddr, req.UserAgent(), r.InhibitLogin, r.InitialDisplayName, r.DeviceID, ) } @@ -762,10 +764,10 @@ func LegacyRegister( return util.MessageResponse(http.StatusForbidden, "HMAC incorrect") } - return completeRegistration(req.Context(), userAPI, r.Username, r.Password, "", false, nil, nil) + return completeRegistration(req.Context(), userAPI, r.Username, r.Password, "", req.RemoteAddr, req.UserAgent(), false, nil, nil) case authtypes.LoginTypeDummy: // there is nothing to do - return completeRegistration(req.Context(), userAPI, r.Username, r.Password, "", false, nil, nil) + return completeRegistration(req.Context(), userAPI, r.Username, r.Password, "", req.RemoteAddr, req.UserAgent(), false, nil, nil) default: return util.JSONResponse{ Code: http.StatusNotImplemented, @@ -812,7 +814,7 @@ func parseAndValidateLegacyLogin(req *http.Request, r *legacyRegisterRequest) *u func completeRegistration( ctx context.Context, userAPI userapi.UserInternalAPI, - username, password, appserviceID string, + username, password, appserviceID, ipAddr, userAgent string, inhibitLogin eventutil.WeakBoolean, displayName, deviceID *string, ) util.JSONResponse { @@ -880,6 +882,8 @@ func completeRegistration( AccessToken: token, DeviceDisplayName: displayName, DeviceID: deviceID, + IPAddr: ipAddr, + UserAgent: userAgent, }, &devRes) if err != nil { return util.JSONResponse{ diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index 73e223d61..a9bd92794 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -92,7 +92,7 @@ func main() { } device, err := deviceDB.CreateDevice( - context.Background(), *username, nil, *accessToken, nil, + context.Background(), *username, nil, *accessToken, nil, "127.0.0.1", "", ) if err != nil { fmt.Println(err.Error()) diff --git a/userapi/api/api.go b/userapi/api/api.go index d384c5b18..6c3f3c69c 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -192,6 +192,10 @@ type PerformDeviceCreationRequest struct { DeviceID *string // optional: if nil no display name will be associated with this device. DeviceDisplayName *string + // IP address of this device + IPAddr string + // Useragent for this device + UserAgent string } // PerformDeviceCreationResponse is the response for PerformDeviceCreation @@ -222,6 +226,9 @@ type Device struct { // associated with access tokens. SessionID int64 DisplayName string + LastSeenTS int64 + LastSeenIP string + UserAgent string } // Account represents a Matrix account on this home server. diff --git a/userapi/internal/api.go b/userapi/internal/api.go index ec8284397..81d002414 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -113,7 +113,7 @@ func (a *UserInternalAPI) PerformDeviceCreation(ctx context.Context, req *api.Pe "device_id": req.DeviceID, "display_name": req.DeviceDisplayName, }).Info("PerformDeviceCreation") - dev, err := a.DeviceDB.CreateDevice(ctx, req.Localpart, req.DeviceID, req.AccessToken, req.DeviceDisplayName) + dev, err := a.DeviceDB.CreateDevice(ctx, req.Localpart, req.DeviceID, req.AccessToken, req.DeviceDisplayName, req.IPAddr, req.UserAgent) if err != nil { return err } diff --git a/userapi/storage/devices/interface.go b/userapi/storage/devices/interface.go index 168c84c5c..9953ba062 100644 --- a/userapi/storage/devices/interface.go +++ b/userapi/storage/devices/interface.go @@ -31,10 +31,11 @@ type Database interface { // an error will be returned. // If no device ID is given one is generated. // Returns the device on success. - CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string) (dev *api.Device, returnErr error) + CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string, ipAddr, userAgent string) (dev *api.Device, returnErr error) UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error RemoveDevice(ctx context.Context, deviceID, localpart string) error RemoveDevices(ctx context.Context, localpart string, devices []string) error // RemoveAllDevices deleted all devices for this user. Returns the devices deleted. RemoveAllDevices(ctx context.Context, localpart, exceptDeviceID string) (devices []api.Device, err error) + UpdateDeviceLastSeen(ctx context.Context, deviceID, ipAddr string) error } diff --git a/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql b/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql new file mode 100644 index 000000000..4f5f2b172 --- /dev/null +++ b/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql @@ -0,0 +1,13 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS last_seen_ts BIGINT NOT NULL; +ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS ip TEXT; +ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS user_agent TEXT; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE device_devices DROP COLUMN last_seen_ts; +ALTER TABLE device_devices DROP COLUMN ip; +ALTER TABLE device_devices DROP COLUMN user_agent; +-- +goose StatementEnd diff --git a/userapi/storage/devices/postgres/devices_table.go b/userapi/storage/devices/postgres/devices_table.go index c06af7549..2a4d337c7 100644 --- a/userapi/storage/devices/postgres/devices_table.go +++ b/userapi/storage/devices/postgres/devices_table.go @@ -51,8 +51,15 @@ CREATE TABLE IF NOT EXISTS device_devices ( -- When this devices was first recognised on the network, as a unix timestamp (ms resolution). created_ts BIGINT NOT NULL, -- The display name, human friendlier than device_id and updatable - display_name TEXT - -- TODO: device keys, device display names, last used ts and IP address?, token restrictions (if 3rd-party OAuth app) + display_name TEXT, + -- The time the device was last used, as a unix timestamp (ms resolution). + last_seen_ts BIGINT NOT NULL, + -- The last seen IP address of this device + ip TEXT, + -- User agent of this device + user_agent TEXT + + -- TODO: device keys, device display names, token restrictions (if 3rd-party OAuth app) ); -- Device IDs must be unique for a given user. @@ -60,7 +67,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS device_localpart_id_idx ON device_devices(loca ` const insertDeviceSQL = "" + - "INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name) VALUES ($1, $2, $3, $4, $5)" + + "INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name, last_seen_ts, ip, user_agent) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)" + " RETURNING session_id" const selectDeviceByTokenSQL = "" + @@ -87,6 +94,9 @@ const deleteDevicesSQL = "" + const selectDevicesByIDSQL = "" + "SELECT device_id, localpart, display_name FROM device_devices WHERE device_id = ANY($1)" +const updateDeviceLastSeen = "" + + "UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE device_id = $3" + type devicesStatements struct { insertDeviceStmt *sql.Stmt selectDeviceByTokenStmt *sql.Stmt @@ -94,6 +104,7 @@ type devicesStatements struct { selectDevicesByLocalpartStmt *sql.Stmt selectDevicesByIDStmt *sql.Stmt updateDeviceNameStmt *sql.Stmt + updateDeviceLastSeenStmt *sql.Stmt deleteDeviceStmt *sql.Stmt deleteDevicesByLocalpartStmt *sql.Stmt deleteDevicesStmt *sql.Stmt @@ -132,6 +143,9 @@ func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerN if s.selectDevicesByIDStmt, err = db.Prepare(selectDevicesByIDSQL); err != nil { return } + if s.updateDeviceLastSeenStmt, err = db.Prepare(updateDeviceLastSeen); err != nil { + return + } s.serverName = server return } @@ -141,12 +155,12 @@ func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerN // Returns the device on success. func (s *devicesStatements) insertDevice( ctx context.Context, txn *sql.Tx, id, localpart, accessToken string, - displayName *string, + displayName *string, ipAddr, userAgent string, ) (*api.Device, error) { createdTimeMS := time.Now().UnixNano() / 1000000 var sessionID int64 stmt := sqlutil.TxStmt(txn, s.insertDeviceStmt) - if err := stmt.QueryRowContext(ctx, id, localpart, accessToken, createdTimeMS, displayName).Scan(&sessionID); err != nil { + if err := stmt.QueryRowContext(ctx, id, localpart, accessToken, createdTimeMS, displayName, createdTimeMS, ipAddr, userAgent).Scan(&sessionID); err != nil { return nil, err } return &api.Device{ @@ -154,6 +168,9 @@ func (s *devicesStatements) insertDevice( UserID: userutil.MakeUserID(localpart, s.serverName), AccessToken: accessToken, SessionID: sessionID, + LastSeenTS: createdTimeMS, + LastSeenIP: ipAddr, + UserAgent: userAgent, }, nil } @@ -280,3 +297,10 @@ func (s *devicesStatements) selectDevicesByLocalpart( return devices, rows.Err() } + +func (s *devicesStatements) updateDeviceLastSeen(ctx context.Context, txn *sql.Tx, deviceID, ipAddr string) error { + lastSeenTs := time.Now().UnixNano() / 1000000 + stmt := sqlutil.TxStmt(txn, s.updateDeviceLastSeenStmt) + _, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, deviceID) + return err +} diff --git a/userapi/storage/devices/postgres/storage.go b/userapi/storage/devices/postgres/storage.go index c5bd5b6cf..faa5796b0 100644 --- a/userapi/storage/devices/postgres/storage.go +++ b/userapi/storage/devices/postgres/storage.go @@ -83,7 +83,7 @@ func (d *Database) GetDevicesByID(ctx context.Context, deviceIDs []string) ([]ap // Returns the device on success. func (d *Database) CreateDevice( ctx context.Context, localpart string, deviceID *string, accessToken string, - displayName *string, + displayName *string, ipAddr, userAgent string, ) (dev *api.Device, returnErr error) { if deviceID != nil { returnErr = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { @@ -93,7 +93,7 @@ func (d *Database) CreateDevice( return err } - dev, err = d.devices.insertDevice(ctx, txn, *deviceID, localpart, accessToken, displayName) + dev, err = d.devices.insertDevice(ctx, txn, *deviceID, localpart, accessToken, displayName, ipAddr, userAgent) return err }) } else { @@ -108,7 +108,7 @@ func (d *Database) CreateDevice( returnErr = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { var err error - dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName) + dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName, ipAddr, userAgent) return err }) if returnErr == nil { @@ -189,3 +189,10 @@ func (d *Database) RemoveAllDevices( }) return } + +// UpdateDeviceLastSeen updates a the last seen timestamp and the ip address +func (d *Database) UpdateDeviceLastSeen(ctx context.Context, deviceID, ipAddr string) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { + return d.devices.updateDeviceLastSeen(ctx, txn, deviceID, ipAddr) + }) +} diff --git a/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.sql b/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.sql new file mode 100644 index 000000000..887f90e0d --- /dev/null +++ b/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.sql @@ -0,0 +1,44 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE device_devices RENAME TO device_devices_tmp; +CREATE TABLE device_devices ( + access_token TEXT PRIMARY KEY, + session_id INTEGER, + device_id TEXT , + localpart TEXT , + created_ts BIGINT, + display_name TEXT, + last_seen_ts BIGINT, + ip TEXT, + user_agent TEXT, + UNIQUE (localpart, device_id) +); +INSERT +INTO device_devices ( + access_token, session_id, device_id, localpart, created_ts, display_name, last_seen_ts, ip, user_agent +) SELECT + access_token, session_id, device_id, localpart, created_ts, display_name, created_ts, '', '' +FROM device_devices_tmp; +DROP TABLE device_devices_tmp; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE device_devices RENAME TO device_devices_tmp; +CREATE TABLE IF NOT EXISTS device_devices ( + access_token TEXT PRIMARY KEY, + session_id INTEGER, + device_id TEXT , + localpart TEXT , + created_ts BIGINT, + display_name TEXT, + UNIQUE (localpart, device_id) +); +INSERT +INTO device_devices ( + access_token, session_id, device_id, localpart, created_ts, display_name +) SELECT + access_token, session_id, device_id, localpart, created_ts, display_name +FROM device_devices_tmp; +DROP TABLE device_devices_tmp; +-- +goose StatementEnd \ No newline at end of file diff --git a/userapi/storage/devices/sqlite3/devices_table.go b/userapi/storage/devices/sqlite3/devices_table.go index c75e19825..6b0de10ee 100644 --- a/userapi/storage/devices/sqlite3/devices_table.go +++ b/userapi/storage/devices/sqlite3/devices_table.go @@ -40,14 +40,17 @@ CREATE TABLE IF NOT EXISTS device_devices ( localpart TEXT , created_ts BIGINT, display_name TEXT, + last_seen_ts BIGINT, + ip TEXT, + user_agent TEXT, UNIQUE (localpart, device_id) ); ` const insertDeviceSQL = "" + - "INSERT INTO device_devices (device_id, localpart, access_token, created_ts, display_name, session_id)" + - " VALUES ($1, $2, $3, $4, $5, $6)" + "INSERT INTO device_devices (device_id, localpart, access_token, created_ts, display_name, session_id, last_seen_ts, ip, user_agent)" + + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" const selectDevicesCountSQL = "" + "SELECT COUNT(access_token) FROM device_devices" @@ -76,6 +79,9 @@ const deleteDevicesSQL = "" + const selectDevicesByIDSQL = "" + "SELECT device_id, localpart, display_name FROM device_devices WHERE device_id IN ($1)" +const updateDeviceLastSeen = "" + + "UPDATE device_devices SET last_seen_ts = $1, ip = $2 WHERE device_id = $3" + type devicesStatements struct { db *sql.DB writer sqlutil.Writer @@ -86,6 +92,7 @@ type devicesStatements struct { selectDevicesByIDStmt *sql.Stmt selectDevicesByLocalpartStmt *sql.Stmt updateDeviceNameStmt *sql.Stmt + updateDeviceLastSeenStmt *sql.Stmt deleteDeviceStmt *sql.Stmt deleteDevicesByLocalpartStmt *sql.Stmt serverName gomatrixserverlib.ServerName @@ -125,6 +132,9 @@ func (s *devicesStatements) prepare(db *sql.DB, writer sqlutil.Writer, server go if s.selectDevicesByIDStmt, err = db.Prepare(selectDevicesByIDSQL); err != nil { return } + if s.updateDeviceLastSeenStmt, err = db.Prepare(updateDeviceLastSeen); err != nil { + return + } s.serverName = server return } @@ -134,7 +144,7 @@ func (s *devicesStatements) prepare(db *sql.DB, writer sqlutil.Writer, server go // Returns the device on success. func (s *devicesStatements) insertDevice( ctx context.Context, txn *sql.Tx, id, localpart, accessToken string, - displayName *string, + displayName *string, ipAddr, userAgent string, ) (*api.Device, error) { createdTimeMS := time.Now().UnixNano() / 1000000 var sessionID int64 @@ -144,7 +154,7 @@ func (s *devicesStatements) insertDevice( return nil, err } sessionID++ - if _, err := insertStmt.ExecContext(ctx, id, localpart, accessToken, createdTimeMS, displayName, sessionID); err != nil { + if _, err := insertStmt.ExecContext(ctx, id, localpart, accessToken, createdTimeMS, displayName, sessionID, createdTimeMS, ipAddr, userAgent); err != nil { return nil, err } return &api.Device{ @@ -152,6 +162,9 @@ func (s *devicesStatements) insertDevice( UserID: userutil.MakeUserID(localpart, s.serverName), AccessToken: accessToken, SessionID: sessionID, + LastSeenTS: createdTimeMS, + LastSeenIP: ipAddr, + UserAgent: userAgent, }, nil } @@ -288,3 +301,10 @@ func (s *devicesStatements) selectDevicesByID(ctx context.Context, deviceIDs []s } return devices, rows.Err() } + +func (s *devicesStatements) updateDeviceLastSeen(ctx context.Context, txn *sql.Tx, deviceID, ipAddr string) error { + lastSeenTs := time.Now().UnixNano() / 1000000 + stmt := sqlutil.TxStmt(txn, s.updateDeviceLastSeenStmt) + _, err := stmt.ExecContext(ctx, lastSeenTs, ipAddr, deviceID) + return err +} diff --git a/userapi/storage/devices/sqlite3/storage.go b/userapi/storage/devices/sqlite3/storage.go index 7c6645dd6..cfaf4fd99 100644 --- a/userapi/storage/devices/sqlite3/storage.go +++ b/userapi/storage/devices/sqlite3/storage.go @@ -87,7 +87,7 @@ func (d *Database) GetDevicesByID(ctx context.Context, deviceIDs []string) ([]ap // Returns the device on success. func (d *Database) CreateDevice( ctx context.Context, localpart string, deviceID *string, accessToken string, - displayName *string, + displayName *string, ipAddr, userAgent string, ) (dev *api.Device, returnErr error) { if deviceID != nil { returnErr = d.writer.Do(d.db, nil, func(txn *sql.Tx) error { @@ -97,7 +97,7 @@ func (d *Database) CreateDevice( return err } - dev, err = d.devices.insertDevice(ctx, txn, *deviceID, localpart, accessToken, displayName) + dev, err = d.devices.insertDevice(ctx, txn, *deviceID, localpart, accessToken, displayName, ipAddr, userAgent) return err }) } else { @@ -112,7 +112,7 @@ func (d *Database) CreateDevice( returnErr = d.writer.Do(d.db, nil, func(txn *sql.Tx) error { var err error - dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName) + dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName, ipAddr, userAgent) return err }) if returnErr == nil { @@ -193,3 +193,10 @@ func (d *Database) RemoveAllDevices( }) return } + +// UpdateDeviceLastSeen updates a the last seen timestamp and the ip address +func (d *Database) UpdateDeviceLastSeen(ctx context.Context, deviceID, ipAddr string) error { + return d.writer.Do(d.db, nil, func(txn *sql.Tx) error { + return d.devices.updateDeviceLastSeen(ctx, txn, deviceID, ipAddr) + }) +} From 2bd0449c5b2b5cbefffbe145f6c4c183e4ff0552 Mon Sep 17 00:00:00 2001 From: S7evinK Date: Fri, 9 Oct 2020 13:41:53 +0200 Subject: [PATCH 066/104] Use default value when adding column which is not null (#1501) Signed-off-by: Till Faelligen --- .../devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql b/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql index 4f5f2b172..e7900b0b3 100644 --- a/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql +++ b/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql @@ -1,6 +1,6 @@ -- +goose Up -- +goose StatementBegin -ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS last_seen_ts BIGINT NOT NULL; +ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS last_seen_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)*1000; ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS ip TEXT; ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS user_agent TEXT; -- +goose StatementEnd From 4df7e345bb8a8182c50666e40023909f27f5d607 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 9 Oct 2020 15:06:43 +0100 Subject: [PATCH 067/104] Only return 500 on /send if a database error occurs (#1503) --- federationapi/routing/send.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index e2ab9b334..fe4295213 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -16,6 +16,7 @@ package routing import ( "context" + "database/sql" "encoding/json" "fmt" "net/http" @@ -234,17 +235,10 @@ func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.Res // we should stop processing the transaction, and returns false if it // is just some less serious error about a specific event. func isProcessingErrorFatal(err error) bool { - switch err.(type) { - case roomNotFoundError: - case *gomatrixserverlib.NotAllowed: - case missingPrevEventsError: - default: - switch err { - case context.Canceled: - case context.DeadlineExceeded: - default: - return true - } + switch err { + case sql.ErrConnDone: + case sql.ErrTxDone: + return true } return false } From fe5d1400bf426d1fea8a6c833088d1e491e9dddc Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 9 Oct 2020 17:08:32 +0100 Subject: [PATCH 068/104] Update federation timeouts (#1504) * Update to matrix-org/gomatrixserverlib#234 * Update gomatrixserverlib * Update federation timeouts * Fix dendritejs * Increase /send context time in destination queue --- cmd/dendrite-demo-libp2p/main.go | 2 +- cmd/dendrite-demo-yggdrasil/yggconn/client.go | 2 +- cmd/dendritejs/main.go | 4 ++-- federationsender/internal/api.go | 16 ++++++++++++++++ federationsender/internal/perform.go | 2 +- federationsender/queue/destinationqueue.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- internal/setup/base.go | 4 ++-- keyserver/internal/device_list_update_test.go | 2 +- 10 files changed, 28 insertions(+), 12 deletions(-) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 0f30e8d30..b5386325c 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -89,7 +89,7 @@ func createClient( "matrix", p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")), ) - return gomatrixserverlib.NewClientWithTransport(true, tr) + return gomatrixserverlib.NewClientWithTransport(tr) } func main() { diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/client.go b/cmd/dendrite-demo-yggdrasil/yggconn/client.go index 1236c5530..a5f89439d 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/client.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/client.go @@ -33,7 +33,7 @@ func (n *Node) CreateClient( }, }, ) - return gomatrixserverlib.NewClientWithTransport(true, tr) + return gomatrixserverlib.NewClientWithTransport(tr) } func (n *Node) CreateFederationClient( diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 267259c78..2d7f8b02b 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -141,14 +141,14 @@ func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLoc fed := gomatrixserverlib.NewFederationClient( cfg.Global.ServerName, cfg.Global.KeyID, cfg.Global.PrivateKey, true, ) - fed.Client = *gomatrixserverlib.NewClientWithTransport(true, tr) + fed.Client = *gomatrixserverlib.NewClientWithTransport(tr) return fed } func createClient(node *go_http_js_libp2p.P2pLocalNode) *gomatrixserverlib.Client { tr := go_http_js_libp2p.NewP2pTransport(node) - return gomatrixserverlib.NewClientWithTransport(true, tr) + return gomatrixserverlib.NewClientWithTransport(tr) } func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http_js_libp2p.P2pLocalNode) { diff --git a/federationsender/internal/api.go b/federationsender/internal/api.go index f9d353572..31617045e 100644 --- a/federationsender/internal/api.go +++ b/federationsender/internal/api.go @@ -109,6 +109,8 @@ func (a *FederationSenderInternalAPI) doRequest( func (a *FederationSenderInternalAPI) GetUserDevices( ctx context.Context, s gomatrixserverlib.ServerName, userID string, ) (gomatrixserverlib.RespUserDevices, error) { + ctx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() ires, err := a.doRequest(s, func() (interface{}, error) { return a.federation.GetUserDevices(ctx, s, userID) }) @@ -121,6 +123,8 @@ func (a *FederationSenderInternalAPI) GetUserDevices( func (a *FederationSenderInternalAPI) ClaimKeys( ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string, ) (gomatrixserverlib.RespClaimKeys, error) { + ctx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() ires, err := a.doRequest(s, func() (interface{}, error) { return a.federation.ClaimKeys(ctx, s, oneTimeKeys) }) @@ -145,6 +149,8 @@ func (a *FederationSenderInternalAPI) QueryKeys( func (a *FederationSenderInternalAPI) Backfill( ctx context.Context, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string, ) (res gomatrixserverlib.Transaction, err error) { + ctx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() ires, err := a.doRequest(s, func() (interface{}, error) { return a.federation.Backfill(ctx, s, roomID, limit, eventIDs) }) @@ -157,6 +163,8 @@ func (a *FederationSenderInternalAPI) Backfill( func (a *FederationSenderInternalAPI) LookupState( ctx context.Context, s gomatrixserverlib.ServerName, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion, ) (res gomatrixserverlib.RespState, err error) { + ctx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() ires, err := a.doRequest(s, func() (interface{}, error) { return a.federation.LookupState(ctx, s, roomID, eventID, roomVersion) }) @@ -169,6 +177,8 @@ func (a *FederationSenderInternalAPI) LookupState( func (a *FederationSenderInternalAPI) LookupStateIDs( ctx context.Context, s gomatrixserverlib.ServerName, roomID, eventID string, ) (res gomatrixserverlib.RespStateIDs, err error) { + ctx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() ires, err := a.doRequest(s, func() (interface{}, error) { return a.federation.LookupStateIDs(ctx, s, roomID, eventID) }) @@ -181,6 +191,8 @@ func (a *FederationSenderInternalAPI) LookupStateIDs( func (a *FederationSenderInternalAPI) GetEvent( ctx context.Context, s gomatrixserverlib.ServerName, eventID string, ) (res gomatrixserverlib.Transaction, err error) { + ctx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() ires, err := a.doRequest(s, func() (interface{}, error) { return a.federation.GetEvent(ctx, s, eventID) }) @@ -193,6 +205,8 @@ func (a *FederationSenderInternalAPI) GetEvent( func (a *FederationSenderInternalAPI) GetServerKeys( ctx context.Context, s gomatrixserverlib.ServerName, ) (gomatrixserverlib.ServerKeys, error) { + ctx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() ires, err := a.doRequest(s, func() (interface{}, error) { return a.federation.GetServerKeys(ctx, s) }) @@ -205,6 +219,8 @@ func (a *FederationSenderInternalAPI) GetServerKeys( func (a *FederationSenderInternalAPI) LookupServerKeys( ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, ) ([]gomatrixserverlib.ServerKeys, error) { + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() ires, err := a.doRequest(s, func() (interface{}, error) { return a.federation.LookupServerKeys(ctx, s, keyRequests) }) diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index 0c9dd2572..d014ca5c7 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -42,7 +42,7 @@ type federatedJoin struct { RoomID string } -// PerformJoinRequest implements api.FederationSenderInternalAPI +// PerformJoin implements api.FederationSenderInternalAPI func (r *FederationSenderInternalAPI) PerformJoin( ctx context.Context, request *api.PerformJoinRequest, diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index e87f00634..29fef7059 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -349,7 +349,7 @@ func (oq *destinationQueue) nextTransaction() (bool, error) { // TODO: we should check for 500-ish fails vs 400-ish here, // since we shouldn't queue things indefinitely in response // to a 400-ish error - ctx, cancel = context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel = context.WithTimeout(context.Background(), time.Minute*5) defer cancel() _, err = oq.client.SendTransaction(ctx, t) switch err.(type) { diff --git a/go.mod b/go.mod index c98aa61e9..c02463832 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20201006143701-222e7423a5e3 + github.com/matrix-org/gomatrixserverlib v0.0.0-20201009153043-8d27a9f0e350 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index c92774bd0..101b8e18f 100644 --- a/go.sum +++ b/go.sum @@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201006143701-222e7423a5e3 h1:lWR/w6rXKZJJU1yGHb2zem/EK7+aYhUcRgAOiouZAxk= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201006143701-222e7423a5e3/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201009153043-8d27a9f0e350 h1:G9K8k5KIzbeBdd0bMk+4itdZU3JGHgV+z0FNUsTEhkE= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201009153043-8d27a9f0e350/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/internal/setup/base.go b/internal/setup/base.go index 77fdb04a1..24a0d6aa6 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -254,9 +254,9 @@ func (b *BaseDendrite) CreateClient() *gomatrixserverlib.Client { // CreateFederationClient creates a new federation client. Should only be called // once per component. func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationClient { - client := gomatrixserverlib.NewFederationClient( + client := gomatrixserverlib.NewFederationClientWithTimeout( b.Cfg.Global.ServerName, b.Cfg.Global.KeyID, b.Cfg.Global.PrivateKey, - b.Cfg.FederationSender.DisableTLSValidation, + b.Cfg.FederationSender.DisableTLSValidation, time.Minute*5, ) client.SetUserAgent(fmt.Sprintf("Dendrite/%s", internal.VersionString())) return client diff --git a/keyserver/internal/device_list_update_test.go b/keyserver/internal/device_list_update_test.go index 56bb4888c..9c4cc1165 100644 --- a/keyserver/internal/device_list_update_test.go +++ b/keyserver/internal/device_list_update_test.go @@ -108,7 +108,7 @@ func newFedClient(tripper func(*http.Request) (*http.Response, error)) *gomatrix fedClient := gomatrixserverlib.NewFederationClient( gomatrixserverlib.ServerName("example.test"), gomatrixserverlib.KeyID("ed25519:test"), pkey, true, ) - fedClient.Client = *gomatrixserverlib.NewClientWithTransport(true, &roundTripper{tripper}) + fedClient.Client = *gomatrixserverlib.NewClientWithTransport(&roundTripper{tripper}) return fedClient } From 9096bfcee8b22b99b4ddd1f1f56ee7aa59280268 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Sat, 10 Oct 2020 00:21:15 +0100 Subject: [PATCH 069/104] Validate m.room.create events in send_join responses (#1505) * Validate m.room.create events in send_join responses For sytest compliance, refs #1315 and #1317 Fixes #1317 * Linting --- federationsender/internal/perform.go | 58 +++++++++++++++++++++++++++- sytest-whitelist | 2 + 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index d014ca5c7..bff6833ca 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -2,6 +2,7 @@ package internal import ( "context" + "encoding/json" "errors" "fmt" "time" @@ -174,8 +175,10 @@ func (r *FederationSenderInternalAPI) performJoinUsingServer( // Work out if we support the room version that has been supplied in // the make_join response. + // "If not provided, the room version is assumed to be either "1" or "2"." + // https://matrix.org/docs/spec/server_server/unstable#get-matrix-federation-v1-make-join-roomid-userid if respMakeJoin.RoomVersion == "" { - respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1 + respMakeJoin.RoomVersion = setDefaultRoomVersionFromJoinEvent(respMakeJoin.JoinEvent) } if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil { return fmt.Errorf("respMakeJoin.RoomVersion.EventFormat: %w", err) @@ -205,6 +208,9 @@ func (r *FederationSenderInternalAPI) performJoinUsingServer( return fmt.Errorf("r.federation.SendJoin: %w", err) } r.statistics.ForServer(serverName).Success() + if err := sanityCheckSendJoinResponse(respSendJoin); err != nil { + return err + } // Process the join response in a goroutine. The idea here is // that we'll try and wait for as long as possible for the work @@ -424,3 +430,53 @@ func (r *FederationSenderInternalAPI) PerformBroadcastEDU( return nil } + +func sanityCheckSendJoinResponse(respSendJoin gomatrixserverlib.RespSendJoin) error { + // sanity check we have a create event and it has a known room version + for _, ev := range respSendJoin.AuthEvents { + if ev.Type() == gomatrixserverlib.MRoomCreate && ev.StateKeyEquals("") { + // make sure the room version is known + content := ev.Content() + verBody := struct { + Version string `json:"room_version"` + }{} + err := json.Unmarshal(content, &verBody) + if err != nil { + return err + } + if verBody.Version == "" { + // https://matrix.org/docs/spec/client_server/r0.6.0#m-room-create + // The version of the room. Defaults to "1" if the key does not exist. + verBody.Version = "1" + } + knownVersions := gomatrixserverlib.RoomVersions() + if _, ok := knownVersions[gomatrixserverlib.RoomVersion(verBody.Version)]; !ok { + return fmt.Errorf("send_join m.room.create event has an unknown room version: %s", verBody.Version) + } + return nil + } + } + return fmt.Errorf("send_join response is missing m.room.create event") +} + +func setDefaultRoomVersionFromJoinEvent(joinEvent gomatrixserverlib.EventBuilder) gomatrixserverlib.RoomVersion { + // if auth events are not event references we know it must be v3+ + // we have to do these shenanigans to satisfy sytest, specifically for: + // "Outbound federation rejects m.room.create events with an unknown room version" + hasEventRefs := true + authEvents, ok := joinEvent.AuthEvents.([]interface{}) + if ok { + if len(authEvents) > 0 { + _, ok = authEvents[0].(string) + if ok { + // event refs are objects, not strings, so we know we must be dealing with a v3+ room. + hasEventRefs = false + } + } + } + + if hasEventRefs { + return gomatrixserverlib.RoomVersionV1 + } + return gomatrixserverlib.RoomVersionV4 +} diff --git a/sytest-whitelist b/sytest-whitelist index 099fc6cbd..805f0e4dd 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -68,6 +68,8 @@ Request to logout with invalid an access token is rejected Request to logout without an access token is rejected Room creation reports m.room.create to myself Room creation reports m.room.member to myself +Outbound federation rejects send_join responses with no m.room.create event +Outbound federation rejects m.room.create events with an unknown room version Invited user can see room metadata # Blacklisted because these tests call /r0/events which we don't implement # New room members see their own join event From 6b579e6abadd71f0e367b98f4988758fc07e5f54 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 12 Oct 2020 10:02:20 +0100 Subject: [PATCH 070/104] Don't ignore local keys from fetchers (#1513) --- signingkeyserver/internal/api.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/signingkeyserver/internal/api.go b/signingkeyserver/internal/api.go index 54c41b52f..4a1dd29e7 100644 --- a/signingkeyserver/internal/api.go +++ b/signingkeyserver/internal/api.go @@ -224,10 +224,6 @@ func (s *ServerKeyAPI) handleFetcherKeys( // Now let's look at the results that we got from this fetcher. for req, res := range fetcherResults { - if req.ServerName == s.ServerName { - continue - } - if prev, ok := results[req]; ok { // We've already got a previous entry for this request // so let's see if the newly retrieved one contains a more From 0804594a614998fd115fb99224183c66af34ecfa Mon Sep 17 00:00:00 2001 From: Pieter Date: Mon, 12 Oct 2020 11:06:56 +0200 Subject: [PATCH 071/104] Add systemd example service to docs (#1512) Signed-off-by: Pieter Hollander --- docs/systemd/monolith-example.service | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 docs/systemd/monolith-example.service diff --git a/docs/systemd/monolith-example.service b/docs/systemd/monolith-example.service new file mode 100644 index 000000000..7dd7755db --- /dev/null +++ b/docs/systemd/monolith-example.service @@ -0,0 +1,17 @@ +[Unit] +Description=Dendrite (Matrix Homeserver) +After=syslog.target +After=network.target +After=postgresql.service + +[Service] +RestartSec=2s +Type=simple +User=dendrite +Group=dendrite +WorkingDirectory=/opt/dendrite/ +ExecStart=/opt/dendrite/bin/dendrite-monolith-server +Restart=always + +[Install] +WantedBy=multi-user.target From 8001627cfca89e52389d943f7f148a1a86c71715 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 12 Oct 2020 15:56:15 +0100 Subject: [PATCH 072/104] Get missing event tweaks (#1514) * Adjust backfill to send backward extremity with state before other backfilled events, include prev_events with no state amongst missing events * Not finished refactor * Fix test * Remove isInboundTxn * Remove debug logging --- federationapi/routing/send.go | 90 +++++++++++++++++++++--------- federationapi/routing/send_test.go | 17 ++++++ roomserver/internal/query/query.go | 2 +- 3 files changed, 81 insertions(+), 28 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index fe4295213..a6e39821a 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -183,7 +183,7 @@ func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.Res // Process the events. for _, e := range pdus { - if err := t.processEvent(ctx, e.Unwrap(), true); err != nil { + if err := t.processEvent(ctx, e.Unwrap()); err != nil { // If the error is due to the event itself being bad then we skip // it and move onto the next event. We report an error so that the // sender knows that we have skipped processing it. @@ -338,11 +338,15 @@ func (t *txnReq) processDeviceListUpdate(ctx context.Context, e gomatrixserverli } } -func (t *txnReq) processEvent(ctx context.Context, e gomatrixserverlib.Event, isInboundTxn bool) error { +func (t *txnReq) processEvent(ctx context.Context, e gomatrixserverlib.Event) error { logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID()) // Work out if the roomserver knows everything it needs to know to auth - // the event. + // the event. This includes the prev_events and auth_events. + // NOTE! This is going to include prev_events that have an empty state + // snapshot. This is because we will need to re-request the event, and + // it's /state_ids, in order for it to exist in the roomserver correctly + // before the roomserver tries to work out stateReq := api.QueryMissingAuthPrevEventsRequest{ RoomID: e.RoomID(), AuthEventIDs: e.AuthEventIDs(), @@ -410,7 +414,7 @@ func (t *txnReq) processEvent(ctx context.Context, e gomatrixserverlib.Event, is if len(stateResp.MissingPrevEventIDs) > 0 { logger.Infof("Event refers to %d unknown prev_events", len(stateResp.MissingPrevEventIDs)) - return t.processEventWithMissingState(ctx, e, stateResp.RoomVersion, isInboundTxn) + return t.processEventWithMissingState(ctx, e, stateResp.RoomVersion) } // pass the event to the roomserver which will do auth checks @@ -438,7 +442,7 @@ func checkAllowedByState(e gomatrixserverlib.Event, stateEvents []gomatrixserver return gomatrixserverlib.Allowed(e, &authUsingState) } -func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion, isInboundTxn bool) error { +func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) error { // Do this with a fresh context, so that we keep working even if the // original request times out. With any luck, by the time the remote // side retries, we'll have fetched the missing state. @@ -464,39 +468,82 @@ func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixser // - fill in the gap completely then process event `e` returning no backwards extremity // - fail to fill in the gap and tell us to terminate the transaction err=not nil // - fail to fill in the gap and tell us to fetch state at the new backwards extremity, and to not terminate the transaction - backwardsExtremity, err := t.getMissingEvents(gmectx, e, roomVersion, isInboundTxn) + newEvents, err := t.getMissingEvents(gmectx, e, roomVersion) if err != nil { return err } - if backwardsExtremity == nil { - // we filled in the gap! + if len(newEvents) == 0 { return nil } + backwardsExtremity := &newEvents[0] + newEvents = newEvents[1:] + // at this point we know we're going to have a gap: we need to work out the room state at the new backwards extremity. - // security: we have to do state resolution on the new backwards extremity (TODO: WHY) // Therefore, we cannot just query /state_ids with this event to get the state before. Instead, we need to query // the state AFTER all the prev_events for this event, then apply state resolution to that to get the state before the event. var states []*gomatrixserverlib.RespState needed := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{*backwardsExtremity}).Tuples() for _, prevEventID := range backwardsExtremity.PrevEventIDs() { + // Look up what the state is after the backward extremity. This will either + // come from the roomserver, if we know all the required events, or it will + // come from a remote server via /state_ids if not. var prevState *gomatrixserverlib.RespState prevState, err = t.lookupStateAfterEvent(gmectx, roomVersion, backwardsExtremity.RoomID(), prevEventID, needed) if err != nil { util.GetLogger(ctx).WithError(err).Errorf("Failed to lookup state after prev_event: %s", prevEventID) return err } + // Append the state onto the collected state. We'll run this through the + // state resolution next. states = append(states, prevState) } + + // Now that we have collected all of the state from the prev_events, we'll + // run the state through the appropriate state resolution algorithm for the + // room. This does a couple of things: + // 1. Ensures that the state is deduplicated fully for each state-key tuple + // 2. Ensures that we pick the latest events from both sets, in the case that + // one of the prev_events is quite a bit older than the others resolvedState, err := t.resolveStatesAndCheck(gmectx, roomVersion, states, backwardsExtremity) if err != nil { util.GetLogger(ctx).WithError(err).Errorf("Failed to resolve state conflicts for event %s", backwardsExtremity.EventID()) return err } - // pass the event along with the state to the roomserver using a background context so we don't - // needlessly expire - return api.SendEventWithState(context.Background(), t.rsAPI, resolvedState, e.Headered(roomVersion), t.haveEventIDs()) + // First of all, send the backward extremity into the roomserver with the + // newly resolved state. This marks the "oldest" point in the backfill and + // sets the baseline state for any new events after this. + err = api.SendEventWithState( + context.Background(), + t.rsAPI, + resolvedState, + backwardsExtremity.Headered(roomVersion), + t.haveEventIDs(), + ) + if err != nil { + return fmt.Errorf("api.SendEventWithState: %w", err) + } + + // Then send all of the newer backfilled events, of which will all be newer + // than the backward extremity, into the roomserver without state. This way + // they will automatically fast-forward based on the room state at the + // extremity in the last step. + headeredNewEvents := make([]gomatrixserverlib.HeaderedEvent, len(newEvents)) + for i, newEvent := range newEvents { + headeredNewEvents[i] = newEvent.Headered(roomVersion) + } + if err = api.SendEvents( + context.Background(), + t.rsAPI, + append(headeredNewEvents, e.Headered(roomVersion)), + api.DoNotSendToOtherServers, + nil, + ); err != nil { + return fmt.Errorf("api.SendEvents: %w", err) + } + + return nil } // lookupStateAfterEvent returns the room state after `eventID`, which is the state before eventID with the state of `eventID` (if it's a state event) @@ -652,11 +699,7 @@ retryAllowedState: // This function recursively calls txnReq.processEvent with the missing events, which will be processed before this function returns. // This means that we may recursively call this function, as we spider back up prev_events. // nolint:gocyclo -func (t *txnReq) getMissingEvents(ctx context.Context, e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion, isInboundTxn bool) (backwardsExtremity *gomatrixserverlib.Event, err error) { - if !isInboundTxn { - // we've recursed here, so just take a state snapshot please! - return &e, nil - } +func (t *txnReq) getMissingEvents(ctx context.Context, e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) (newEvents []gomatrixserverlib.Event, err error) { logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID()) needed := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{e}) // query latest events (our trusted forward extremities) @@ -667,7 +710,7 @@ func (t *txnReq) getMissingEvents(ctx context.Context, e gomatrixserverlib.Event var res api.QueryLatestEventsAndStateResponse if err = t.rsAPI.QueryLatestEventsAndState(ctx, &req, &res); err != nil { logger.WithError(err).Warn("Failed to query latest events") - return &e, nil + return nil, err } latestEvents := make([]string, len(res.LatestEvents)) for i := range res.LatestEvents { @@ -726,7 +769,7 @@ func (t *txnReq) getMissingEvents(ctx context.Context, e gomatrixserverlib.Event logger.Infof("get_missing_events returned %d events", len(missingResp.Events)) // topologically sort and sanity check that we are making forward progress - newEvents := gomatrixserverlib.ReverseTopologicalOrdering(missingResp.Events, gomatrixserverlib.TopologicalOrderByPrevEvents) + newEvents = gomatrixserverlib.ReverseTopologicalOrdering(missingResp.Events, gomatrixserverlib.TopologicalOrderByPrevEvents) shouldHaveSomeEventIDs := e.PrevEventIDs() hasPrevEvent := false Event: @@ -749,16 +792,9 @@ Event: err: err, } } - // process the missing events then the event which started this whole thing - for _, ev := range append(newEvents, e) { - err := t.processEvent(ctx, ev, false) - if err != nil { - return nil, err - } - } // we processed everything! - return nil, nil + return newEvents, nil } func (t *txnReq) lookupMissingStateViaState(ctx context.Context, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion) ( diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index ba653c1e8..d7e422479 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -516,6 +516,23 @@ func TestTransactionFetchMissingPrevEvents(t *testing.T) { var rsAPI *testRoomserverAPI // ref here so we can refer to inputRoomEvents inside these functions rsAPI = &testRoomserverAPI{ + queryEventsByID: func(req *api.QueryEventsByIDRequest) api.QueryEventsByIDResponse { + res := api.QueryEventsByIDResponse{} + for _, ev := range testEvents { + for _, id := range req.EventIDs { + if ev.EventID() == id { + res.Events = append(res.Events, ev) + } + } + } + return res + }, + queryStateAfterEvents: func(req *api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse { + return api.QueryStateAfterEventsResponse{ + PrevEventsExist: true, + StateEvents: testEvents[:5], + } + }, queryMissingAuthPrevEvents: func(req *api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse { missingPrevEvent := []string{"missing_prev_event"} if len(req.PrevEventIDs) == 1 { diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 736604217..810511505 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -122,7 +122,7 @@ func (r *Queryer) QueryMissingAuthPrevEvents( } for _, prevEventID := range request.PrevEventIDs { - if nids, err := r.DB.EventNIDs(ctx, []string{prevEventID}); err != nil || len(nids) == 0 { + if state, err := r.DB.StateAtEventIDs(ctx, []string{prevEventID}); err != nil || len(state) == 0 { response.MissingPrevEventIDs = append(response.MissingPrevEventIDs, prevEventID) } } From 2f578531baf5c033ae785501ee955381862ed314 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 13 Oct 2020 09:50:11 +0100 Subject: [PATCH 073/104] Update Docker samples --- build/docker/README.md | 27 +++++++++++++++++++----- build/docker/config/dendrite-config.yaml | 2 +- build/docker/postgres/create_db.sh | 2 +- 3 files changed, 24 insertions(+), 7 deletions(-) mode change 100644 => 100755 build/docker/postgres/create_db.sh diff --git a/build/docker/README.md b/build/docker/README.md index 45d96d1cb..7bf72e156 100644 --- a/build/docker/README.md +++ b/build/docker/README.md @@ -38,21 +38,38 @@ go run github.com/matrix-org/dendrite/cmd/generate-keys \ --tls-key=server.key ``` -## Starting Dendrite +## Starting Dendrite as a monolith deployment -Once in place, start the dependencies: +Create your config based on the `dendrite.yaml` configuration file in the `docker/config` +folder in the [Dendrite repository](https://github.com/matrix-org/dendrite). Additionally, +make the following changes to the configuration: + +- Enable Naffka: `use_naffka: true` + +Once in place, start the PostgreSQL dependency: ``` -docker-compose -f docker-compose.deps.yml up +docker-compose -f docker-compose.deps.yml up postgres ``` -Wait a few seconds for Kafka and Postgres to finish starting up, and then start a monolith: +Wait a few seconds for PostgreSQL to finish starting up, and then start a monolith: ``` docker-compose -f docker-compose.monolith.yml up ``` -... or start the polylith components: +## Starting Dendrite as a polylith deployment + +Create your config based on the `dendrite.yaml` configuration file in the `docker/config` +folder in the [Dendrite repository](https://github.com/matrix-org/dendrite). + +Once in place, start all the dependencies: + +``` +docker-compose -f docker-compose.deps.yml up +``` + +Wait a few seconds for PostgreSQL and Kafka to finish starting up, and then start a polylith: ``` docker-compose -f docker-compose.polylith.yml up diff --git a/build/docker/config/dendrite-config.yaml b/build/docker/config/dendrite-config.yaml index 2dc2f3b7d..106ab20dd 100644 --- a/build/docker/config/dendrite-config.yaml +++ b/build/docker/config/dendrite-config.yaml @@ -76,7 +76,7 @@ global: # Naffka database options. Not required when using Kafka. naffka_database: - connection_string: file:naffka.db + connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_naffka?sslmode=disable max_open_conns: 100 max_idle_conns: 2 conn_max_lifetime: -1 diff --git a/build/docker/postgres/create_db.sh b/build/docker/postgres/create_db.sh old mode 100644 new mode 100755 index f8ee715a9..97514467b --- a/build/docker/postgres/create_db.sh +++ b/build/docker/postgres/create_db.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh for db in account device mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice e2ekey naffka; do createdb -U dendrite -O dendrite dendrite_$db From 73bc28b11f10a7ff1582b9e5b9cffe53cd029104 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 13 Oct 2020 09:54:07 +0100 Subject: [PATCH 074/104] Update docker-compose.deps.yml --- build/docker/docker-compose.deps.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/docker/docker-compose.deps.yml b/build/docker/docker-compose.deps.yml index facfc01b3..afc572d0c 100644 --- a/build/docker/docker-compose.deps.yml +++ b/build/docker/docker-compose.deps.yml @@ -6,6 +6,9 @@ services: restart: always volumes: - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh + # To persist your PostgreSQL databases outside of the Docker image, to + # prevent data loss, you will need to add something like this: + # - ./path/to/persistent/storage:/var/lib/postgresql/data environment: POSTGRES_PASSWORD: itsasecret POSTGRES_USER: dendrite From d7ea814fa80ea2aba671be17c7d985d1191fbf6a Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 13 Oct 2020 10:20:27 +0100 Subject: [PATCH 075/104] Wrap NewMembershipUpdater in a db writer (#1515) --- internal/sqlutil/trace.go | 2 +- .../storage/shared/membership_updater.go | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/internal/sqlutil/trace.go b/internal/sqlutil/trace.go index 23359b500..0684e92e1 100644 --- a/internal/sqlutil/trace.go +++ b/internal/sqlutil/trace.go @@ -93,7 +93,7 @@ func trackGoID(query string) { if strings.HasPrefix(q, "SELECT") { return // SELECTs can go on other goroutines } - logrus.Warnf("unsafe goid: SQL executed not on an ExclusiveWriter: %s", q) + logrus.Warnf("unsafe goid %d: SQL executed not on an ExclusiveWriter: %s", thisGoID, q) } // Open opens a database specified by its database driver name and a driver-specific data source name, diff --git a/roomserver/storage/shared/membership_updater.go b/roomserver/storage/shared/membership_updater.go index 834af6069..7abddd018 100644 --- a/roomserver/storage/shared/membership_updater.go +++ b/roomserver/storage/shared/membership_updater.go @@ -22,12 +22,21 @@ func NewMembershipUpdater( ctx context.Context, d *Database, txn *sql.Tx, roomID, targetUserID string, targetLocal bool, roomVersion gomatrixserverlib.RoomVersion, ) (*MembershipUpdater, error) { - roomNID, err := d.assignRoomNID(ctx, txn, roomID, roomVersion) - if err != nil { - return nil, err - } + var roomNID types.RoomNID + var targetUserNID types.EventStateKeyNID + var err error + err = d.Writer.Do(d.DB, txn, func(txn *sql.Tx) error { + roomNID, err = d.assignRoomNID(ctx, txn, roomID, roomVersion) + if err != nil { + return err + } - targetUserNID, err := d.assignStateKeyNID(ctx, txn, targetUserID) + targetUserNID, err = d.assignStateKeyNID(ctx, txn, targetUserID) + if err != nil { + return err + } + return nil + }) if err != nil { return nil, err } From 9d6b77c58afd64be07ae82f05e11f5671d936ba8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 13 Oct 2020 11:53:20 +0100 Subject: [PATCH 076/104] Try to retrieve missing auth events from multiple servers (#1516) * Recursively fetch auth events if needed * Fix processEvent call * Ask more servers in lookupEvent * Don't panic! * Panic at the Disco * Find servers more aggressively * Add getServers * Fix number of servers to 5, don't bail making RespState if auth events missing * Fix panic * Ignore missing state events too * Report number of servers correctly * Don't reuse request context for /send_join * Update federation API tests * Don't recurse processEvents * Implement getEvents differently --- federationapi/routing/send.go | 189 +++++++++++++++++---------- federationapi/routing/send_test.go | 2 +- federationsender/internal/perform.go | 14 +- 3 files changed, 130 insertions(+), 75 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index a6e39821a..24e29a18d 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -246,9 +246,6 @@ func isProcessingErrorFatal(err error) bool { type roomNotFoundError struct { roomID string } -type unmarshalError struct { - err error -} type verifySigError struct { eventID string err error @@ -259,7 +256,6 @@ type missingPrevEventsError struct { } func (e roomNotFoundError) Error() string { return fmt.Sprintf("room %q not found", e.roomID) } -func (e unmarshalError) Error() string { return fmt.Sprintf("unable to parse event: %s", e.err) } func (e verifySigError) Error() string { return fmt.Sprintf("unable to verify signature of event %q: %s", e.eventID, e.err) } @@ -338,6 +334,19 @@ func (t *txnReq) processDeviceListUpdate(ctx context.Context, e gomatrixserverli } } +func (t *txnReq) getServers(ctx context.Context, roomID string) []gomatrixserverlib.ServerName { + servers := []gomatrixserverlib.ServerName{t.Origin} + serverReq := &api.QueryServerJoinedToRoomRequest{ + RoomID: roomID, + } + serverRes := &api.QueryServerJoinedToRoomResponse{} + if err := t.rsAPI.QueryServerJoinedToRoom(ctx, serverReq, serverRes); err == nil { + servers = append(servers, serverRes.ServerNames...) + util.GetLogger(ctx).Infof("Found %d server(s) to query for missing events in %q", len(servers), roomID) + } + return servers +} + func (t *txnReq) processEvent(ctx context.Context, e gomatrixserverlib.Event) error { logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID()) @@ -354,7 +363,7 @@ func (t *txnReq) processEvent(ctx context.Context, e gomatrixserverlib.Event) er } var stateResp api.QueryMissingAuthPrevEventsResponse if err := t.rsAPI.QueryMissingAuthPrevEvents(ctx, &stateReq, &stateResp); err != nil { - return err + return fmt.Errorf("t.rsAPI.QueryMissingAuthPrevEvents: %w", err) } if !stateResp.RoomExists { @@ -369,46 +378,8 @@ func (t *txnReq) processEvent(ctx context.Context, e gomatrixserverlib.Event) er if len(stateResp.MissingAuthEventIDs) > 0 { logger.Infof("Event refers to %d unknown auth_events", len(stateResp.MissingAuthEventIDs)) - - servers := []gomatrixserverlib.ServerName{t.Origin} - serverReq := &api.QueryServerJoinedToRoomRequest{ - RoomID: e.RoomID(), - } - serverRes := &api.QueryServerJoinedToRoomResponse{} - if err := t.rsAPI.QueryServerJoinedToRoom(ctx, serverReq, serverRes); err == nil { - servers = append(servers, serverRes.ServerNames...) - logger.Infof("Found %d server(s) to query for missing events", len(servers)) - } - - getAuthEvent: - for _, missingAuthEventID := range stateResp.MissingAuthEventIDs { - for _, server := range servers { - logger.Infof("Retrieving missing auth event %q from %q", missingAuthEventID, server) - tx, err := t.federation.GetEvent(ctx, server, missingAuthEventID) - if err != nil { - continue // try the next server - } - ev, err := gomatrixserverlib.NewEventFromUntrustedJSON(tx.PDUs[0], stateResp.RoomVersion) - if err != nil { - logger.WithError(err).Errorf("Failed to unmarshal auth event %q", missingAuthEventID) - continue // try the next server - } - if err = api.SendInputRoomEvents( - context.Background(), - t.rsAPI, - []api.InputRoomEvent{ - { - Kind: api.KindOutlier, - Event: ev.Headered(stateResp.RoomVersion), - AuthEventIDs: ev.AuthEventIDs(), - SendAsServer: api.DoNotSendToOtherServers, - }, - }, - ); err != nil { - logger.WithError(err).Errorf("Failed to send auth event %q to roomserver", missingAuthEventID) - continue getAuthEvent // move onto the next event - } - } + if err := t.retrieveMissingAuthEvents(ctx, e, &stateResp); err != nil { + return fmt.Errorf("t.retrieveMissingAuthEvents: %w", err) } } @@ -431,6 +402,60 @@ func (t *txnReq) processEvent(ctx context.Context, e gomatrixserverlib.Event) er ) } +func (t *txnReq) retrieveMissingAuthEvents( + ctx context.Context, e gomatrixserverlib.Event, stateResp *api.QueryMissingAuthPrevEventsResponse, +) error { + logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID()) + + missingAuthEvents := make(map[string]struct{}) + for _, missingAuthEventID := range stateResp.MissingAuthEventIDs { + missingAuthEvents[missingAuthEventID] = struct{}{} + } + + servers := t.getServers(ctx, e.RoomID()) + if len(servers) > 5 { + servers = servers[:5] + } +withNextEvent: + for missingAuthEventID := range missingAuthEvents { + withNextServer: + for _, server := range servers { + logger.Infof("Retrieving missing auth event %q from %q", missingAuthEventID, server) + tx, err := t.federation.GetEvent(ctx, server, missingAuthEventID) + if err != nil { + logger.WithError(err).Warnf("Failed to retrieve auth event %q", missingAuthEventID) + continue withNextServer + } + ev, err := gomatrixserverlib.NewEventFromUntrustedJSON(tx.PDUs[0], stateResp.RoomVersion) + if err != nil { + logger.WithError(err).Warnf("Failed to unmarshal auth event %q", missingAuthEventID) + continue withNextServer + } + if err = api.SendInputRoomEvents( + context.Background(), + t.rsAPI, + []api.InputRoomEvent{ + { + Kind: api.KindOutlier, + Event: ev.Headered(stateResp.RoomVersion), + AuthEventIDs: ev.AuthEventIDs(), + SendAsServer: api.DoNotSendToOtherServers, + }, + }, + ); err != nil { + return fmt.Errorf("api.SendEvents: %w", err) + } + delete(missingAuthEvents, missingAuthEventID) + continue withNextEvent + } + } + + if missing := len(missingAuthEvents); missing > 0 { + return fmt.Errorf("Event refers to %d auth_events which we failed to fetch", missing) + } + return nil +} + func checkAllowedByState(e gomatrixserverlib.Event, stateEvents []gomatrixserverlib.Event) error { authUsingState := gomatrixserverlib.NewAuthEvents(nil) for i := range stateEvents { @@ -557,18 +582,23 @@ func (t *txnReq) lookupStateAfterEvent(ctx context.Context, roomVersion gomatrix respState, err := t.lookupStateBeforeEvent(ctx, roomVersion, roomID, eventID) if err != nil { - return nil, err + return nil, fmt.Errorf("t.lookupStateBeforeEvent: %w", err) + } + + servers := t.getServers(ctx, roomID) + if len(servers) > 5 { + servers = servers[:5] } // fetch the event we're missing and add it to the pile - h, err := t.lookupEvent(ctx, roomVersion, eventID, false) + h, err := t.lookupEvent(ctx, roomVersion, eventID, false, servers) switch err.(type) { case verifySigError: return respState, nil case nil: // do nothing default: - return nil, err + return nil, fmt.Errorf("t.lookupEvent: %w", err) } t.haveEvents[h.EventID()] = h if h.StateKey() != nil { @@ -669,7 +699,11 @@ retryAllowedState: if err = checkAllowedByState(*backwardsExtremity, resolvedStateEvents); err != nil { switch missing := err.(type) { case gomatrixserverlib.MissingAuthEventError: - h, err2 := t.lookupEvent(ctx, roomVersion, missing.AuthEventID, true) + servers := t.getServers(ctx, backwardsExtremity.RoomID()) + if len(servers) > 5 { + servers = servers[:5] + } + h, err2 := t.lookupEvent(ctx, roomVersion, missing.AuthEventID, true, servers) switch err2.(type) { case verifySigError: return &gomatrixserverlib.RespState{ @@ -874,6 +908,12 @@ func (t *txnReq) lookupMissingStateViaStateIDs(ctx context.Context, roomID, even "concurrent_requests": concurrentRequests, }).Info("Fetching missing state at event") + // Get a list of servers to fetch from. + servers := t.getServers(ctx, roomID) + if len(servers) > 5 { + servers = servers[:5] + } + // Create a queue containing all of the missing event IDs that we want // to retrieve. pending := make(chan string, missingCount) @@ -899,7 +939,7 @@ func (t *txnReq) lookupMissingStateViaStateIDs(ctx context.Context, roomID, even // Define what we'll do in order to fetch the missing event ID. fetch := func(missingEventID string) { var h *gomatrixserverlib.HeaderedEvent - h, err = t.lookupEvent(ctx, roomVersion, missingEventID, false) + h, err = t.lookupEvent(ctx, roomVersion, missingEventID, false, servers) switch err.(type) { case verifySigError: return @@ -937,26 +977,25 @@ func (t *txnReq) lookupMissingStateViaStateIDs(ctx context.Context, roomID, even } func (t *txnReq) createRespStateFromStateIDs(stateIDs gomatrixserverlib.RespStateIDs) ( - *gomatrixserverlib.RespState, error) { + *gomatrixserverlib.RespState, error) { // nolint:unparam // create a RespState response using the response to /state_ids as a guide - respState := gomatrixserverlib.RespState{ - AuthEvents: make([]gomatrixserverlib.Event, len(stateIDs.AuthEventIDs)), - StateEvents: make([]gomatrixserverlib.Event, len(stateIDs.StateEventIDs)), - } + respState := gomatrixserverlib.RespState{} for i := range stateIDs.StateEventIDs { ev, ok := t.haveEvents[stateIDs.StateEventIDs[i]] if !ok { - return nil, fmt.Errorf("missing state event %s", stateIDs.StateEventIDs[i]) + logrus.Warnf("Missing state event in createRespStateFromStateIDs: %s", stateIDs.StateEventIDs[i]) + continue } - respState.StateEvents[i] = ev.Unwrap() + respState.StateEvents = append(respState.StateEvents, ev.Unwrap()) } for i := range stateIDs.AuthEventIDs { ev, ok := t.haveEvents[stateIDs.AuthEventIDs[i]] if !ok { - return nil, fmt.Errorf("missing auth event %s", stateIDs.AuthEventIDs[i]) + logrus.Warnf("Missing auth event in createRespStateFromStateIDs: %s", stateIDs.AuthEventIDs[i]) + continue } - respState.AuthEvents[i] = ev.Unwrap() + respState.AuthEvents = append(respState.AuthEvents, ev.Unwrap()) } // We purposefully do not do auth checks on the returned events, as they will still // be processed in the exact same way, just as a 'rejected' event @@ -964,7 +1003,7 @@ func (t *txnReq) createRespStateFromStateIDs(stateIDs gomatrixserverlib.RespStat return &respState, nil } -func (t *txnReq) lookupEvent(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, missingEventID string, localFirst bool) (*gomatrixserverlib.HeaderedEvent, error) { +func (t *txnReq) lookupEvent(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, missingEventID string, localFirst bool, servers []gomatrixserverlib.ServerName) (*gomatrixserverlib.HeaderedEvent, error) { if localFirst { // fetch from the roomserver queryReq := api.QueryEventsByIDRequest{ @@ -977,19 +1016,27 @@ func (t *txnReq) lookupEvent(ctx context.Context, roomVersion gomatrixserverlib. return &queryRes.Events[0], nil } } - txn, err := t.federation.GetEvent(ctx, t.Origin, missingEventID) - if err != nil || len(txn.PDUs) == 0 { - util.GetLogger(ctx).WithError(err).WithField("event_id", missingEventID).Warn("failed to get missing /event for event ID") - return nil, err - } - pdu := txn.PDUs[0] var event gomatrixserverlib.Event - event, err = gomatrixserverlib.NewEventFromUntrustedJSON(pdu, roomVersion) - if err != nil { - util.GetLogger(ctx).WithError(err).Warnf("Transaction: Failed to parse event JSON of event %q", event.EventID()) - return nil, unmarshalError{err} + found := false + for _, serverName := range servers { + txn, err := t.federation.GetEvent(ctx, serverName, missingEventID) + if err != nil || len(txn.PDUs) == 0 { + util.GetLogger(ctx).WithError(err).WithField("event_id", missingEventID).Warn("Failed to get missing /event for event ID") + continue + } + event, err = gomatrixserverlib.NewEventFromUntrustedJSON(txn.PDUs[0], roomVersion) + if err != nil { + util.GetLogger(ctx).WithError(err).WithField("event_id", missingEventID).Warnf("Transaction: Failed to parse event JSON of event") + continue + } + found = true + break } - if err = gomatrixserverlib.VerifyAllEventSignatures(ctx, []gomatrixserverlib.Event{event}, t.keys); err != nil { + if !found { + util.GetLogger(ctx).WithField("event_id", missingEventID).Warnf("Failed to get missing /event for event ID from %d server(s)", len(servers)) + return nil, fmt.Errorf("wasn't able to find event via %d server(s)", len(servers)) + } + if err := gomatrixserverlib.VerifyAllEventSignatures(ctx, []gomatrixserverlib.Event{event}, t.keys); err != nil { util.GetLogger(ctx).WithError(err).Warnf("Transaction: Couldn't validate signature of event %q", event.EventID()) return nil, verifySigError{event.EventID(), err} } diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index d7e422479..0a462433c 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -491,7 +491,7 @@ func TestTransactionFailAuthChecks(t *testing.T) { queryMissingAuthPrevEvents: func(req *api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse { return api.QueryMissingAuthPrevEventsResponse{ RoomExists: true, - MissingAuthEventIDs: []string{"create_event"}, + MissingAuthEventIDs: []string{}, MissingPrevEventIDs: []string{}, } }, diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index bff6833ca..254883e63 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -196,6 +196,11 @@ func (r *FederationSenderInternalAPI) performJoinUsingServer( return fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err) } + // No longer reuse the request context from this point forward. + // We don't want the client timing out to interrupt the join. + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(context.Background()) + // Try to perform a send_join using the newly built event. respSendJoin, err := r.federation.SendJoin( ctx, @@ -205,11 +210,16 @@ func (r *FederationSenderInternalAPI) performJoinUsingServer( ) if err != nil { r.statistics.ForServer(serverName).Failure() + cancel() return fmt.Errorf("r.federation.SendJoin: %w", err) } r.statistics.ForServer(serverName).Success() + + // Sanity-check the join response to ensure that it has a create + // event, that the room version is known, etc. if err := sanityCheckSendJoinResponse(respSendJoin); err != nil { - return err + cancel() + return fmt.Errorf("sanityCheckSendJoinResponse: %w", err) } // Process the join response in a goroutine. The idea here is @@ -217,8 +227,6 @@ func (r *FederationSenderInternalAPI) performJoinUsingServer( // to complete, but if the client does give up waiting, we'll // still continue to process the join anyway so that we don't // waste the effort. - var cancel context.CancelFunc - ctx, cancel = context.WithCancel(context.Background()) go func() { defer cancel() From 20aec70eadca81de61701bfda506319d593376d2 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 14 Oct 2020 09:57:27 +0100 Subject: [PATCH 077/104] Send cumulative state when creating room (#1519) * Send state with new room events * lookupEvent sends outliers * Revert "lookupEvent sends outliers" This reverts commit 3e1655644105a542b806e28d6d2536fbd23ecc83. --- clientapi/routing/createroom.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index af43064fe..9655339cd 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -339,12 +339,21 @@ func createRoom( util.GetLogger(req.Context()).WithError(err).Error("authEvents.AddEvent failed") return jsonerror.InternalServerError() } - } - // send events to the room server - if err = roomserverAPI.SendEvents(req.Context(), rsAPI, builtEvents, cfg.Matrix.ServerName, nil); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") - return jsonerror.InternalServerError() + accumulated := gomatrixserverlib.UnwrapEventHeaders(builtEvents) + if err = roomserverAPI.SendEventWithState( + req.Context(), + rsAPI, + &gomatrixserverlib.RespState{ + StateEvents: accumulated, + AuthEvents: accumulated, + }, + ev.Headered(roomVersion), + nil, + ); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("SendEventWithState failed") + return jsonerror.InternalServerError() + } } // TODO(#269): Reserve room alias while we create the room. This stops us From 7a1fd123debad64bf806503d519efa7bf3c85066 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 14 Oct 2020 12:39:37 +0100 Subject: [PATCH 078/104] Improved state handling in /send (#1521) * Capture errors * Don't request only state key tuples needed for auth (we end up discarding room state this way) * QueryStateAfterEvent returns all state when no tuples supplied * Resolve state * Comments --- federationapi/routing/send.go | 10 +++--- roomserver/api/query.go | 3 +- .../internal/input/input_latest_events.go | 18 ++++++---- roomserver/internal/query/query.go | 35 ++++++++++++++++--- 4 files changed, 48 insertions(+), 18 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 24e29a18d..fa2a7bbb6 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -508,13 +508,12 @@ func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixser // Therefore, we cannot just query /state_ids with this event to get the state before. Instead, we need to query // the state AFTER all the prev_events for this event, then apply state resolution to that to get the state before the event. var states []*gomatrixserverlib.RespState - needed := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{*backwardsExtremity}).Tuples() for _, prevEventID := range backwardsExtremity.PrevEventIDs() { // Look up what the state is after the backward extremity. This will either // come from the roomserver, if we know all the required events, or it will // come from a remote server via /state_ids if not. var prevState *gomatrixserverlib.RespState - prevState, err = t.lookupStateAfterEvent(gmectx, roomVersion, backwardsExtremity.RoomID(), prevEventID, needed) + prevState, err = t.lookupStateAfterEvent(gmectx, roomVersion, backwardsExtremity.RoomID(), prevEventID) if err != nil { util.GetLogger(ctx).WithError(err).Errorf("Failed to lookup state after prev_event: %s", prevEventID) return err @@ -573,9 +572,9 @@ func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixser // lookupStateAfterEvent returns the room state after `eventID`, which is the state before eventID with the state of `eventID` (if it's a state event) // added into the mix. -func (t *txnReq) lookupStateAfterEvent(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string, needed []gomatrixserverlib.StateKeyTuple) (*gomatrixserverlib.RespState, error) { +func (t *txnReq) lookupStateAfterEvent(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (*gomatrixserverlib.RespState, error) { // try doing all this locally before we resort to querying federation - respState := t.lookupStateAfterEventLocally(ctx, roomID, eventID, needed) + respState := t.lookupStateAfterEventLocally(ctx, roomID, eventID) if respState != nil { return respState, nil } @@ -619,12 +618,11 @@ func (t *txnReq) lookupStateAfterEvent(ctx context.Context, roomVersion gomatrix return respState, nil } -func (t *txnReq) lookupStateAfterEventLocally(ctx context.Context, roomID, eventID string, needed []gomatrixserverlib.StateKeyTuple) *gomatrixserverlib.RespState { +func (t *txnReq) lookupStateAfterEventLocally(ctx context.Context, roomID, eventID string) *gomatrixserverlib.RespState { var res api.QueryStateAfterEventsResponse err := t.rsAPI.QueryStateAfterEvents(ctx, &api.QueryStateAfterEventsRequest{ RoomID: roomID, PrevEventIDs: []string{eventID}, - StateToFetch: needed, }, &res) if err != nil || !res.PrevEventsExist { util.GetLogger(ctx).WithError(err).Warnf("failed to query state after %s locally", eventID) diff --git a/roomserver/api/query.go b/roomserver/api/query.go index aff6ee07a..3afca7e81 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -63,7 +63,8 @@ type QueryStateAfterEventsRequest struct { RoomID string `json:"room_id"` // The list of previous events to return the events after. PrevEventIDs []string `json:"prev_event_ids"` - // The state key tuples to fetch from the state + // The state key tuples to fetch from the state. If none are specified then + // the entire resolved room state will be returned. StateToFetch []gomatrixserverlib.StateKeyTuple `json:"state_to_fetch"` } diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index 229665a0b..ca5d214d7 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -133,8 +133,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { // If the event has already been written to the output log then we // don't need to do anything, as we've handled it already. - hasBeenSent, err := u.updater.HasEventBeenSent(u.stateAtEvent.EventNID) - if err != nil { + if hasBeenSent, err := u.updater.HasEventBeenSent(u.stateAtEvent.EventNID); err != nil { return fmt.Errorf("u.updater.HasEventBeenSent: %w", err) } else if hasBeenSent { return nil @@ -142,17 +141,19 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { // Work out what the latest events are. This will include the new // event if it is not already referenced. - u.calculateLatest( + if err := u.calculateLatest( oldLatest, types.StateAtEventAndReference{ EventReference: u.event.EventReference(), StateAtEvent: u.stateAtEvent, }, - ) + ); err != nil { + return fmt.Errorf("u.calculateLatest: %w", err) + } // Now that we know what the latest events are, it's time to get the // latest state. - if err = u.latestState(); err != nil { + if err := u.latestState(); err != nil { return fmt.Errorf("u.latestState: %w", err) } @@ -261,7 +262,7 @@ func (u *latestEventsUpdater) latestState() error { func (u *latestEventsUpdater) calculateLatest( oldLatest []types.StateAtEventAndReference, newEvent types.StateAtEventAndReference, -) { +) error { var newLatest []types.StateAtEventAndReference // First of all, let's see if any of the existing forward extremities @@ -271,6 +272,7 @@ func (u *latestEventsUpdater) calculateLatest( referenced, err := u.updater.IsReferenced(l.EventReference) if err != nil { logrus.WithError(err).Errorf("Failed to retrieve event reference for %q", l.EventID) + return fmt.Errorf("u.updater.IsReferenced (old): %w", err) } else if !referenced { newLatest = append(newLatest, l) } @@ -285,7 +287,7 @@ func (u *latestEventsUpdater) calculateLatest( // We've already referenced this new event so we can just return // the newly completed extremities at this point. u.latest = newLatest - return + return nil } } @@ -296,11 +298,13 @@ func (u *latestEventsUpdater) calculateLatest( referenced, err := u.updater.IsReferenced(newEvent.EventReference) if err != nil { logrus.WithError(err).Errorf("Failed to retrieve event reference for %q", newEvent.EventReference.EventID) + return fmt.Errorf("u.updater.IsReferenced (new): %w", err) } else if !referenced || len(newLatest) == 0 { newLatest = append(newLatest, newEvent) } u.latest = newLatest + return nil } func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) { diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 810511505..ecfb580f2 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -49,6 +49,7 @@ func (r *Queryer) QueryLatestEventsAndState( } // QueryStateAfterEvents implements api.RoomserverInternalAPI +// nolint:gocyclo func (r *Queryer) QueryStateAfterEvents( ctx context.Context, request *api.QueryStateAfterEventsRequest, @@ -78,10 +79,18 @@ func (r *Queryer) QueryStateAfterEvents( } response.PrevEventsExist = true - // Look up the currrent state for the requested tuples. - stateEntries, err := roomState.LoadStateAfterEventsForStringTuples( - ctx, prevStates, request.StateToFetch, - ) + var stateEntries []types.StateEntry + if len(request.StateToFetch) == 0 { + // Look up all of the current room state. + stateEntries, err = roomState.LoadCombinedStateAfterEvents( + ctx, prevStates, + ) + } else { + // Look up the current state for the requested tuples. + stateEntries, err = roomState.LoadStateAfterEventsForStringTuples( + ctx, prevStates, request.StateToFetch, + ) + } if err != nil { return err } @@ -91,6 +100,24 @@ func (r *Queryer) QueryStateAfterEvents( return err } + if len(request.PrevEventIDs) > 1 && len(request.StateToFetch) == 0 { + var authEventIDs []string + for _, e := range stateEvents { + authEventIDs = append(authEventIDs, e.AuthEventIDs()...) + } + authEventIDs = util.UniqueStrings(authEventIDs) + + authEvents, err := getAuthChain(ctx, r.DB.EventsFromIDs, authEventIDs) + if err != nil { + return fmt.Errorf("getAuthChain: %w", err) + } + + stateEvents, err = state.ResolveConflictsAdhoc(info.RoomVersion, stateEvents, authEvents) + if err != nil { + return fmt.Errorf("state.ResolveConflictsAdhoc: %w", err) + } + } + for _, event := range stateEvents { response.StateEvents = append(response.StateEvents, event.Headered(info.RoomVersion)) } From 286dd408ae623b218f9c5a97d53f174c0ea19cc2 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 14 Oct 2020 12:51:06 +0100 Subject: [PATCH 079/104] Better semver compliance for version reporting --- internal/version.go | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/internal/version.go b/internal/version.go index 2ffd7c90e..a9e245d44 100644 --- a/internal/version.go +++ b/internal/version.go @@ -1,6 +1,12 @@ package internal -import "fmt" +import ( + "fmt" + "strings" +) + +// the final version string +var version string // -ldflags "-X github.com/matrix-org/dendrite/internal.branch=master" var branch string @@ -16,12 +22,22 @@ const ( ) func VersionString() string { - version := fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionTag) - if branch != "" { - version += fmt.Sprintf("-%s", branch) - } - if build != "" { - version += fmt.Sprintf("+%s", build) - } return version } + +func init() { + version = fmt.Sprintf("%d.%d.%d", VersionMajor, VersionMinor, VersionPatch) + if VersionTag != "" { + version += "-" + VersionTag + } + parts := []string{} + if build != "" { + parts = append(parts, build) + } + if branch != "" { + parts = append(parts, branch) + } + if len(parts) > 0 { + version += "+" + strings.Join(parts, ".") + } +} From 8d9ecb3996730ebf8eb75191f17bddb3f96332c8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 14 Oct 2020 15:24:43 +0100 Subject: [PATCH 080/104] Ignore duplicate redaction entries (#1522) --- roomserver/storage/postgres/redactions_table.go | 3 ++- roomserver/storage/sqlite3/redactions_table.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/roomserver/storage/postgres/redactions_table.go b/roomserver/storage/postgres/redactions_table.go index 289e1320f..42aba5985 100644 --- a/roomserver/storage/postgres/redactions_table.go +++ b/roomserver/storage/postgres/redactions_table.go @@ -39,7 +39,8 @@ CREATE INDEX IF NOT EXISTS roomserver_redactions_redacts_event_id ON roomserver_ const insertRedactionSQL = "" + "INSERT INTO roomserver_redactions (redaction_event_id, redacts_event_id, validated)" + - " VALUES ($1, $2, $3)" + " VALUES ($1, $2, $3)" + + " ON CONFLICT DO NOTHING" const selectRedactionInfoByRedactionEventIDSQL = "" + "SELECT redaction_event_id, redacts_event_id, validated FROM roomserver_redactions" + diff --git a/roomserver/storage/sqlite3/redactions_table.go b/roomserver/storage/sqlite3/redactions_table.go index a2179357c..e64714862 100644 --- a/roomserver/storage/sqlite3/redactions_table.go +++ b/roomserver/storage/sqlite3/redactions_table.go @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS roomserver_redactions ( ` const insertRedactionSQL = "" + - "INSERT INTO roomserver_redactions (redaction_event_id, redacts_event_id, validated)" + + "INSERT OR IGNORE INTO roomserver_redactions (redaction_event_id, redacts_event_id, validated)" + " VALUES ($1, $2, $3)" const selectRedactionInfoByRedactionEventIDSQL = "" + From e3a3908654428c3b164159d3af5b1bddf3411df5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 14 Oct 2020 16:07:15 +0100 Subject: [PATCH 081/104] Update sytest-whitelist (closes #1315) --- sytest-whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/sytest-whitelist b/sytest-whitelist index 805f0e4dd..f4fb993af 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -482,3 +482,4 @@ m.room.history_visibility == "joined" allows/forbids appropriately for Real user POST rejects invalid utf-8 in JSON Users cannot kick users who have already left a room A prev_batch token from incremental sync can be used in the v1 messages API +Event with an invalid signature in the send_join response should not cause room join to fail From 6f12b8f85c6e244b026df8016e60315e99603d9d Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 14 Oct 2020 16:49:25 +0100 Subject: [PATCH 082/104] Ignore typing events where sender doesn't match origin (#1523) * Ignore typing notifications where the sender doesn't match the origin * Update sytest-whitelist * Fix formatting directives --- federationapi/routing/send.go | 9 +++++++++ sytest-whitelist | 1 + 2 files changed, 10 insertions(+) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index fa2a7bbb6..611a90a7c 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -289,6 +289,15 @@ func (t *txnReq) processEDUs(ctx context.Context) { util.GetLogger(ctx).WithError(err).Error("Failed to unmarshal typing event") continue } + _, domain, err := gomatrixserverlib.SplitID('@', typingPayload.UserID) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("Failed to split domain from typing event sender") + continue + } + if domain != t.Origin { + util.GetLogger(ctx).Warnf("Dropping typing event where sender domain (%q) doesn't match origin (%q)", domain, t.Origin) + continue + } if err := eduserverAPI.SendTyping(ctx, t.eduAPI, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil { util.GetLogger(ctx).WithError(err).Error("Failed to send typing event to edu server") } diff --git a/sytest-whitelist b/sytest-whitelist index f4fb993af..2ba0a88b2 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -483,3 +483,4 @@ POST rejects invalid utf-8 in JSON Users cannot kick users who have already left a room A prev_batch token from incremental sync can be used in the v1 messages API Event with an invalid signature in the send_join response should not cause room join to fail +Inbound federation rejects typing notifications from wrong remote From e3c2b081c7c197cad931d6525931dc9e960b93c3 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 14 Oct 2020 17:05:09 +0100 Subject: [PATCH 083/104] txn nil guard when rolling back LatestEventsUpdater (#1524) * txn nil guard when rolling back LatestEventsUpdater * Spell lint correctly --- roomserver/storage/shared/latest_events_updater.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/roomserver/storage/shared/latest_events_updater.go b/roomserver/storage/shared/latest_events_updater.go index 29eab0c98..b316f639d 100644 --- a/roomserver/storage/shared/latest_events_updater.go +++ b/roomserver/storage/shared/latest_events_updater.go @@ -18,23 +18,30 @@ type LatestEventsUpdater struct { currentStateSnapshotNID types.StateSnapshotNID } +func rollback(txn *sql.Tx) { + if txn == nil { + return + } + txn.Rollback() // nolint: errcheck +} + func NewLatestEventsUpdater(ctx context.Context, d *Database, txn *sql.Tx, roomInfo types.RoomInfo) (*LatestEventsUpdater, error) { eventNIDs, lastEventNIDSent, currentStateSnapshotNID, err := d.RoomsTable.SelectLatestEventsNIDsForUpdate(ctx, txn, roomInfo.RoomNID) if err != nil { - txn.Rollback() // nolint: errcheck + rollback(txn) return nil, err } stateAndRefs, err := d.EventsTable.BulkSelectStateAtEventAndReference(ctx, txn, eventNIDs) if err != nil { - txn.Rollback() // nolint: errcheck + rollback(txn) return nil, err } var lastEventIDSent string if lastEventNIDSent != 0 { lastEventIDSent, err = d.EventsTable.SelectEventID(ctx, txn, lastEventNIDSent) if err != nil { - txn.Rollback() // nolint: errcheck + rollback(txn) return nil, err } } From 10f1beb0de7a52ccdd122b05b4adffdbdab4ea2e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 15 Oct 2020 12:08:49 +0100 Subject: [PATCH 084/104] Don't re-run state resolution on a single trusted state snapshot (#1526) * Don't re-run state resolution on a single trusted state snapshot * Lint * Check if backward extremity is create event before checking missing state --- federationapi/routing/send.go | 72 ++++++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 18 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 611a90a7c..783fdc3b8 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -476,6 +476,7 @@ func checkAllowedByState(e gomatrixserverlib.Event, stateEvents []gomatrixserver return gomatrixserverlib.Allowed(e, &authUsingState) } +// nolint:gocyclo func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) error { // Do this with a fresh context, so that we keep working even if the // original request times out. With any luck, by the time the remote @@ -513,35 +514,70 @@ func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixser backwardsExtremity := &newEvents[0] newEvents = newEvents[1:] + type respState struct { + // A snapshot is considered trustworthy if it came from our own roomserver. + // That's because the state will have been through state resolution once + // already in QueryStateAfterEvent. + trustworthy bool + *gomatrixserverlib.RespState + } + // at this point we know we're going to have a gap: we need to work out the room state at the new backwards extremity. // Therefore, we cannot just query /state_ids with this event to get the state before. Instead, we need to query // the state AFTER all the prev_events for this event, then apply state resolution to that to get the state before the event. - var states []*gomatrixserverlib.RespState + var states []*respState for _, prevEventID := range backwardsExtremity.PrevEventIDs() { // Look up what the state is after the backward extremity. This will either // come from the roomserver, if we know all the required events, or it will // come from a remote server via /state_ids if not. - var prevState *gomatrixserverlib.RespState - prevState, err = t.lookupStateAfterEvent(gmectx, roomVersion, backwardsExtremity.RoomID(), prevEventID) - if err != nil { - util.GetLogger(ctx).WithError(err).Errorf("Failed to lookup state after prev_event: %s", prevEventID) - return err + prevState, trustworthy, lerr := t.lookupStateAfterEvent(gmectx, roomVersion, backwardsExtremity.RoomID(), prevEventID) + if lerr != nil { + util.GetLogger(ctx).WithError(lerr).Errorf("Failed to lookup state after prev_event: %s", prevEventID) + return lerr } // Append the state onto the collected state. We'll run this through the // state resolution next. - states = append(states, prevState) + states = append(states, &respState{trustworthy, prevState}) } // Now that we have collected all of the state from the prev_events, we'll // run the state through the appropriate state resolution algorithm for the - // room. This does a couple of things: + // room if needed. This does a couple of things: // 1. Ensures that the state is deduplicated fully for each state-key tuple // 2. Ensures that we pick the latest events from both sets, in the case that // one of the prev_events is quite a bit older than the others - resolvedState, err := t.resolveStatesAndCheck(gmectx, roomVersion, states, backwardsExtremity) - if err != nil { - util.GetLogger(ctx).WithError(err).Errorf("Failed to resolve state conflicts for event %s", backwardsExtremity.EventID()) - return err + resolvedState := &gomatrixserverlib.RespState{} + switch len(states) { + case 0: + extremityIsCreate := backwardsExtremity.Type() == gomatrixserverlib.MRoomCreate && backwardsExtremity.StateKeyEquals("") + if !extremityIsCreate { + // There are no previous states and this isn't the beginning of the + // room - this is an error condition! + util.GetLogger(ctx).Errorf("Failed to lookup any state after prev_events") + return fmt.Errorf("expected %d states but got %d", len(backwardsExtremity.PrevEventIDs()), len(states)) + } + case 1: + // There's only one previous state - if it's trustworthy (came from a + // local state snapshot which will already have been through state res), + // use it as-is. There's no point in resolving it again. + if states[0].trustworthy { + resolvedState = states[0].RespState + break + } + // Otherwise, if it isn't trustworthy (came from federation), run it through + // state resolution anyway for safety, in case there are duplicates. + fallthrough + default: + respStates := make([]*gomatrixserverlib.RespState, len(states)) + for i := range states { + respStates[i] = states[i].RespState + } + // There's more than one previous state - run them all through state res + resolvedState, err = t.resolveStatesAndCheck(gmectx, roomVersion, respStates, backwardsExtremity) + if err != nil { + util.GetLogger(ctx).WithError(err).Errorf("Failed to resolve state conflicts for event %s", backwardsExtremity.EventID()) + return err + } } // First of all, send the backward extremity into the roomserver with the @@ -581,16 +617,16 @@ func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixser // lookupStateAfterEvent returns the room state after `eventID`, which is the state before eventID with the state of `eventID` (if it's a state event) // added into the mix. -func (t *txnReq) lookupStateAfterEvent(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (*gomatrixserverlib.RespState, error) { +func (t *txnReq) lookupStateAfterEvent(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (*gomatrixserverlib.RespState, bool, error) { // try doing all this locally before we resort to querying federation respState := t.lookupStateAfterEventLocally(ctx, roomID, eventID) if respState != nil { - return respState, nil + return respState, true, nil } respState, err := t.lookupStateBeforeEvent(ctx, roomVersion, roomID, eventID) if err != nil { - return nil, fmt.Errorf("t.lookupStateBeforeEvent: %w", err) + return nil, false, fmt.Errorf("t.lookupStateBeforeEvent: %w", err) } servers := t.getServers(ctx, roomID) @@ -602,11 +638,11 @@ func (t *txnReq) lookupStateAfterEvent(ctx context.Context, roomVersion gomatrix h, err := t.lookupEvent(ctx, roomVersion, eventID, false, servers) switch err.(type) { case verifySigError: - return respState, nil + return respState, false, nil case nil: // do nothing default: - return nil, fmt.Errorf("t.lookupEvent: %w", err) + return nil, false, fmt.Errorf("t.lookupEvent: %w", err) } t.haveEvents[h.EventID()] = h if h.StateKey() != nil { @@ -624,7 +660,7 @@ func (t *txnReq) lookupStateAfterEvent(ctx context.Context, roomVersion gomatrix } } - return respState, nil + return respState, false, nil } func (t *txnReq) lookupStateAfterEventLocally(ctx context.Context, roomID, eventID string) *gomatrixserverlib.RespState { From 49abe359e6a2b0c3f214190b73404c5cf9a0e051 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 15 Oct 2020 13:27:13 +0100 Subject: [PATCH 085/104] Start Kafka connections for each component that needs them (#1527) * Start Kafka connection for each component that needs one * Fix roomserver unit tests * Rename to naffkaInstance (@Kegsay review comment) * Fix import cycle --- appservice/appservice.go | 5 ++- build/docker/docker-compose.deps.yml | 2 + build/gobind/monolith.go | 14 +++---- clientapi/clientapi.go | 5 ++- cmd/dendrite-client-api-server/main.go | 2 +- cmd/dendrite-demo-libp2p/main.go | 14 +++---- cmd/dendrite-demo-yggdrasil/main.go | 14 +++---- cmd/dendrite-key-server/main.go | 2 +- cmd/dendrite-monolith-server/main.go | 14 +++---- cmd/dendrite-sync-api-server/main.go | 3 +- cmd/dendritejs/main.go | 14 +++---- eduserver/eduserver.go | 6 ++- federationsender/federationsender.go | 9 +++-- internal/setup/base.go | 46 +--------------------- internal/setup/kafka/kafka.go | 53 ++++++++++++++++++++++++++ internal/setup/monolith.go | 17 ++++----- keyserver/keyserver.go | 6 ++- roomserver/roomserver.go | 5 ++- roomserver/roomserver_test.go | 22 ++++++++--- syncapi/syncapi.go | 5 ++- 20 files changed, 143 insertions(+), 115 deletions(-) create mode 100644 internal/setup/kafka/kafka.go diff --git a/appservice/appservice.go b/appservice/appservice.go index e356f68ee..cf9a47b74 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -30,6 +30,7 @@ import ( "github.com/matrix-org/dendrite/appservice/workers" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/dendrite/internal/setup/kafka" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/sirupsen/logrus" @@ -47,6 +48,8 @@ func NewInternalAPI( userAPI userapi.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, ) appserviceAPI.AppServiceQueryAPI { + consumer, _ := kafka.SetupConsumerProducer(&base.Cfg.Global.Kafka) + // Create a connection to the appservice postgres DB appserviceDB, err := storage.NewDatabase(&base.Cfg.AppServiceAPI.Database) if err != nil { @@ -86,7 +89,7 @@ func NewInternalAPI( // We can't add ASes at runtime so this is safe to do. if len(workerStates) > 0 { consumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, appserviceDB, + base.Cfg, consumer, appserviceDB, rsAPI, workerStates, ) if err := consumer.Start(); err != nil { diff --git a/build/docker/docker-compose.deps.yml b/build/docker/docker-compose.deps.yml index afc572d0c..74e478a8d 100644 --- a/build/docker/docker-compose.deps.yml +++ b/build/docker/docker-compose.deps.yml @@ -29,6 +29,8 @@ services: KAFKA_ADVERTISED_HOST_NAME: "kafka" KAFKA_DELETE_TOPIC_ENABLE: "true" KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" + ports: + - 9092:9092 depends_on: - zookeeper networks: diff --git a/build/gobind/monolith.go b/build/gobind/monolith.go index 7d10b87e4..fd010809c 100644 --- a/build/gobind/monolith.go +++ b/build/gobind/monolith.go @@ -112,7 +112,7 @@ func (m *DendriteMonolith) Start() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation, base.KafkaProducer) + keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation) userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI) keyAPI.SetUserAPI(userAPI) @@ -146,13 +146,11 @@ func (m *DendriteMonolith) Start() { rsAPI.SetFederationSenderAPI(fsAPI) monolith := setup.Monolith{ - Config: base.Cfg, - AccountDB: accountDB, - Client: ygg.CreateClient(base), - FedClient: federation, - KeyRing: keyRing, - KafkaConsumer: base.KafkaConsumer, - KafkaProducer: base.KafkaProducer, + Config: base.Cfg, + AccountDB: accountDB, + Client: ygg.CreateClient(base), + FedClient: federation, + KeyRing: keyRing, AppserviceAPI: asAPI, EDUInternalAPI: eduInputAPI, diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 2ab92ed4e..ebe55aec9 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -15,7 +15,6 @@ package clientapi import ( - "github.com/Shopify/sarama" "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/api" @@ -24,6 +23,7 @@ import ( eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/setup/kafka" "github.com/matrix-org/dendrite/internal/transactions" keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" @@ -36,7 +36,6 @@ import ( func AddPublicRoutes( router *mux.Router, cfg *config.ClientAPI, - producer sarama.SyncProducer, accountsDB accounts.Database, federation *gomatrixserverlib.FederationClient, rsAPI roomserverAPI.RoomserverInternalAPI, @@ -48,6 +47,8 @@ func AddPublicRoutes( keyAPI keyserverAPI.KeyInternalAPI, extRoomsProvider api.ExtraPublicRoomsProvider, ) { + _, producer := kafka.SetupConsumerProducer(&cfg.Matrix.Kafka) + syncProducer := &producers.SyncAPIProducer{ Producer: producer, Topic: cfg.Matrix.Kafka.TopicFor(config.TopicOutputClientData), diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 0fdc6679f..0061de74f 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -37,7 +37,7 @@ func main() { keyAPI := base.KeyServerHTTPClient() clientapi.AddPublicRoutes( - base.PublicClientAPIMux, &base.Cfg.ClientAPI, base.KafkaProducer, accountDB, federation, + base.PublicClientAPIMux, &base.Cfg.ClientAPI, accountDB, federation, rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI, keyAPI, nil, ) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index b5386325c..61fdd801a 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -139,7 +139,7 @@ func main() { accountDB := base.Base.CreateAccountsDB() federation := createFederationClient(base) - keyAPI := keyserver.NewInternalAPI(&base.Base.Cfg.KeyServer, federation, base.Base.KafkaProducer) + keyAPI := keyserver.NewInternalAPI(&base.Base.Cfg.KeyServer, federation) userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI) keyAPI.SetUserAPI(userAPI) @@ -169,13 +169,11 @@ func main() { } monolith := setup.Monolith{ - Config: base.Base.Cfg, - AccountDB: accountDB, - Client: createClient(base), - FedClient: federation, - KeyRing: keyRing, - KafkaConsumer: base.Base.KafkaConsumer, - KafkaProducer: base.Base.KafkaProducer, + Config: base.Base.Cfg, + AccountDB: accountDB, + Client: createClient(base), + FedClient: federation, + KeyRing: keyRing, AppserviceAPI: asAPI, EDUInternalAPI: eduInputAPI, diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 5e8b92318..a40973638 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -96,7 +96,7 @@ func main() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation, base.KafkaProducer) + keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation) userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI) keyAPI.SetUserAPI(userAPI) @@ -129,13 +129,11 @@ func main() { rsComponent.SetFederationSenderAPI(fsAPI) monolith := setup.Monolith{ - Config: base.Cfg, - AccountDB: accountDB, - Client: ygg.CreateClient(base), - FedClient: federation, - KeyRing: keyRing, - KafkaConsumer: base.KafkaConsumer, - KafkaProducer: base.KafkaProducer, + Config: base.Cfg, + AccountDB: accountDB, + Client: ygg.CreateClient(base), + FedClient: federation, + KeyRing: keyRing, AppserviceAPI: asAPI, EDUInternalAPI: eduInputAPI, diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-key-server/main.go index 92d18ac38..ff5b22236 100644 --- a/cmd/dendrite-key-server/main.go +++ b/cmd/dendrite-key-server/main.go @@ -24,7 +24,7 @@ func main() { base := setup.NewBaseDendrite(cfg, "KeyServer", true) defer base.Close() // nolint: errcheck - intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient(), base.KafkaProducer) + intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient()) intAPI.SetUserAPI(base.UserAPIClient()) keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 0fe70ca8c..e935805f6 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -108,7 +108,7 @@ func main() { // This is different to rsAPI which can be the http client which doesn't need this dependency rsImpl.SetFederationSenderAPI(fsAPI) - keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI, base.KafkaProducer) + keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI) userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI) keyAPI.SetUserAPI(userAPI) @@ -127,13 +127,11 @@ func main() { } monolith := setup.Monolith{ - Config: base.Cfg, - AccountDB: accountDB, - Client: base.CreateClient(), - FedClient: federation, - KeyRing: keyRing, - KafkaConsumer: base.KafkaConsumer, - KafkaProducer: base.KafkaProducer, + Config: base.Cfg, + AccountDB: accountDB, + Client: base.CreateClient(), + FedClient: federation, + KeyRing: keyRing, AppserviceAPI: asAPI, EDUInternalAPI: eduInputAPI, diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index b879f842f..351dbc5f4 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -21,6 +21,7 @@ import ( func main() { cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "SyncAPI", true) defer base.Close() // nolint: errcheck @@ -30,7 +31,7 @@ func main() { rsAPI := base.RoomserverHTTPClient() syncapi.AddPublicRoutes( - base.PublicClientAPIMux, base.KafkaConsumer, userAPI, rsAPI, + base.PublicClientAPIMux, userAPI, rsAPI, base.KeyServerHTTPClient(), federation, &cfg.SyncAPI, ) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 2d7f8b02b..85cc8a9fb 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -190,7 +190,7 @@ func main() { accountDB := base.CreateAccountsDB() federation := createFederationClient(cfg, node) - keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation, base.KafkaProducer) + keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation) userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI) keyAPI.SetUserAPI(userAPI) @@ -212,13 +212,11 @@ func main() { p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI, federation) monolith := setup.Monolith{ - Config: base.Cfg, - AccountDB: accountDB, - Client: createClient(node), - FedClient: federation, - KeyRing: &keyRing, - KafkaConsumer: base.KafkaConsumer, - KafkaProducer: base.KafkaProducer, + Config: base.Cfg, + AccountDB: accountDB, + Client: createClient(node), + FedClient: federation, + KeyRing: &keyRing, AppserviceAPI: asQuery, EDUInternalAPI: eduInputAPI, diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go index b6196c269..098ac0248 100644 --- a/eduserver/eduserver.go +++ b/eduserver/eduserver.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/eduserver/inthttp" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/dendrite/internal/setup/kafka" userapi "github.com/matrix-org/dendrite/userapi/api" ) @@ -41,10 +42,13 @@ func NewInternalAPI( userAPI userapi.UserInternalAPI, ) api.EDUServerInputAPI { cfg := &base.Cfg.EDUServer + + _, producer := kafka.SetupConsumerProducer(&cfg.Matrix.Kafka) + return &input.EDUServerInputAPI{ Cache: eduCache, UserAPI: userAPI, - Producer: base.KafkaProducer, + Producer: producer, OutputTypingEventTopic: string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputTypingEvent)), OutputSendToDeviceEventTopic: string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputSendToDeviceEvent)), ServerName: cfg.Matrix.ServerName, diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 2f1223284..78791140e 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/federationsender/statistics" "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/dendrite/internal/setup/kafka" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -55,6 +56,8 @@ func NewInternalAPI( FailuresUntilBlacklist: cfg.FederationMaxRetries, } + consumer, _ := kafka.SetupConsumerProducer(&cfg.Matrix.Kafka) + queues := queue.NewOutgoingQueues( federationSenderDB, cfg.Matrix.ServerName, federation, rsAPI, stats, @@ -66,7 +69,7 @@ func NewInternalAPI( ) rsConsumer := consumers.NewOutputRoomEventConsumer( - cfg, base.KafkaConsumer, queues, + cfg, consumer, queues, federationSenderDB, rsAPI, ) if err = rsConsumer.Start(); err != nil { @@ -74,13 +77,13 @@ func NewInternalAPI( } tsConsumer := consumers.NewOutputEDUConsumer( - cfg, base.KafkaConsumer, queues, federationSenderDB, + cfg, consumer, queues, federationSenderDB, ) if err := tsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start typing server consumer") } keyConsumer := consumers.NewKeyChangeConsumer( - &base.Cfg.KeyServer, base.KafkaConsumer, queues, federationSenderDB, rsAPI, + &base.Cfg.KeyServer, consumer, queues, federationSenderDB, rsAPI, ) if err := keyConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start key server consumer") diff --git a/internal/setup/base.go b/internal/setup/base.go index 24a0d6aa6..8bc4ae17a 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -26,13 +26,9 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/matrix-org/naffka" - naffkaStorage "github.com/matrix-org/naffka/storage" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/userapi/storage/accounts" - "github.com/Shopify/sarama" "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" @@ -73,8 +69,8 @@ type BaseDendrite struct { httpClient *http.Client Cfg *config.Dendrite Caches *caching.Caches - KafkaConsumer sarama.Consumer - KafkaProducer sarama.SyncProducer + // KafkaConsumer sarama.Consumer + // KafkaProducer sarama.SyncProducer } const HTTPServerTimeout = time.Minute * 5 @@ -106,14 +102,6 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo logrus.WithError(err).Panicf("failed to start opentracing") } - var kafkaConsumer sarama.Consumer - var kafkaProducer sarama.SyncProducer - if cfg.Global.Kafka.UseNaffka { - kafkaConsumer, kafkaProducer = setupNaffka(cfg) - } else { - kafkaConsumer, kafkaProducer = setupKafka(cfg) - } - cache, err := caching.NewInMemoryLRUCache(true) if err != nil { logrus.WithError(err).Warnf("Failed to create cache") @@ -152,8 +140,6 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo InternalAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(), apiHttpClient: &apiClient, httpClient: &client, - KafkaConsumer: kafkaConsumer, - KafkaProducer: kafkaProducer, } } @@ -334,31 +320,3 @@ func (b *BaseDendrite) SetupAndServeHTTP( select {} } - -// setupKafka creates kafka consumer/producer pair from the config. -func setupKafka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { - consumer, err := sarama.NewConsumer(cfg.Global.Kafka.Addresses, nil) - if err != nil { - logrus.WithError(err).Panic("failed to start kafka consumer") - } - - producer, err := sarama.NewSyncProducer(cfg.Global.Kafka.Addresses, nil) - if err != nil { - logrus.WithError(err).Panic("failed to setup kafka producers") - } - - return consumer, producer -} - -// setupNaffka creates kafka consumer/producer pair from the config. -func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { - naffkaDB, err := naffkaStorage.NewDatabase(string(cfg.Global.Kafka.Database.ConnectionString)) - if err != nil { - logrus.WithError(err).Panic("Failed to setup naffka database") - } - naff, err := naffka.New(naffkaDB) - if err != nil { - logrus.WithError(err).Panic("Failed to setup naffka") - } - return naff, naff -} diff --git a/internal/setup/kafka/kafka.go b/internal/setup/kafka/kafka.go new file mode 100644 index 000000000..9855ae156 --- /dev/null +++ b/internal/setup/kafka/kafka.go @@ -0,0 +1,53 @@ +package kafka + +import ( + "github.com/Shopify/sarama" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/naffka" + naffkaStorage "github.com/matrix-org/naffka/storage" + "github.com/sirupsen/logrus" +) + +func SetupConsumerProducer(cfg *config.Kafka) (sarama.Consumer, sarama.SyncProducer) { + if cfg.UseNaffka { + return setupNaffka(cfg) + } + return setupKafka(cfg) +} + +// setupKafka creates kafka consumer/producer pair from the config. +func setupKafka(cfg *config.Kafka) (sarama.Consumer, sarama.SyncProducer) { + consumer, err := sarama.NewConsumer(cfg.Addresses, nil) + if err != nil { + logrus.WithError(err).Panic("failed to start kafka consumer") + } + + producer, err := sarama.NewSyncProducer(cfg.Addresses, nil) + if err != nil { + logrus.WithError(err).Panic("failed to setup kafka producers") + } + + return consumer, producer +} + +// In monolith mode with Naffka, we don't have the same constraints about +// consuming the same topic from more than one place like we do with Kafka. +// Therefore, we will only open one Naffka connection in case Naffka is +// running on SQLite. +var naffkaInstance *naffka.Naffka + +// setupNaffka creates kafka consumer/producer pair from the config. +func setupNaffka(cfg *config.Kafka) (sarama.Consumer, sarama.SyncProducer) { + if naffkaInstance != nil { + return naffkaInstance, naffkaInstance + } + naffkaDB, err := naffkaStorage.NewDatabase(string(cfg.Database.ConnectionString)) + if err != nil { + logrus.WithError(err).Panic("Failed to setup naffka database") + } + naffkaInstance, err = naffka.New(naffkaDB) + if err != nil { + logrus.WithError(err).Panic("Failed to setup naffka") + } + return naffkaInstance, naffkaInstance +} diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index a0675d61f..9d3625d2f 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -15,7 +15,6 @@ package setup import ( - "github.com/Shopify/sarama" "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi" @@ -38,13 +37,11 @@ import ( // Monolith represents an instantiation of all dependencies required to build // all components of Dendrite, for use in monolith mode. type Monolith struct { - Config *config.Dendrite - AccountDB accounts.Database - KeyRing *gomatrixserverlib.KeyRing - Client *gomatrixserverlib.Client - FedClient *gomatrixserverlib.FederationClient - KafkaConsumer sarama.Consumer - KafkaProducer sarama.SyncProducer + Config *config.Dendrite + AccountDB accounts.Database + KeyRing *gomatrixserverlib.KeyRing + Client *gomatrixserverlib.Client + FedClient *gomatrixserverlib.FederationClient AppserviceAPI appserviceAPI.AppServiceQueryAPI EDUInternalAPI eduServerAPI.EDUServerInputAPI @@ -61,7 +58,7 @@ type Monolith struct { // AddAllPublicRoutes attaches all public paths to the given router func (m *Monolith) AddAllPublicRoutes(csMux, ssMux, keyMux, mediaMux *mux.Router) { clientapi.AddPublicRoutes( - csMux, &m.Config.ClientAPI, m.KafkaProducer, m.AccountDB, + csMux, &m.Config.ClientAPI, m.AccountDB, m.FedClient, m.RoomserverAPI, m.EDUInternalAPI, m.AppserviceAPI, transactions.New(), m.FederationSenderAPI, m.UserAPI, m.KeyAPI, m.ExtPublicRoomsProvider, @@ -73,7 +70,7 @@ func (m *Monolith) AddAllPublicRoutes(csMux, ssMux, keyMux, mediaMux *mux.Router ) mediaapi.AddPublicRoutes(mediaMux, &m.Config.MediaAPI, m.UserAPI, m.Client) syncapi.AddPublicRoutes( - csMux, m.KafkaConsumer, m.UserAPI, m.RoomserverAPI, + csMux, m.UserAPI, m.RoomserverAPI, m.KeyAPI, m.FedClient, &m.Config.SyncAPI, ) } diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index 78420db1f..6c54d2a08 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -15,10 +15,10 @@ package keyserver import ( - "github.com/Shopify/sarama" "github.com/gorilla/mux" fedsenderapi "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/setup/kafka" "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/keyserver/internal" "github.com/matrix-org/dendrite/keyserver/inthttp" @@ -36,8 +36,10 @@ func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI) { // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - cfg *config.KeyServer, fedClient fedsenderapi.FederationClient, producer sarama.SyncProducer, + cfg *config.KeyServer, fedClient fedsenderapi.FederationClient, ) api.KeyInternalAPI { + _, producer := kafka.SetupConsumerProducer(&cfg.Matrix.Kafka) + db, err := storage.NewDatabase(&cfg.Database) if err != nil { logrus.WithError(err).Panicf("failed to connect to key server database") diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 4c138116f..b2cc0728c 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/dendrite/internal/setup/kafka" "github.com/matrix-org/dendrite/roomserver/internal" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/sirupsen/logrus" @@ -41,6 +42,8 @@ func NewInternalAPI( ) api.RoomserverInternalAPI { cfg := &base.Cfg.RoomServer + _, producer := kafka.SetupConsumerProducer(&cfg.Matrix.Kafka) + var perspectiveServerNames []gomatrixserverlib.ServerName for _, kp := range base.Cfg.SigningKeyServer.KeyPerspectives { perspectiveServerNames = append(perspectiveServerNames, kp.ServerName) @@ -52,7 +55,7 @@ func NewInternalAPI( } return internal.NewRoomserverAPI( - cfg, roomserverDB, base.KafkaProducer, string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputRoomEvent)), + cfg, roomserverDB, producer, string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputRoomEvent)), base.Caches, keyRing, perspectiveServerNames, ) } diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index 2a03195c9..1b692a098 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -17,7 +17,10 @@ import ( "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/internal" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" ) const ( @@ -160,7 +163,9 @@ func mustCreateRoomserverAPI(t *testing.T) (api.RoomserverInternalAPI, *dummyPro cfg.Defaults() cfg.Global.ServerName = testOrigin cfg.Global.Kafka.UseNaffka = true - cfg.RoomServer.Database.ConnectionString = config.DataSource(roomserverDBFileURI) + cfg.RoomServer.Database = config.DatabaseOptions{ + ConnectionString: roomserverDBFileURI, + } dp := &dummyProducer{ topic: cfg.Global.Kafka.TopicFor(config.TopicOutputRoomEvent), } @@ -169,12 +174,17 @@ func mustCreateRoomserverAPI(t *testing.T) (api.RoomserverInternalAPI, *dummyPro t.Fatalf("failed to make caches: %s", err) } base := &setup.BaseDendrite{ - KafkaProducer: dp, - Caches: cache, - Cfg: cfg, + Caches: cache, + Cfg: cfg, } - - return NewInternalAPI(base, &test.NopJSONVerifier{}), dp + roomserverDB, err := storage.Open(&cfg.RoomServer.Database, base.Caches) + if err != nil { + logrus.WithError(err).Panicf("failed to connect to room server db") + } + return internal.NewRoomserverAPI( + &cfg.RoomServer, roomserverDB, dp, string(cfg.Global.Kafka.TopicFor(config.TopicOutputRoomEvent)), + base.Caches, &test.NopJSONVerifier{}, nil, + ), dp } func mustSendEvents(t *testing.T, ver gomatrixserverlib.RoomVersion, events []json.RawMessage) (api.RoomserverInternalAPI, *dummyProducer, []gomatrixserverlib.HeaderedEvent) { diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 43e2455b6..de0bb434b 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -17,11 +17,11 @@ package syncapi import ( "context" - "github.com/Shopify/sarama" "github.com/gorilla/mux" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/setup/kafka" keyapi "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/roomserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -37,13 +37,14 @@ import ( // component. func AddPublicRoutes( router *mux.Router, - consumer sarama.Consumer, userAPI userapi.UserInternalAPI, rsAPI api.RoomserverInternalAPI, keyAPI keyapi.KeyInternalAPI, federation *gomatrixserverlib.FederationClient, cfg *config.SyncAPI, ) { + consumer, _ := kafka.SetupConsumerProducer(&cfg.Matrix.Kafka) + syncDB, err := storage.NewSyncServerDatasource(&cfg.Database) if err != nil { logrus.WithError(err).Panicf("failed to connect to sync db") From e154c45b5188846f8f8abf56d6e7979fc7bb98b7 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 15 Oct 2020 14:14:17 +0100 Subject: [PATCH 086/104] Better logging around db.StoreEvent --- roomserver/storage/shared/storage.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index e96eab71b..f2be8b3cf 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -458,7 +458,7 @@ func (d *Database) StoreEvent( eventNID, stateNID, err = d.EventsTable.SelectEvent(ctx, txn, event.EventID()) } if err != nil { - return err + return fmt.Errorf("d.EventsTable.SelectEvent: %w", err) } } @@ -467,6 +467,9 @@ func (d *Database) StoreEvent( } if !isRejected { // ignore rejected redaction events redactionEvent, redactedEventID, err = d.handleRedactions(ctx, txn, eventNID, event) + if err != nil { + return fmt.Errorf("d.handleRedactions: %w", err) + } } return nil }) @@ -627,6 +630,7 @@ func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( // to cross-reference with other tables when loading. // // Returns the redaction event and the event ID of the redacted event if this call resulted in a redaction. +// nolint:gocyclo func (d *Database) handleRedactions( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, event gomatrixserverlib.Event, ) (*gomatrixserverlib.Event, string, error) { @@ -644,13 +648,13 @@ func (d *Database) handleRedactions( RedactsEventID: event.Redacts(), }) if err != nil { - return nil, "", err + return nil, "", fmt.Errorf("d.RedactionsTable.InsertRedaction: %w", err) } } redactionEvent, redactedEvent, validated, err := d.loadRedactionPair(ctx, txn, eventNID, event) if err != nil { - return nil, "", err + return nil, "", fmt.Errorf("d.loadRedactionPair: %w", err) } if validated || redactedEvent == nil || redactionEvent == nil { // we've seen this redaction before or there is nothing to redact @@ -664,7 +668,7 @@ func (d *Database) handleRedactions( // mark the event as redacted err = redactedEvent.SetUnsignedField("redacted_because", redactionEvent) if err != nil { - return nil, "", err + return nil, "", fmt.Errorf("redactedEvent.SetUnsignedField: %w", err) } if redactionsArePermanent { redactedEvent.Event = redactedEvent.Redact() @@ -672,10 +676,15 @@ func (d *Database) handleRedactions( // overwrite the eventJSON table err = d.EventJSONTable.InsertEventJSON(ctx, txn, redactedEvent.EventNID, redactedEvent.JSON()) if err != nil { - return nil, "", err + return nil, "", fmt.Errorf("d.EventJSONTable.InsertEventJSON: %w", err) } - return &redactionEvent.Event, redactedEvent.EventID(), d.RedactionsTable.MarkRedactionValidated(ctx, txn, redactionEvent.EventID(), true) + err = d.RedactionsTable.MarkRedactionValidated(ctx, txn, redactionEvent.EventID(), true) + if err != nil { + err = fmt.Errorf("d.RedactionsTable.MarkRedactionValidated: %w", err) + } + + return &redactionEvent.Event, redactedEvent.EventID(), err } // loadRedactionPair returns both the redaction event and the redacted event, else nil. From 3e5d38e2849816e00297dbd41d748620deaf3a95 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 15 Oct 2020 16:20:19 +0100 Subject: [PATCH 087/104] Improve state resolution v2 performance (matrix-org/gomatrixserverlib#237) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c02463832..d3060fa0f 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20201009153043-8d27a9f0e350 + github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index 101b8e18f..377a2e093 100644 --- a/go.sum +++ b/go.sum @@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201009153043-8d27a9f0e350 h1:G9K8k5KIzbeBdd0bMk+4itdZU3JGHgV+z0FNUsTEhkE= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201009153043-8d27a9f0e350/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8 h1:GF1PxbvImWDoz1DQZNMoaYtIqQXtyLAtmQOzwwmw1OI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= From 4a7fb9c045211c54c13610119a0f5ed0df355a0f Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 15 Oct 2020 18:09:41 +0100 Subject: [PATCH 088/104] Automatically upgrade databases on startup (#1529) * Support auto-upgrading accounts DB * Auto-upgrade device DB deltas * Support up/downgrading from cmd/goose * Linting * Create tables then do migrations then prepare statements To avoid failing due to some things not existing * Linting --- cmd/goose/main.go | 86 ++++++++++-- internal/sqlutil/migrate.go | 130 ++++++++++++++++++ .../accounts/postgres/accounts_table.go | 9 +- .../deltas/20200929203058_is_active.go | 33 +++++ .../deltas/20200929203058_is_active.sql | 9 -- userapi/storage/accounts/postgres/storage.go | 15 ++ .../accounts/sqlite3/accounts_table.go | 10 +- .../deltas/20200929203058_is_active.go | 64 +++++++++ .../deltas/20200929203058_is_active.sql | 38 ----- userapi/storage/accounts/sqlite3/storage.go | 14 ++ .../deltas/20201001204705_last_seen_ts_ip.go | 39 ++++++ .../deltas/20201001204705_last_seen_ts_ip.sql | 13 -- .../storage/devices/postgres/devices_table.go | 9 +- userapi/storage/devices/postgres/storage.go | 14 ++ .../deltas/20201001204705_last_seen_ts_ip.go | 70 ++++++++++ .../deltas/20201001204705_last_seen_ts_ip.sql | 44 ------ .../storage/devices/sqlite3/devices_table.go | 9 +- userapi/storage/devices/sqlite3/storage.go | 12 ++ 18 files changed, 485 insertions(+), 133 deletions(-) create mode 100644 internal/sqlutil/migrate.go create mode 100644 userapi/storage/accounts/postgres/deltas/20200929203058_is_active.go delete mode 100644 userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql create mode 100644 userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.go delete mode 100644 userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql create mode 100644 userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.go delete mode 100644 userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql create mode 100644 userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.go delete mode 100644 userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.sql diff --git a/cmd/goose/main.go b/cmd/goose/main.go index ef3942d90..83c97a729 100644 --- a/cmd/goose/main.go +++ b/cmd/goose/main.go @@ -8,19 +8,38 @@ import ( "log" "os" - // Example complex Go migration import: - // _ "github.com/matrix-org/dendrite/serverkeyapi/storage/postgres/deltas" + pgaccounts "github.com/matrix-org/dendrite/userapi/storage/accounts/postgres/deltas" + slaccounts "github.com/matrix-org/dendrite/userapi/storage/accounts/sqlite3/deltas" + pgdevices "github.com/matrix-org/dendrite/userapi/storage/devices/postgres/deltas" + sldevices "github.com/matrix-org/dendrite/userapi/storage/devices/sqlite3/deltas" "github.com/pressly/goose" _ "github.com/lib/pq" _ "github.com/mattn/go-sqlite3" ) -var ( - flags = flag.NewFlagSet("goose", flag.ExitOnError) - dir = flags.String("dir", ".", "directory with migration files") +const ( + AppService = "appservice" + FederationSender = "federationsender" + KeyServer = "keyserver" + MediaAPI = "mediaapi" + RoomServer = "roomserver" + SigningKeyServer = "signingkeyserver" + SyncAPI = "syncapi" + UserAPIAccounts = "userapi_accounts" + UserAPIDevices = "userapi_devices" ) +var ( + dir = flags.String("dir", "", "directory with migration files") + flags = flag.NewFlagSet("goose", flag.ExitOnError) + component = flags.String("component", "", "dendrite component name") + knownDBs = []string{ + AppService, FederationSender, KeyServer, MediaAPI, RoomServer, SigningKeyServer, SyncAPI, UserAPIAccounts, UserAPIDevices, + } +) + +// nolint: gocyclo func main() { err := flags.Parse(os.Args[1:]) if err != nil { @@ -37,19 +56,20 @@ Drivers: sqlite3 Examples: - goose -d roomserver/storage/sqlite3/deltas sqlite3 ./roomserver.db status - goose -d roomserver/storage/sqlite3/deltas sqlite3 ./roomserver.db up + goose -component roomserver sqlite3 ./roomserver.db status + goose -component roomserver sqlite3 ./roomserver.db up - goose -d roomserver/storage/postgres/deltas postgres "user=dendrite dbname=dendrite sslmode=disable" status + goose -component roomserver postgres "user=dendrite dbname=dendrite sslmode=disable" status Options: - - -dir string - directory with migration files (default ".") + -component string + Dendrite component name e.g roomserver, signingkeyserver, clientapi, syncapi -table string migrations table name (default "goose_db_version") -h print help -v enable verbose mode + -dir string + directory with migration files, only relevant when creating new migrations. -version print version @@ -74,6 +94,25 @@ Commands: fmt.Println("engine must be one of 'sqlite3' or 'postgres'") return } + + knownComponent := false + for _, c := range knownDBs { + if c == *component { + knownComponent = true + break + } + } + if !knownComponent { + fmt.Printf("component must be one of %v\n", knownDBs) + return + } + + if engine == "sqlite3" { + loadSQLiteDeltas(*component) + } else { + loadPostgresDeltas(*component) + } + dbstring, command := args[1], args[2] db, err := goose.OpenDBWithDriver(engine, dbstring) @@ -92,7 +131,30 @@ Commands: arguments = append(arguments, args[3:]...) } - if err := goose.Run(command, db, *dir, arguments...); err != nil { + // goose demands a directory even though we don't use it for upgrades + d := *dir + if d == "" { + d = os.TempDir() + } + if err := goose.Run(command, db, d, arguments...); err != nil { log.Fatalf("goose %v: %v", command, err) } } + +func loadSQLiteDeltas(component string) { + switch component { + case UserAPIAccounts: + slaccounts.LoadFromGoose() + case UserAPIDevices: + sldevices.LoadFromGoose() + } +} + +func loadPostgresDeltas(component string) { + switch component { + case UserAPIAccounts: + pgaccounts.LoadFromGoose() + case UserAPIDevices: + pgdevices.LoadFromGoose() + } +} diff --git a/internal/sqlutil/migrate.go b/internal/sqlutil/migrate.go new file mode 100644 index 000000000..833977ba4 --- /dev/null +++ b/internal/sqlutil/migrate.go @@ -0,0 +1,130 @@ +package sqlutil + +import ( + "database/sql" + "fmt" + "runtime" + "sort" + + "github.com/matrix-org/dendrite/internal/config" + "github.com/pressly/goose" +) + +type Migrations struct { + registeredGoMigrations map[int64]*goose.Migration +} + +func NewMigrations() *Migrations { + return &Migrations{ + registeredGoMigrations: make(map[int64]*goose.Migration), + } +} + +// Copy-pasted from goose directly to store migrations into a map we control + +// AddMigration adds a migration. +func (m *Migrations) AddMigration(up func(*sql.Tx) error, down func(*sql.Tx) error) { + _, filename, _, _ := runtime.Caller(1) + m.AddNamedMigration(filename, up, down) +} + +// AddNamedMigration : Add a named migration. +func (m *Migrations) AddNamedMigration(filename string, up func(*sql.Tx) error, down func(*sql.Tx) error) { + v, _ := goose.NumericComponent(filename) + migration := &goose.Migration{Version: v, Next: -1, Previous: -1, Registered: true, UpFn: up, DownFn: down, Source: filename} + + if existing, ok := m.registeredGoMigrations[v]; ok { + panic(fmt.Sprintf("failed to add migration %q: version conflicts with %q", filename, existing.Source)) + } + + m.registeredGoMigrations[v] = migration +} + +// RunDeltas up to the latest version. +func (m *Migrations) RunDeltas(db *sql.DB, props *config.DatabaseOptions) error { + maxVer := goose.MaxVersion + minVer := int64(0) + migrations, err := m.collect(minVer, maxVer) + if err != nil { + return fmt.Errorf("RunDeltas: Failed to collect migrations: %w", err) + } + if props.ConnectionString.IsPostgres() { + if err = goose.SetDialect("postgres"); err != nil { + return err + } + } else if props.ConnectionString.IsSQLite() { + if err = goose.SetDialect("sqlite3"); err != nil { + return err + } + } else { + return fmt.Errorf("Unknown connection string: %s", props.ConnectionString) + } + for { + current, err := goose.EnsureDBVersion(db) + if err != nil { + return fmt.Errorf("RunDeltas: Failed to EnsureDBVersion: %w", err) + } + + next, err := migrations.Next(current) + if err != nil { + if err == goose.ErrNoNextVersion { + return nil + } + + return fmt.Errorf("RunDeltas: Failed to load next migration to %+v : %w", next, err) + } + + if err = next.Up(db); err != nil { + return fmt.Errorf("RunDeltas: Failed run migration: %w", err) + } + } +} + +func (m *Migrations) collect(current, target int64) (goose.Migrations, error) { + var migrations goose.Migrations + + // Go migrations registered via goose.AddMigration(). + for _, migration := range m.registeredGoMigrations { + v, err := goose.NumericComponent(migration.Source) + if err != nil { + return nil, err + } + if versionFilter(v, current, target) { + migrations = append(migrations, migration) + } + } + + migrations = sortAndConnectMigrations(migrations) + + return migrations, nil +} + +func sortAndConnectMigrations(migrations goose.Migrations) goose.Migrations { + sort.Sort(migrations) + + // now that we're sorted in the appropriate direction, + // populate next and previous for each migration + for i, m := range migrations { + prev := int64(-1) + if i > 0 { + prev = migrations[i-1].Version + migrations[i-1].Next = m.Version + } + migrations[i].Previous = prev + } + + return migrations +} + +func versionFilter(v, current, target int64) bool { + + if target > current { + return v > current && v <= target + } + + if target < current { + return v <= current && v > target + } + + return false +} diff --git a/userapi/storage/accounts/postgres/accounts_table.go b/userapi/storage/accounts/postgres/accounts_table.go index 254da84c3..4eaa5b581 100644 --- a/userapi/storage/accounts/postgres/accounts_table.go +++ b/userapi/storage/accounts/postgres/accounts_table.go @@ -75,11 +75,12 @@ type accountsStatements struct { serverName gomatrixserverlib.ServerName } +func (s *accountsStatements) execSchema(db *sql.DB) error { + _, err := db.Exec(accountsSchema) + return err +} + func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) { - _, err = db.Exec(accountsSchema) - if err != nil { - return - } if s.insertAccountStmt, err = db.Prepare(insertAccountSQL); err != nil { return } diff --git a/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.go b/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.go new file mode 100644 index 000000000..9e14286e0 --- /dev/null +++ b/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.go @@ -0,0 +1,33 @@ +package deltas + +import ( + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/pressly/goose" +) + +func LoadFromGoose() { + goose.AddMigration(UpIsActive, DownIsActive) +} + +func LoadIsActive(m *sqlutil.Migrations) { + m.AddMigration(UpIsActive, DownIsActive) +} + +func UpIsActive(tx *sql.Tx) error { + _, err := tx.Exec("ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS is_deactivated BOOLEAN DEFAULT FALSE;") + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} + +func DownIsActive(tx *sql.Tx) error { + _, err := tx.Exec("ALTER TABLE account_accounts DROP COLUMN is_deactivated;") + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + return nil +} diff --git a/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql b/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql deleted file mode 100644 index 32e6e1664..000000000 --- a/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql +++ /dev/null @@ -1,9 +0,0 @@ --- +goose Up --- +goose StatementBegin -ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS is_deactivated BOOLEAN DEFAULT FALSE; --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin -ALTER TABLE account_accounts DROP COLUMN is_deactivated; --- +goose StatementEnd diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go index 2230f7e79..40c4b8ff5 100644 --- a/userapi/storage/accounts/postgres/storage.go +++ b/userapi/storage/accounts/postgres/storage.go @@ -25,6 +25,8 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts/postgres/deltas" + _ "github.com/matrix-org/dendrite/userapi/storage/accounts/postgres/deltas" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" @@ -55,6 +57,18 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver db: db, writer: sqlutil.NewDummyWriter(), } + + // Create tables before executing migrations so we don't fail if the table is missing, + // and THEN prepare statements so we don't fail due to referencing new columns + if err = d.accounts.execSchema(db); err != nil { + return nil, err + } + m := sqlutil.NewMigrations() + deltas.LoadIsActive(m) + if err = m.RunDeltas(db, dbProperties); err != nil { + return nil, err + } + if err = d.PartitionOffsetStatements.Prepare(db, d.writer, "account"); err != nil { return nil, err } @@ -70,6 +84,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err = d.threepids.prepare(db); err != nil { return nil, err } + return d, nil } diff --git a/userapi/storage/accounts/sqlite3/accounts_table.go b/userapi/storage/accounts/sqlite3/accounts_table.go index d0ea8a8bc..50f07237e 100644 --- a/userapi/storage/accounts/sqlite3/accounts_table.go +++ b/userapi/storage/accounts/sqlite3/accounts_table.go @@ -74,13 +74,13 @@ type accountsStatements struct { serverName gomatrixserverlib.ServerName } +func (s *accountsStatements) execSchema(db *sql.DB) error { + _, err := db.Exec(accountsSchema) + return err +} + func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) { s.db = db - - _, err = db.Exec(accountsSchema) - if err != nil { - return - } if s.insertAccountStmt, err = db.Prepare(insertAccountSQL); err != nil { return } diff --git a/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.go b/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.go new file mode 100644 index 000000000..9fddb05a1 --- /dev/null +++ b/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.go @@ -0,0 +1,64 @@ +package deltas + +import ( + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/pressly/goose" +) + +func LoadFromGoose() { + goose.AddMigration(UpIsActive, DownIsActive) +} + +func LoadIsActive(m *sqlutil.Migrations) { + m.AddMigration(UpIsActive, DownIsActive) +} + +func UpIsActive(tx *sql.Tx) error { + _, err := tx.Exec(` + ALTER TABLE account_accounts RENAME TO account_accounts_tmp; +CREATE TABLE account_accounts ( + localpart TEXT NOT NULL PRIMARY KEY, + created_ts BIGINT NOT NULL, + password_hash TEXT, + appservice_id TEXT, + is_deactivated BOOLEAN DEFAULT 0 +); +INSERT + INTO account_accounts ( + localpart, created_ts, password_hash, appservice_id + ) SELECT + localpart, created_ts, password_hash, appservice_id + FROM account_accounts_tmp +; +DROP TABLE account_accounts_tmp;`) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} + +func DownIsActive(tx *sql.Tx) error { + _, err := tx.Exec(` + ALTER TABLE account_accounts RENAME TO account_accounts_tmp; +CREATE TABLE account_accounts ( + localpart TEXT NOT NULL PRIMARY KEY, + created_ts BIGINT NOT NULL, + password_hash TEXT, + appservice_id TEXT +); +INSERT + INTO account_accounts ( + localpart, created_ts, password_hash, appservice_id + ) SELECT + localpart, created_ts, password_hash, appservice_id + FROM account_accounts_tmp +; +DROP TABLE account_accounts_tmp;`) + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + return nil +} diff --git a/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql b/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql deleted file mode 100644 index 51e9bae3c..000000000 --- a/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql +++ /dev/null @@ -1,38 +0,0 @@ --- +goose Up --- +goose StatementBegin -ALTER TABLE account_accounts RENAME TO account_accounts_tmp; -CREATE TABLE account_accounts ( - localpart TEXT NOT NULL PRIMARY KEY, - created_ts BIGINT NOT NULL, - password_hash TEXT, - appservice_id TEXT, - is_deactivated BOOLEAN DEFAULT 0 -); -INSERT - INTO account_accounts ( - localpart, created_ts, password_hash, appservice_id - ) SELECT - localpart, created_ts, password_hash, appservice_id - FROM account_accounts_tmp -; -DROP TABLE account_accounts_tmp; --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin -ALTER TABLE account_accounts RENAME TO account_accounts_tmp; -CREATE TABLE account_accounts ( - localpart TEXT NOT NULL PRIMARY KEY, - created_ts BIGINT NOT NULL, - password_hash TEXT, - appservice_id TEXT -); -INSERT - INTO account_accounts ( - localpart, created_ts, password_hash, appservice_id - ) SELECT - localpart, created_ts, password_hash, appservice_id - FROM account_accounts_tmp -; -DROP TABLE account_accounts_tmp; --- +goose StatementEnd diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go index 7a2830a93..0be7bcbe7 100644 --- a/userapi/storage/accounts/sqlite3/storage.go +++ b/userapi/storage/accounts/sqlite3/storage.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts/sqlite3/deltas" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" // Import the sqlite3 database driver. @@ -60,6 +61,18 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver db: db, writer: sqlutil.NewExclusiveWriter(), } + + // Create tables before executing migrations so we don't fail if the table is missing, + // and THEN prepare statements so we don't fail due to referencing new columns + if err = d.accounts.execSchema(db); err != nil { + return nil, err + } + m := sqlutil.NewMigrations() + deltas.LoadIsActive(m) + if err = m.RunDeltas(db, dbProperties); err != nil { + return nil, err + } + partitions := sqlutil.PartitionOffsetStatements{} if err = partitions.Prepare(db, d.writer, "account"); err != nil { return nil, err @@ -76,6 +89,7 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err = d.threepids.prepare(db); err != nil { return nil, err } + return d, nil } diff --git a/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.go b/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.go new file mode 100644 index 000000000..290f854c8 --- /dev/null +++ b/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.go @@ -0,0 +1,39 @@ +package deltas + +import ( + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/pressly/goose" +) + +func LoadFromGoose() { + goose.AddMigration(UpLastSeenTSIP, DownLastSeenTSIP) +} + +func LoadLastSeenTSIP(m *sqlutil.Migrations) { + m.AddMigration(UpLastSeenTSIP, DownLastSeenTSIP) +} + +func UpLastSeenTSIP(tx *sql.Tx) error { + _, err := tx.Exec(` +ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS last_seen_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)*1000; +ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS ip TEXT; +ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS user_agent TEXT;`) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} + +func DownLastSeenTSIP(tx *sql.Tx) error { + _, err := tx.Exec(` + ALTER TABLE device_devices DROP COLUMN last_seen_ts; + ALTER TABLE device_devices DROP COLUMN ip; + ALTER TABLE device_devices DROP COLUMN user_agent;`) + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + return nil +} diff --git a/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql b/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql deleted file mode 100644 index e7900b0b3..000000000 --- a/userapi/storage/devices/postgres/deltas/20201001204705_last_seen_ts_ip.sql +++ /dev/null @@ -1,13 +0,0 @@ --- +goose Up --- +goose StatementBegin -ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS last_seen_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)*1000; -ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS ip TEXT; -ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS user_agent TEXT; --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin -ALTER TABLE device_devices DROP COLUMN last_seen_ts; -ALTER TABLE device_devices DROP COLUMN ip; -ALTER TABLE device_devices DROP COLUMN user_agent; --- +goose StatementEnd diff --git a/userapi/storage/devices/postgres/devices_table.go b/userapi/storage/devices/postgres/devices_table.go index 2a4d337c7..379fed794 100644 --- a/userapi/storage/devices/postgres/devices_table.go +++ b/userapi/storage/devices/postgres/devices_table.go @@ -111,11 +111,12 @@ type devicesStatements struct { serverName gomatrixserverlib.ServerName } +func (s *devicesStatements) execSchema(db *sql.DB) error { + _, err := db.Exec(devicesSchema) + return err +} + func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) { - _, err = db.Exec(devicesSchema) - if err != nil { - return - } if s.insertDeviceStmt, err = db.Prepare(insertDeviceSQL); err != nil { return } diff --git a/userapi/storage/devices/postgres/storage.go b/userapi/storage/devices/postgres/storage.go index faa5796b0..e318b260b 100644 --- a/userapi/storage/devices/postgres/storage.go +++ b/userapi/storage/devices/postgres/storage.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/devices/postgres/deltas" "github.com/matrix-org/gomatrixserverlib" ) @@ -42,9 +43,22 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver return nil, err } d := devicesStatements{} + + // Create tables before executing migrations so we don't fail if the table is missing, + // and THEN prepare statements so we don't fail due to referencing new columns + if err = d.execSchema(db); err != nil { + return nil, err + } + m := sqlutil.NewMigrations() + deltas.LoadLastSeenTSIP(m) + if err = m.RunDeltas(db, dbProperties); err != nil { + return nil, err + } + if err = d.prepare(db, serverName); err != nil { return nil, err } + return &Database{db, d}, nil } diff --git a/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.go b/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.go new file mode 100644 index 000000000..262098265 --- /dev/null +++ b/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.go @@ -0,0 +1,70 @@ +package deltas + +import ( + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/pressly/goose" +) + +func LoadFromGoose() { + goose.AddMigration(UpLastSeenTSIP, DownLastSeenTSIP) +} + +func LoadLastSeenTSIP(m *sqlutil.Migrations) { + m.AddMigration(UpLastSeenTSIP, DownLastSeenTSIP) +} + +func UpLastSeenTSIP(tx *sql.Tx) error { + _, err := tx.Exec(` + ALTER TABLE device_devices RENAME TO device_devices_tmp; + CREATE TABLE device_devices ( + access_token TEXT PRIMARY KEY, + session_id INTEGER, + device_id TEXT , + localpart TEXT , + created_ts BIGINT, + display_name TEXT, + last_seen_ts BIGINT, + ip TEXT, + user_agent TEXT, + UNIQUE (localpart, device_id) + ); + INSERT + INTO device_devices ( + access_token, session_id, device_id, localpart, created_ts, display_name, last_seen_ts, ip, user_agent + ) SELECT + access_token, session_id, device_id, localpart, created_ts, display_name, created_ts, '', '' + FROM device_devices_tmp; + DROP TABLE device_devices_tmp;`) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} + +func DownLastSeenTSIP(tx *sql.Tx) error { + _, err := tx.Exec(` +ALTER TABLE device_devices RENAME TO device_devices_tmp; +CREATE TABLE IF NOT EXISTS device_devices ( + access_token TEXT PRIMARY KEY, + session_id INTEGER, + device_id TEXT , + localpart TEXT , + created_ts BIGINT, + display_name TEXT, + UNIQUE (localpart, device_id) +); +INSERT +INTO device_devices ( + access_token, session_id, device_id, localpart, created_ts, display_name +) SELECT + access_token, session_id, device_id, localpart, created_ts, display_name +FROM device_devices_tmp; +DROP TABLE device_devices_tmp;`) + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + return nil +} diff --git a/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.sql b/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.sql deleted file mode 100644 index 887f90e0d..000000000 --- a/userapi/storage/devices/sqlite3/deltas/20201001204705_last_seen_ts_ip.sql +++ /dev/null @@ -1,44 +0,0 @@ --- +goose Up --- +goose StatementBegin -ALTER TABLE device_devices RENAME TO device_devices_tmp; -CREATE TABLE device_devices ( - access_token TEXT PRIMARY KEY, - session_id INTEGER, - device_id TEXT , - localpart TEXT , - created_ts BIGINT, - display_name TEXT, - last_seen_ts BIGINT, - ip TEXT, - user_agent TEXT, - UNIQUE (localpart, device_id) -); -INSERT -INTO device_devices ( - access_token, session_id, device_id, localpart, created_ts, display_name, last_seen_ts, ip, user_agent -) SELECT - access_token, session_id, device_id, localpart, created_ts, display_name, created_ts, '', '' -FROM device_devices_tmp; -DROP TABLE device_devices_tmp; --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin -ALTER TABLE device_devices RENAME TO device_devices_tmp; -CREATE TABLE IF NOT EXISTS device_devices ( - access_token TEXT PRIMARY KEY, - session_id INTEGER, - device_id TEXT , - localpart TEXT , - created_ts BIGINT, - display_name TEXT, - UNIQUE (localpart, device_id) -); -INSERT -INTO device_devices ( - access_token, session_id, device_id, localpart, created_ts, display_name -) SELECT - access_token, session_id, device_id, localpart, created_ts, display_name -FROM device_devices_tmp; -DROP TABLE device_devices_tmp; --- +goose StatementEnd \ No newline at end of file diff --git a/userapi/storage/devices/sqlite3/devices_table.go b/userapi/storage/devices/sqlite3/devices_table.go index 6b0de10ee..26c03222a 100644 --- a/userapi/storage/devices/sqlite3/devices_table.go +++ b/userapi/storage/devices/sqlite3/devices_table.go @@ -98,13 +98,14 @@ type devicesStatements struct { serverName gomatrixserverlib.ServerName } +func (s *devicesStatements) execSchema(db *sql.DB) error { + _, err := db.Exec(devicesSchema) + return err +} + func (s *devicesStatements) prepare(db *sql.DB, writer sqlutil.Writer, server gomatrixserverlib.ServerName) (err error) { s.db = db s.writer = writer - _, err = db.Exec(devicesSchema) - if err != nil { - return - } if s.insertDeviceStmt, err = db.Prepare(insertDeviceSQL); err != nil { return } diff --git a/userapi/storage/devices/sqlite3/storage.go b/userapi/storage/devices/sqlite3/storage.go index cfaf4fd99..25888eae4 100644 --- a/userapi/storage/devices/sqlite3/storage.go +++ b/userapi/storage/devices/sqlite3/storage.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/devices/sqlite3/deltas" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -46,6 +47,17 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver } writer := sqlutil.NewExclusiveWriter() d := devicesStatements{} + + // Create tables before executing migrations so we don't fail if the table is missing, + // and THEN prepare statements so we don't fail due to referencing new columns + if err = d.execSchema(db); err != nil { + return nil, err + } + m := sqlutil.NewMigrations() + deltas.LoadLastSeenTSIP(m) + if err = m.RunDeltas(db, dbProperties); err != nil { + return nil, err + } if err = d.prepare(db, writer, serverName); err != nil { return nil, err } From 640e8c50ec1b7ebb54c57a59bf1d6a7716c328cf Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 16 Oct 2020 15:44:39 +0100 Subject: [PATCH 089/104] Take write lock for rate limit map (#1532) * Take write lock for rate limit map * Fix potential race condition --- clientapi/routing/rate_limiting.go | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/clientapi/routing/rate_limiting.go b/clientapi/routing/rate_limiting.go index 16e3c0565..9d3f817a2 100644 --- a/clientapi/routing/rate_limiting.go +++ b/clientapi/routing/rate_limiting.go @@ -13,6 +13,7 @@ import ( type rateLimits struct { limits map[string]chan struct{} limitsMutex sync.RWMutex + cleanMutex sync.RWMutex enabled bool requestThreshold int64 cooloffDuration time.Duration @@ -38,6 +39,7 @@ func (l *rateLimits) clean() { // empty. If they are then we will close and delete them, // freeing up memory. time.Sleep(time.Second * 30) + l.cleanMutex.Lock() l.limitsMutex.Lock() for k, c := range l.limits { if len(c) == 0 { @@ -46,6 +48,7 @@ func (l *rateLimits) clean() { } } l.limitsMutex.Unlock() + l.cleanMutex.Unlock() } } @@ -55,12 +58,12 @@ func (l *rateLimits) rateLimit(req *http.Request) *util.JSONResponse { return nil } - // Lock the map long enough to check for rate limiting. We hold it - // for longer here than we really need to but it makes sure that we - // also don't conflict with the cleaner goroutine which might clean - // up a channel after we have retrieved it otherwise. - l.limitsMutex.RLock() - defer l.limitsMutex.RUnlock() + // Take a read lock out on the cleaner mutex. The cleaner expects to + // be able to take a write lock, which isn't possible while there are + // readers, so this has the effect of blocking the cleaner goroutine + // from doing its work until there are no requests in flight. + l.cleanMutex.RLock() + defer l.cleanMutex.RUnlock() // First of all, work out if X-Forwarded-For was sent to us. If not // then we'll just use the IP address of the caller. @@ -69,12 +72,19 @@ func (l *rateLimits) rateLimit(req *http.Request) *util.JSONResponse { caller = forwardedFor } - // Look up the caller's channel, if they have one. If they don't then - // let's create one. + // Look up the caller's channel, if they have one. + l.limitsMutex.RLock() rateLimit, ok := l.limits[caller] + l.limitsMutex.RUnlock() + + // If the caller doesn't have a channel, create one and write it + // back to the map. if !ok { - l.limits[caller] = make(chan struct{}, l.requestThreshold) - rateLimit = l.limits[caller] + rateLimit = make(chan struct{}, l.requestThreshold) + + l.limitsMutex.Lock() + l.limits[caller] = rateLimit + l.limitsMutex.Unlock() } // Check if the user has got free resource slots for this request. From 0974f6e2c055d8d06b5ea9c175252b22b2399fe2 Mon Sep 17 00:00:00 2001 From: S7evinK Date: Mon, 19 Oct 2020 10:38:10 +0200 Subject: [PATCH 090/104] Fix internal http api (#1535) Signed-off-by: Till Faelligen --- userapi/inthttp/server.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/userapi/inthttp/server.go b/userapi/inthttp/server.go index e24aad3a9..81e936e58 100644 --- a/userapi/inthttp/server.go +++ b/userapi/inthttp/server.go @@ -39,7 +39,7 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - internalAPIMux.Handle(PerformAccountCreationPath, + internalAPIMux.Handle(PerformPasswordUpdatePath, httputil.MakeInternalAPI("performPasswordUpdate", func(req *http.Request) util.JSONResponse { request := api.PerformPasswordUpdateRequest{} response := api.PerformPasswordUpdateResponse{} @@ -169,7 +169,7 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - internalAPIMux.Handle(QueryDeviceInfosPath, + internalAPIMux.Handle(QuerySearchProfilesPath, httputil.MakeInternalAPI("querySearchProfiles", func(req *http.Request) util.JSONResponse { request := api.QuerySearchProfilesRequest{} response := api.QuerySearchProfilesResponse{} @@ -182,4 +182,17 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(InputAccountDataPath, + httputil.MakeInternalAPI("inputAccountDataPath", func(req *http.Request) util.JSONResponse { + request := api.InputAccountDataRequest{} + response := api.InputAccountDataResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.InputAccountData(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) } From 6e63df1d9a3eadf924d518a1a02f04dfd03ad6b1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 19 Oct 2020 14:59:13 +0100 Subject: [PATCH 091/104] KindOld (#1531) * Add KindOld * Don't process latest events/memberships for old events * Allow federationsender to ignore duplicate key entries when LatestEventIDs is duplicated by RS output events * Signal to downstream components if an event has become a forward extremity * Don't exclude from sync * Soft-fail checks on KindNew * Don't run the latest events updater at all for KindOld * Don't make federation sender change after all * Kind in federation sender join * Don't send isForwardExtremity * Fix syncapi * Update comments * Fix SendEventWithState * Update sytest-whitelist * Generate old output events * Sync API consumes old room events * Update comments --- clientapi/routing/createroom.go | 1 + clientapi/routing/membership.go | 1 + clientapi/routing/profile.go | 4 +- clientapi/routing/redaction.go | 2 +- clientapi/routing/sendevent.go | 1 + clientapi/threepid/invites.go | 1 + federationapi/routing/join.go | 1 + federationapi/routing/leave.go | 1 + federationapi/routing/send.go | 3 ++ federationapi/routing/threepid.go | 3 +- federationsender/internal/perform.go | 1 + roomserver/api/input.go | 16 ++++++-- roomserver/api/output.go | 18 +++++++++ roomserver/api/wrapper.go | 14 ++++--- roomserver/internal/input/input_events.go | 39 +++++++++++++------ .../internal/input/input_latest_events.go | 11 +++--- roomserver/roomserver_test.go | 4 +- syncapi/consumers/roomserver.go | 37 +++++++++++++++++- sytest-whitelist | 1 + 19 files changed, 125 insertions(+), 34 deletions(-) diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 9655339cd..cff3c9813 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -344,6 +344,7 @@ func createRoom( if err = roomserverAPI.SendEventWithState( req.Context(), rsAPI, + roomserverAPI.KindNew, &gomatrixserverlib.RespState{ StateEvents: accumulated, AuthEvents: accumulated, diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 88cb23647..fe0795577 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -76,6 +76,7 @@ func sendMembership(ctx context.Context, accountDB accounts.Database, device *us if err = roomserverAPI.SendEvents( ctx, rsAPI, + api.KindNew, []gomatrixserverlib.HeaderedEvent{event.Event.Headered(roomVer)}, cfg.Matrix.ServerName, nil, diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 60669a0c8..bbe35facd 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -170,7 +170,7 @@ func SetAvatarURL( return jsonerror.InternalServerError() } - if err := api.SendEvents(req.Context(), rsAPI, events, cfg.Matrix.ServerName, nil); err != nil { + if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, events, cfg.Matrix.ServerName, nil); err != nil { util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") return jsonerror.InternalServerError() } @@ -288,7 +288,7 @@ func SetDisplayName( return jsonerror.InternalServerError() } - if err := api.SendEvents(req.Context(), rsAPI, events, cfg.Matrix.ServerName, nil); err != nil { + if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, events, cfg.Matrix.ServerName, nil); err != nil { util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go index 9701685e0..266c0aff2 100644 --- a/clientapi/routing/redaction.go +++ b/clientapi/routing/redaction.go @@ -121,7 +121,7 @@ func SendRedaction( JSON: jsonerror.NotFound("Room does not exist"), } } - if err = roomserverAPI.SendEvents(context.Background(), rsAPI, []gomatrixserverlib.HeaderedEvent{*e}, cfg.Matrix.ServerName, nil); err != nil { + if err = roomserverAPI.SendEvents(context.Background(), rsAPI, api.KindNew, []gomatrixserverlib.HeaderedEvent{*e}, cfg.Matrix.ServerName, nil); err != nil { util.GetLogger(req.Context()).WithError(err).Errorf("failed to SendEvents") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 9744a5640..1303663ff 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -92,6 +92,7 @@ func SendEvent( // event ID in case of duplicate transaction is discarded if err := api.SendEvents( req.Context(), rsAPI, + api.KindNew, []gomatrixserverlib.HeaderedEvent{ e.Headered(verRes.RoomVersion), }, diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index b9575a284..272d3407d 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -361,6 +361,7 @@ func emit3PIDInviteEvent( return api.SendEvents( ctx, rsAPI, + api.KindNew, []gomatrixserverlib.HeaderedEvent{ (*event).Headered(queryRes.RoomVersion), }, diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index c637116f7..12f205366 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -290,6 +290,7 @@ func SendJoin( if !alreadyJoined { if err = api.SendEvents( httpReq.Context(), rsAPI, + api.KindNew, []gomatrixserverlib.HeaderedEvent{ event.Headered(stateAndAuthChainResponse.RoomVersion), }, diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index e16dfcc2e..fb81d9319 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -256,6 +256,7 @@ func SendLeave( // the room, so set SendAsServer to cfg.Matrix.ServerName if err = api.SendEvents( httpReq.Context(), rsAPI, + api.KindNew, []gomatrixserverlib.HeaderedEvent{ event.Headered(verRes.RoomVersion), }, diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 783fdc3b8..76dc3a2ee 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -403,6 +403,7 @@ func (t *txnReq) processEvent(ctx context.Context, e gomatrixserverlib.Event) er return api.SendEvents( context.Background(), t.rsAPI, + api.KindNew, []gomatrixserverlib.HeaderedEvent{ e.Headered(stateResp.RoomVersion), }, @@ -586,6 +587,7 @@ func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixser err = api.SendEventWithState( context.Background(), t.rsAPI, + api.KindOld, resolvedState, backwardsExtremity.Headered(roomVersion), t.haveEventIDs(), @@ -605,6 +607,7 @@ func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixser if err = api.SendEvents( context.Background(), t.rsAPI, + api.KindOld, append(headeredNewEvents, e.Headered(roomVersion)), api.DoNotSendToOtherServers, nil, diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index ec6cc1488..4db5273af 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -89,7 +89,7 @@ func CreateInvitesFrom3PIDInvites( } // Send all the events - if err := api.SendEvents(req.Context(), rsAPI, evs, cfg.Matrix.ServerName, nil); err != nil { + if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, evs, cfg.Matrix.ServerName, nil); err != nil { util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") return jsonerror.InternalServerError() } @@ -174,6 +174,7 @@ func ExchangeThirdPartyInvite( // Send the event to the roomserver if err = api.SendEvents( httpReq.Context(), rsAPI, + api.KindNew, []gomatrixserverlib.HeaderedEvent{ signedEvent.Event.Headered(verRes.RoomVersion), }, diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index 254883e63..3904ab856 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -248,6 +248,7 @@ func (r *FederationSenderInternalAPI) performJoinUsingServer( // returned state to the roomserver to update our local view. if err = roomserverAPI.SendEventWithState( ctx, r.rsAPI, + roomserverAPI.KindNew, respState, event.Headered(respMakeJoin.RoomVersion), nil, diff --git a/roomserver/api/input.go b/roomserver/api/input.go index dd693203b..e1a8afa00 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -21,17 +21,25 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) +type Kind int + const ( // KindOutlier event fall outside the contiguous event graph. // We do not have the state for these events. // These events are state events used to authenticate other events. // They can become part of the contiguous event graph via backfill. - KindOutlier = 1 + KindOutlier Kind = iota + 1 // KindNew event extend the contiguous graph going forwards. // They usually don't need state, but may include state if the // there was a new event that references an event that we don't - // have a copy of. - KindNew = 2 + // have a copy of. New events will influence the fwd extremities + // of the room and output events will be generated as a result. + KindNew + // KindOld event extend the graph backwards, or fill gaps in + // history. They may or may not include state. They will not be + // considered for forward extremities, and output events will NOT + // be generated for them. + KindOld ) // DoNotSendToOtherServers tells us not to send the event to other matrix @@ -43,7 +51,7 @@ const DoNotSendToOtherServers = "" type InputRoomEvent struct { // Whether this event is new, backfilled or an outlier. // This controls how the event is processed. - Kind int `json:"kind"` + Kind Kind `json:"kind"` // The event JSON for the event to add. Event gomatrixserverlib.HeaderedEvent `json:"event"` // List of state event IDs that authenticate this event. diff --git a/roomserver/api/output.go b/roomserver/api/output.go index d57f3b04c..9cb814a47 100644 --- a/roomserver/api/output.go +++ b/roomserver/api/output.go @@ -24,6 +24,8 @@ type OutputType string const ( // OutputTypeNewRoomEvent indicates that the event is an OutputNewRoomEvent OutputTypeNewRoomEvent OutputType = "new_room_event" + // OutputTypeOldRoomEvent indicates that the event is an OutputOldRoomEvent + OutputTypeOldRoomEvent OutputType = "old_room_event" // OutputTypeNewInviteEvent indicates that the event is an OutputNewInviteEvent OutputTypeNewInviteEvent OutputType = "new_invite_event" // OutputTypeRetireInviteEvent indicates that the event is an OutputRetireInviteEvent @@ -58,6 +60,8 @@ type OutputEvent struct { Type OutputType `json:"type"` // The content of event with type OutputTypeNewRoomEvent NewRoomEvent *OutputNewRoomEvent `json:"new_room_event,omitempty"` + // The content of event with type OutputTypeOldRoomEvent + OldRoomEvent *OutputOldRoomEvent `json:"old_room_event,omitempty"` // The content of event with type OutputTypeNewInviteEvent NewInviteEvent *OutputNewInviteEvent `json:"new_invite_event,omitempty"` // The content of event with type OutputTypeRetireInviteEvent @@ -178,6 +182,20 @@ func (ore *OutputNewRoomEvent) AddsState() []gomatrixserverlib.HeaderedEvent { return append(ore.AddStateEvents, ore.Event) } +// An OutputOldRoomEvent is written when the roomserver receives an old event. +// This will typically happen as a result of getting either missing events +// or backfilling. Downstream components may wish to send these events to +// clients when it is advantageous to do so, but with the consideration that +// the event is likely a historic event. +// +// Old events do not update forward extremities or the current room state, +// therefore they must not be treated as if they do. Downstream components +// should build their current room state up from OutputNewRoomEvents only. +type OutputOldRoomEvent struct { + // The Event. + Event gomatrixserverlib.HeaderedEvent `json:"event"` +} + // An OutputNewInviteEvent is written whenever an invite becomes active. // Invite events can be received outside of an existing room so have to be // tracked separately from the room events themselves. diff --git a/roomserver/api/wrapper.go b/roomserver/api/wrapper.go index a38c00df7..9e8219103 100644 --- a/roomserver/api/wrapper.go +++ b/roomserver/api/wrapper.go @@ -24,13 +24,14 @@ import ( // SendEvents to the roomserver The events are written with KindNew. func SendEvents( - ctx context.Context, rsAPI RoomserverInternalAPI, events []gomatrixserverlib.HeaderedEvent, + ctx context.Context, rsAPI RoomserverInternalAPI, + kind Kind, events []gomatrixserverlib.HeaderedEvent, sendAsServer gomatrixserverlib.ServerName, txnID *TransactionID, ) error { ires := make([]InputRoomEvent, len(events)) for i, event := range events { ires[i] = InputRoomEvent{ - Kind: KindNew, + Kind: kind, Event: event, AuthEventIDs: event.AuthEventIDs(), SendAsServer: string(sendAsServer), @@ -40,12 +41,13 @@ func SendEvents( return SendInputRoomEvents(ctx, rsAPI, ires) } -// SendEventWithState writes an event with KindNew to the roomserver +// SendEventWithState writes an event with the specified kind to the roomserver // with the state at the event as KindOutlier before it. Will not send any event that is // marked as `true` in haveEventIDs func SendEventWithState( - ctx context.Context, rsAPI RoomserverInternalAPI, state *gomatrixserverlib.RespState, - event gomatrixserverlib.HeaderedEvent, haveEventIDs map[string]bool, + ctx context.Context, rsAPI RoomserverInternalAPI, kind Kind, + state *gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent, + haveEventIDs map[string]bool, ) error { outliers, err := state.Events() if err != nil { @@ -70,7 +72,7 @@ func SendEventWithState( } ires = append(ires, InputRoomEvent{ - Kind: KindNew, + Kind: kind, Event: event, AuthEventIDs: event.AuthEventIDs(), HasState: true, diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 113341591..67031609f 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -119,7 +119,7 @@ func (r *Inputer) processRoomEvent( // We haven't calculated a state for this event yet. // Lets calculate one. err = r.calculateAndSetState(ctx, input, *roomInfo, &stateAtEvent, event, isRejected) - if err != nil { + if err != nil && input.Kind != api.KindOld { return "", fmt.Errorf("r.calculateAndSetState: %w", err) } } @@ -136,16 +136,31 @@ func (r *Inputer) processRoomEvent( return event.EventID(), rejectionErr } - if err = r.updateLatestEvents( - ctx, // context - roomInfo, // room info for the room being updated - stateAtEvent, // state at event (below) - event, // event - input.SendAsServer, // send as server - input.TransactionID, // transaction ID - input.HasState, // rewrites state? - ); err != nil { - return "", fmt.Errorf("r.updateLatestEvents: %w", err) + switch input.Kind { + case api.KindNew: + if err = r.updateLatestEvents( + ctx, // context + roomInfo, // room info for the room being updated + stateAtEvent, // state at event (below) + event, // event + input.SendAsServer, // send as server + input.TransactionID, // transaction ID + input.HasState, // rewrites state? + ); err != nil { + return "", fmt.Errorf("r.updateLatestEvents: %w", err) + } + case api.KindOld: + err = r.WriteOutputEvents(event.RoomID(), []api.OutputEvent{ + { + Type: api.OutputTypeOldRoomEvent, + OldRoomEvent: &api.OutputOldRoomEvent{ + Event: headered, + }, + }, + }) + if err != nil { + return "", fmt.Errorf("r.WriteOutputEvents (old): %w", err) + } } // processing this event resulted in an event (which may not be the one we're processing) @@ -163,7 +178,7 @@ func (r *Inputer) processRoomEvent( }, }) if err != nil { - return "", fmt.Errorf("r.WriteOutputEvents: %w", err) + return "", fmt.Errorf("r.WriteOutputEvents (redactions): %w", err) } } diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index ca5d214d7..5adcd0877 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -164,8 +164,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { return fmt.Errorf("u.api.updateMemberships: %w", err) } - var update *api.OutputEvent - update, err = u.makeOutputNewRoomEvent() + update, err := u.makeOutputNewRoomEvent() if err != nil { return fmt.Errorf("u.makeOutputNewRoomEvent: %w", err) } @@ -259,6 +258,8 @@ func (u *latestEventsUpdater) latestState() error { return nil } +// calculateLatest works out the new set of forward extremities. Returns +// true if the new event is included in those extremites, false otherwise. func (u *latestEventsUpdater) calculateLatest( oldLatest []types.StateAtEventAndReference, newEvent types.StateAtEventAndReference, @@ -326,7 +327,6 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) if err != nil { return nil, err } - for _, entry := range u.added { ore.AddsStateEventIDs = append(ore.AddsStateEventIDs, eventIDMap[entry.EventNID]) } @@ -339,13 +339,14 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) for _, entry := range u.stateBeforeEventAdds { ore.StateBeforeAddsEventIDs = append(ore.StateBeforeAddsEventIDs, eventIDMap[entry.EventNID]) } + ore.SendAsServer = u.sendAsServer // include extra state events if they were added as nearly every downstream component will care about it // and we'd rather not have them all hit QueryEventsByID at the same time! if len(ore.AddsStateEventIDs) > 0 { - ore.AddStateEvents, err = u.extraEventsForIDs(u.roomInfo.RoomVersion, ore.AddsStateEventIDs) - if err != nil { + var err error + if ore.AddStateEvents, err = u.extraEventsForIDs(u.roomInfo.RoomVersion, ore.AddsStateEventIDs); err != nil { return nil, fmt.Errorf("failed to load add_state_events from db: %w", err) } } diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index 1b692a098..c8e60efa3 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -191,7 +191,7 @@ func mustSendEvents(t *testing.T, ver gomatrixserverlib.RoomVersion, events []js t.Helper() rsAPI, dp := mustCreateRoomserverAPI(t) hevents := mustLoadRawEvents(t, ver, events) - if err := api.SendEvents(ctx, rsAPI, hevents, testOrigin, nil); err != nil { + if err := api.SendEvents(ctx, rsAPI, api.KindNew, hevents, testOrigin, nil); err != nil { t.Errorf("failed to SendEvents: %s", err) } return rsAPI, dp, hevents @@ -337,7 +337,7 @@ func TestOutputRewritesState(t *testing.T) { deleteDatabase() rsAPI, producer := mustCreateRoomserverAPI(t) defer deleteDatabase() - err := api.SendEvents(context.Background(), rsAPI, originalEvents, testOrigin, nil) + err := api.SendEvents(context.Background(), rsAPI, api.KindNew, originalEvents, testOrigin, nil) if err != nil { t.Fatalf("failed to send original events: %s", err) } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index ca48c8300..373baea54 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -97,6 +97,8 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { } } return s.onNewRoomEvent(context.TODO(), *output.NewRoomEvent) + case api.OutputTypeOldRoomEvent: + return s.onOldRoomEvent(context.TODO(), *output.OldRoomEvent) case api.OutputTypeNewInviteEvent: return s.onNewInviteEvent(context.TODO(), *output.NewInviteEvent) case api.OutputTypeRetireInviteEvent: @@ -168,7 +170,40 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( log.ErrorKey: err, "add": msg.AddsStateEventIDs, "del": msg.RemovesStateEventIDs, - }).Panicf("roomserver output log: write event failure") + }).Panicf("roomserver output log: write new event failure") + return nil + } + + if pduPos, err = s.notifyJoinedPeeks(ctx, &ev, pduPos); err != nil { + logrus.WithError(err).Errorf("Failed to notifyJoinedPeeks for PDU pos %d", pduPos) + return err + } + + s.notifier.OnNewEvent(&ev, "", nil, types.NewStreamToken(pduPos, 0, nil)) + + return nil +} + +func (s *OutputRoomEventConsumer) onOldRoomEvent( + ctx context.Context, msg api.OutputOldRoomEvent, +) error { + ev := msg.Event + + pduPos, err := s.db.WriteEvent( + ctx, + &ev, + []gomatrixserverlib.HeaderedEvent{}, + []string{}, // adds no state + []string{}, // removes no state + nil, // no transaction + false, // not excluded from sync + ) + if err != nil { + // panic rather than continue with an inconsistent database + log.WithFields(log.Fields{ + "event": string(ev.JSON()), + log.ErrorKey: err, + }).Panicf("roomserver output log: write old event failure") return nil } diff --git a/sytest-whitelist b/sytest-whitelist index 2ba0a88b2..2d0323318 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -484,3 +484,4 @@ Users cannot kick users who have already left a room A prev_batch token from incremental sync can be used in the v1 messages API Event with an invalid signature in the send_join response should not cause room join to fail Inbound federation rejects typing notifications from wrong remote +Should not be able to take over the room by pretending there is no PL event From 45abdcaeb967ea54e1edbe3c1f1d73d1a82a2904 Mon Sep 17 00:00:00 2001 From: Devon Johnson Date: Sat, 17 Oct 2020 21:07:34 -0700 Subject: [PATCH 092/104] Send state after event, not current Signed-off-by: Devon Johnson --- clientapi/routing/state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go index 2a424cbe8..f69b54bbc 100644 --- a/clientapi/routing/state.go +++ b/clientapi/routing/state.go @@ -141,7 +141,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a util.GetLogger(ctx).WithError(err).Error("Failed to QueryMembershipForUser") return jsonerror.InternalServerError() } - for _, ev := range stateRes.StateEvents { + for _, ev := range stateAfterRes.StateEvents { stateEvents = append( stateEvents, gomatrixserverlib.HeaderedToClientEvent(ev, gomatrixserverlib.FormatAll), From 5d80ff11a06d404250933d796bd96d05a9828007 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 19 Oct 2020 15:17:28 +0100 Subject: [PATCH 093/104] Update sytest-whitelist --- sytest-whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/sytest-whitelist b/sytest-whitelist index 2d0323318..5c57bb145 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -485,3 +485,4 @@ A prev_batch token from incremental sync can be used in the v1 messages API Event with an invalid signature in the send_join response should not cause room join to fail Inbound federation rejects typing notifications from wrong remote Should not be able to take over the room by pretending there is no PL event +Can get rooms/{roomId}/state for a departed room (SPEC-216) From c7bf122a26a53f4366884b1f912fdafd0501b610 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 19 Oct 2020 15:38:42 +0100 Subject: [PATCH 094/104] Update sytest lists --- sytest-blacklist | 5 ++++- sytest-whitelist | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sytest-blacklist b/sytest-blacklist index ff7fdf7e0..f493f94fe 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -57,4 +57,7 @@ The only membership state included in a gapped incremental sync is for senders i # Blacklisted out of flakiness after #1479 Invited user can reject local invite after originator leaves Invited user can reject invite for empty room -If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes \ No newline at end of file +If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes + +# Blacklisted due to flakiness +A prev_batch token from incremental sync can be used in the v1 messages API \ No newline at end of file diff --git a/sytest-whitelist b/sytest-whitelist index 5c57bb145..cecf24f75 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -481,7 +481,6 @@ m.room.history_visibility == "joined" allows/forbids appropriately for Guest use m.room.history_visibility == "joined" allows/forbids appropriately for Real users POST rejects invalid utf-8 in JSON Users cannot kick users who have already left a room -A prev_batch token from incremental sync can be used in the v1 messages API Event with an invalid signature in the send_join response should not cause room join to fail Inbound federation rejects typing notifications from wrong remote Should not be able to take over the room by pretending there is no PL event From a71360d0992b4bc120e9a98af7755e72e2fc3eb5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 20 Oct 2020 11:22:46 +0100 Subject: [PATCH 095/104] Update INSTALL.md (#1549) --- docs/INSTALL.md | 45 ++++++++++++++++++--------------------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index be12d7b86..d246e9221 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -93,7 +93,7 @@ brew services start kafka ### SQLite database setup Dendrite can use the built-in SQLite database engine for small setups. -The SQLite databases do not need to be preconfigured - Dendrite will +The SQLite databases do not need to be pre-built - Dendrite will create them automatically at startup. ### Postgres database setup @@ -109,7 +109,7 @@ Assuming that Postgres 9.5 (or later) is installed: * Create the component databases: ```bash - for i in account device mediaapi syncapi roomserver signingkeyserver federationsender appservice e2ekey naffka; do + for i in mediaapi syncapi roomserver signingkeyserver federationsender appservice keyserver userapi_account userapi_device naffka; do sudo -u postgres createdb -O dendrite dendrite_$i done ``` @@ -135,8 +135,8 @@ Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Th * The `server_name` entry to reflect the hostname of your Dendrite server * The `database` lines with an updated connection string based on your - desired setup, e.g. replacing `component` with the name of the component: - * For Postgres: `postgres://dendrite:password@localhost/component` + desired setup, e.g. replacing `database` with the name of the database: + * For Postgres: `postgres://dendrite:password@localhost/database` * For SQLite on disk: `file:component.db` or `file:///path/to/component.db` * Postgres and SQLite can be mixed and matched. * The `use_naffka` option if using Naffka in a monolith deployment @@ -147,6 +147,10 @@ then configuring `key_perspectives` (like `matrix.org` in the sample) can help to improve reliability considerably by allowing your homeserver to fetch public keys for dead homeservers from somewhere else. +**WARNING:** Dendrite supports running all components from the same database in +Postgres mode, but this is **NOT** a supported configuration with SQLite. When +using SQLite, all components **MUST** use their own database file. + ## Starting a monolith server It is possible to use Naffka as an in-process replacement to Kafka when using @@ -167,30 +171,17 @@ as shown below, it will also listen for HTTPS connections on port 8448. The following contains scripts which will run all the required processes in order to point a Matrix client at Dendrite. -### Client proxy +### nginx (or other reverse proxy) -This is what Matrix clients will talk to. If you use the script below, point -your client at `http://localhost:8008`. +This is what your clients and federated hosts will talk to. It must forward +requests onto the correct API server based on URL: -```bash -./bin/client-api-proxy \ ---bind-address ":8008" \ ---client-api-server-url "http://localhost:7771" \ ---sync-api-server-url "http://localhost:7773" \ ---media-api-server-url "http://localhost:7774" \ -``` +* `/_matrix/client` to the client API server +* `/_matrix/federation` to the federation API server +* `/_matrix/key` to the federation API server +* `/_matrix/media` to the media API server -### Federation proxy - -This is what Matrix servers will talk to. This is only required if you want -to support federation. - -```bash -./bin/federation-api-proxy \ ---bind-address ":8448" \ ---federation-api-url "http://localhost:7772" \ ---media-api-server-url "http://localhost:7774" \ -``` +See `docs/nginx/polylith-sample.conf` for a sample configuration. ### Client API server @@ -198,7 +189,7 @@ This is what implements CS API endpoints. Clients talk to this via the proxy in order to send messages, create and join rooms, etc. ```bash -./bin/dendrite-client-api-server --config=dendrite.yaml +./bin/dendrite-client-api-server --config dendrite.yaml ``` ### Sync server @@ -239,7 +230,7 @@ contacted by other components. This includes the following components. This is what implements the room DAG. Clients do not talk to this. ```bash -./bin/dendrite-room-server --config=dendrite.yaml +./bin/dendrite-room-server --config dendrite.yaml ``` #### Federation sender From 92982a402ff553fe6d9179859a96ec33e2f09929 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 20 Oct 2020 11:34:22 +0100 Subject: [PATCH 096/104] Update Docker (#1542) * Separate Docker images, rather than tags * Allow specifying tag to build/push/pull * Include goose in Docker builds --- build/docker/Dockerfile.component | 1 + build/docker/docker-compose.monolith.yml | 2 +- build/docker/docker-compose.polylith.yml | 61 +++++------------------- build/docker/images-build.sh | 32 +++++++------ build/docker/images-pull.sh | 30 ++++++------ build/docker/images-push.sh | 30 ++++++------ 6 files changed, 62 insertions(+), 94 deletions(-) diff --git a/build/docker/Dockerfile.component b/build/docker/Dockerfile.component index 13634391a..1acf510fb 100644 --- a/build/docker/Dockerfile.component +++ b/build/docker/Dockerfile.component @@ -6,6 +6,7 @@ ARG component=monolith ENV entrypoint=${component} COPY --from=base /build/bin/${component} /usr/bin +COPY --from=base /build/bin/goose /usr/bin VOLUME /etc/dendrite WORKDIR /etc/dendrite diff --git a/build/docker/docker-compose.monolith.yml b/build/docker/docker-compose.monolith.yml index 336a43984..7d63e1c65 100644 --- a/build/docker/docker-compose.monolith.yml +++ b/build/docker/docker-compose.monolith.yml @@ -2,7 +2,7 @@ version: "3.4" services: monolith: hostname: monolith - image: matrixdotorg/dendrite:monolith + image: matrixdotorg/dendrite-monolith:latest command: [ "--config=dendrite.yaml", "--tls-cert=server.crt", diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index 8a4c50e06..e8da9c24a 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -1,28 +1,8 @@ version: "3.4" services: - client_api_proxy: - hostname: client_api_proxy - image: matrixdotorg/dendrite:clientproxy - command: [ - "--bind-address=:8008", - "--client-api-server-url=http://client_api:8071", - "--sync-api-server-url=http://sync_api:8073", - "--media-api-server-url=http://media_api:8074" - ] - volumes: - - ./config:/etc/dendrite - networks: - - internal - depends_on: - - sync_api - - client_api - - media_api - ports: - - "8008:8008" - client_api: hostname: client_api - image: matrixdotorg/dendrite:clientapi + image: matrixdotorg/dendrite-clientapi:latest command: [ "--config=dendrite.yaml" ] @@ -34,7 +14,7 @@ services: media_api: hostname: media_api - image: matrixdotorg/dendrite:mediaapi + image: matrixdotorg/dendrite-mediaapi:latest command: [ "--config=dendrite.yaml" ] @@ -45,7 +25,7 @@ services: sync_api: hostname: sync_api - image: matrixdotorg/dendrite:syncapi + image: matrixdotorg/dendrite-syncapi:latest command: [ "--config=dendrite.yaml" ] @@ -56,7 +36,7 @@ services: room_server: hostname: room_server - image: matrixdotorg/dendrite:roomserver + image: matrixdotorg/dendrite-roomserver:latest command: [ "--config=dendrite.yaml" ] @@ -67,7 +47,7 @@ services: edu_server: hostname: edu_server - image: matrixdotorg/dendrite:eduserver + image: matrixdotorg/dendrite-eduserver:latest command: [ "--config=dendrite.yaml" ] @@ -76,28 +56,9 @@ services: networks: - internal - federation_api_proxy: - hostname: federation_api_proxy - image: matrixdotorg/dendrite:federationproxy - command: [ - "--bind-address=:8448", - "--federation-api-url=http://federation_api:8072", - "--media-api-server-url=http://media_api:8074" - ] - volumes: - - ./config:/etc/dendrite - depends_on: - - federation_api - - federation_sender - - media_api - networks: - - internal - ports: - - "8448:8448" - federation_api: hostname: federation_api - image: matrixdotorg/dendrite:federationapi + image: matrixdotorg/dendrite-federationapi:latest command: [ "--config=dendrite.yaml" ] @@ -108,7 +69,7 @@ services: federation_sender: hostname: federation_sender - image: matrixdotorg/dendrite:federationsender + image: matrixdotorg/dendrite-federationsender:latest command: [ "--config=dendrite.yaml" ] @@ -119,7 +80,7 @@ services: key_server: hostname: key_server - image: matrixdotorg/dendrite:keyserver + image: matrixdotorg/dendrite-keyserver:latest command: [ "--config=dendrite.yaml" ] @@ -130,7 +91,7 @@ services: signing_key_server: hostname: signing_key_server - image: matrixdotorg/dendrite:signingkeyserver + image: matrixdotorg/dendrite-signingkeyserver:latest command: [ "--config=dendrite.yaml" ] @@ -141,7 +102,7 @@ services: user_api: hostname: user_api - image: matrixdotorg/dendrite:userapi + image: matrixdotorg/dendrite-userapi:latest command: [ "--config=dendrite.yaml" ] @@ -152,7 +113,7 @@ services: appservice_api: hostname: appservice_api - image: matrixdotorg/dendrite:appservice + image: matrixdotorg/dendrite-appservice:latest command: [ "--config=dendrite.yaml" ] diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index d72bac214..daad63be0 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -2,20 +2,22 @@ cd $(git rev-parse --show-toplevel) -docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:latest . +TAG=${1:-latest} -docker build -t matrixdotorg/dendrite:monolith --build-arg component=dendrite-monolith-server -f build/docker/Dockerfile.component . +echo "Building tag '${TAG}'" -docker build -t matrixdotorg/dendrite:appservice --build-arg component=dendrite-appservice-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:clientapi --build-arg component=dendrite-client-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:clientproxy --build-arg component=client-api-proxy -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:eduserver --build-arg component=dendrite-edu-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationapi --build-arg component=dendrite-federation-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationsender --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationproxy --build-arg component=federation-api-proxy -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:keyserver --build-arg component=dendrite-key-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:signingkeyserver --build-arg component=dendrite-signing-key-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:userapi --build-arg component=dendrite-user-api-server -f build/docker/Dockerfile.component . +docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} . + +docker build -t matrixdotorg/dendrite-monolith:${TAG} --build-arg component=dendrite-monolith-server -f build/docker/Dockerfile.component . + +docker build -t matrixdotorg/dendrite-appservice:${TAG} --build-arg component=dendrite-appservice-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-clientapi:${TAG} --build-arg component=dendrite-client-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-eduserver:${TAG} --build-arg component=dendrite-edu-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-federationapi:${TAG} --build-arg component=dendrite-federation-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-federationsender:${TAG} --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-keyserver:${TAG} --build-arg component=dendrite-key-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-mediaapi:${TAG} --build-arg component=dendrite-media-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-roomserver:${TAG} --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-syncapi:${TAG} --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-signingkeyserver:${TAG} --build-arg component=dendrite-signing-key-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-userapi:${TAG} --build-arg component=dendrite-user-api-server -f build/docker/Dockerfile.component . diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh index be9185464..e3284a2a6 100755 --- a/build/docker/images-pull.sh +++ b/build/docker/images-pull.sh @@ -1,17 +1,19 @@ #!/bin/bash -docker pull matrixdotorg/dendrite:monolith +TAG=${1:-latest} -docker pull matrixdotorg/dendrite:appservice -docker pull matrixdotorg/dendrite:clientapi -docker pull matrixdotorg/dendrite:clientproxy -docker pull matrixdotorg/dendrite:eduserver -docker pull matrixdotorg/dendrite:federationapi -docker pull matrixdotorg/dendrite:federationsender -docker pull matrixdotorg/dendrite:federationproxy -docker pull matrixdotorg/dendrite:keyserver -docker pull matrixdotorg/dendrite:mediaapi -docker pull matrixdotorg/dendrite:roomserver -docker pull matrixdotorg/dendrite:syncapi -docker pull matrixdotorg/dendrite:signingkeyserver -docker pull matrixdotorg/dendrite:userapi +echo "Pulling tag '${TAG}'" + +docker pull matrixdotorg/dendrite-monolith:${TAG} + +docker pull matrixdotorg/dendrite-appservice:${TAG} +docker pull matrixdotorg/dendrite-clientapi:${TAG} +docker pull matrixdotorg/dendrite-eduserver:${TAG} +docker pull matrixdotorg/dendrite-federationapi:${TAG} +docker pull matrixdotorg/dendrite-federationsender:${TAG} +docker pull matrixdotorg/dendrite-keyserver:${TAG} +docker pull matrixdotorg/dendrite-mediaapi:${TAG} +docker pull matrixdotorg/dendrite-roomserver:${TAG} +docker pull matrixdotorg/dendrite-syncapi:${TAG} +docker pull matrixdotorg/dendrite-signingkeyserver:${TAG} +docker pull matrixdotorg/dendrite-userapi:${TAG} diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index 64920171b..e4eb773ac 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -1,17 +1,19 @@ #!/bin/bash -docker push matrixdotorg/dendrite:monolith +TAG=${1:-latest} -docker push matrixdotorg/dendrite:appservice -docker push matrixdotorg/dendrite:clientapi -docker push matrixdotorg/dendrite:clientproxy -docker push matrixdotorg/dendrite:eduserver -docker push matrixdotorg/dendrite:federationapi -docker push matrixdotorg/dendrite:federationsender -docker push matrixdotorg/dendrite:federationproxy -docker push matrixdotorg/dendrite:keyserver -docker push matrixdotorg/dendrite:mediaapi -docker push matrixdotorg/dendrite:roomserver -docker push matrixdotorg/dendrite:syncapi -docker push matrixdotorg/dendrite:signingkeyserver -docker push matrixdotorg/dendrite:userapi +echo "Pushing tag '${TAG}'" + +docker push matrixdotorg/dendrite-monolith:${TAG} + +docker push matrixdotorg/dendrite-appservice:${TAG} +docker push matrixdotorg/dendrite-clientapi:${TAG} +docker push matrixdotorg/dendrite-eduserver:${TAG} +docker push matrixdotorg/dendrite-federationapi:${TAG} +docker push matrixdotorg/dendrite-federationsender:${TAG} +docker push matrixdotorg/dendrite-keyserver:${TAG} +docker push matrixdotorg/dendrite-mediaapi:${TAG} +docker push matrixdotorg/dendrite-roomserver:${TAG} +docker push matrixdotorg/dendrite-syncapi:${TAG} +docker push matrixdotorg/dendrite-signingkeyserver:${TAG} +docker push matrixdotorg/dendrite-userapi:${TAG} From eb86e2b336e8feaf54eb7fe688c896aa11bd5329 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 20 Oct 2020 11:42:54 +0100 Subject: [PATCH 097/104] Fix sqlite locking bugs present on sytest (#1543) * Fix sqite locking bugs present on sytest Comments do the explaining. * Fix deadlock in sqlite mode Caused by starting a writer whilst within a writer * Only complain about invalid state deltas for non-overwrite events * Do not re-process outlier unnecessarily --- roomserver/internal/input/input_events.go | 24 ++++++++++++++++++ .../internal/input/input_latest_events.go | 2 +- .../storage/shared/latest_events_updater.go | 14 +++++------ roomserver/storage/shared/storage.go | 25 ++++++++++++++++--- 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 67031609f..6a5d9d264 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -17,6 +17,7 @@ package input import ( + "bytes" "context" "fmt" @@ -26,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) @@ -44,6 +46,28 @@ func (r *Inputer) processRoomEvent( headered := input.Event event := headered.Unwrap() + // if we have already got this event then do not process it again, if the input kind is an outlier. + // Outliers contain no extra information which may warrant a re-processing. + if input.Kind == api.KindOutlier { + evs, err := r.DB.EventsFromIDs(ctx, []string{event.EventID()}) + if err == nil && len(evs) == 1 { + // check hash matches if we're on early room versions where the event ID was a random string + idFormat, err := headered.RoomVersion.EventIDFormat() + if err == nil { + switch idFormat { + case gomatrixserverlib.EventIDFormatV1: + if bytes.Equal(event.EventReference().EventSHA256, evs[0].EventReference().EventSHA256) { + util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring") + return event.EventID(), nil + } + default: + util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring") + return event.EventID(), nil + } + } + } + } + // Check that the event passes authentication checks and work out // the numeric IDs for the auth events. isRejected := false diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index 5adcd0877..f76b0a0b4 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -233,7 +233,7 @@ func (u *latestEventsUpdater) latestState() error { if err != nil { return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err) } - if len(u.removed) > len(u.added) { + if !u.stateAtEvent.Overwrite && len(u.removed) > len(u.added) { // This really shouldn't happen. // TODO: What is ultimately the best way to handle this situation? logrus.Errorf( diff --git a/roomserver/storage/shared/latest_events_updater.go b/roomserver/storage/shared/latest_events_updater.go index b316f639d..8825dc464 100644 --- a/roomserver/storage/shared/latest_events_updater.go +++ b/roomserver/storage/shared/latest_events_updater.go @@ -70,16 +70,14 @@ func (u *LatestEventsUpdater) CurrentStateSnapshotNID() types.StateSnapshotNID { return u.currentStateSnapshotNID } -// StorePreviousEvents implements types.RoomRecentEventsUpdater +// StorePreviousEvents implements types.RoomRecentEventsUpdater - This must be called from a Writer func (u *LatestEventsUpdater) StorePreviousEvents(eventNID types.EventNID, previousEventReferences []gomatrixserverlib.EventReference) error { - return u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error { - for _, ref := range previousEventReferences { - if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { - return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err) - } + for _, ref := range previousEventReferences { + if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, u.txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { + return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err) } - return nil - }) + } + return nil } // IsReferenced implements types.RoomRecentEventsUpdater diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index f2be8b3cf..51dcb8887 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -492,15 +492,32 @@ func (d *Database) StoreEvent( if roomInfo == nil && len(prevEvents) > 0 { return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("expected room %q to exist", event.RoomID()) } + // Create an updater - NB: on sqlite this WILL create a txn as we are directly calling the shared DB form of + // GetLatestEventsForUpdate - not via the SQLiteDatabase form which has `nil` txns. This + // function only does SELECTs though so the created txn (at this point) is just a read txn like + // any other so this is fine. If we ever update GetLatestEventsForUpdate or NewLatestEventsUpdater + // to do writes however then this will need to go inside `Writer.Do`. updater, err = d.GetLatestEventsForUpdate(ctx, *roomInfo) if err != nil { return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("NewLatestEventsUpdater: %w", err) } - if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil { - return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("updater.StorePreviousEvents: %w", err) + // Ensure that we atomically store prev events AND commit them. If we don't wrap StorePreviousEvents + // and EndTransaction in a writer then it's possible for a new write txn to be made between the two + // function calls which will then fail with 'database is locked'. This new write txn would HAVE to be + // something like SetRoomAlias/RemoveRoomAlias as normal input events are already done sequentially due to + // SupportsConcurrentRoomInputs() == false on sqlite, though this does not apply to setting room aliases + // as they don't go via InputRoomEvents + err = d.Writer.Do(d.DB, updater.txn, func(txn *sql.Tx) error { + if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil { + return fmt.Errorf("updater.StorePreviousEvents: %w", err) + } + succeeded := true + err = sqlutil.EndTransaction(updater, &succeeded) + return err + }) + if err != nil { + return 0, types.StateAtEvent{}, nil, "", err } - succeeded := true - err = sqlutil.EndTransaction(updater, &succeeded) } return roomNID, types.StateAtEvent{ From 53a745f3332c4bb35dddd80ff8f83e7eb11456ca Mon Sep 17 00:00:00 2001 From: Pika <15848969+ThatNerdyPikachu@users.noreply.github.com> Date: Tue, 20 Oct 2020 06:47:37 -0400 Subject: [PATCH 098/104] fix create-account (#1546) --- cmd/create-account/main.go | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index a9bd92794..f6de2d0d4 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -22,7 +22,6 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/userapi/storage/accounts" - "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/matrix-org/gomatrixserverlib" ) @@ -39,7 +38,6 @@ var ( username = flag.String("username", "", "The user ID localpart to register e.g 'alice' in '@alice:localhost'.") password = flag.String("password", "", "Optional. The password to register with. If not specified, this account will be password-less.") serverNameStr = flag.String("servername", "localhost", "The Matrix server domain which will form the domain part of the user ID.") - accessToken = flag.String("token", "", "Optional. The desired access_token to have. If not specified, a random access_token will be made.") ) func main() { @@ -78,29 +76,5 @@ func main() { os.Exit(1) } - deviceDB, err := devices.NewDatabase(&config.DatabaseOptions{ - ConnectionString: config.DataSource(*database), - }, serverName) - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - - if *accessToken == "" { - t := "token_" + *username - accessToken = &t - } - - device, err := deviceDB.CreateDevice( - context.Background(), *username, nil, *accessToken, nil, "127.0.0.1", "", - ) - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - - fmt.Println("Created account:") - fmt.Printf("user_id = %s\n", device.UserID) - fmt.Printf("device_id = %s\n", device.ID) - fmt.Printf("access_token = %s\n", device.AccessToken) + fmt.Println("Created account") } From 837c295c26aa42faf1087368d05ceca2e42d95a0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 20 Oct 2020 12:29:53 +0100 Subject: [PATCH 099/104] Linting --- roomserver/internal/input/input_events.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 6a5d9d264..c055289c9 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -49,11 +49,11 @@ func (r *Inputer) processRoomEvent( // if we have already got this event then do not process it again, if the input kind is an outlier. // Outliers contain no extra information which may warrant a re-processing. if input.Kind == api.KindOutlier { - evs, err := r.DB.EventsFromIDs(ctx, []string{event.EventID()}) - if err == nil && len(evs) == 1 { + evs, err2 := r.DB.EventsFromIDs(ctx, []string{event.EventID()}) + if err2 == nil && len(evs) == 1 { // check hash matches if we're on early room versions where the event ID was a random string - idFormat, err := headered.RoomVersion.EventIDFormat() - if err == nil { + idFormat, err2 := headered.RoomVersion.EventIDFormat() + if err2 == nil { switch idFormat { case gomatrixserverlib.EventIDFormatV1: if bytes.Equal(event.EventReference().EventSHA256, evs[0].EventReference().EventSHA256) { From 6c3c621de09a62c7ec8d89b2d8e6a9d73833fd0b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 20 Oct 2020 12:36:16 +0100 Subject: [PATCH 100/104] Remove invalid state delta check (#1550) --- roomserver/internal/input/input_latest_events.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index f76b0a0b4..5631959b7 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -233,18 +233,6 @@ func (u *latestEventsUpdater) latestState() error { if err != nil { return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err) } - if !u.stateAtEvent.Overwrite && len(u.removed) > len(u.added) { - // This really shouldn't happen. - // TODO: What is ultimately the best way to handle this situation? - logrus.Errorf( - "Invalid state delta on event %q wants to remove %d state but only add %d state (between state snapshots %d and %d)", - u.event.EventID(), len(u.removed), len(u.added), u.oldStateNID, u.newStateNID, - ) - u.added = u.added[:0] - u.removed = u.removed[:0] - u.newStateNID = u.oldStateNID - return nil - } // Also work out the state before the event removes and the event // adds. From 39c7a8915ccba3ac4d281ae7938f2e912ad966b4 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 20 Oct 2020 16:11:24 +0100 Subject: [PATCH 101/104] Multi-personality polylith binary (#1552) * Initial work oon multipersonality binary * Remove old binaries * Monolith and polylith binaries * Better logging * dendrite-poly-multi * Fix path * Copyright notices etc * Tweaks * Update Docker, INSTALL.md * Take first argument if flags package doesn't find any args * Postgres 9.6 or later, fix some more Docker stuff * Don't create unnecessary e2ekey DB * Run go mod tidy --- build/docker/Dockerfile | 2 +- build/docker/Dockerfile.component | 14 ---- build/docker/Dockerfile.monolith | 13 ++++ build/docker/Dockerfile.polylith | 13 ++++ build/docker/docker-compose.deps.yml | 2 +- build/docker/docker-compose.monolith.yml | 1 - build/docker/docker-compose.polylith.yml | 67 ++++++---------- build/docker/images-build.sh | 15 +--- build/docker/images-pull.sh | 13 +--- build/docker/images-push.sh | 13 +--- build/docker/postgres/create_db.sh | 2 +- cmd/dendrite-polylith-multi/main.go | 78 +++++++++++++++++++ .../personalities/appservice.go} | 11 +-- .../personalities/clientapi.go} | 12 +-- .../personalities/eduserver.go} | 18 ++--- .../personalities/federationapi.go} | 11 +-- .../personalities/federationsender.go} | 11 +-- .../personalities/keyserver.go} | 9 +-- .../personalities/mediaapi.go} | 11 +-- .../personalities/roomserver.go} | 11 +-- .../personalities/signingkeyserver.go} | 9 +-- .../personalities/syncapi.go} | 12 +-- .../personalities/userapi.go} | 11 +-- docs/INSTALL.md | 26 +++---- 24 files changed, 189 insertions(+), 196 deletions(-) delete mode 100644 build/docker/Dockerfile.component create mode 100644 build/docker/Dockerfile.monolith create mode 100644 build/docker/Dockerfile.polylith create mode 100644 cmd/dendrite-polylith-multi/main.go rename cmd/{dendrite-appservice-server/main.go => dendrite-polylith-multi/personalities/appservice.go} (83%) rename cmd/{dendrite-client-api-server/main.go => dendrite-polylith-multi/personalities/clientapi.go} (87%) rename cmd/{dendrite-edu-server/main.go => dendrite-polylith-multi/personalities/eduserver.go} (77%) rename cmd/{dendrite-federation-api-server/main.go => dendrite-polylith-multi/personalities/federationapi.go} (86%) rename cmd/{dendrite-federation-sender-server/main.go => dendrite-polylith-multi/personalities/federationsender.go} (85%) rename cmd/{dendrite-key-server/main.go => dendrite-polylith-multi/personalities/keyserver.go} (87%) rename cmd/{dendrite-media-api-server/main.go => dendrite-polylith-multi/personalities/mediaapi.go} (82%) rename cmd/{dendrite-room-server/main.go => dendrite-polylith-multi/personalities/roomserver.go} (84%) rename cmd/{dendrite-signing-key-server/main.go => dendrite-polylith-multi/personalities/signingkeyserver.go} (86%) rename cmd/{dendrite-sync-api-server/main.go => dendrite-polylith-multi/personalities/syncapi.go} (84%) rename cmd/{dendrite-user-api-server/main.go => dendrite-polylith-multi/personalities/userapi.go} (84%) diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index d8e07681f..5cab0530f 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.13.7-alpine3.11 AS builder +FROM docker.io/golang:1.15-alpine AS builder RUN apk --update --no-cache add bash build-base diff --git a/build/docker/Dockerfile.component b/build/docker/Dockerfile.component deleted file mode 100644 index 1acf510fb..000000000 --- a/build/docker/Dockerfile.component +++ /dev/null @@ -1,14 +0,0 @@ -FROM matrixdotorg/dendrite:latest AS base - -FROM alpine:latest - -ARG component=monolith -ENV entrypoint=${component} - -COPY --from=base /build/bin/${component} /usr/bin -COPY --from=base /build/bin/goose /usr/bin - -VOLUME /etc/dendrite -WORKDIR /etc/dendrite - -ENTRYPOINT /usr/bin/${entrypoint} $@ \ No newline at end of file diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith new file mode 100644 index 000000000..3e9d0cba4 --- /dev/null +++ b/build/docker/Dockerfile.monolith @@ -0,0 +1,13 @@ +FROM matrixdotorg/dendrite:latest AS base + +FROM alpine:latest + +COPY --from=base /build/bin/dendrite-monolith-server /usr/bin +COPY --from=base /build/bin/goose /usr/bin +COPY --from=base /build/bin/create-account /usr/bin +COPY --from=base /build/bin/generate-keys /usr/bin + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] \ No newline at end of file diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith new file mode 100644 index 000000000..dd4cbd38f --- /dev/null +++ b/build/docker/Dockerfile.polylith @@ -0,0 +1,13 @@ +FROM matrixdotorg/dendrite:latest AS base + +FROM alpine:latest + +COPY --from=base /build/bin/dendrite-polylith-multi /usr/bin +COPY --from=base /build/bin/goose /usr/bin +COPY --from=base /build/bin/create-account /usr/bin +COPY --from=base /build/bin/generate-keys /usr/bin + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] \ No newline at end of file diff --git a/build/docker/docker-compose.deps.yml b/build/docker/docker-compose.deps.yml index 74e478a8d..1a27ffac0 100644 --- a/build/docker/docker-compose.deps.yml +++ b/build/docker/docker-compose.deps.yml @@ -2,7 +2,7 @@ version: "3.4" services: postgres: hostname: postgres - image: postgres:9.5 + image: postgres:9.6 restart: always volumes: - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh diff --git a/build/docker/docker-compose.monolith.yml b/build/docker/docker-compose.monolith.yml index 7d63e1c65..8fb798343 100644 --- a/build/docker/docker-compose.monolith.yml +++ b/build/docker/docker-compose.monolith.yml @@ -4,7 +4,6 @@ services: hostname: monolith image: matrixdotorg/dendrite-monolith:latest command: [ - "--config=dendrite.yaml", "--tls-cert=server.crt", "--tls-key=server.key" ] diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index e8da9c24a..f377e36fc 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -2,22 +2,17 @@ version: "3.4" services: client_api: hostname: client_api - image: matrixdotorg/dendrite-clientapi:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: clientapi volumes: - ./config:/etc/dendrite - - room_server networks: - internal media_api: hostname: media_api - image: matrixdotorg/dendrite-mediaapi:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: mediaapi volumes: - ./config:/etc/dendrite networks: @@ -25,10 +20,8 @@ services: sync_api: hostname: sync_api - image: matrixdotorg/dendrite-syncapi:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: syncapi volumes: - ./config:/etc/dendrite networks: @@ -36,10 +29,8 @@ services: room_server: hostname: room_server - image: matrixdotorg/dendrite-roomserver:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: roomserver volumes: - ./config:/etc/dendrite networks: @@ -47,10 +38,8 @@ services: edu_server: hostname: edu_server - image: matrixdotorg/dendrite-eduserver:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: eduserver volumes: - ./config:/etc/dendrite networks: @@ -58,10 +47,8 @@ services: federation_api: hostname: federation_api - image: matrixdotorg/dendrite-federationapi:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: federationapi volumes: - ./config:/etc/dendrite networks: @@ -69,10 +56,8 @@ services: federation_sender: hostname: federation_sender - image: matrixdotorg/dendrite-federationsender:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: federationsender volumes: - ./config:/etc/dendrite networks: @@ -80,10 +65,8 @@ services: key_server: hostname: key_server - image: matrixdotorg/dendrite-keyserver:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: keyserver volumes: - ./config:/etc/dendrite networks: @@ -91,10 +74,8 @@ services: signing_key_server: hostname: signing_key_server - image: matrixdotorg/dendrite-signingkeyserver:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: signingkeyserver volumes: - ./config:/etc/dendrite networks: @@ -102,10 +83,8 @@ services: user_api: hostname: user_api - image: matrixdotorg/dendrite-userapi:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: userapi volumes: - ./config:/etc/dendrite networks: @@ -113,10 +92,8 @@ services: appservice_api: hostname: appservice_api - image: matrixdotorg/dendrite-appservice:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: appservice volumes: - ./config:/etc/dendrite networks: diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index daad63be0..f80f6bed2 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -8,16 +8,5 @@ echo "Building tag '${TAG}'" docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} . -docker build -t matrixdotorg/dendrite-monolith:${TAG} --build-arg component=dendrite-monolith-server -f build/docker/Dockerfile.component . - -docker build -t matrixdotorg/dendrite-appservice:${TAG} --build-arg component=dendrite-appservice-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-clientapi:${TAG} --build-arg component=dendrite-client-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-eduserver:${TAG} --build-arg component=dendrite-edu-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-federationapi:${TAG} --build-arg component=dendrite-federation-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-federationsender:${TAG} --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-keyserver:${TAG} --build-arg component=dendrite-key-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-mediaapi:${TAG} --build-arg component=dendrite-media-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-roomserver:${TAG} --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-syncapi:${TAG} --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-signingkeyserver:${TAG} --build-arg component=dendrite-signing-key-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-userapi:${TAG} --build-arg component=dendrite-user-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-monolith:${TAG} -f build/docker/Dockerfile.monolith . +docker build -t matrixdotorg/dendrite-polylith:${TAG} -f build/docker/Dockerfile.polylith . \ No newline at end of file diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh index e3284a2a6..496e80067 100755 --- a/build/docker/images-pull.sh +++ b/build/docker/images-pull.sh @@ -5,15 +5,4 @@ TAG=${1:-latest} echo "Pulling tag '${TAG}'" docker pull matrixdotorg/dendrite-monolith:${TAG} - -docker pull matrixdotorg/dendrite-appservice:${TAG} -docker pull matrixdotorg/dendrite-clientapi:${TAG} -docker pull matrixdotorg/dendrite-eduserver:${TAG} -docker pull matrixdotorg/dendrite-federationapi:${TAG} -docker pull matrixdotorg/dendrite-federationsender:${TAG} -docker pull matrixdotorg/dendrite-keyserver:${TAG} -docker pull matrixdotorg/dendrite-mediaapi:${TAG} -docker pull matrixdotorg/dendrite-roomserver:${TAG} -docker pull matrixdotorg/dendrite-syncapi:${TAG} -docker pull matrixdotorg/dendrite-signingkeyserver:${TAG} -docker pull matrixdotorg/dendrite-userapi:${TAG} +docker pull matrixdotorg/dendrite-polylith:${TAG} \ No newline at end of file diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index e4eb773ac..fd9b999ea 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -5,15 +5,4 @@ TAG=${1:-latest} echo "Pushing tag '${TAG}'" docker push matrixdotorg/dendrite-monolith:${TAG} - -docker push matrixdotorg/dendrite-appservice:${TAG} -docker push matrixdotorg/dendrite-clientapi:${TAG} -docker push matrixdotorg/dendrite-eduserver:${TAG} -docker push matrixdotorg/dendrite-federationapi:${TAG} -docker push matrixdotorg/dendrite-federationsender:${TAG} -docker push matrixdotorg/dendrite-keyserver:${TAG} -docker push matrixdotorg/dendrite-mediaapi:${TAG} -docker push matrixdotorg/dendrite-roomserver:${TAG} -docker push matrixdotorg/dendrite-syncapi:${TAG} -docker push matrixdotorg/dendrite-signingkeyserver:${TAG} -docker push matrixdotorg/dendrite-userapi:${TAG} +docker push matrixdotorg/dendrite-polylith:${TAG} \ No newline at end of file diff --git a/build/docker/postgres/create_db.sh b/build/docker/postgres/create_db.sh index 97514467b..7495a3978 100755 --- a/build/docker/postgres/create_db.sh +++ b/build/docker/postgres/create_db.sh @@ -1,5 +1,5 @@ #!/bin/sh -for db in account device mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice e2ekey naffka; do +for db in account device mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice naffka; do createdb -U dendrite -O dendrite dendrite_$db done diff --git a/cmd/dendrite-polylith-multi/main.go b/cmd/dendrite-polylith-multi/main.go new file mode 100644 index 000000000..0d6406c01 --- /dev/null +++ b/cmd/dendrite-polylith-multi/main.go @@ -0,0 +1,78 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "os" + "strings" + + "github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/setup" + "github.com/sirupsen/logrus" +) + +type entrypoint func(base *setup.BaseDendrite, cfg *config.Dendrite) + +func main() { + cfg := setup.ParseFlags(true) + + component := "" + if flag.NFlag() > 0 { + component = flag.Arg(0) // ./dendrite-polylith-multi --config=... clientapi + } else if len(os.Args) > 1 { + component = os.Args[1] // ./dendrite-polylith-multi clientapi + } + + components := map[string]entrypoint{ + "appservice": personalities.Appservice, + "clientapi": personalities.ClientAPI, + "eduserver": personalities.EDUServer, + "federationapi": personalities.FederationAPI, + "federationsender": personalities.FederationSender, + "keyserver": personalities.KeyServer, + "mediaapi": personalities.MediaAPI, + "roomserver": personalities.RoomServer, + "signingkeyserver": personalities.SigningKeyServer, + "syncapi": personalities.SyncAPI, + "userapi": personalities.UserAPI, + } + + start, ok := components[component] + if !ok { + if component == "" { + logrus.Errorf("No component specified") + logrus.Info("The first argument on the command line must be the name of the component to run") + } else { + logrus.Errorf("Unknown component %q specified", component) + } + + var list []string + for c := range components { + list = append(list, c) + } + logrus.Infof("Valid components: %s", strings.Join(list, ", ")) + + os.Exit(1) + } + + logrus.Infof("Starting %q component", component) + + base := setup.NewBaseDendrite(cfg, component, false) // TODO + defer base.Close() // nolint: errcheck + + start(base, cfg) +} diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-polylith-multi/personalities/appservice.go similarity index 83% rename from cmd/dendrite-appservice-server/main.go rename to cmd/dendrite-polylith-multi/personalities/appservice.go index 6adbdb17c..7fa87b115 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/appservice.go @@ -1,4 +1,4 @@ -// Copyright 2018 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "AppServiceAPI", true) - - defer base.Close() // nolint: errcheck +func Appservice(base *setup.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() rsAPI := base.RoomserverHTTPClient() diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/clientapi.go similarity index 87% rename from cmd/dendrite-client-api-server/main.go rename to cmd/dendrite-polylith-multi/personalities/clientapi.go index 0061de74f..09fc63ab3 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/clientapi.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,20 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( "github.com/matrix-org/dendrite/clientapi" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/internal/transactions" ) -func main() { - cfg := setup.ParseFlags(false) - - base := setup.NewBaseDendrite(cfg, "ClientAPI", true) - defer base.Close() // nolint: errcheck - +func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { accountDB := base.CreateAccountsDB() federation := base.CreateFederationClient() diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-polylith-multi/personalities/eduserver.go similarity index 77% rename from cmd/dendrite-edu-server/main.go rename to cmd/dendrite-polylith-multi/personalities/eduserver.go index 3a34b9a68..a5d2926f1 100644 --- a/cmd/dendrite-edu-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/eduserver.go @@ -1,3 +1,5 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -10,26 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( - _ "net/http/pprof" - "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" - "github.com/sirupsen/logrus" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "EDUServerAPI", true) - defer func() { - if err := base.Close(); err != nil { - logrus.WithError(err).Warn("BaseDendrite close failed") - } - }() - +func EDUServer(base *setup.BaseDendrite, cfg *config.Dendrite) { intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient()) eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI) diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/federationapi.go similarity index 86% rename from cmd/dendrite-federation-api-server/main.go rename to cmd/dendrite-polylith-multi/personalities/federationapi.go index 3ebb16f4b..a1bbeafad 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/federationapi.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "FederationAPI", true) - defer base.Close() // nolint: errcheck - +func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() federation := base.CreateFederationClient() serverKeyAPI := base.SigningKeyServerHTTPClient() diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-polylith-multi/personalities/federationsender.go similarity index 85% rename from cmd/dendrite-federation-sender-server/main.go rename to cmd/dendrite-polylith-multi/personalities/federationsender.go index 99b416c45..052523789 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/federationsender.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "FederationSender", true) - defer base.Close() // nolint: errcheck - +func FederationSender(base *setup.BaseDendrite, cfg *config.Dendrite) { federation := base.CreateFederationClient() serverKeyAPI := base.SigningKeyServerHTTPClient() diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-polylith-multi/personalities/keyserver.go similarity index 87% rename from cmd/dendrite-key-server/main.go rename to cmd/dendrite-polylith-multi/personalities/keyserver.go index ff5b22236..8c159ad06 100644 --- a/cmd/dendrite-key-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/keyserver.go @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/keyserver" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "KeyServer", true) - defer base.Close() // nolint: errcheck - +func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) { intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient()) intAPI.SetUserAPI(base.UserAPIClient()) diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/mediaapi.go similarity index 82% rename from cmd/dendrite-media-api-server/main.go rename to cmd/dendrite-polylith-multi/personalities/mediaapi.go index 2c2fe3b36..64e5bc312 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/mediaapi.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/mediaapi" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "MediaAPI", true) - defer base.Close() // nolint: errcheck - +func MediaAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() client := base.CreateClient() diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-polylith-multi/personalities/roomserver.go similarity index 84% rename from cmd/dendrite-room-server/main.go rename to cmd/dendrite-polylith-multi/personalities/roomserver.go index d3f145745..91027506d 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/roomserver.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/roomserver" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "RoomServerAPI", true) - defer base.Close() // nolint: errcheck - +func RoomServer(base *setup.BaseDendrite, cfg *config.Dendrite) { serverKeyAPI := base.SigningKeyServerHTTPClient() keyRing := serverKeyAPI.KeyRing() diff --git a/cmd/dendrite-signing-key-server/main.go b/cmd/dendrite-polylith-multi/personalities/signingkeyserver.go similarity index 86% rename from cmd/dendrite-signing-key-server/main.go rename to cmd/dendrite-polylith-multi/personalities/signingkeyserver.go index a4d48d361..a7bfff10b 100644 --- a/cmd/dendrite-signing-key-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/signingkeyserver.go @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/signingkeyserver" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "SigningKeyServer", true) - defer base.Close() // nolint: errcheck - +func SigningKeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) { federation := base.CreateFederationClient() intAPI := signingkeyserver.NewInternalAPI(&base.Cfg.SigningKeyServer, federation, base.Caches) diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/syncapi.go similarity index 84% rename from cmd/dendrite-sync-api-server/main.go rename to cmd/dendrite-polylith-multi/personalities/syncapi.go index 351dbc5f4..2d5c0b525 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/syncapi.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,19 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/syncapi" ) -func main() { - cfg := setup.ParseFlags(false) - - base := setup.NewBaseDendrite(cfg, "SyncAPI", true) - defer base.Close() // nolint: errcheck - +func SyncAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() federation := base.CreateFederationClient() diff --git a/cmd/dendrite-user-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/userapi.go similarity index 84% rename from cmd/dendrite-user-api-server/main.go rename to cmd/dendrite-polylith-multi/personalities/userapi.go index fb65fefbc..fe5e4fbd0 100644 --- a/cmd/dendrite-user-api-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/userapi.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/userapi" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "UserAPI", true) - defer base.Close() // nolint: errcheck - +func UserAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { accountDB := base.CreateAccountsDB() userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient()) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index d246e9221..f804193cd 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -24,7 +24,7 @@ use in production environments just yet! Dendrite requires: * Go 1.13 or higher -* Postgres 9.5 or higher (if using Postgres databases, not needed for SQLite) +* Postgres 9.6 or higher (if using Postgres databases, not needed for SQLite) If you want to run a polylith deployment, you also need: @@ -98,7 +98,7 @@ create them automatically at startup. ### Postgres database setup -Assuming that Postgres 9.5 (or later) is installed: +Assuming that Postgres 9.6 (or later) is installed: * Create role, choosing a new password when prompted: @@ -189,7 +189,7 @@ This is what implements CS API endpoints. Clients talk to this via the proxy in order to send messages, create and join rooms, etc. ```bash -./bin/dendrite-client-api-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml clientapi ``` ### Sync server @@ -198,7 +198,7 @@ This is what implements `/sync` requests. Clients talk to this via the proxy in order to receive messages. ```bash -./bin/dendrite-sync-api-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml syncapi ``` ### Media server @@ -207,7 +207,7 @@ This implements `/media` requests. Clients talk to this via the proxy in order to upload and retrieve media. ```bash -./bin/dendrite-media-api-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml mediaapi ``` ### Federation API server @@ -217,7 +217,7 @@ order to send transactions. This is only required if you want to support federation. ```bash -./bin/dendrite-federation-api-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml federationapi ``` ### Internal components @@ -230,7 +230,7 @@ contacted by other components. This includes the following components. This is what implements the room DAG. Clients do not talk to this. ```bash -./bin/dendrite-room-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml roomserver ``` #### Federation sender @@ -239,7 +239,7 @@ This sends events from our users to other servers. This is only required if you want to support federation. ```bash -./bin/dendrite-federation-sender-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml federationsender ``` #### Appservice server @@ -250,7 +250,7 @@ running locally. This is only required if you want to support running application services on your homeserver. ```bash -./bin/dendrite-appservice-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml appservice ``` #### Key server @@ -258,7 +258,7 @@ application services on your homeserver. This manages end-to-end encryption keys for users. ```bash -./bin/dendrite-key-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml keyserver ``` #### Signing key server @@ -266,7 +266,7 @@ This manages end-to-end encryption keys for users. This manages signing keys for servers. ```bash -./bin/dendrite-signing-key-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml signingkeyserver ``` #### EDU server @@ -274,7 +274,7 @@ This manages signing keys for servers. This manages processing EDUs such as typing, send-to-device events and presence. Clients do not talk to ```bash -./bin/dendrite-edu-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml eduserver ``` #### User server @@ -283,6 +283,6 @@ This manages user accounts, device access tokens and user account data, amongst other things. ```bash -./bin/dendrite-user-api-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml userapi ``` From 6a16d46fba9edf82203b2b3606d9a16ed6ef4ae0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 20 Oct 2020 16:46:53 +0100 Subject: [PATCH 102/104] Version 0.2.0 (#1551) * v0.2.0-rc2 * Update CHANGES.md (also with some markdown lint suggestions) * Update version number * Update CHANGES.md * Update CHANGES.md * Update CHANGES.md * Add known issue * Update CHANGES.md --- CHANGES.md | 194 +++++++++++++++++++++++++++++--------------- internal/version.go | 2 +- 2 files changed, 128 insertions(+), 68 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 17fb75bed..d05b871ac 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,89 +1,149 @@ -# Dendrite 0.1.0 (2020-10-08) +# Changelog + +## Dendrite 0.2.0 (2020-10-20) + +### Important + +* This release makes breaking changes for polylith deployments, since they now use the multi-personality binary rather than separate binary files + * Users of polylith deployments should revise their setups to use the new binary - see the Features section below +* This release also makes breaking changes for Docker deployments, as are now publishing images to Docker Hub in separate repositories for monolith and polylith + * New repositories are as follows: [matrixdotorg/dendrite-monolith](https://hub.docker.com/repository/docker/matrixdotorg/dendrite-monolith) and [matrixdotorg/dendrite-polylith](https://hub.docker.com/repository/docker/matrixdotorg/dendrite-polylith) + * The new `latest` tag will be updated with the latest release, and new versioned tags, e.g. `v0.2.0`, will preserve specific release versions + * [Sample Compose configs](https://github.com/matrix-org/dendrite/tree/master/build/docker) have been updated - if you are running a Docker deployment, please review the changes + * Images for the client API proxy and federation API proxy are no longer provided as they are unsupported - please use [nginx](docs/nginx/) (or another reverse proxy) instead + +### Features + +* Dendrite polylith deployments now use a special multi-personality binary, rather than separate binaries + * This is cleaner, builds faster and simplifies deployment + * The first command line argument states the component to run, e.g. `./dendrite-polylith-multi roomserver` +* Database migrations are now run at startup +* Invalid UTF-8 in requests is now rejected (contributed by [Pestdoktor](https://github.com/Pestdoktor)) +* Fully read markers are now implemented in the client API (contributed by [Lesterpig](https://github.com/Lesterpig)) +* Missing auth events are now retrieved from other servers in the room, rather than just the event origin +* `m.room.create` events are now validated properly when processing a `/send_join` response +* The roomserver now implements `KindOld` for handling historic events without them becoming forward extremity candidates, i.e. for backfilled or missing events + +### Fixes + +* State resolution v2 performance has been improved dramatically when dealing with large state sets +* The roomserver no longer processes outlier events if they are already known +* A SQLite locking issue in the previous events updater has been fixed +* The client API `/state` endpoint now correctly returns state after the leave event, if the user has left the room +* The client API `/createRoom` endpoint now sends cumulative state to the roomserver for the initial room events +* The federation API `/send` endpoint now correctly requests the entire room state from the roomserver when needed +* Some internal HTTP API paths have been fixed in the user API (contributed by [S7evinK](https://github.com/S7evinK)) +* A race condition in the rate limiting code resulting in concurrent map writes has been fixed +* Each component now correctly starts a consumer/producer connection in monolith mode (when using Kafka) +* State resolution is no longer run for single trusted state snapshots that have been verified before +* A crash when rolling back the transaction in the latest events updater has been fixed +* Typing events are now ignored when the sender domain does not match the origin server +* Duplicate redaction entries no longer result in database errors +* Recursion has been removed from the code path for retrieving missing events +* `QueryMissingAuthPrevEvents` now returns events that have no associated state as if they are missing +* Signing key fetchers no longer ignore keys for the local domain, if retrieving a key that is not known in the local config +* Federation timeouts have been adjusted so we don't give up on remote requests so quickly +* `create-account` no longer relies on the device database (contributed by [ThatNerdyPikachu](https://github.com/ThatNerdyPikachu)) + +### Known issues + +* Old events can incorrectly appear in `/sync` as if they are new when retrieving missing events from federated servers, causing them to appear at the bottom of the timeline in clients + +## Dendrite 0.1.0 (2020-10-08) First versioned release of Dendrite. ## Client-Server API Features ### Account registration and management -- Registration: By password only. -- Login: By password only. No fallback. -- Logout: Yes. -- Change password: Yes. -- Link email/msisdn to account: No. -- Deactivate account: Yes. -- Check if username is available: Yes. -- Account data: Yes. -- OpenID: No. + +* Registration: By password only. +* Login: By password only. No fallback. +* Logout: Yes. +* Change password: Yes. +* Link email/msisdn to account: No. +* Deactivate account: Yes. +* Check if username is available: Yes. +* Account data: Yes. +* OpenID: No. ### Rooms -- Room creation: Yes, including presets. -- Joining rooms: Yes, including by alias or `?server_name=`. -- Event sending: Yes, including transaction IDs. -- Aliases: Yes. -- Published room directory: Yes. -- Kicking users: Yes. -- Banning users: Yes. -- Inviting users: Yes, but not third-party invites. -- Forgetting rooms: No. -- Room versions: All (v1 - v6) -- Tagging: Yes. + +* Room creation: Yes, including presets. +* Joining rooms: Yes, including by alias or `?server_name=`. +* Event sending: Yes, including transaction IDs. +* Aliases: Yes. +* Published room directory: Yes. +* Kicking users: Yes. +* Banning users: Yes. +* Inviting users: Yes, but not third-party invites. +* Forgetting rooms: No. +* Room versions: All (v1 * v6) +* Tagging: Yes. ### User management -- User directory: Basic support. -- Ignoring users: No. -- Groups/Communities: No. + +* User directory: Basic support. +* Ignoring users: No. +* Groups/Communities: No. ### Device management -- Creating devices: Yes. -- Deleting devices: Yes. -- Send-to-device messaging: Yes. + +* Creating devices: Yes. +* Deleting devices: Yes. +* Send-to-device messaging: Yes. ### Sync -- Filters: Timeline limit only. Rest unimplemented. -- Deprecated `/events` and `/initialSync`: No. + +* Filters: Timeline limit only. Rest unimplemented. +* Deprecated `/events` and `/initialSync`: No. ### Room events -- Typing: Yes. -- Receipts: No. -- Read Markers: No. -- Presence: No. -- Content repository (attachments): Yes. -- History visibility: No, defaults to `joined`. -- Push notifications: No. -- Event context: No. -- Reporting content: No. + +* Typing: Yes. +* Receipts: No. +* Read Markers: No. +* Presence: No. +* Content repository (attachments): Yes. +* History visibility: No, defaults to `joined`. +* Push notifications: No. +* Event context: No. +* Reporting content: No. ### End-to-End Encryption -- Uploading device keys: Yes. -- Downloading device keys: Yes. -- Claiming one-time keys: Yes. -- Querying key changes: Yes. -- Cross-Signing: No. + +* Uploading device keys: Yes. +* Downloading device keys: Yes. +* Claiming one-time keys: Yes. +* Querying key changes: Yes. +* Cross-Signing: No. ### Misc -- Server-side search: No. -- Guest access: Partial. -- Room previews: No, partial support for Peeking via MSC2753. -- Third-Party networks: No. -- Server notices: No. -- Policy lists: No. + +* Server-side search: No. +* Guest access: Partial. +* Room previews: No, partial support for Peeking via MSC2753. +* Third-Party networks: No. +* Server notices: No. +* Policy lists: No. ## Federation Features -- Querying keys (incl. notary): Yes. -- Server ACLs: Yes. -- Sending transactions: Yes. -- Joining rooms: Yes. -- Inviting to rooms: Yes, but not third-party invites. -- Leaving rooms: Yes. -- Content repository: Yes. -- Backfilling / get_missing_events: Yes. -- Retrieving state of the room (`/state` and `/state_ids`): Yes. -- Public rooms: Yes. -- Querying profile data: Yes. -- Device management: Yes. -- Send-to-Device messaging: Yes. -- Querying/Claiming E2E Keys: Yes. -- Typing: Yes. -- Presence: No. -- Receipts: No. -- OpenID: No. \ No newline at end of file + +* Querying keys (incl. notary): Yes. +* Server ACLs: Yes. +* Sending transactions: Yes. +* Joining rooms: Yes. +* Inviting to rooms: Yes, but not third-party invites. +* Leaving rooms: Yes. +* Content repository: Yes. +* Backfilling / get_missing_events: Yes. +* Retrieving state of the room (`/state` and `/state_ids`): Yes. +* Public rooms: Yes. +* Querying profile data: Yes. +* Device management: Yes. +* Send-to-Device messaging: Yes. +* Querying/Claiming E2E Keys: Yes. +* Typing: Yes. +* Presence: No. +* Receipts: No. +* OpenID: No. \ No newline at end of file diff --git a/internal/version.go b/internal/version.go index a9e245d44..040ffa32a 100644 --- a/internal/version.go +++ b/internal/version.go @@ -16,7 +16,7 @@ var build string const ( VersionMajor = 0 - VersionMinor = 1 + VersionMinor = 2 VersionPatch = 0 VersionTag = "" // example: "rc1" ) From 7ca89ef5116bd918b50c275f6a52d171ea1ccd96 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 20 Oct 2020 17:10:37 +0100 Subject: [PATCH 103/104] Update gomatrixserverlib --- go.mod | 2 +- go.sum | 4 ++-- sytest-whitelist | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d3060fa0f..afbe15f14 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8 + github.com/matrix-org/gomatrixserverlib v0.0.0-20201020160945-66a6d4926351 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index 377a2e093..eaec548ba 100644 --- a/go.sum +++ b/go.sum @@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8 h1:GF1PxbvImWDoz1DQZNMoaYtIqQXtyLAtmQOzwwmw1OI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201020160945-66a6d4926351 h1:Yg+fvSEvDLjMFkkE4W0e1yBTtCMVmH3tztw/RhTLPcA= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201020160945-66a6d4926351/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/sytest-whitelist b/sytest-whitelist index cecf24f75..1a12b591b 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -485,3 +485,4 @@ Event with an invalid signature in the send_join response should not cause room Inbound federation rejects typing notifications from wrong remote Should not be able to take over the room by pretending there is no PL event Can get rooms/{roomId}/state for a departed room (SPEC-216) +Users cannot set notifications powerlevel higher than their own From 24e38c413567355a88c0b92f5a325d7b77325ad1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 20 Oct 2020 17:13:12 +0100 Subject: [PATCH 104/104] Internal HTTP APIs over H2C (#1541) * H2C on internal HTTP because SCIENCE * Update comments --- go.mod | 1 + internal/setup/base.go | 30 ++++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index afbe15f14..d12efd4e0 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20201006093556-760d9a7fd5ee go.uber.org/atomic v1.6.0 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a + golang.org/x/net v0.0.0-20200528225125-3c3fba18258b gopkg.in/h2non/bimg.v1 v1.1.4 gopkg.in/yaml.v2 v2.3.0 ) diff --git a/internal/setup/base.go b/internal/setup/base.go index 8bc4ae17a..4e1cee479 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -15,8 +15,10 @@ package setup import ( + "crypto/tls" "fmt" "io" + "net" "net/http" "net/url" "time" @@ -25,6 +27,8 @@ import ( "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/gomatrixserverlib" "github.com/prometheus/client_golang/prometheus/promhttp" + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/userapi/storage/accounts" @@ -107,7 +111,22 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo logrus.WithError(err).Warnf("Failed to create cache") } - apiClient := http.Client{Timeout: time.Minute * 10} + apiClient := http.Client{ + Timeout: time.Minute * 10, + Transport: &http2.Transport{ + AllowHTTP: true, + DialTLS: func(network, addr string, _ *tls.Config) (net.Conn, error) { + // Ordinarily HTTP/2 would expect TLS, but the remote listener is + // H2C-enabled (HTTP/2 without encryption). Overriding the DialTLS + // function with a plain Dial allows us to trick the HTTP client + // into establishing a HTTP/2 connection without TLS. + // TODO: Eventually we will want to look at authenticating and + // encrypting these internal HTTP APIs, at which point we will have + // to reconsider H2C and change all this anyway. + return net.Dial(network, addr) + }, + }, + } client := http.Client{Timeout: HTTPClientTimeout} if cfg.FederationSender.Proxy.Enabled { client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{ @@ -269,10 +288,17 @@ func (b *BaseDendrite) SetupAndServeHTTP( internalServ := externalServ if internalAddr != NoListener && externalAddr != internalAddr { + // H2C allows us to accept HTTP/2 connections without TLS + // encryption. Since we don't currently require any form of + // authentication or encryption on these internal HTTP APIs, + // H2C gives us all of the advantages of HTTP/2 (such as + // stream multiplexing and avoiding head-of-line blocking) + // without enabling TLS. + internalH2S := &http2.Server{} internalRouter = mux.NewRouter().SkipClean(true).UseEncodedPath() internalServ = &http.Server{ Addr: string(internalAddr), - Handler: internalRouter, + Handler: h2c.NewHandler(internalRouter, internalH2S), } }