diff --git a/appservice/appservice.go b/appservice/appservice.go index 4f89d64bd..e52db2c28 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -44,8 +44,7 @@ func SetupAppServiceAPIComponent( accountsDB accounts.Database, deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, - roomserverAliasAPI roomserverAPI.RoomserverAliasAPI, - roomserverQueryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, transactionsCache *transactions.Cache, ) appserviceAPI.AppServiceQueryAPI { // Create a connection to the appservice postgres DB @@ -87,7 +86,7 @@ func SetupAppServiceAPIComponent( consumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB, - roomserverQueryAPI, roomserverAliasAPI, workerStates, + rsAPI, workerStates, ) if err := consumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start appservice roomserver consumer") @@ -100,7 +99,7 @@ func SetupAppServiceAPIComponent( // Set up HTTP Endpoints routing.Setup( - base.APIMux, base.Cfg, roomserverQueryAPI, roomserverAliasAPI, + base.APIMux, base.Cfg, rsAPI, accountsDB, federation, transactionsCache, ) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 3bd364c58..b7f689249 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -35,8 +35,7 @@ type OutputRoomEventConsumer struct { roomServerConsumer *common.ContinualConsumer db accounts.Database asDB storage.Database - query api.RoomserverQueryAPI - alias api.RoomserverAliasAPI + rsAPI api.RoomserverInternalAPI serverName string workerStates []types.ApplicationServiceWorkerState } @@ -48,8 +47,7 @@ func NewOutputRoomEventConsumer( kafkaConsumer sarama.Consumer, store accounts.Database, appserviceDB storage.Database, - queryAPI api.RoomserverQueryAPI, - aliasAPI api.RoomserverAliasAPI, + rsAPI api.RoomserverInternalAPI, workerStates []types.ApplicationServiceWorkerState, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ @@ -61,8 +59,7 @@ func NewOutputRoomEventConsumer( roomServerConsumer: &consumer, db: store, asDB: appserviceDB, - query: queryAPI, - alias: aliasAPI, + rsAPI: rsAPI, serverName: string(cfg.Matrix.ServerName), workerStates: workerStates, } @@ -139,7 +136,7 @@ func (s *OutputRoomEventConsumer) lookupMissingStateEvents( // Request the missing events from the roomserver eventReq := api.QueryEventsByIDRequest{EventIDs: missing} var eventResp api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { return nil, err } @@ -200,7 +197,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont // Check all known room aliases of the room the event came from queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()} var queryRes api.GetAliasesForRoomIDResponse - if err := s.alias.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil { + if err := s.rsAPI.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil { for _, alias := range queryRes.Aliases { if appservice.IsInterestedInRoomAlias(alias) { return true diff --git a/appservice/routing/routing.go b/appservice/routing/routing.go index 42fa80520..9f59e05f7 100644 --- a/appservice/routing/routing.go +++ b/appservice/routing/routing.go @@ -37,7 +37,7 @@ const pathPrefixApp = "/_matrix/app/v1" // nolint: gocyclo func Setup( apiMux *mux.Router, cfg *config.Dendrite, // nolint: unparam - queryAPI api.RoomserverQueryAPI, aliasAPI api.RoomserverAliasAPI, // nolint: unparam + rsAPI api.RoomserverInternalAPI, // nolint: unparam accountDB accounts.Database, // nolint: unparam federation *gomatrixserverlib.FederationClient, // nolint: unparam transactionsCache *transactions.Cache, // nolint: unparam diff --git a/build.sh b/build.sh index 3ef148891..087f4ae72 100755 --- a/build.sh +++ b/build.sh @@ -3,4 +3,6 @@ # Put installed packages into ./bin export GOBIN=$PWD/`dirname $0`/bin -go install -v $PWD/`dirname $0`/cmd/... \ No newline at end of file +go install -v $PWD/`dirname $0`/cmd/... + +GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 1339f7c8c..f81e0242c 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -38,15 +38,13 @@ func SetupClientAPIComponent( accountsDB accounts.Database, federation *gomatrixserverlib.FederationClient, keyRing *gomatrixserverlib.KeyRing, - aliasAPI roomserverAPI.RoomserverAliasAPI, - inputAPI roomserverAPI.RoomserverInputAPI, - queryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, eduInputAPI eduServerAPI.EDUServerInputAPI, asAPI appserviceAPI.AppServiceQueryAPI, transactionsCache *transactions.Cache, - fedSenderAPI federationSenderAPI.FederationSenderQueryAPI, + fsAPI federationSenderAPI.FederationSenderInternalAPI, ) { - roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI) + roomserverProducer := producers.NewRoomserverProducer(rsAPI) eduProducer := producers.NewEDUServerProducer(eduInputAPI) userUpdateProducer := &producers.UserUpdateProducer{ @@ -60,15 +58,15 @@ func SetupClientAPIComponent( } consumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, accountsDB, queryAPI, + base.Cfg, base.KafkaConsumer, accountsDB, rsAPI, ) if err := consumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start room server consumer") } routing.Setup( - base.APIMux, base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI, + base.APIMux, base.Cfg, roomserverProducer, rsAPI, asAPI, accountsDB, deviceDB, federation, *keyRing, userUpdateProducer, - syncProducer, eduProducer, transactionsCache, fedSenderAPI, + syncProducer, eduProducer, transactionsCache, fsAPI, ) } diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go index 3c7905721..d0c91e88f 100644 --- a/clientapi/consumers/roomserver.go +++ b/clientapi/consumers/roomserver.go @@ -30,10 +30,10 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - roomServerConsumer *common.ContinualConsumer - db accounts.Database - query api.RoomserverQueryAPI - serverName string + rsAPI api.RoomserverInternalAPI + rsConsumer *common.ContinualConsumer + db accounts.Database + serverName string } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -41,7 +41,7 @@ func NewOutputRoomEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, store accounts.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ @@ -50,10 +50,10 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ - roomServerConsumer: &consumer, - db: store, - query: queryAPI, - serverName: string(cfg.Matrix.ServerName), + rsConsumer: &consumer, + db: store, + rsAPI: rsAPI, + serverName: string(cfg.Matrix.ServerName), } consumer.ProcessMessage = s.onMessage @@ -62,7 +62,7 @@ func NewOutputRoomEventConsumer( // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return s.roomServerConsumer.Start() + return s.rsConsumer.Start() } // onMessage is called when the sync server receives a new event from the room server output log. @@ -134,7 +134,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // Request the missing events from the roomserver eventReq := api.QueryEventsByIDRequest{EventIDs: missing} var eventResp api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { return nil, err } diff --git a/clientapi/producers/roomserver.go b/clientapi/producers/roomserver.go index fac1e3c7c..a804abfe9 100644 --- a/clientapi/producers/roomserver.go +++ b/clientapi/producers/roomserver.go @@ -23,15 +23,13 @@ import ( // RoomserverProducer produces events for the roomserver to consume. type RoomserverProducer struct { - InputAPI api.RoomserverInputAPI - QueryAPI api.RoomserverQueryAPI + RsAPI api.RoomserverInternalAPI } // NewRoomserverProducer creates a new RoomserverProducer -func NewRoomserverProducer(inputAPI api.RoomserverInputAPI, queryAPI api.RoomserverQueryAPI) *RoomserverProducer { +func NewRoomserverProducer(rsAPI api.RoomserverInternalAPI) *RoomserverProducer { return &RoomserverProducer{ - InputAPI: inputAPI, - QueryAPI: queryAPI, + RsAPI: rsAPI, } } @@ -95,7 +93,7 @@ func (c *RoomserverProducer) SendInputRoomEvents( ) (eventID string, err error) { request := api.InputRoomEventsRequest{InputRoomEvents: ires} var response api.InputRoomEventsResponse - err = c.InputAPI.InputRoomEvents(ctx, &request, &response) + err = c.RsAPI.InputRoomEvents(ctx, &request, &response) eventID = response.EventID return } @@ -118,5 +116,5 @@ func (c *RoomserverProducer) SendInvite( }}, } var response api.InputRoomEventsResponse - return c.InputAPI.InputRoomEvents(ctx, &request, &response) + return c.RsAPI.InputRoomEvents(ctx, &request, &response) } diff --git a/clientapi/routing/capabilities.go b/clientapi/routing/capabilities.go index 1792c6308..199b15240 100644 --- a/clientapi/routing/capabilities.go +++ b/clientapi/routing/capabilities.go @@ -26,11 +26,11 @@ import ( // SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite) // by building a m.room.member event then sending it to the room server func GetCapabilities( - req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI, + req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { roomVersionsQueryReq := roomserverAPI.QueryRoomVersionCapabilitiesRequest{} roomVersionsQueryRes := roomserverAPI.QueryRoomVersionCapabilitiesResponse{} - if err := queryAPI.QueryRoomVersionCapabilities( + if err := rsAPI.QueryRoomVersionCapabilities( req.Context(), &roomVersionsQueryReq, &roomVersionsQueryRes, diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index ef11e8b3e..28e2b1514 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -137,13 +137,13 @@ type fledglingEvent struct { func CreateRoom( req *http.Request, device *authtypes.Device, cfg *config.Dendrite, producer *producers.RoomserverProducer, - accountDB accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI, + accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { // TODO (#267): Check room ID doesn't clash with an existing one, and we // probably shouldn't be using pseudo-random strings, maybe GUIDs? roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.Matrix.ServerName) - return createRoom(req, device, cfg, roomID, producer, accountDB, aliasAPI, asAPI) + return createRoom(req, device, cfg, roomID, producer, accountDB, rsAPI, asAPI) } // createRoom implements /createRoom @@ -151,7 +151,7 @@ func CreateRoom( func createRoom( req *http.Request, device *authtypes.Device, cfg *config.Dendrite, roomID string, producer *producers.RoomserverProducer, - accountDB accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI, + accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { logger := util.GetLogger(req.Context()) @@ -340,7 +340,7 @@ func createRoom( } var aliasResp roomserverAPI.SetRoomAliasResponse - err = aliasAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp) + err = rsAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed") return jsonerror.InternalServerError() diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 248696ab2..a0a60a47a 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -46,8 +46,8 @@ func DirectoryRoom( roomAlias string, federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, - rsAPI roomserverAPI.RoomserverAliasAPI, - fedSenderAPI federationSenderAPI.FederationSenderQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, + fedSenderAPI federationSenderAPI.FederationSenderInternalAPI, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('#', roomAlias) if err != nil { @@ -115,7 +115,7 @@ func SetLocalAlias( device *authtypes.Device, alias string, cfg *config.Dendrite, - aliasAPI roomserverAPI.RoomserverAliasAPI, + aliasAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('#', alias) if err != nil { @@ -190,7 +190,7 @@ func RemoveLocalAlias( req *http.Request, device *authtypes.Device, alias string, - aliasAPI roomserverAPI.RoomserverAliasAPI, + aliasAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { creatorQueryReq := roomserverAPI.GetCreatorIDForAliasRequest{ diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go index 2d3152510..bf49968de 100644 --- a/clientapi/routing/getevent.go +++ b/clientapi/routing/getevent.go @@ -44,7 +44,7 @@ func GetEvent( roomID string, eventID string, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, keyRing gomatrixserverlib.KeyRing, ) util.JSONResponse { @@ -52,7 +52,7 @@ func GetEvent( EventIDs: []string{eventID}, } var eventsResp api.QueryEventsByIDResponse - err := queryAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp) + err := rsAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryEventsByID failed") return jsonerror.InternalServerError() @@ -88,7 +88,7 @@ func GetEvent( }}, } var stateResp api.QueryStateAfterEventsResponse - if err := queryAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil { + if err := rsAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil { util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryStateAfterEvents failed") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index d0dee7c2a..df83c2a9f 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -27,12 +27,11 @@ import ( "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" - "github.com/matrix-org/dendrite/roomserver/api" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" - "github.com/sirupsen/logrus" ) // JoinRoomByIDOrAlias implements the "/join/{roomIDOrAlias}" API. @@ -44,8 +43,8 @@ func JoinRoomByIDOrAlias( cfg *config.Dendrite, federation *gomatrixserverlib.FederationClient, producer *producers.RoomserverProducer, - queryAPI roomserverAPI.RoomserverQueryAPI, - aliasAPI roomserverAPI.RoomserverAliasAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, + fsAPI federationSenderAPI.FederationSenderInternalAPI, keyRing gomatrixserverlib.KeyRing, accountDB accounts.Database, ) util.JSONResponse { @@ -79,7 +78,8 @@ func JoinRoomByIDOrAlias( content["avatar_url"] = profile.AvatarURL r := joinRoomReq{ - req, evTime, content, device.UserID, cfg, federation, producer, queryAPI, aliasAPI, keyRing, + req, evTime, content, device.UserID, cfg, federation, producer, + rsAPI, fsAPI, keyRing, } if strings.HasPrefix(roomIDOrAlias, "!") { @@ -105,8 +105,8 @@ type joinRoomReq struct { cfg *config.Dendrite federation *gomatrixserverlib.FederationClient producer *producers.RoomserverProducer - queryAPI roomserverAPI.RoomserverQueryAPI - aliasAPI roomserverAPI.RoomserverAliasAPI + rsAPI roomserverAPI.RoomserverInternalAPI + fsAPI federationSenderAPI.FederationSenderInternalAPI keyRing gomatrixserverlib.KeyRing } @@ -122,7 +122,7 @@ func (r joinRoomReq) joinRoomByID(roomID string) util.JSONResponse { RoomID: roomID, TargetUserID: r.userID, } var queryRes roomserverAPI.QueryInvitesForUserResponse - if err := r.queryAPI.QueryInvitesForUser(r.req.Context(), &queryReq, &queryRes); err != nil { + if err := r.rsAPI.QueryInvitesForUser(r.req.Context(), &queryReq, &queryRes); err != nil { util.GetLogger(r.req.Context()).WithError(err).Error("r.queryAPI.QueryInvitesForUser failed") return jsonerror.InternalServerError() } @@ -170,7 +170,7 @@ func (r joinRoomReq) joinRoomByAlias(roomAlias string) util.JSONResponse { if domain == r.cfg.Matrix.ServerName { queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias} var queryRes roomserverAPI.GetRoomIDForAliasResponse - if err = r.aliasAPI.GetRoomIDForAlias(r.req.Context(), &queryReq, &queryRes); err != nil { + if err = r.rsAPI.GetRoomIDForAlias(r.req.Context(), &queryReq, &queryRes); err != nil { util.GetLogger(r.req.Context()).WithError(err).Error("r.aliasAPI.GetRoomIDForAlias failed") return jsonerror.InternalServerError() } @@ -241,7 +241,7 @@ func (r joinRoomReq) joinRoomUsingServers( } queryRes := roomserverAPI.QueryLatestEventsAndStateResponse{} - event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.queryAPI, &queryRes) + event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.rsAPI, &queryRes) if err == nil { // If we have successfully built an event at this point then we can // assert that the room is a local room, as BuildEvent was able to @@ -326,71 +326,15 @@ func (r joinRoomReq) joinRoomUsingServers( // server was invalid this returns an error. // Otherwise this returns a JSONResponse. func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib.ServerName) (*util.JSONResponse, error) { - // Ask the room server for information about room versions. - var request api.QueryRoomVersionCapabilitiesRequest - var response api.QueryRoomVersionCapabilitiesResponse - if err := r.queryAPI.QueryRoomVersionCapabilities(r.req.Context(), &request, &response); err != nil { + fedJoinReq := federationSenderAPI.PerformJoinRequest{ + RoomID: roomID, + UserID: r.userID, + ServerName: server, + } + fedJoinRes := federationSenderAPI.PerformJoinResponse{} + if err := r.fsAPI.PerformJoin(r.req.Context(), &fedJoinReq, &fedJoinRes); err != nil { return nil, err } - var supportedVersions []gomatrixserverlib.RoomVersion - for version := range response.AvailableRoomVersions { - supportedVersions = append(supportedVersions, version) - } - respMakeJoin, err := r.federation.MakeJoin(r.req.Context(), server, roomID, r.userID, supportedVersions) - if err != nil { - // TODO: Check if the user was not allowed to join the room. - return nil, fmt.Errorf("r.federation.MakeJoin: %w", err) - } - - // Set all the fields to be what they should be, this should be a no-op - // but it's possible that the remote server returned us something "odd" - err = r.writeToBuilder(&respMakeJoin.JoinEvent, roomID) - if err != nil { - return nil, fmt.Errorf("r.writeToBuilder: %w", err) - } - - if respMakeJoin.RoomVersion == "" { - respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1 - } - if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil { - return &util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.UnsupportedRoomVersion( - fmt.Sprintf("Room version '%s' is not supported", respMakeJoin.RoomVersion), - ), - }, nil - } - - event, err := respMakeJoin.JoinEvent.Build( - r.evTime, r.cfg.Matrix.ServerName, r.cfg.Matrix.KeyID, - r.cfg.Matrix.PrivateKey, respMakeJoin.RoomVersion, - ) - if err != nil { - return nil, fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err) - } - - respSendJoin, err := r.federation.SendJoin(r.req.Context(), server, event, respMakeJoin.RoomVersion) - if err != nil { - return nil, fmt.Errorf("r.federation.SendJoin: %w", err) - } - - if err = r.checkSendJoinResponse(event, server, respMakeJoin, respSendJoin); err != nil { - return nil, err - } - - util.GetLogger(r.req.Context()).WithFields(logrus.Fields{ - "room_id": roomID, - "num_auth_events": len(respSendJoin.AuthEvents), - "num_state_events": len(respSendJoin.StateEvents), - }).Info("Room join signature and auth verification passed") - - if err = r.producer.SendEventWithState( - r.req.Context(), - respSendJoin.ToRespState(), - event.Headered(respMakeJoin.RoomVersion), - ); err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("r.producer.SendEventWithState") - } return &util.JSONResponse{ Code: http.StatusOK, @@ -400,49 +344,3 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib }{roomID}, }, nil } - -// checkSendJoinResponse checks that all of the signatures are correct -// and that the join is allowed by the supplied state. -func (r joinRoomReq) checkSendJoinResponse( - event gomatrixserverlib.Event, - server gomatrixserverlib.ServerName, - respMakeJoin gomatrixserverlib.RespMakeJoin, - respSendJoin gomatrixserverlib.RespSendJoin, -) error { - // A list of events that we have retried, if they were not included in - // the auth events supplied in the send_join. - retries := map[string]bool{} - -retryCheck: - // TODO: Can we expand Check here to return a list of missing auth - // events rather than failing one at a time? - if err := respSendJoin.Check(r.req.Context(), r.keyRing, event); err != nil { - switch e := err.(type) { - case gomatrixserverlib.MissingAuthEventError: - // Check that we haven't already retried for this event, prevents - // us from ending up in endless loops - if !retries[e.AuthEventID] { - // Ask the server that we're talking to right now for the event - tx, txerr := r.federation.GetEvent(r.req.Context(), server, e.AuthEventID) - if txerr != nil { - return fmt.Errorf("r.federation.GetEvent: %w", txerr) - } - // For each event returned, add it to the auth events. - for _, pdu := range tx.PDUs { - ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, respMakeJoin.RoomVersion) - if everr != nil { - return fmt.Errorf("gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr) - } - respSendJoin.AuthEvents = append(respSendJoin.AuthEvents, ev) - } - // Mark the event as retried and then give the check another go. - retries[e.AuthEventID] = true - goto retryCheck - } - return fmt.Errorf("respSendJoin (after retries): %w", e) - default: - return fmt.Errorf("respSendJoin: %w", err) - } - } - return nil -} diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index dff194dd3..9030f9f7e 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -45,12 +45,12 @@ var errMissingUserID = errors.New("'user_id' must be supplied") func SendMembership( req *http.Request, accountDB accounts.Database, device *authtypes.Device, roomID string, membership string, cfg *config.Dendrite, - queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -71,7 +71,7 @@ func SendMembership( } inviteStored, jsonErrResp := checkAndProcessThreepid( - req, device, &body, cfg, queryAPI, accountDB, producer, + req, device, &body, cfg, rsAPI, accountDB, producer, membership, roomID, evTime, ) if jsonErrResp != nil { @@ -89,7 +89,7 @@ func SendMembership( } event, err := buildMembershipEvent( - req.Context(), body, accountDB, device, membership, roomID, cfg, evTime, queryAPI, asAPI, + req.Context(), body, accountDB, device, membership, roomID, cfg, evTime, rsAPI, asAPI, ) if err == errMissingUserID { return util.JSONResponse{ @@ -153,7 +153,7 @@ func buildMembershipEvent( device *authtypes.Device, membership, roomID string, cfg *config.Dendrite, evTime time.Time, - queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) (*gomatrixserverlib.Event, error) { stateKey, reason, err := getMembershipStateKey(body, device, membership) if err != nil { @@ -188,7 +188,7 @@ func buildMembershipEvent( return nil, err } - return common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil) + return common.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) } // loadProfile lookups the profile of a given user from the database and returns @@ -248,7 +248,7 @@ func checkAndProcessThreepid( device *authtypes.Device, body *threepid.MembershipRequest, cfg *config.Dendrite, - queryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, accountDB accounts.Database, producer *producers.RoomserverProducer, membership, roomID string, @@ -256,7 +256,7 @@ func checkAndProcessThreepid( ) (inviteStored bool, errRes *util.JSONResponse) { inviteStored, err := threepid.CheckAndProcessInvite( - req.Context(), device, body, cfg, queryAPI, accountDB, producer, + req.Context(), device, body, cfg, rsAPI, accountDB, producer, membership, roomID, evTime, ) if err == threepid.ErrMissingParameter { diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index 0b846e5e3..f5d9bc4c5 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -39,7 +39,7 @@ type getJoinedRoomsResponse struct { func GetMemberships( req *http.Request, device *authtypes.Device, roomID string, joinedOnly bool, _ *config.Dendrite, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { queryReq := api.QueryMembershipsForRoomRequest{ JoinedOnly: joinedOnly, @@ -47,8 +47,8 @@ func GetMemberships( Sender: device.UserID, } var queryRes api.QueryMembershipsForRoomResponse - if err := queryAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryMembershipsForRoom failed") + if err := rsAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryMembershipsForRoom failed") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index a51c55ea5..b51533e45 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -94,7 +94,7 @@ func GetAvatarURL( func SetAvatarURL( req *http.Request, accountDB accounts.Database, device *authtypes.Device, userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite, - rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI, + rsProducer *producers.RoomserverProducer, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -154,7 +154,7 @@ func SetAvatarURL( } events, err := buildMembershipEvents( - req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI, + req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI, ) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed") @@ -208,7 +208,7 @@ func GetDisplayName( func SetDisplayName( req *http.Request, accountDB accounts.Database, device *authtypes.Device, userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite, - rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI, + rsProducer *producers.RoomserverProducer, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -268,7 +268,7 @@ func SetDisplayName( } events, err := buildMembershipEvents( - req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI, + req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI, ) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed") @@ -337,14 +337,14 @@ func buildMembershipEvents( ctx context.Context, memberships []authtypes.Membership, newProfile authtypes.Profile, userID string, cfg *config.Dendrite, - evTime time.Time, queryAPI api.RoomserverQueryAPI, + evTime time.Time, rsAPI api.RoomserverInternalAPI, ) ([]gomatrixserverlib.HeaderedEvent, error) { evs := []gomatrixserverlib.HeaderedEvent{} for _, membership := range memberships { verReq := api.QueryRoomVersionForRoomRequest{RoomID: membership.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { return []gomatrixserverlib.HeaderedEvent{}, err } @@ -366,7 +366,7 @@ func buildMembershipEvents( return nil, err } - event, err := common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil) + event, err := common.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) if err != nil { return nil, err } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 5dc6d7db9..42b391de6 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -49,8 +49,7 @@ const pathPrefixUnstable = "/_matrix/client/unstable" func Setup( apiMux *mux.Router, cfg *config.Dendrite, producer *producers.RoomserverProducer, - queryAPI roomserverAPI.RoomserverQueryAPI, - aliasAPI roomserverAPI.RoomserverAliasAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, accountDB accounts.Database, deviceDB devices.Database, @@ -60,7 +59,7 @@ func Setup( syncProducer *producers.SyncAPIProducer, eduProducer *producers.EDUServerProducer, transactionsCache *transactions.Cache, - federationSender federationSenderAPI.FederationSenderQueryAPI, + federationSender federationSenderAPI.FederationSenderInternalAPI, ) { apiMux.Handle("/_matrix/client/versions", @@ -91,7 +90,7 @@ func Setup( r0mux.Handle("/createRoom", common.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - return CreateRoom(req, device, cfg, producer, accountDB, aliasAPI, asAPI) + return CreateRoom(req, device, cfg, producer, accountDB, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", @@ -101,7 +100,8 @@ func Setup( return util.ErrorResponse(err) } return JoinRoomByIDOrAlias( - req, device, vars["roomIDOrAlias"], cfg, federation, producer, queryAPI, aliasAPI, keyRing, accountDB, + req, device, vars["roomIDOrAlias"], cfg, federation, producer, + rsAPI, federationSender, keyRing, accountDB, ) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -117,7 +117,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, queryAPI, asAPI, producer) + return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, rsAPI, asAPI, producer) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}", @@ -126,7 +126,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, queryAPI, producer, nil) + return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, rsAPI, producer, nil) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}", @@ -137,7 +137,7 @@ func Setup( } txnID := vars["txnID"] return SendEvent(req, device, vars["roomID"], vars["eventType"], &txnID, - nil, cfg, queryAPI, producer, transactionsCache) + nil, cfg, rsAPI, producer, transactionsCache) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/event/{eventID}", @@ -146,7 +146,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, queryAPI, federation, keyRing) + return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation, keyRing) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -155,7 +155,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingStateRequest(req.Context(), queryAPI, vars["roomID"]) + return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"]) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { @@ -163,7 +163,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], "") + return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], "") })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { @@ -171,7 +171,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], vars["stateKey"]) + return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], vars["stateKey"]) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}", @@ -186,7 +186,7 @@ func Setup( if strings.HasSuffix(eventType, "/") { eventType = eventType[:len(eventType)-1] } - return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, queryAPI, producer, nil) + return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, rsAPI, producer, nil) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -197,7 +197,7 @@ func Setup( return util.ErrorResponse(err) } stateKey := vars["stateKey"] - return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, queryAPI, producer, nil) + return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, rsAPI, producer, nil) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -219,7 +219,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return DirectoryRoom(req, vars["roomAlias"], federation, cfg, aliasAPI, federationSender) + return DirectoryRoom(req, vars["roomAlias"], federation, cfg, rsAPI, federationSender) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -229,7 +229,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetLocalAlias(req, device, vars["roomAlias"], cfg, aliasAPI) + return SetLocalAlias(req, device, vars["roomAlias"], cfg, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -239,7 +239,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return RemoveLocalAlias(req, device, vars["roomAlias"], aliasAPI) + return RemoveLocalAlias(req, device, vars["roomAlias"], rsAPI) }), ).Methods(http.MethodDelete, http.MethodOptions) @@ -353,7 +353,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, queryAPI) + return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows @@ -375,7 +375,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, queryAPI) + return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows @@ -488,7 +488,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetMemberships(req, device, vars["roomID"], false, cfg, queryAPI) + return GetMemberships(req, device, vars["roomID"], false, cfg, rsAPI) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -498,7 +498,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetMemberships(req, device, vars["roomID"], true, cfg, queryAPI) + return GetMemberships(req, device, vars["roomID"], true, cfg, rsAPI) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -602,7 +602,7 @@ func Setup( r0mux.Handle("/capabilities", common.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - return GetCapabilities(req, queryAPI) + return GetCapabilities(req, rsAPI) }), ).Methods(http.MethodGet) } diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 5b2cd8ad4..7280dcd94 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -45,13 +45,13 @@ func SendEvent( device *authtypes.Device, roomID, eventType string, txnID, stateKey *string, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, txnCache *transactions.Cache, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -65,7 +65,7 @@ func SendEvent( } } - e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, queryAPI) + e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, rsAPI) if resErr != nil { return *resErr } @@ -115,7 +115,7 @@ func generateSendEvent( device *authtypes.Device, roomID, eventType string, stateKey *string, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) (*gomatrixserverlib.Event, *util.JSONResponse) { // parse the incoming http request userID := device.UserID @@ -148,7 +148,7 @@ func generateSendEvent( } var queryRes api.QueryLatestEventsAndStateResponse - e, err := common.BuildEvent(req.Context(), &builder, cfg, evTime, queryAPI, &queryRes) + e, err := common.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes) if err == common.ErrRoomNoExists { return nil, &util.JSONResponse{ Code: http.StatusNotFound, diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go index c243eec0f..e3e5bdb6d 100644 --- a/clientapi/routing/state.go +++ b/clientapi/routing/state.go @@ -40,7 +40,7 @@ type stateEventInStateResp struct { // TODO: Check if the user is in the room. If not, check if the room's history // is publicly visible. Current behaviour is returning an empty array if the // user cannot see the room's history. -func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string) util.JSONResponse { +func OnIncomingStateRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID string) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) stateReq := api.QueryLatestEventsAndStateRequest{ @@ -48,7 +48,7 @@ func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI } stateRes := api.QueryLatestEventsAndStateResponse{} - if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { + if err := rsAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed") return jsonerror.InternalServerError() } @@ -98,7 +98,7 @@ func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI // /rooms/{roomID}/state/{type}/{statekey} request. It will look in current // state to see if there is an event with that type and state key, if there // is then (by default) we return the content, otherwise a 404. -func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string, evType, stateKey string) util.JSONResponse { +func OnIncomingStateTypeRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID string, evType, stateKey string) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) util.GetLogger(ctx).WithFields(log.Fields{ @@ -118,7 +118,7 @@ func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQuer } stateRes := api.QueryLatestEventsAndStateResponse{} - if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { + if err := rsAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed") return jsonerror.InternalServerError() } diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index e34e91b56..5e7e4f2b4 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -87,7 +87,7 @@ var ( func CheckAndProcessInvite( ctx context.Context, device *authtypes.Device, body *MembershipRequest, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, db accounts.Database, + rsAPI api.RoomserverInternalAPI, db accounts.Database, producer *producers.RoomserverProducer, membership string, roomID string, evTime time.Time, ) (inviteStoredOnIDServer bool, err error) { @@ -112,7 +112,7 @@ func CheckAndProcessInvite( // "m.room.third_party_invite" have to be emitted from the data in // storeInviteRes. err = emit3PIDInviteEvent( - ctx, body, storeInviteRes, device, roomID, cfg, queryAPI, producer, evTime, + ctx, body, storeInviteRes, device, roomID, cfg, rsAPI, producer, evTime, ) inviteStoredOnIDServer = err == nil @@ -331,7 +331,7 @@ func emit3PIDInviteEvent( ctx context.Context, body *MembershipRequest, res *idServerStoreInviteResponse, device *authtypes.Device, roomID string, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, producer *producers.RoomserverProducer, + rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, evTime time.Time, ) error { builder := &gomatrixserverlib.EventBuilder{ @@ -354,7 +354,7 @@ func emit3PIDInviteEvent( } queryRes := api.QueryLatestEventsAndStateResponse{} - event, err := common.BuildEvent(ctx, builder, cfg, evTime, queryAPI, &queryRes) + event, err := common.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes) if err != nil { return err } diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index f203969f4..91ca707e7 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -28,11 +28,11 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() federation := base.CreateFederationClient() - alias, _, query := base.CreateHTTPRoomserverAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() cache := transactions.New() appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, alias, query, cache, + base, accountDB, deviceDB, federation, rsAPI, cache, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI)) diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 815a978a8..1ce66db2c 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -36,13 +36,14 @@ func main() { keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) asQuery := base.CreateHTTPAppServiceAPIs() - alias, input, query := base.CreateHTTPRoomserverAPIs() - fedSenderAPI := base.CreateHTTPFederationSenderAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() + fsAPI := base.CreateHTTPFederationSenderAPIs() + rsAPI.SetFederationSenderAPI(fsAPI) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, &keyRing, - alias, input, query, eduInputAPI, asQuery, transactions.New(), fedSenderAPI, + rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI)) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 183a3c991..0ff610a5b 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -149,27 +149,34 @@ func main() { federation := createFederationClient(base) keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := roomserver.SetupRoomServerComponent(&base.Base, keyRing) - eduInputAPI := eduserver.SetupEDUServerComponent(&base.Base, cache.New()) - asQuery := appservice.SetupAppServiceAPIComponent( - &base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(), + rsAPI := roomserver.SetupRoomServerComponent( + &base.Base, keyRing, federation, ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query, input) + eduInputAPI := eduserver.SetupEDUServerComponent( + &base.Base, cache.New(), + ) + asAPI := appservice.SetupAppServiceAPIComponent( + &base.Base, accountDB, deviceDB, federation, rsAPI, transactions.New(), + ) + fsAPI := federationsender.SetupFederationSenderComponent( + &base.Base, federation, rsAPI, &keyRing, + ) + rsAPI.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( &base.Base, deviceDB, accountDB, - federation, &keyRing, alias, input, query, - eduInputAPI, asQuery, transactions.New(), fedSenderAPI, + federation, &keyRing, rsAPI, + eduInputAPI, asAPI, transactions.New(), fsAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) + federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(&base.Base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, query, federation, nil) // Check this later - syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, query, federation, &cfg) + publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, rsAPI, federation, nil) // Check this later + syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, rsAPI, federation, &cfg) httpHandler := common.WrapHandlerInCORS(base.Base.APIMux) diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index dd06cd3f9..d829326a7 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -32,17 +32,18 @@ func main() { deviceDB := base.CreateDeviceDB() keyDB := base.CreateKeyDB() federation := base.CreateFederationClient() - federationSender := base.CreateHTTPFederationSenderAPIs() + fsAPI := base.CreateHTTPFederationSenderAPIs() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := base.CreateHTTPRoomserverAPIs() - asQuery := base.CreateHTTPAppServiceAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() + asAPI := base.CreateHTTPAppServiceAPIs() + rsAPI.SetFederationSenderAPI(fsAPI) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) eduProducer := producers.NewEDUServerProducer(eduInputAPI) federationapi.SetupFederationAPIComponent( base, accountDB, deviceDB, federation, &keyRing, - alias, input, query, asQuery, federationSender, eduProducer, + rsAPI, asAPI, fsAPI, eduProducer, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 1593afaa5..0daac1bcb 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -16,6 +16,7 @@ package main import ( "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/common/keydb" "github.com/matrix-org/dendrite/federationsender" ) @@ -25,12 +26,13 @@ func main() { defer base.Close() // nolint: errcheck federation := base.CreateFederationClient() - - _, input, query := base.CreateHTTPRoomserverAPIs() - - federationsender.SetupFederationSenderComponent( - base, federation, query, input, + keyDB := base.CreateKeyDB() + keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) + rsAPI := base.CreateHTTPRoomserverAPIs() + fsAPI := federationsender.SetupFederationSenderComponent( + base, federation, rsAPI, &keyRing, ) + rsAPI.SetFederationSenderAPI(fsAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 5193288a4..e004bc12e 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -57,27 +57,34 @@ func main() { federation := base.CreateFederationClient() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing) - eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) - asQuery := appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, alias, query, transactions.New(), + rsAPI := roomserver.SetupRoomServerComponent( + base, keyRing, federation, ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input) + eduInputAPI := eduserver.SetupEDUServerComponent( + base, cache.New(), + ) + asAPI := appservice.SetupAppServiceAPIComponent( + base, accountDB, deviceDB, federation, rsAPI, transactions.New(), + ) + fsAPI := federationsender.SetupFederationSenderComponent( + base, federation, rsAPI, &keyRing, + ) + rsAPI.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, - federation, &keyRing, alias, input, query, - eduInputAPI, asQuery, transactions.New(), fedSenderAPI, + federation, &keyRing, rsAPI, + eduInputAPI, asAPI, transactions.New(), fsAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, federation, nil) - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, nil) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) httpHandler := common.WrapHandlerInCORS(base.APIMux) diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index 4cef1e6e5..fca39a2fe 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -28,12 +28,15 @@ func main() { deviceDB := base.CreateDeviceDB() - _, _, query := base.CreateHTTPRoomserverAPIs() - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties()) + fsAPI := base.CreateHTTPFederationSenderAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() + rsAPI.SetFederationSenderAPI(fsAPI) + + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, nil, nil) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, nil, nil) base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI)) diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index f33a2b88f..41149ad90 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -30,7 +30,9 @@ func main() { federation := base.CreateFederationClient() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - roomserver.SetupRoomServerComponent(base, keyRing) + fsAPI := base.CreateHTTPFederationSenderAPIs() + rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation) + rsAPI.SetFederationSenderAPI(fsAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer)) diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index 55e9faeef..259447af2 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -28,9 +28,9 @@ func main() { accountDB := base.CreateAccountsDB() federation := base.CreateFederationClient() - _, _, query := base.CreateHTTPRoomserverAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI)) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 9bd8f2ee2..5b7ed4807 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -123,27 +123,28 @@ func main() { } p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node) - alias, input, query := roomserver.SetupRoomServerComponent(base) + rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) asQuery := appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, alias, query, transactions.New(), + base, accountDB, deviceDB, federation, rsAPI, transactions.New(), ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query) + fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, rsAPI, &keyRing) + rsAPI.SetFederationSenderAPI(fedSenderAPI) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, - federation, &keyRing, alias, input, query, + federation, &keyRing, rsAPI, eduInputAPI, asQuery, transactions.New(), fedSenderAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asQuery, fedSenderAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, federation, p2pPublicRoomProvider) - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, p2pPublicRoomProvider) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) httpHandler := common.WrapHandlerInCORS(base.APIMux) diff --git a/cmd/roomserver-integration-tests/main.go b/cmd/roomserver-integration-tests/main.go index 682fc6224..7126405e5 100644 --- a/cmd/roomserver-integration-tests/main.go +++ b/cmd/roomserver-integration-tests/main.go @@ -209,7 +209,7 @@ func writeToRoomServer(input []string, roomserverURL string) error { return err } } - x, err := api.NewRoomserverInputAPIHTTP(roomserverURL, &http.Client{Timeout: timeoutHTTP}) + x, err := api.NewRoomserverInternalAPIHTTP(roomserverURL, &http.Client{Timeout: timeoutHTTP}, nil) if err != nil { return err } @@ -225,7 +225,7 @@ func writeToRoomServer(input []string, roomserverURL string) error { // Once those messages have been written it runs the checkQueries function passing // a api.RoomserverQueryAPI client. The caller can use this function to check the // behaviour of the query API. -func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverQueryAPI)) { +func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverInternalAPI)) { dir, err := ioutil.TempDir("", "room-server-test") if err != nil { panic(err) @@ -276,7 +276,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)} gotOutput, err := runAndReadFromTopic(cmd, cfg.RoomServerURL()+"/metrics", doInput, outputTopic, len(wantOutput), func() { - queryAPI, _ := api.NewRoomserverQueryAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache) + queryAPI, _ := api.NewRoomserverInternalAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache) checkQueries(queryAPI) }) if err != nil { @@ -410,7 +410,7 @@ func main() { }}`, } - testRoomserver(input, want, func(q api.RoomserverQueryAPI) { + testRoomserver(input, want, func(q api.RoomserverInternalAPI) { var response api.QueryLatestEventsAndStateResponse if err := q.QueryLatestEventsAndState( context.Background(), diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 8a55b8e33..a7e6736a6 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -119,24 +119,12 @@ func (b *BaseDendrite) CreateHTTPAppServiceAPIs() appserviceAPI.AppServiceQueryA // CreateHTTPRoomserverAPIs returns the AliasAPI, InputAPI and QueryAPI for hitting // the roomserver over HTTP. -func (b *BaseDendrite) CreateHTTPRoomserverAPIs() ( - roomserverAPI.RoomserverAliasAPI, - roomserverAPI.RoomserverInputAPI, - roomserverAPI.RoomserverQueryAPI, -) { - alias, err := roomserverAPI.NewRoomserverAliasAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient) +func (b *BaseDendrite) CreateHTTPRoomserverAPIs() roomserverAPI.RoomserverInternalAPI { + rsAPI, err := roomserverAPI.NewRoomserverInternalAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache) if err != nil { - logrus.WithError(err).Panic("NewRoomserverAliasAPIHTTP failed") + logrus.WithError(err).Panic("NewRoomserverInternalAPIHTTP failed", b.httpClient) } - input, err := roomserverAPI.NewRoomserverInputAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient) - if err != nil { - logrus.WithError(err).Panic("NewRoomserverInputAPIHTTP failed", b.httpClient) - } - query, err := roomserverAPI.NewRoomserverQueryAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache) - if err != nil { - logrus.WithError(err).Panic("NewRoomserverQueryAPIHTTP failed", b.httpClient) - } - return alias, input, query + return rsAPI } // CreateHTTPEDUServerAPIs returns eduInputAPI for hitting the EDU @@ -149,12 +137,12 @@ func (b *BaseDendrite) CreateHTTPEDUServerAPIs() eduServerAPI.EDUServerInputAPI return e } -// CreateHTTPFederationSenderAPIs returns FederationSenderQueryAPI for hitting +// CreateHTTPFederationSenderAPIs returns FederationSenderInternalAPI for hitting // the federation sender over HTTP -func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderQueryAPI { - f, err := federationSenderAPI.NewFederationSenderQueryAPIHTTP(b.Cfg.FederationSenderURL(), b.httpClient) +func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderInternalAPI { + f, err := federationSenderAPI.NewFederationSenderInternalAPIHTTP(b.Cfg.FederationSenderURL(), b.httpClient) if err != nil { - logrus.WithError(err).Panic("NewFederationSenderQueryAPIHTTP failed", b.httpClient) + logrus.WithError(err).Panic("NewFederationSenderInternalAPIHTTP failed", b.httpClient) } return f } diff --git a/common/events.go b/common/events.go index 556b7b671..0c8ead3a7 100644 --- a/common/events.go +++ b/common/events.go @@ -39,13 +39,13 @@ var ErrRoomNoExists = errors.New("Room does not exist") func BuildEvent( ctx context.Context, builder *gomatrixserverlib.EventBuilder, cfg *config.Dendrite, evTime time.Time, - queryAPI api.RoomserverQueryAPI, queryRes *api.QueryLatestEventsAndStateResponse, + rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse, ) (*gomatrixserverlib.Event, error) { if queryRes == nil { queryRes = &api.QueryLatestEventsAndStateResponse{} } - err := AddPrevEventsToEvent(ctx, builder, queryAPI, queryRes) + err := AddPrevEventsToEvent(ctx, builder, rsAPI, queryRes) if err != nil { // This can pass through a ErrRoomNoExists to the caller return nil, err @@ -66,7 +66,7 @@ func BuildEvent( func AddPrevEventsToEvent( ctx context.Context, builder *gomatrixserverlib.EventBuilder, - queryAPI api.RoomserverQueryAPI, queryRes *api.QueryLatestEventsAndStateResponse, + rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse, ) error { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) if err != nil { @@ -82,8 +82,8 @@ func AddPrevEventsToEvent( RoomID: builder.RoomID, StateToFetch: eventsNeeded.Tuples(), } - if err = queryAPI.QueryLatestEventsAndState(ctx, &queryReq, queryRes); err != nil { - return fmt.Errorf("queryAPI.QueryLatestEventsAndState: %w", err) + if err = rsAPI.QueryLatestEventsAndState(ctx, &queryReq, queryRes); err != nil { + return fmt.Errorf("rsAPI.QueryLatestEventsAndState: %w", err) } if !queryRes.RoomExists { diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index ed96322b8..d458b8537 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -36,18 +36,16 @@ func SetupFederationAPIComponent( deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, keyRing *gomatrixserverlib.KeyRing, - aliasAPI roomserverAPI.RoomserverAliasAPI, - inputAPI roomserverAPI.RoomserverInputAPI, - queryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, - federationSenderAPI federationSenderAPI.FederationSenderQueryAPI, + federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, eduProducer *producers.EDUServerProducer, ) { - roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI) + roomserverProducer := producers.NewRoomserverProducer(rsAPI) routing.Setup( - base.APIMux, base.Cfg, queryAPI, aliasAPI, asAPI, - roomserverProducer, eduProducer, federationSenderAPI, *keyRing, + base.APIMux, base.Cfg, rsAPI, asAPI, roomserverProducer, + eduProducer, federationSenderAPI, *keyRing, federation, accountsDB, deviceDB, ) } diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 6f49b9a81..651a4a2d2 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -33,7 +33,7 @@ import ( func Backfill( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, cfg *config.Dendrite, ) util.JSONResponse { @@ -82,7 +82,7 @@ func Backfill( } // Query the roomserver. - if err = query.QueryBackfill(httpReq.Context(), &req, &res); err != nil { + if err = rsAPI.QueryBackfill(httpReq.Context(), &req, &res); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryBackfill failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/eventauth.go b/federationapi/routing/eventauth.go index 003165c85..34eaad1c5 100644 --- a/federationapi/routing/eventauth.go +++ b/federationapi/routing/eventauth.go @@ -25,13 +25,13 @@ import ( func GetEventAuth( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, eventID string, ) util.JSONResponse { // TODO: Optimisation: we shouldn't be querying all the room state // that is in state.StateEvents - we just ignore it. - state, err := getState(ctx, request, query, roomID, eventID) + state, err := getState(ctx, request, rsAPI, roomID, eventID) if err != nil { return *err } diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go index a91528b3d..ced9e3d53 100644 --- a/federationapi/routing/events.go +++ b/federationapi/routing/events.go @@ -16,7 +16,9 @@ package routing import ( "context" + "encoding/json" "net/http" + "time" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -27,15 +29,22 @@ import ( func GetEvent( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, eventID string, + origin gomatrixserverlib.ServerName, ) util.JSONResponse { - event, err := getEvent(ctx, request, query, eventID) + event, err := getEvent(ctx, request, rsAPI, eventID) if err != nil { return *err } - return util.JSONResponse{Code: http.StatusOK, JSON: event} + return util.JSONResponse{Code: http.StatusOK, JSON: gomatrixserverlib.Transaction{ + Origin: origin, + OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), + PDUs: []json.RawMessage{ + event.JSON(), + }, + }} } // getEvent returns the requested event, @@ -43,11 +52,11 @@ func GetEvent( func getEvent( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, eventID string, ) (*gomatrixserverlib.Event, *util.JSONResponse) { var authResponse api.QueryServerAllowedToSeeEventResponse - err := query.QueryServerAllowedToSeeEvent( + err := rsAPI.QueryServerAllowedToSeeEvent( ctx, &api.QueryServerAllowedToSeeEventRequest{ EventID: eventID, @@ -66,7 +75,7 @@ func getEvent( } var eventsResponse api.QueryEventsByIDResponse - err = query.QueryEventsByID( + err = rsAPI.QueryEventsByID( ctx, &api.QueryEventsByIDRequest{EventIDs: []string{eventID}}, &eventsResponse, diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 0c899ab9f..be5e988ab 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -33,13 +33,13 @@ func MakeJoin( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, cfg *config.Dendrite, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID, userID string, remoteVersions []gomatrixserverlib.RoomVersion, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := query.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusInternalServerError, JSON: jsonerror.InternalServerError(), @@ -97,7 +97,7 @@ func MakeJoin( queryRes := api.QueryLatestEventsAndStateResponse{ RoomVersion: verRes.RoomVersion, } - event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), query, &queryRes) + event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) if err == common.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, @@ -136,15 +136,15 @@ func SendJoin( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, cfg *config.Dendrite, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, keys gomatrixserverlib.KeyRing, roomID, eventID string, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := query.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryRoomVersionForRoom failed") + if err := rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryRoomVersionForRoom failed") return util.JSONResponse{ Code: http.StatusInternalServerError, JSON: jsonerror.InternalServerError(), @@ -216,14 +216,14 @@ func SendJoin( // Fetch the state and auth chain. We do this before we send the events // on, in case this fails. var stateAndAuthChainResponse api.QueryStateAndAuthChainResponse - err = query.QueryStateAndAuthChain(httpReq.Context(), &api.QueryStateAndAuthChainRequest{ + err = rsAPI.QueryStateAndAuthChain(httpReq.Context(), &api.QueryStateAndAuthChainRequest{ PrevEventIDs: event.PrevEventIDs(), AuthEventIDs: event.AuthEventIDs(), RoomID: roomID, ResolveState: true, }, &stateAndAuthChainResponse) if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryStateAndAuthChain failed") + util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryStateAndAuthChain failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 6fc3b12ed..1124bfa27 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -30,7 +30,7 @@ func MakeLeave( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, cfg *config.Dendrite, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID, userID string, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('@', userID) @@ -61,7 +61,7 @@ func MakeLeave( } var queryRes api.QueryLatestEventsAndStateResponse - event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), query, &queryRes) + event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) if err == common.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, @@ -102,7 +102,7 @@ func SendLeave( ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := producer.QueryAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + if err := producer.RsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go index 069bff3dd..ae91c5891 100644 --- a/federationapi/routing/missingevents.go +++ b/federationapi/routing/missingevents.go @@ -34,7 +34,7 @@ type getMissingEventRequest struct { func GetMissingEvents( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, ) util.JSONResponse { var gme getMissingEventRequest @@ -46,7 +46,7 @@ func GetMissingEvents( } var eventsResponse api.QueryMissingEventsResponse - if err := query.QueryMissingEvents( + if err := rsAPI.QueryMissingEvents( httpReq.Context(), &api.QueryMissingEventsRequest{ EarliestEvents: gme.EarliestEvents, LatestEvents: gme.LatestEvents, diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 7cb50e525..c58690c63 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -32,8 +32,8 @@ func RoomAliasToID( httpReq *http.Request, federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, - aliasAPI roomserverAPI.RoomserverAliasAPI, - senderAPI federationSenderAPI.FederationSenderQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, + senderAPI federationSenderAPI.FederationSenderInternalAPI, ) util.JSONResponse { roomAlias := httpReq.FormValue("room_alias") if roomAlias == "" { @@ -55,7 +55,7 @@ func RoomAliasToID( if domain == cfg.Matrix.ServerName { queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias} var queryRes roomserverAPI.GetRoomIDForAliasResponse - if err = aliasAPI.GetRoomIDForAlias(httpReq.Context(), &queryReq, &queryRes); err != nil { + if err = rsAPI.GetRoomIDForAlias(httpReq.Context(), &queryReq, &queryRes); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 83bac5550..a5b8ce24e 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -44,12 +44,11 @@ const ( func Setup( apiMux *mux.Router, cfg *config.Dendrite, - query roomserverAPI.RoomserverQueryAPI, - aliasAPI roomserverAPI.RoomserverAliasAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, eduProducer *producers.EDUServerProducer, - federationSenderAPI federationSenderAPI.FederationSenderQueryAPI, + federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, keys gomatrixserverlib.KeyRing, federation *gomatrixserverlib.FederationClient, accountDB accounts.Database, @@ -80,7 +79,7 @@ func Setup( } return Send( httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]), - cfg, query, producer, eduProducer, keys, federation, + cfg, rsAPI, producer, eduProducer, keys, federation, ) }, )).Methods(http.MethodPut, http.MethodOptions) @@ -101,7 +100,7 @@ func Setup( v1fedmux.Handle("/3pid/onbind", common.MakeExternalAPI("3pid_onbind", func(req *http.Request) util.JSONResponse { - return CreateInvitesFrom3PIDInvites(req, query, asAPI, cfg, producer, federation, accountDB) + return CreateInvitesFrom3PIDInvites(req, rsAPI, asAPI, cfg, producer, federation, accountDB) }, )).Methods(http.MethodPost, http.MethodOptions) @@ -113,7 +112,7 @@ func Setup( return util.ErrorResponse(err) } return ExchangeThirdPartyInvite( - httpReq, request, vars["roomID"], query, cfg, federation, producer, + httpReq, request, vars["roomID"], rsAPI, cfg, federation, producer, ) }, )).Methods(http.MethodPut, http.MethodOptions) @@ -126,7 +125,7 @@ func Setup( return util.ErrorResponse(err) } return GetEvent( - httpReq.Context(), request, query, vars["eventID"], + httpReq.Context(), request, rsAPI, vars["eventID"], cfg.Matrix.ServerName, ) }, )).Methods(http.MethodGet) @@ -139,7 +138,7 @@ func Setup( return util.ErrorResponse(err) } return GetState( - httpReq.Context(), request, query, vars["roomID"], + httpReq.Context(), request, rsAPI, vars["roomID"], ) }, )).Methods(http.MethodGet) @@ -152,7 +151,7 @@ func Setup( return util.ErrorResponse(err) } return GetStateIDs( - httpReq.Context(), request, query, vars["roomID"], + httpReq.Context(), request, rsAPI, vars["roomID"], ) }, )).Methods(http.MethodGet) @@ -162,7 +161,7 @@ func Setup( func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars := mux.Vars(httpReq) return GetEventAuth( - httpReq.Context(), request, query, vars["roomID"], vars["eventID"], + httpReq.Context(), request, rsAPI, vars["roomID"], vars["eventID"], ) }, )).Methods(http.MethodGet) @@ -171,7 +170,7 @@ func Setup( "federation_query_room_alias", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { return RoomAliasToID( - httpReq, federation, cfg, aliasAPI, federationSenderAPI, + httpReq, federation, cfg, rsAPI, federationSenderAPI, ) }, )).Methods(http.MethodGet) @@ -222,7 +221,7 @@ func Setup( remoteVersions = append(remoteVersions, gomatrixserverlib.RoomVersionV1) } return MakeJoin( - httpReq, request, cfg, query, roomID, eventID, remoteVersions, + httpReq, request, cfg, rsAPI, roomID, eventID, remoteVersions, ) }, )).Methods(http.MethodGet) @@ -237,7 +236,7 @@ func Setup( roomID := vars["roomID"] eventID := vars["eventID"] return SendJoin( - httpReq, request, cfg, query, producer, keys, roomID, eventID, + httpReq, request, cfg, rsAPI, producer, keys, roomID, eventID, ) }, )).Methods(http.MethodPut) @@ -252,7 +251,7 @@ func Setup( roomID := vars["roomID"] eventID := vars["eventID"] return MakeLeave( - httpReq, request, cfg, query, roomID, eventID, + httpReq, request, cfg, rsAPI, roomID, eventID, ) }, )).Methods(http.MethodGet) @@ -286,7 +285,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetMissingEvents(httpReq, request, query, vars["roomID"]) + return GetMissingEvents(httpReq, request, rsAPI, vars["roomID"]) }, )).Methods(http.MethodPost) @@ -297,7 +296,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return Backfill(httpReq, request, query, vars["roomID"], cfg) + return Backfill(httpReq, request, rsAPI, vars["roomID"], cfg) }, )).Methods(http.MethodGet) } diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 5a9766f81..88411b81a 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -34,7 +34,7 @@ func Send( request *gomatrixserverlib.FederationRequest, txnID gomatrixserverlib.TransactionID, cfg *config.Dendrite, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, eduProducer *producers.EDUServerProducer, keys gomatrixserverlib.KeyRing, @@ -42,7 +42,7 @@ func Send( ) util.JSONResponse { t := txnReq{ context: httpReq.Context(), - query: query, + rsAPI: rsAPI, producer: producer, eduProducer: eduProducer, keys: keys, @@ -99,7 +99,7 @@ func Send( type txnReq struct { gomatrixserverlib.Transaction context context.Context - query api.RoomserverQueryAPI + rsAPI api.RoomserverInternalAPI producer *producers.RoomserverProducer eduProducer *producers.EDUServerProducer keys gomatrixserverlib.KeyRing @@ -120,7 +120,7 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { } verReq := api.QueryRoomVersionForRoomRequest{RoomID: header.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := t.query.QueryRoomVersionForRoom(t.context, &verReq, &verRes); err != nil { + if err := t.rsAPI.QueryRoomVersionForRoom(t.context, &verReq, &verRes); err != nil { util.GetLogger(t.context).WithError(err).Warn("Transaction: Failed to query room version for room", verReq.RoomID) return nil, roomNotFoundError{verReq.RoomID} } @@ -228,7 +228,7 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event) error { StateToFetch: needed.Tuples(), } var stateResp api.QueryStateAfterEventsResponse - if err := t.query.QueryStateAfterEvents(t.context, &stateReq, &stateResp); err != nil { + if err := t.rsAPI.QueryStateAfterEvents(t.context, &stateReq, &stateResp); err != nil { return err } diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 548598dd7..f90c494c3 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -27,7 +27,7 @@ import ( func GetState( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, ) util.JSONResponse { eventID, err := parseEventIDParam(request) @@ -35,7 +35,7 @@ func GetState( return *err } - state, err := getState(ctx, request, query, roomID, eventID) + state, err := getState(ctx, request, rsAPI, roomID, eventID) if err != nil { return *err } @@ -47,7 +47,7 @@ func GetState( func GetStateIDs( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, ) util.JSONResponse { eventID, err := parseEventIDParam(request) @@ -55,7 +55,7 @@ func GetStateIDs( return *err } - state, err := getState(ctx, request, query, roomID, eventID) + state, err := getState(ctx, request, rsAPI, roomID, eventID) if err != nil { return *err } @@ -94,11 +94,11 @@ func parseEventIDParam( func getState( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, eventID string, ) (*gomatrixserverlib.RespState, *util.JSONResponse) { - event, resErr := getEvent(ctx, request, query, eventID) + event, resErr := getEvent(ctx, request, rsAPI, eventID) if resErr != nil { return nil, resErr } @@ -110,7 +110,7 @@ func getState( authEventIDs := getIDsFromEventRef(event.AuthEvents()) var response api.QueryStateAndAuthChainResponse - err := query.QueryStateAndAuthChain( + err := rsAPI.QueryStateAndAuthChain( ctx, &api.QueryStateAndAuthChainRequest{ RoomID: roomID, diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index f93d934ed..3c1d09e11 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -58,7 +58,7 @@ var ( // CreateInvitesFrom3PIDInvites implements POST /_matrix/federation/v1/3pid/onbind func CreateInvitesFrom3PIDInvites( - req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI, + req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite, producer *producers.RoomserverProducer, federation *gomatrixserverlib.FederationClient, accountDB accounts.Database, @@ -72,7 +72,7 @@ func CreateInvitesFrom3PIDInvites( for _, inv := range body.Invites { verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -80,7 +80,7 @@ func CreateInvitesFrom3PIDInvites( } event, err := createInviteFrom3PIDInvite( - req.Context(), queryAPI, asAPI, cfg, inv, federation, accountDB, + req.Context(), rsAPI, asAPI, cfg, inv, federation, accountDB, ) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("createInviteFrom3PIDInvite failed") @@ -108,7 +108,7 @@ func ExchangeThirdPartyInvite( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, roomID string, - queryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, cfg *config.Dendrite, federation *gomatrixserverlib.FederationClient, producer *producers.RoomserverProducer, @@ -148,7 +148,7 @@ func ExchangeThirdPartyInvite( verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err = queryAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + if err = rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -156,7 +156,7 @@ func ExchangeThirdPartyInvite( } // Auth and build the event from what the remote server sent us - event, err := buildMembershipEvent(httpReq.Context(), &builder, queryAPI, cfg) + event, err := buildMembershipEvent(httpReq.Context(), &builder, rsAPI, cfg) if err == errNotInRoom { return util.JSONResponse{ Code: http.StatusNotFound, @@ -199,14 +199,14 @@ func ExchangeThirdPartyInvite( // Returns an error if there was a problem building the event or fetching the // necessary data to do so. func createInviteFrom3PIDInvite( - ctx context.Context, queryAPI roomserverAPI.RoomserverQueryAPI, + ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite, inv invite, federation *gomatrixserverlib.FederationClient, accountDB accounts.Database, ) (*gomatrixserverlib.Event, error) { verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { return nil, err } @@ -245,7 +245,7 @@ func createInviteFrom3PIDInvite( return nil, err } - event, err := buildMembershipEvent(ctx, builder, queryAPI, cfg) + event, err := buildMembershipEvent(ctx, builder, rsAPI, cfg) if err == errNotInRoom { return nil, sendToRemoteServer(ctx, inv, federation, cfg, *builder) } @@ -263,7 +263,7 @@ func createInviteFrom3PIDInvite( // Returns an error if something failed during the process. func buildMembershipEvent( ctx context.Context, - builder *gomatrixserverlib.EventBuilder, queryAPI roomserverAPI.RoomserverQueryAPI, + builder *gomatrixserverlib.EventBuilder, rsAPI roomserverAPI.RoomserverInternalAPI, cfg *config.Dendrite, ) (*gomatrixserverlib.Event, error) { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) @@ -281,7 +281,7 @@ func buildMembershipEvent( StateToFetch: eventsNeeded.Tuples(), } var queryRes roomserverAPI.QueryLatestEventsAndStateResponse - if err = queryAPI.QueryLatestEventsAndState(ctx, &queryReq, &queryRes); err != nil { + if err = rsAPI.QueryLatestEventsAndState(ctx, &queryReq, &queryRes); err != nil { return nil, err } diff --git a/federationsender/api/api.go b/federationsender/api/api.go new file mode 100644 index 000000000..10dc66da2 --- /dev/null +++ b/federationsender/api/api.go @@ -0,0 +1,53 @@ +package api + +import ( + "context" + "errors" + "net/http" +) + +// FederationSenderInternalAPI is used to query information from the federation sender. +type FederationSenderInternalAPI interface { + // Query the joined hosts and the membership events accounting for their participation in a room. + // Note that if a server has multiple users in the room, it will have multiple entries in the returned slice. + // See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version. + QueryJoinedHostsInRoom( + ctx context.Context, + request *QueryJoinedHostsInRoomRequest, + response *QueryJoinedHostsInRoomResponse, + ) error + // Query the server names of the joined hosts in a room. + // Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice + // containing only the server names (without information for membership events). + QueryJoinedHostServerNamesInRoom( + ctx context.Context, + request *QueryJoinedHostServerNamesInRoomRequest, + response *QueryJoinedHostServerNamesInRoomResponse, + ) error + // Handle an instruction to make_join & send_join with a remote server. + PerformJoin( + ctx context.Context, + request *PerformJoinRequest, + response *PerformJoinResponse, + ) error + // Handle an instruction to make_leave & send_leave with a remote server. + PerformLeave( + ctx context.Context, + request *PerformLeaveRequest, + response *PerformLeaveResponse, + ) error +} + +// NewFederationSenderInternalAPIHTTP creates a FederationSenderInternalAPI implemented by talking to a HTTP POST API. +// If httpClient is nil an error is returned +func NewFederationSenderInternalAPIHTTP(federationSenderURL string, httpClient *http.Client) (FederationSenderInternalAPI, error) { + if httpClient == nil { + return nil, errors.New("NewFederationSenderInternalAPIHTTP: httpClient is ") + } + return &httpFederationSenderInternalAPI{federationSenderURL, httpClient}, nil +} + +type httpFederationSenderInternalAPI struct { + federationSenderURL string + httpClient *http.Client +} diff --git a/federationsender/api/perform.go b/federationsender/api/perform.go new file mode 100644 index 000000000..87736f294 --- /dev/null +++ b/federationsender/api/perform.go @@ -0,0 +1,60 @@ +package api + +import ( + "context" + + commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/matrix-org/gomatrixserverlib" + "github.com/opentracing/opentracing-go" +) + +const ( + // FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API. + FederationSenderPerformJoinRequestPath = "/api/federationsender/performJoinRequest" + + // FederationSenderPerformLeaveRequestPath is the HTTP path for the PerformLeaveRequest API. + FederationSenderPerformLeaveRequestPath = "/api/federationsender/performLeaveRequest" +) + +type PerformJoinRequest struct { + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + ServerName gomatrixserverlib.ServerName `json:"server_name"` + Content map[string]interface{} `json:"content"` +} + +type PerformJoinResponse struct { +} + +// Handle an instruction to make_join & send_join with a remote server. +func (h *httpFederationSenderInternalAPI) PerformJoin( + ctx context.Context, + request *PerformJoinRequest, + response *PerformJoinResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoinRequest") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +type PerformLeaveRequest struct { + RoomID string `json:"room_id"` +} + +type PerformLeaveResponse struct { +} + +// Handle an instruction to make_leave & send_leave with a remote server. +func (h *httpFederationSenderInternalAPI) PerformLeave( + ctx context.Context, + request *PerformLeaveRequest, + response *PerformLeaveResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLeaveRequest") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderPerformLeaveRequestPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/federationsender/api/query.go b/federationsender/api/query.go index 7c0ca7ff2..7a58cc863 100644 --- a/federationsender/api/query.go +++ b/federationsender/api/query.go @@ -2,16 +2,20 @@ package api import ( "context" - "errors" - "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/federationsender/types" "github.com/opentracing/opentracing-go" ) +// FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API. +const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom" + +// FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API. +const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom" + // QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom type QueryJoinedHostsInRoomRequest struct { RoomID string `json:"room_id"` @@ -22,6 +26,19 @@ type QueryJoinedHostsInRoomResponse struct { JoinedHosts []types.JoinedHost `json:"joined_hosts"` } +// QueryJoinedHostsInRoom implements FederationSenderInternalAPI +func (h *httpFederationSenderInternalAPI) QueryJoinedHostsInRoom( + ctx context.Context, + request *QueryJoinedHostsInRoomRequest, + response *QueryJoinedHostsInRoomResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + // QueryJoinedHostServerNamesRequest is a request to QueryJoinedHostServerNames type QueryJoinedHostServerNamesInRoomRequest struct { RoomID string `json:"room_id"` @@ -32,61 +49,8 @@ type QueryJoinedHostServerNamesInRoomResponse struct { ServerNames []gomatrixserverlib.ServerName `json:"server_names"` } -// FederationSenderQueryAPI is used to query information from the federation sender. -type FederationSenderQueryAPI interface { - // Query the joined hosts and the membership events accounting for their participation in a room. - // Note that if a server has multiple users in the room, it will have multiple entries in the returned slice. - // See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version. - QueryJoinedHostsInRoom( - ctx context.Context, - request *QueryJoinedHostsInRoomRequest, - response *QueryJoinedHostsInRoomResponse, - ) error - // Query the server names of the joined hosts in a room. - // Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice - // containing only the server names (without information for membership events). - QueryJoinedHostServerNamesInRoom( - ctx context.Context, - request *QueryJoinedHostServerNamesInRoomRequest, - response *QueryJoinedHostServerNamesInRoomResponse, - ) error -} - -// FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API. -const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom" - -// FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API. -const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom" - -// NewFederationSenderQueryAPIHTTP creates a FederationSenderQueryAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewFederationSenderQueryAPIHTTP(federationSenderURL string, httpClient *http.Client) (FederationSenderQueryAPI, error) { - if httpClient == nil { - return nil, errors.New("NewFederationSenderQueryAPIHTTP: httpClient is ") - } - return &httpFederationSenderQueryAPI{federationSenderURL, httpClient}, nil -} - -type httpFederationSenderQueryAPI struct { - federationSenderURL string - httpClient *http.Client -} - -// QueryJoinedHostsInRoom implements FederationSenderQueryAPI -func (h *httpFederationSenderQueryAPI) QueryJoinedHostsInRoom( - ctx context.Context, - request *QueryJoinedHostsInRoomRequest, - response *QueryJoinedHostsInRoomResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom") - defer span.Finish() - - apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryJoinedHostServerNamesInRoom implements FederationSenderQueryAPI -func (h *httpFederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( +// QueryJoinedHostServerNamesInRoom implements FederationSenderInternalAPI +func (h *httpFederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( ctx context.Context, request *QueryJoinedHostServerNamesInRoomRequest, response *QueryJoinedHostServerNamesInRoomResponse, diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 18c8324b4..67d08b339 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -33,11 +33,11 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - cfg *config.Dendrite - roomServerConsumer *common.ContinualConsumer - db storage.Database - queues *queue.OutgoingQueues - query api.RoomserverQueryAPI + cfg *config.Dendrite + rsAPI api.RoomserverInternalAPI + rsConsumer *common.ContinualConsumer + db storage.Database + queues *queue.OutgoingQueues } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -46,7 +46,7 @@ func NewOutputRoomEventConsumer( kafkaConsumer sarama.Consumer, queues *queue.OutgoingQueues, store storage.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputRoomEvent), @@ -54,11 +54,11 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ - cfg: cfg, - roomServerConsumer: &consumer, - db: store, - queues: queues, - query: queryAPI, + cfg: cfg, + rsConsumer: &consumer, + db: store, + queues: queues, + rsAPI: rsAPI, } consumer.ProcessMessage = s.onMessage @@ -67,7 +67,7 @@ func NewOutputRoomEventConsumer( // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return s.roomServerConsumer.Start() + return s.rsConsumer.Start() } // onMessage is called when the federation server receives a new event from the room server output log. @@ -369,7 +369,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // from the roomserver using the query API. eventReq := api.QueryEventsByIDRequest{EventIDs: missing} var eventResp api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { return nil, err } diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 4ddffeb49..bf9d326bb 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -34,21 +34,21 @@ import ( func SetupFederationSenderComponent( base *basecomponent.BaseDendrite, federation *gomatrixserverlib.FederationClient, - rsQueryAPI roomserverAPI.RoomserverQueryAPI, - rsInputAPI roomserverAPI.RoomserverInputAPI, -) api.FederationSenderQueryAPI { - federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender), base.Cfg.DbProperties()) + rsAPI roomserverAPI.RoomserverInternalAPI, + keyRing *gomatrixserverlib.KeyRing, +) api.FederationSenderInternalAPI { + federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender)) if err != nil { logrus.WithError(err).Panic("failed to connect to federation sender db") } - roomserverProducer := producers.NewRoomserverProducer(rsInputAPI, base.Cfg.Matrix.ServerName) + roomserverProducer := producers.NewRoomserverProducer(rsAPI, base.Cfg.Matrix.ServerName) queues := queue.NewOutgoingQueues(base.Cfg.Matrix.ServerName, federation, roomserverProducer) rsConsumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, queues, - federationSenderDB, rsQueryAPI, + federationSenderDB, rsAPI, ) if err = rsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start room server consumer") @@ -61,10 +61,10 @@ func SetupFederationSenderComponent( logrus.WithError(err).Panic("failed to start typing server consumer") } - queryAPI := query.FederationSenderQueryAPI{ - DB: federationSenderDB, - } + queryAPI := query.NewFederationSenderInternalAPI( + federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing, + ) queryAPI.SetupHTTP(http.DefaultServeMux) - return &queryAPI + return queryAPI } diff --git a/federationsender/producers/roomserver.go b/federationsender/producers/roomserver.go index 0395f9628..48aeed8cc 100644 --- a/federationsender/producers/roomserver.go +++ b/federationsender/producers/roomserver.go @@ -23,16 +23,16 @@ import ( // RoomserverProducer produces events for the roomserver to consume. type RoomserverProducer struct { - InputAPI api.RoomserverInputAPI + InputAPI api.RoomserverInternalAPI serverName gomatrixserverlib.ServerName } // NewRoomserverProducer creates a new RoomserverProducer func NewRoomserverProducer( - inputAPI api.RoomserverInputAPI, serverName gomatrixserverlib.ServerName, + rsAPI api.RoomserverInternalAPI, serverName gomatrixserverlib.ServerName, ) *RoomserverProducer { return &RoomserverProducer{ - InputAPI: inputAPI, + InputAPI: rsAPI, serverName: serverName, } } @@ -54,6 +54,42 @@ func (c *RoomserverProducer) SendInviteResponse( return c.SendInputRoomEvents(ctx, []api.InputRoomEvent{ire}) } +// SendEventWithState writes an event with KindNew to the roomserver input log +// with the state at the event as KindOutlier before it. +func (c *RoomserverProducer) SendEventWithState( + ctx context.Context, state gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent, +) error { + outliers, err := state.Events() + if err != nil { + return err + } + + var ires []api.InputRoomEvent + for _, outlier := range outliers { + ires = append(ires, api.InputRoomEvent{ + Kind: api.KindOutlier, + Event: outlier.Headered(event.RoomVersion), + AuthEventIDs: outlier.AuthEventIDs(), + }) + } + + stateEventIDs := make([]string, len(state.StateEvents)) + for i := range state.StateEvents { + stateEventIDs[i] = state.StateEvents[i].EventID() + } + + ires = append(ires, api.InputRoomEvent{ + Kind: api.KindNew, + Event: event, + AuthEventIDs: event.AuthEventIDs(), + HasState: true, + StateEventIDs: stateEventIDs, + }) + + _, err = c.SendInputRoomEvents(ctx, ires) + return err +} + // SendInputRoomEvents writes the given input room events to the roomserver input API. func (c *RoomserverProducer) SendInputRoomEvents( ctx context.Context, ires []api.InputRoomEvent, diff --git a/federationsender/query/api.go b/federationsender/query/api.go new file mode 100644 index 000000000..e92453f98 --- /dev/null +++ b/federationsender/query/api.go @@ -0,0 +1,97 @@ +package query + +import ( + "encoding/json" + "net/http" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationsender/producers" + "github.com/matrix-org/dendrite/federationsender/storage" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +// FederationSenderInternalAPI is an implementation of api.FederationSenderInternalAPI +type FederationSenderInternalAPI struct { + api.FederationSenderInternalAPI + db storage.Database + cfg *config.Dendrite + producer *producers.RoomserverProducer + federation *gomatrixserverlib.FederationClient + keyRing *gomatrixserverlib.KeyRing +} + +func NewFederationSenderInternalAPI( + db storage.Database, cfg *config.Dendrite, + producer *producers.RoomserverProducer, + federation *gomatrixserverlib.FederationClient, + keyRing *gomatrixserverlib.KeyRing, +) *FederationSenderInternalAPI { + return &FederationSenderInternalAPI{ + db: db, + cfg: cfg, + producer: producer, + federation: federation, + keyRing: keyRing, + } +} + +// SetupHTTP adds the FederationSenderInternalAPI handlers to the http.ServeMux. +func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { + servMux.Handle( + api.FederationSenderQueryJoinedHostsInRoomPath, + common.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryJoinedHostsInRoomRequest + var response api.QueryJoinedHostsInRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := f.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.FederationSenderQueryJoinedHostServerNamesInRoomPath, + common.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryJoinedHostServerNamesInRoomRequest + var response api.QueryJoinedHostServerNamesInRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := f.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle(api.FederationSenderPerformJoinRequestPath, + common.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse { + var request api.PerformJoinRequest + var response api.PerformJoinResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := f.PerformJoin(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle(api.FederationSenderPerformLeaveRequestPath, + common.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse { + var request api.PerformLeaveRequest + var response api.PerformLeaveResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := f.PerformLeave(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/federationsender/query/perform.go b/federationsender/query/perform.go new file mode 100644 index 000000000..d39fef5ed --- /dev/null +++ b/federationsender/query/perform.go @@ -0,0 +1,120 @@ +package query + +import ( + "context" + "fmt" + "time" + + "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationsender/query/perform" + "github.com/matrix-org/dendrite/roomserver/version" + "github.com/matrix-org/gomatrixserverlib" +) + +// PerformJoinRequest implements api.FederationSenderInternalAPI +func (r *FederationSenderInternalAPI) PerformJoin( + ctx context.Context, + request *api.PerformJoinRequest, + response *api.PerformJoinResponse, +) (err error) { + // Look up the supported room versions. + var supportedVersions []gomatrixserverlib.RoomVersion + for version := range version.SupportedRoomVersions() { + supportedVersions = append(supportedVersions, version) + } + + // Try to perform a make_join using the information supplied in the + // request. + respMakeJoin, err := r.federation.MakeJoin( + ctx, + request.ServerName, + request.RoomID, + request.UserID, + supportedVersions, + ) + if err != nil { + // TODO: Check if the user was not allowed to join the room. + return fmt.Errorf("r.federation.MakeJoin: %w", err) + } + + // Set all the fields to be what they should be, this should be a no-op + // but it's possible that the remote server returned us something "odd" + respMakeJoin.JoinEvent.Type = "m.room.member" + respMakeJoin.JoinEvent.Sender = request.UserID + respMakeJoin.JoinEvent.StateKey = &request.UserID + respMakeJoin.JoinEvent.RoomID = request.RoomID + respMakeJoin.JoinEvent.Redacts = "" + if request.Content == nil { + request.Content = map[string]interface{}{} + } + request.Content["membership"] = "join" + if err = respMakeJoin.JoinEvent.SetContent(request.Content); err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.SetContent: %w", err) + } + if err = respMakeJoin.JoinEvent.SetUnsigned(struct{}{}); err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.SetUnsigned: %w", err) + } + + // Work out if we support the room version that has been supplied in + // the make_join response. + if respMakeJoin.RoomVersion == "" { + respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1 + } + if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil { + return fmt.Errorf("respMakeJoin.RoomVersion.EventFormat: %w", err) + } + + // Build the join event. + event, err := respMakeJoin.JoinEvent.Build( + time.Now(), + r.cfg.Matrix.ServerName, + r.cfg.Matrix.KeyID, + r.cfg.Matrix.PrivateKey, + respMakeJoin.RoomVersion, + ) + if err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err) + } + + // Try to perform a send_join using the newly built event. + respSendJoin, err := r.federation.SendJoin( + ctx, + request.ServerName, + event, + respMakeJoin.RoomVersion, + ) + if err != nil { + return fmt.Errorf("r.federation.SendJoin: %w", err) + } + + // Check that the send_join response was valid. + joinCtx := perform.JoinContext(r.federation, r.keyRing) + if err = joinCtx.CheckSendJoinResponse( + ctx, event, request.ServerName, respMakeJoin, respSendJoin, + ); err != nil { + return fmt.Errorf("perform.JoinRequest.CheckSendJoinResponse: %w", err) + } + + // If we successfully performed a send_join above then the other + // server now thinks we're a part of the room. Send the newly + // returned state to the roomserver to update our local view. + if err = r.producer.SendEventWithState( + ctx, + respSendJoin.ToRespState(), + event.Headered(respMakeJoin.RoomVersion), + ); err != nil { + return fmt.Errorf("r.producer.SendEventWithState: %w", err) + } + + // Everything went to plan. + return nil +} + +// PerformLeaveRequest implements api.FederationSenderInternalAPI +func (r *FederationSenderInternalAPI) PerformLeave( + ctx context.Context, + request *api.PerformLeaveRequest, + response *api.PerformLeaveResponse, +) (err error) { + return nil +} diff --git a/federationsender/query/perform/join.go b/federationsender/query/perform/join.go new file mode 100644 index 000000000..3c7ef0765 --- /dev/null +++ b/federationsender/query/perform/join.go @@ -0,0 +1,70 @@ +package perform + +import ( + "context" + "fmt" + + "github.com/matrix-org/gomatrixserverlib" +) + +// This file contains helpers for the PerformJoin function. + +type joinContext struct { + federation *gomatrixserverlib.FederationClient + keyRing *gomatrixserverlib.KeyRing +} + +// Returns a new join context. +func JoinContext(f *gomatrixserverlib.FederationClient, k *gomatrixserverlib.KeyRing) *joinContext { + return &joinContext{ + federation: f, + keyRing: k, + } +} + +// checkSendJoinResponse checks that all of the signatures are correct +// and that the join is allowed by the supplied state. +func (r joinContext) CheckSendJoinResponse( + ctx context.Context, + event gomatrixserverlib.Event, + server gomatrixserverlib.ServerName, + respMakeJoin gomatrixserverlib.RespMakeJoin, + respSendJoin gomatrixserverlib.RespSendJoin, +) error { + // A list of events that we have retried, if they were not included in + // the auth events supplied in the send_join. + retries := map[string]bool{} + +retryCheck: + // TODO: Can we expand Check here to return a list of missing auth + // events rather than failing one at a time? + if err := respSendJoin.Check(ctx, r.keyRing, event); err != nil { + switch e := err.(type) { + case gomatrixserverlib.MissingAuthEventError: + // Check that we haven't already retried for this event, prevents + // us from ending up in endless loops + if !retries[e.AuthEventID] { + // Ask the server that we're talking to right now for the event + tx, txerr := r.federation.GetEvent(ctx, server, e.AuthEventID) + if txerr != nil { + return fmt.Errorf("r.federation.GetEvent: %w", txerr) + } + // For each event returned, add it to the auth events. + for _, pdu := range tx.PDUs { + ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, respMakeJoin.RoomVersion) + if everr != nil { + return fmt.Errorf("gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr) + } + respSendJoin.AuthEvents = append(respSendJoin.AuthEvents, ev) + } + // Mark the event as retried and then give the check another go. + retries[e.AuthEventID] = true + goto retryCheck + } + return fmt.Errorf("respSendJoin (after retries): %w", e) + default: + return fmt.Errorf("respSendJoin: %w", err) + } + } + return nil +} diff --git a/federationsender/query/query.go b/federationsender/query/query.go index 8c35bb29e..004ad156d 100644 --- a/federationsender/query/query.go +++ b/federationsender/query/query.go @@ -2,45 +2,28 @@ package query import ( "context" - "encoding/json" - "net/http" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) -// FederationSenderQueryDatabase has the APIs needed to implement the query API. -type FederationSenderQueryDatabase interface { - GetJoinedHosts( - ctx context.Context, roomID string, - ) ([]types.JoinedHost, error) -} - -// FederationSenderQueryAPI is an implementation of api.FederationSenderQueryAPI -type FederationSenderQueryAPI struct { - DB FederationSenderQueryDatabase -} - -// QueryJoinedHostsInRoom implements api.FederationSenderQueryAPI -func (f *FederationSenderQueryAPI) QueryJoinedHostsInRoom( +// QueryJoinedHostsInRoom implements api.FederationSenderInternalAPI +func (f *FederationSenderInternalAPI) QueryJoinedHostsInRoom( ctx context.Context, request *api.QueryJoinedHostsInRoomRequest, response *api.QueryJoinedHostsInRoomResponse, ) (err error) { - response.JoinedHosts, err = f.DB.GetJoinedHosts(ctx, request.RoomID) + response.JoinedHosts, err = f.db.GetJoinedHosts(ctx, request.RoomID) return } -// QueryJoinedHostServerNamesInRoom implements api.FederationSenderQueryAPI -func (f *FederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( +// QueryJoinedHostServerNamesInRoom implements api.FederationSenderInternalAPI +func (f *FederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( ctx context.Context, request *api.QueryJoinedHostServerNamesInRoomRequest, response *api.QueryJoinedHostServerNamesInRoomResponse, ) (err error) { - joinedHosts, err := f.DB.GetJoinedHosts(ctx, request.RoomID) + joinedHosts, err := f.db.GetJoinedHosts(ctx, request.RoomID) if err != nil { return } @@ -54,35 +37,3 @@ func (f *FederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( return } - -// SetupHTTP adds the FederationSenderQueryAPI handlers to the http.ServeMux. -func (f *FederationSenderQueryAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle( - api.FederationSenderQueryJoinedHostsInRoomPath, - common.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryJoinedHostsInRoomRequest - var response api.QueryJoinedHostsInRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := f.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.FederationSenderQueryJoinedHostServerNamesInRoomPath, - common.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryJoinedHostServerNamesInRoomRequest - var response api.QueryJoinedHostServerNamesInRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := f.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/go.mod b/go.mod index 56c58c6c8..55c1179c2 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200428112024-9f47f9bfa4b2 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index de844b2a1..8c1d15ad7 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200428112024-9f47f9bfa4b2 h1:sy2QOqJhb4WXzq8bJhsCntAUYb64Dl6txsFtXWtxxSg= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200428112024-9f47f9bfa4b2/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec h1:9MvZSZzBKvCWqM5KXMGZ1PBDrSLcxs5zfc561UPgcYA= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= diff --git a/publicroomsapi/consumers/roomserver.go b/publicroomsapi/consumers/roomserver.go index 853761c36..efd093b7e 100644 --- a/publicroomsapi/consumers/roomserver.go +++ b/publicroomsapi/consumers/roomserver.go @@ -29,9 +29,9 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - roomServerConsumer *common.ContinualConsumer - db storage.Database - query api.RoomserverQueryAPI + rsAPI api.RoomserverInternalAPI + rsConsumer *common.ContinualConsumer + db storage.Database } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -39,7 +39,7 @@ func NewOutputRoomEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, store storage.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputRoomEvent), @@ -47,9 +47,9 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ - roomServerConsumer: &consumer, - db: store, - query: queryAPI, + rsConsumer: &consumer, + db: store, + rsAPI: rsAPI, } consumer.ProcessMessage = s.onMessage @@ -58,7 +58,7 @@ func NewOutputRoomEventConsumer( // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return s.roomServerConsumer.Start() + return s.rsConsumer.Start() } // onMessage is called when the sync server receives a new event from the room server output log. @@ -87,14 +87,14 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { addQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.AddsStateEventIDs} var addQueryRes api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &addQueryReq, &addQueryRes); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &addQueryReq, &addQueryRes); err != nil { log.Warn(err) return err } remQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.RemovesStateEventIDs} var remQueryRes api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil { log.Warn(err) return err } diff --git a/publicroomsapi/directory/directory.go b/publicroomsapi/directory/directory.go index 837018e64..fe7a67932 100644 --- a/publicroomsapi/directory/directory.go +++ b/publicroomsapi/directory/directory.go @@ -59,7 +59,7 @@ func GetVisibility( // SetVisibility implements PUT /directory/list/room/{roomID} // TODO: Allow admin users to edit the room visibility func SetVisibility( - req *http.Request, publicRoomsDatabase storage.Database, queryAPI api.RoomserverQueryAPI, dev *authtypes.Device, + req *http.Request, publicRoomsDatabase storage.Database, rsAPI api.RoomserverInternalAPI, dev *authtypes.Device, roomID string, ) util.JSONResponse { queryMembershipReq := api.QueryMembershipForUserRequest{ @@ -67,7 +67,7 @@ func SetVisibility( UserID: dev.UserID, } var queryMembershipRes api.QueryMembershipForUserResponse - err := queryAPI.QueryMembershipForUser(req.Context(), &queryMembershipReq, &queryMembershipRes) + err := rsAPI.QueryMembershipForUser(req.Context(), &queryMembershipReq, &queryMembershipRes) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("could not query membership for user") return jsonerror.InternalServerError() @@ -87,7 +87,7 @@ func SetVisibility( }}, } var queryEventsRes api.QueryLatestEventsAndStateResponse - err = queryAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes) + err = rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes) if err != nil || len(queryEventsRes.StateEvents) == 0 { util.GetLogger(req.Context()).WithError(err).Error("could not query events from room") return jsonerror.InternalServerError() diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index 6efb54bd9..6a4e65674 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -32,16 +32,16 @@ func SetupPublicRoomsAPIComponent( base *basecomponent.BaseDendrite, deviceDB devices.Database, publicRoomsDB storage.Database, - rsQueryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, ) { rsConsumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, publicRoomsDB, rsQueryAPI, + base.Cfg, base.KafkaConsumer, publicRoomsDB, rsAPI, ) if err := rsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start public rooms server consumer") } - routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsQueryAPI, fedClient, extRoomsProvider) + routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsAPI, fedClient, extRoomsProvider) } diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index da5ea90d6..09a8eff76 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -39,7 +39,7 @@ const pathPrefixR0 = "/_matrix/client/r0" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, queryAPI api.RoomserverQueryAPI, + apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI, fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, ) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() @@ -66,7 +66,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return directory.SetVisibility(req, publicRoomsDB, queryAPI, device, vars["roomID"]) + return directory.SetVisibility(req, publicRoomsDB, rsAPI, device, vars["roomID"]) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/publicRooms", diff --git a/roomserver/alias/alias_test.go b/roomserver/alias/alias_test.go deleted file mode 100644 index 0aefa19d9..000000000 --- a/roomserver/alias/alias_test.go +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2019 Serra Allgood -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package alias - -import ( - "context" - "fmt" - "strings" - "testing" - - appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrixserverlib" -) - -type MockRoomserverAliasAPIDatabase struct { - mode string - attempts int -} - -// These methods can be essentially noop -func (db MockRoomserverAliasAPIDatabase) SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error { - return nil -} - -func (db MockRoomserverAliasAPIDatabase) GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) { - aliases := make([]string, 0) - return aliases, nil -} - -func (db MockRoomserverAliasAPIDatabase) RemoveRoomAlias(ctx context.Context, alias string) error { - return nil -} - -func (db *MockRoomserverAliasAPIDatabase) GetCreatorIDForAlias( - ctx context.Context, alias string, -) (string, error) { - return "", nil -} - -func (db *MockRoomserverAliasAPIDatabase) GetRoomVersionForRoom( - ctx context.Context, roomID string, -) (gomatrixserverlib.RoomVersion, error) { - return gomatrixserverlib.RoomVersionV1, nil -} - -// This method needs to change depending on test case -func (db *MockRoomserverAliasAPIDatabase) GetRoomIDForAlias( - ctx context.Context, - alias string, -) (string, error) { - switch db.mode { - case "empty": - return "", nil - case "error": - return "", fmt.Errorf("found an error from GetRoomIDForAlias") - case "found": - return "123", nil - case "emptyFound": - switch db.attempts { - case 0: - db.attempts = 1 - return "", nil - case 1: - db.attempts = 0 - return "123", nil - default: - return "", nil - } - default: - return "", fmt.Errorf("unknown option used") - } -} - -type MockAppServiceQueryAPI struct { - mode string -} - -// This method can be noop -func (q MockAppServiceQueryAPI) UserIDExists( - ctx context.Context, - req *appserviceAPI.UserIDExistsRequest, - resp *appserviceAPI.UserIDExistsResponse, -) error { - return nil -} - -func (q MockAppServiceQueryAPI) RoomAliasExists( - ctx context.Context, - req *appserviceAPI.RoomAliasExistsRequest, - resp *appserviceAPI.RoomAliasExistsResponse, -) error { - switch q.mode { - case "error": - return fmt.Errorf("found an error from RoomAliasExists") - case "found": - resp.AliasExists = true - return nil - case "empty": - resp.AliasExists = false - return nil - default: - return fmt.Errorf("Unknown option used") - } -} - -func TestGetRoomIDForAlias(t *testing.T) { - type arguments struct { - ctx context.Context - request *roomserverAPI.GetRoomIDForAliasRequest - response *roomserverAPI.GetRoomIDForAliasResponse - } - args := arguments{ - context.Background(), - &roomserverAPI.GetRoomIDForAliasRequest{}, - &roomserverAPI.GetRoomIDForAliasResponse{}, - } - type testCase struct { - name string - dbMode string - queryMode string - wantError bool - errorMsg string - want string - } - tt := []testCase{ - { - "found local alias", - "found", - "error", - false, - "", - "123", - }, - { - "found appservice alias", - "emptyFound", - "found", - false, - "", - "123", - }, - { - "error returned from DB", - "error", - "", - true, - "GetRoomIDForAlias", - "", - }, - { - "error returned from appserviceAPI", - "empty", - "error", - true, - "RoomAliasExists", - "", - }, - { - "no errors but no alias", - "empty", - "empty", - false, - "", - "", - }, - } - - setup := func(dbMode, queryMode string) *RoomserverAliasAPI { - mockAliasAPIDB := &MockRoomserverAliasAPIDatabase{dbMode, 0} - mockAppServiceQueryAPI := MockAppServiceQueryAPI{queryMode} - - return &RoomserverAliasAPI{ - DB: mockAliasAPIDB, - AppserviceAPI: mockAppServiceQueryAPI, - } - } - - for _, tc := range tt { - t.Run(tc.name, func(t *testing.T) { - aliasAPI := setup(tc.dbMode, tc.queryMode) - - err := aliasAPI.GetRoomIDForAlias(args.ctx, args.request, args.response) - if tc.wantError { - if err == nil { - t.Fatalf("Got no error; wanted error from %s", tc.errorMsg) - } else if !strings.Contains(err.Error(), tc.errorMsg) { - t.Fatalf("Got %s; wanted error from %s", err, tc.errorMsg) - } - } else if err != nil { - t.Fatalf("Got %s; wanted no error", err) - } else if args.response.RoomID != tc.want { - t.Errorf("Got '%s'; wanted '%s'", args.response.RoomID, tc.want) - } - }) - } -} diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go index ad375a830..488e99ab8 100644 --- a/roomserver/api/alias.go +++ b/roomserver/api/alias.go @@ -16,8 +16,6 @@ package api import ( "context" - "errors" - "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" opentracing "github.com/opentracing/opentracing-go" @@ -86,44 +84,6 @@ type RemoveRoomAliasRequest struct { // RemoveRoomAliasResponse is a response to RemoveRoomAlias type RemoveRoomAliasResponse struct{} -// RoomserverAliasAPI is used to save, lookup or remove a room alias -type RoomserverAliasAPI interface { - // Set a room alias - SetRoomAlias( - ctx context.Context, - req *SetRoomAliasRequest, - response *SetRoomAliasResponse, - ) error - - // Get the room ID for an alias - GetRoomIDForAlias( - ctx context.Context, - req *GetRoomIDForAliasRequest, - response *GetRoomIDForAliasResponse, - ) error - - // Get all known aliases for a room ID - GetAliasesForRoomID( - ctx context.Context, - req *GetAliasesForRoomIDRequest, - response *GetAliasesForRoomIDResponse, - ) error - - // Get the user ID of the creator of an alias - GetCreatorIDForAlias( - ctx context.Context, - req *GetCreatorIDForAliasRequest, - response *GetCreatorIDForAliasResponse, - ) error - - // Remove a room alias - RemoveRoomAlias( - ctx context.Context, - req *RemoveRoomAliasRequest, - response *RemoveRoomAliasResponse, - ) error -} - // RoomserverSetRoomAliasPath is the HTTP path for the SetRoomAlias API. const RoomserverSetRoomAliasPath = "/api/roomserver/setRoomAlias" @@ -139,22 +99,8 @@ const RoomserverGetCreatorIDForAliasPath = "/api/roomserver/GetCreatorIDForAlias // RoomserverRemoveRoomAliasPath is the HTTP path for the RemoveRoomAlias API. const RoomserverRemoveRoomAliasPath = "/api/roomserver/removeRoomAlias" -// NewRoomserverAliasAPIHTTP creates a RoomserverAliasAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewRoomserverAliasAPIHTTP(roomserverURL string, httpClient *http.Client) (RoomserverAliasAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is ") - } - return &httpRoomserverAliasAPI{roomserverURL, httpClient}, nil -} - -type httpRoomserverAliasAPI struct { - roomserverURL string - httpClient *http.Client -} - // SetRoomAlias implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) SetRoomAlias( +func (h *httpRoomserverInternalAPI) SetRoomAlias( ctx context.Context, request *SetRoomAliasRequest, response *SetRoomAliasResponse, @@ -167,7 +113,7 @@ func (h *httpRoomserverAliasAPI) SetRoomAlias( } // GetRoomIDForAlias implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) GetRoomIDForAlias( +func (h *httpRoomserverInternalAPI) GetRoomIDForAlias( ctx context.Context, request *GetRoomIDForAliasRequest, response *GetRoomIDForAliasResponse, @@ -180,7 +126,7 @@ func (h *httpRoomserverAliasAPI) GetRoomIDForAlias( } // GetAliasesForRoomID implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) GetAliasesForRoomID( +func (h *httpRoomserverInternalAPI) GetAliasesForRoomID( ctx context.Context, request *GetAliasesForRoomIDRequest, response *GetAliasesForRoomIDResponse, @@ -193,7 +139,7 @@ func (h *httpRoomserverAliasAPI) GetAliasesForRoomID( } // GetCreatorIDForAlias implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) GetCreatorIDForAlias( +func (h *httpRoomserverInternalAPI) GetCreatorIDForAlias( ctx context.Context, request *GetCreatorIDForAliasRequest, response *GetCreatorIDForAliasResponse, @@ -206,7 +152,7 @@ func (h *httpRoomserverAliasAPI) GetCreatorIDForAlias( } // RemoveRoomAlias implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) RemoveRoomAlias( +func (h *httpRoomserverInternalAPI) RemoveRoomAlias( ctx context.Context, request *RemoveRoomAliasRequest, response *RemoveRoomAliasResponse, diff --git a/roomserver/api/api.go b/roomserver/api/api.go new file mode 100644 index 000000000..c12dbddde --- /dev/null +++ b/roomserver/api/api.go @@ -0,0 +1,141 @@ +package api + +import ( + "context" + + fsAPI "github.com/matrix-org/dendrite/federationsender/api" +) + +// RoomserverInputAPI is used to write events to the room server. +type RoomserverInternalAPI interface { + // needed to avoid chicken and egg scenario when setting up the + // interdependencies between the roomserver and other input APIs + SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) + + InputRoomEvents( + ctx context.Context, + request *InputRoomEventsRequest, + response *InputRoomEventsResponse, + ) error + + // Query the latest events and state for a room from the room server. + QueryLatestEventsAndState( + ctx context.Context, + request *QueryLatestEventsAndStateRequest, + response *QueryLatestEventsAndStateResponse, + ) error + + // Query the state after a list of events in a room from the room server. + QueryStateAfterEvents( + ctx context.Context, + request *QueryStateAfterEventsRequest, + response *QueryStateAfterEventsResponse, + ) error + + // Query a list of events by event ID. + QueryEventsByID( + ctx context.Context, + request *QueryEventsByIDRequest, + response *QueryEventsByIDResponse, + ) error + + // Query the membership event for an user for a room. + QueryMembershipForUser( + ctx context.Context, + request *QueryMembershipForUserRequest, + response *QueryMembershipForUserResponse, + ) error + + // Query a list of membership events for a room + QueryMembershipsForRoom( + ctx context.Context, + request *QueryMembershipsForRoomRequest, + response *QueryMembershipsForRoomResponse, + ) error + + // Query a list of invite event senders for a user in a room. + QueryInvitesForUser( + ctx context.Context, + request *QueryInvitesForUserRequest, + response *QueryInvitesForUserResponse, + ) error + + // Query whether a server is allowed to see an event + QueryServerAllowedToSeeEvent( + ctx context.Context, + request *QueryServerAllowedToSeeEventRequest, + response *QueryServerAllowedToSeeEventResponse, + ) error + + // Query missing events for a room from roomserver + QueryMissingEvents( + ctx context.Context, + request *QueryMissingEventsRequest, + response *QueryMissingEventsResponse, + ) error + + // Query to get state and auth chain for a (potentially hypothetical) event. + // Takes lists of PrevEventIDs and AuthEventsIDs and uses them to calculate + // the state and auth chain to return. + QueryStateAndAuthChain( + ctx context.Context, + request *QueryStateAndAuthChainRequest, + response *QueryStateAndAuthChainResponse, + ) error + + // Query a given amount (or less) of events prior to a given set of events. + QueryBackfill( + ctx context.Context, + request *QueryBackfillRequest, + response *QueryBackfillResponse, + ) error + + // Asks for the default room version as preferred by the server. + QueryRoomVersionCapabilities( + ctx context.Context, + request *QueryRoomVersionCapabilitiesRequest, + response *QueryRoomVersionCapabilitiesResponse, + ) error + + // Asks for the room version for a given room. + QueryRoomVersionForRoom( + ctx context.Context, + request *QueryRoomVersionForRoomRequest, + response *QueryRoomVersionForRoomResponse, + ) error + + // Set a room alias + SetRoomAlias( + ctx context.Context, + req *SetRoomAliasRequest, + response *SetRoomAliasResponse, + ) error + + // Get the room ID for an alias + GetRoomIDForAlias( + ctx context.Context, + req *GetRoomIDForAliasRequest, + response *GetRoomIDForAliasResponse, + ) error + + // Get all known aliases for a room ID + GetAliasesForRoomID( + ctx context.Context, + req *GetAliasesForRoomIDRequest, + response *GetAliasesForRoomIDResponse, + ) error + + // Get the user ID of the creator of an alias + GetCreatorIDForAlias( + ctx context.Context, + req *GetCreatorIDForAliasRequest, + response *GetCreatorIDForAliasResponse, + ) error + + // Remove a room alias + RemoveRoomAlias( + ctx context.Context, + req *RemoveRoomAliasRequest, + response *RemoveRoomAliasResponse, + ) error +} diff --git a/roomserver/api/http.go b/roomserver/api/http.go new file mode 100644 index 000000000..d643526bd --- /dev/null +++ b/roomserver/api/http.go @@ -0,0 +1,41 @@ +package api + +import ( + "errors" + "net/http" + + "github.com/matrix-org/dendrite/common/caching" + fsInputAPI "github.com/matrix-org/dendrite/federationsender/api" +) + +type httpRoomserverInternalAPI struct { + roomserverURL string + httpClient *http.Client + fsAPI fsInputAPI.FederationSenderInternalAPI + immutableCache caching.ImmutableCache +} + +// NewRoomserverInputAPIHTTP creates a RoomserverInputAPI implemented by talking to a HTTP POST API. +// If httpClient is nil an error is returned +func NewRoomserverInternalAPIHTTP( + roomserverURL string, + httpClient *http.Client, + //fsInputAPI fsAPI.FederationSenderInternalAPI, + immutableCache caching.ImmutableCache, +) (RoomserverInternalAPI, error) { + if httpClient == nil { + return nil, errors.New("NewRoomserverInternalAPIHTTP: httpClient is ") + } + return &httpRoomserverInternalAPI{ + roomserverURL: roomserverURL, + httpClient: httpClient, + immutableCache: immutableCache, + }, nil +} + +// SetFederationSenderInputAPI passes in a federation sender input API reference +// so that we can avoid the chicken-and-egg problem of both the roomserver input API +// and the federation sender input API being interdependent. +func (h *httpRoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsInputAPI.FederationSenderInternalAPI) { + h.fsAPI = fsAPI +} diff --git a/roomserver/api/input.go b/roomserver/api/input.go index bb4e040de..8e8fdae4e 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -17,8 +17,6 @@ package api import ( "context" - "errors" - "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" "github.com/matrix-org/gomatrixserverlib" @@ -104,34 +102,11 @@ type InputRoomEventsResponse struct { EventID string `json:"event_id"` } -// RoomserverInputAPI is used to write events to the room server. -type RoomserverInputAPI interface { - InputRoomEvents( - ctx context.Context, - request *InputRoomEventsRequest, - response *InputRoomEventsResponse, - ) error -} - // RoomserverInputRoomEventsPath is the HTTP path for the InputRoomEvents API. const RoomserverInputRoomEventsPath = "/api/roomserver/inputRoomEvents" -// NewRoomserverInputAPIHTTP creates a RoomserverInputAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewRoomserverInputAPIHTTP(roomserverURL string, httpClient *http.Client) (RoomserverInputAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverInputAPIHTTP: httpClient is ") - } - return &httpRoomserverInputAPI{roomserverURL, httpClient}, nil -} - -type httpRoomserverInputAPI struct { - roomserverURL string - httpClient *http.Client -} - // InputRoomEvents implements RoomserverInputAPI -func (h *httpRoomserverInputAPI) InputRoomEvents( +func (h *httpRoomserverInternalAPI) InputRoomEvents( ctx context.Context, request *InputRoomEventsRequest, response *InputRoomEventsResponse, diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 11fa5c9ca..cb7cbb86c 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -18,10 +18,7 @@ package api import ( "context" - "errors" - "net/http" - "github.com/matrix-org/dendrite/common/caching" commonHTTP "github.com/matrix-org/dendrite/common/http" "github.com/matrix-org/gomatrixserverlib" opentracing "github.com/opentracing/opentracing-go" @@ -264,95 +261,6 @@ type QueryRoomVersionForRoomResponse struct { RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` } -// RoomserverQueryAPI is used to query information from the room server. -type RoomserverQueryAPI interface { - // Query the latest events and state for a room from the room server. - QueryLatestEventsAndState( - ctx context.Context, - request *QueryLatestEventsAndStateRequest, - response *QueryLatestEventsAndStateResponse, - ) error - - // Query the state after a list of events in a room from the room server. - QueryStateAfterEvents( - ctx context.Context, - request *QueryStateAfterEventsRequest, - response *QueryStateAfterEventsResponse, - ) error - - // Query a list of events by event ID. - QueryEventsByID( - ctx context.Context, - request *QueryEventsByIDRequest, - response *QueryEventsByIDResponse, - ) error - - // Query the membership event for an user for a room. - QueryMembershipForUser( - ctx context.Context, - request *QueryMembershipForUserRequest, - response *QueryMembershipForUserResponse, - ) error - - // Query a list of membership events for a room - QueryMembershipsForRoom( - ctx context.Context, - request *QueryMembershipsForRoomRequest, - response *QueryMembershipsForRoomResponse, - ) error - - // Query a list of invite event senders for a user in a room. - QueryInvitesForUser( - ctx context.Context, - request *QueryInvitesForUserRequest, - response *QueryInvitesForUserResponse, - ) error - - // Query whether a server is allowed to see an event - QueryServerAllowedToSeeEvent( - ctx context.Context, - request *QueryServerAllowedToSeeEventRequest, - response *QueryServerAllowedToSeeEventResponse, - ) error - - // Query missing events for a room from roomserver - QueryMissingEvents( - ctx context.Context, - request *QueryMissingEventsRequest, - response *QueryMissingEventsResponse, - ) error - - // Query to get state and auth chain for a (potentially hypothetical) event. - // Takes lists of PrevEventIDs and AuthEventsIDs and uses them to calculate - // the state and auth chain to return. - QueryStateAndAuthChain( - ctx context.Context, - request *QueryStateAndAuthChainRequest, - response *QueryStateAndAuthChainResponse, - ) error - - // Query a given amount (or less) of events prior to a given set of events. - QueryBackfill( - ctx context.Context, - request *QueryBackfillRequest, - response *QueryBackfillResponse, - ) error - - // Asks for the default room version as preferred by the server. - QueryRoomVersionCapabilities( - ctx context.Context, - request *QueryRoomVersionCapabilitiesRequest, - response *QueryRoomVersionCapabilitiesResponse, - ) error - - // Asks for the room version for a given room. - QueryRoomVersionForRoom( - ctx context.Context, - request *QueryRoomVersionForRoomRequest, - response *QueryRoomVersionForRoomResponse, - ) error -} - // RoomserverQueryLatestEventsAndStatePath is the HTTP path for the QueryLatestEventsAndState API. const RoomserverQueryLatestEventsAndStatePath = "/api/roomserver/queryLatestEventsAndState" @@ -389,23 +297,8 @@ const RoomserverQueryRoomVersionCapabilitiesPath = "/api/roomserver/queryRoomVer // RoomserverQueryRoomVersionForRoomPath is the HTTP path for the QueryRoomVersionForRoom API const RoomserverQueryRoomVersionForRoomPath = "/api/roomserver/queryRoomVersionForRoom" -// NewRoomserverQueryAPIHTTP creates a RoomserverQueryAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewRoomserverQueryAPIHTTP(roomserverURL string, httpClient *http.Client, cache caching.ImmutableCache) (RoomserverQueryAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverQueryAPIHTTP: httpClient is ") - } - return &httpRoomserverQueryAPI{roomserverURL, httpClient, cache}, nil -} - -type httpRoomserverQueryAPI struct { - roomserverURL string - httpClient *http.Client - immutableCache caching.ImmutableCache -} - // QueryLatestEventsAndState implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryLatestEventsAndState( +func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState( ctx context.Context, request *QueryLatestEventsAndStateRequest, response *QueryLatestEventsAndStateResponse, @@ -418,7 +311,7 @@ func (h *httpRoomserverQueryAPI) QueryLatestEventsAndState( } // QueryStateAfterEvents implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryStateAfterEvents( +func (h *httpRoomserverInternalAPI) QueryStateAfterEvents( ctx context.Context, request *QueryStateAfterEventsRequest, response *QueryStateAfterEventsResponse, @@ -431,7 +324,7 @@ func (h *httpRoomserverQueryAPI) QueryStateAfterEvents( } // QueryEventsByID implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryEventsByID( +func (h *httpRoomserverInternalAPI) QueryEventsByID( ctx context.Context, request *QueryEventsByIDRequest, response *QueryEventsByIDResponse, @@ -444,7 +337,7 @@ func (h *httpRoomserverQueryAPI) QueryEventsByID( } // QueryMembershipForUser implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryMembershipForUser( +func (h *httpRoomserverInternalAPI) QueryMembershipForUser( ctx context.Context, request *QueryMembershipForUserRequest, response *QueryMembershipForUserResponse, @@ -457,7 +350,7 @@ func (h *httpRoomserverQueryAPI) QueryMembershipForUser( } // QueryMembershipsForRoom implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryMembershipsForRoom( +func (h *httpRoomserverInternalAPI) QueryMembershipsForRoom( ctx context.Context, request *QueryMembershipsForRoomRequest, response *QueryMembershipsForRoomResponse, @@ -470,7 +363,7 @@ func (h *httpRoomserverQueryAPI) QueryMembershipsForRoom( } // QueryInvitesForUser implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryInvitesForUser( +func (h *httpRoomserverInternalAPI) QueryInvitesForUser( ctx context.Context, request *QueryInvitesForUserRequest, response *QueryInvitesForUserResponse, @@ -483,7 +376,7 @@ func (h *httpRoomserverQueryAPI) QueryInvitesForUser( } // QueryServerAllowedToSeeEvent implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryServerAllowedToSeeEvent( +func (h *httpRoomserverInternalAPI) QueryServerAllowedToSeeEvent( ctx context.Context, request *QueryServerAllowedToSeeEventRequest, response *QueryServerAllowedToSeeEventResponse, @@ -496,7 +389,7 @@ func (h *httpRoomserverQueryAPI) QueryServerAllowedToSeeEvent( } // QueryMissingEvents implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryMissingEvents( +func (h *httpRoomserverInternalAPI) QueryMissingEvents( ctx context.Context, request *QueryMissingEventsRequest, response *QueryMissingEventsResponse, @@ -509,7 +402,7 @@ func (h *httpRoomserverQueryAPI) QueryMissingEvents( } // QueryStateAndAuthChain implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryStateAndAuthChain( +func (h *httpRoomserverInternalAPI) QueryStateAndAuthChain( ctx context.Context, request *QueryStateAndAuthChainRequest, response *QueryStateAndAuthChainResponse, @@ -522,7 +415,7 @@ func (h *httpRoomserverQueryAPI) QueryStateAndAuthChain( } // QueryBackfill implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryBackfill( +func (h *httpRoomserverInternalAPI) QueryBackfill( ctx context.Context, request *QueryBackfillRequest, response *QueryBackfillResponse, @@ -535,7 +428,7 @@ func (h *httpRoomserverQueryAPI) QueryBackfill( } // QueryRoomVersionCapabilities implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryRoomVersionCapabilities( +func (h *httpRoomserverInternalAPI) QueryRoomVersionCapabilities( ctx context.Context, request *QueryRoomVersionCapabilitiesRequest, response *QueryRoomVersionCapabilitiesResponse, @@ -548,7 +441,7 @@ func (h *httpRoomserverQueryAPI) QueryRoomVersionCapabilities( } // QueryRoomVersionForRoom implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryRoomVersionForRoom( +func (h *httpRoomserverInternalAPI) QueryRoomVersionForRoom( ctx context.Context, request *QueryRoomVersionForRoomRequest, response *QueryRoomVersionForRoomResponse, diff --git a/roomserver/auth/auth.go b/roomserver/auth/auth.go index 615a94b3c..fdcf9f062 100644 --- a/roomserver/auth/auth.go +++ b/roomserver/auth/auth.go @@ -27,7 +27,7 @@ func IsServerAllowed( serverCurrentlyInRoom bool, authEvents []gomatrixserverlib.Event, ) bool { - historyVisibility := historyVisibilityForRoom(authEvents) + historyVisibility := HistoryVisibilityForRoom(authEvents) // 1. If the history_visibility was set to world_readable, allow. if historyVisibility == "world_readable" { @@ -52,7 +52,7 @@ func IsServerAllowed( return false } -func historyVisibilityForRoom(authEvents []gomatrixserverlib.Event) string { +func HistoryVisibilityForRoom(authEvents []gomatrixserverlib.Event) string { // https://matrix.org/docs/spec/client_server/r0.6.0#id87 // By default if no history_visibility is set, or if the value is not understood, the visibility is assumed to be shared. visibility := "shared" diff --git a/roomserver/alias/alias.go b/roomserver/internal/alias.go similarity index 50% rename from roomserver/alias/alias.go rename to roomserver/internal/alias.go index eb606e5cd..4139582b6 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/internal/alias.go @@ -12,25 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -package alias +package internal import ( "context" "encoding/json" "errors" - "net/http" "time" - appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) -// RoomserverAliasAPIDatabase has the storage APIs needed to implement the alias API. -type RoomserverAliasAPIDatabase interface { +// RoomserverInternalAPIDatabase has the storage APIs needed to implement the alias API. +type RoomserverInternalAPIDatabase interface { // Save a given room alias with the room ID it refers to. // Returns an error if there was a problem talking to the database. SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error @@ -52,20 +47,11 @@ type RoomserverAliasAPIDatabase interface { ) (gomatrixserverlib.RoomVersion, error) } -// RoomserverAliasAPI is an implementation of alias.RoomserverAliasAPI -type RoomserverAliasAPI struct { - DB RoomserverAliasAPIDatabase - Cfg *config.Dendrite - InputAPI roomserverAPI.RoomserverInputAPI - QueryAPI roomserverAPI.RoomserverQueryAPI - AppserviceAPI appserviceAPI.AppServiceQueryAPI -} - -// SetRoomAlias implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) SetRoomAlias( +// SetRoomAlias implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) SetRoomAlias( ctx context.Context, - request *roomserverAPI.SetRoomAliasRequest, - response *roomserverAPI.SetRoomAliasResponse, + request *api.SetRoomAliasRequest, + response *api.SetRoomAliasResponse, ) error { // Check if the alias isn't already referring to a room roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) @@ -91,11 +77,11 @@ func (r *RoomserverAliasAPI) SetRoomAlias( return r.sendUpdatedAliasesEvent(context.TODO(), request.UserID, request.RoomID) } -// GetRoomIDForAlias implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) GetRoomIDForAlias( +// GetRoomIDForAlias implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) GetRoomIDForAlias( ctx context.Context, - request *roomserverAPI.GetRoomIDForAliasRequest, - response *roomserverAPI.GetRoomIDForAliasResponse, + request *api.GetRoomIDForAliasRequest, + response *api.GetRoomIDForAliasResponse, ) error { // Look up the room ID in the database roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) @@ -103,32 +89,38 @@ func (r *RoomserverAliasAPI) GetRoomIDForAlias( return err } - if roomID == "" { - // No room found locally, try our application services by making a call to - // the appservice component - aliasReq := appserviceAPI.RoomAliasExistsRequest{Alias: request.Alias} - var aliasResp appserviceAPI.RoomAliasExistsResponse - if err = r.AppserviceAPI.RoomAliasExists(ctx, &aliasReq, &aliasResp); err != nil { - return err - } + /* + TODO: Why is this here? It creates an unnecessary dependency + from the roomserver to the appservice component, which should be + altogether optional. - if aliasResp.AliasExists { - roomID, err = r.DB.GetRoomIDForAlias(ctx, request.Alias) - if err != nil { + if roomID == "" { + // No room found locally, try our application services by making a call to + // the appservice component + aliasReq := appserviceAPI.RoomAliasExistsRequest{Alias: request.Alias} + var aliasResp appserviceAPI.RoomAliasExistsResponse + if err = r.AppserviceAPI.RoomAliasExists(ctx, &aliasReq, &aliasResp); err != nil { return err } + + if aliasResp.AliasExists { + roomID, err = r.DB.GetRoomIDForAlias(ctx, request.Alias) + if err != nil { + return err + } + } } - } + */ response.RoomID = roomID return nil } -// GetAliasesForRoomID implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) GetAliasesForRoomID( +// GetAliasesForRoomID implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) GetAliasesForRoomID( ctx context.Context, - request *roomserverAPI.GetAliasesForRoomIDRequest, - response *roomserverAPI.GetAliasesForRoomIDResponse, + request *api.GetAliasesForRoomIDRequest, + response *api.GetAliasesForRoomIDResponse, ) error { // Look up the aliases in the database for the given RoomID aliases, err := r.DB.GetAliasesForRoomID(ctx, request.RoomID) @@ -140,11 +132,11 @@ func (r *RoomserverAliasAPI) GetAliasesForRoomID( return nil } -// GetCreatorIDForAlias implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) GetCreatorIDForAlias( +// GetCreatorIDForAlias implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) GetCreatorIDForAlias( ctx context.Context, - request *roomserverAPI.GetCreatorIDForAliasRequest, - response *roomserverAPI.GetCreatorIDForAliasResponse, + request *api.GetCreatorIDForAliasRequest, + response *api.GetCreatorIDForAliasResponse, ) error { // Look up the aliases in the database for the given RoomID creatorID, err := r.DB.GetCreatorIDForAlias(ctx, request.Alias) @@ -156,11 +148,11 @@ func (r *RoomserverAliasAPI) GetCreatorIDForAlias( return nil } -// RemoveRoomAlias implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) RemoveRoomAlias( +// RemoveRoomAlias implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) RemoveRoomAlias( ctx context.Context, - request *roomserverAPI.RemoveRoomAliasRequest, - response *roomserverAPI.RemoveRoomAliasResponse, + request *api.RemoveRoomAliasRequest, + response *api.RemoveRoomAliasResponse, ) error { // Look up the room ID in the database roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) @@ -186,7 +178,7 @@ type roomAliasesContent struct { // Build the updated m.room.aliases event to send to the room after addition or // removal of an alias -func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( +func (r *RoomserverInternalAPI) sendUpdatedAliasesEvent( ctx context.Context, userID string, roomID string, ) error { serverName := string(r.Cfg.Matrix.ServerName) @@ -222,12 +214,12 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( if len(eventsNeeded.Tuples()) == 0 { return errors.New("expecting state tuples for event builder, got none") } - req := roomserverAPI.QueryLatestEventsAndStateRequest{ + req := api.QueryLatestEventsAndStateRequest{ RoomID: roomID, StateToFetch: eventsNeeded.Tuples(), } - var res roomserverAPI.QueryLatestEventsAndStateResponse - if err = r.QueryAPI.QueryLatestEventsAndState(ctx, &req, &res); err != nil { + var res api.QueryLatestEventsAndStateResponse + if err = r.QueryLatestEventsAndState(ctx, &req, &res); err != nil { return err } builder.Depth = res.Depth @@ -263,91 +255,17 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( } // Create the request - ire := roomserverAPI.InputRoomEvent{ - Kind: roomserverAPI.KindNew, + ire := api.InputRoomEvent{ + Kind: api.KindNew, Event: event.Headered(roomVersion), AuthEventIDs: event.AuthEventIDs(), SendAsServer: serverName, } - inputReq := roomserverAPI.InputRoomEventsRequest{ - InputRoomEvents: []roomserverAPI.InputRoomEvent{ire}, + inputReq := api.InputRoomEventsRequest{ + InputRoomEvents: []api.InputRoomEvent{ire}, } - var inputRes roomserverAPI.InputRoomEventsResponse + var inputRes api.InputRoomEventsResponse // Send the request - return r.InputAPI.InputRoomEvents(ctx, &inputReq, &inputRes) -} - -// SetupHTTP adds the RoomserverAliasAPI handlers to the http.ServeMux. -func (r *RoomserverAliasAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle( - roomserverAPI.RoomserverSetRoomAliasPath, - common.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.SetRoomAliasRequest - var response roomserverAPI.SetRoomAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.SetRoomAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - roomserverAPI.RoomserverGetRoomIDForAliasPath, - common.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.GetRoomIDForAliasRequest - var response roomserverAPI.GetRoomIDForAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.GetRoomIDForAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - roomserverAPI.RoomserverGetCreatorIDForAliasPath, - common.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.GetCreatorIDForAliasRequest - var response roomserverAPI.GetCreatorIDForAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.GetCreatorIDForAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - roomserverAPI.RoomserverGetAliasesForRoomIDPath, - common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.GetAliasesForRoomIDRequest - var response roomserverAPI.GetAliasesForRoomIDResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - roomserverAPI.RoomserverRemoveRoomAliasPath, - common.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.RemoveRoomAliasRequest - var response roomserverAPI.RemoveRoomAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.RemoveRoomAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) + return r.InputRoomEvents(ctx, &inputReq, &inputRes) } diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go new file mode 100644 index 000000000..d1c443f24 --- /dev/null +++ b/roomserver/internal/api.go @@ -0,0 +1,287 @@ +package internal + +import ( + "encoding/json" + "net/http" + "sync" + + "github.com/Shopify/sarama" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/caching" + "github.com/matrix-org/dendrite/common/config" + fsAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +// RoomserverInternalAPI is an implementation of api.RoomserverInternalAPI +type RoomserverInternalAPI struct { + DB storage.Database + Cfg *config.Dendrite + Producer sarama.SyncProducer + ImmutableCache caching.ImmutableCache + ServerName gomatrixserverlib.ServerName + KeyRing gomatrixserverlib.JSONVerifier + FedClient *gomatrixserverlib.FederationClient + OutputRoomEventTopic string // Kafka topic for new output room events + mutex sync.Mutex // Protects calls to processRoomEvent + fsAPI fsAPI.FederationSenderInternalAPI +} + +// SetupHTTP adds the RoomserverInternalAPI handlers to the http.ServeMux. +// nolint: gocyclo +func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { + servMux.Handle(api.RoomserverInputRoomEventsPath, + common.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { + var request api.InputRoomEventsRequest + var response api.InputRoomEventsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := r.InputRoomEvents(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryLatestEventsAndStatePath, + common.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { + var request api.QueryLatestEventsAndStateRequest + var response api.QueryLatestEventsAndStateResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryLatestEventsAndState(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryStateAfterEventsPath, + common.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { + var request api.QueryStateAfterEventsRequest + var response api.QueryStateAfterEventsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryStateAfterEvents(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryEventsByIDPath, + common.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { + var request api.QueryEventsByIDRequest + var response api.QueryEventsByIDResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryEventsByID(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryMembershipForUserPath, + common.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { + var request api.QueryMembershipForUserRequest + var response api.QueryMembershipForUserResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryMembershipForUser(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryMembershipsForRoomPath, + common.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryMembershipsForRoomRequest + var response api.QueryMembershipsForRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryMembershipsForRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryInvitesForUserPath, + common.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse { + var request api.QueryInvitesForUserRequest + var response api.QueryInvitesForUserResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryInvitesForUser(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryServerAllowedToSeeEventPath, + common.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { + var request api.QueryServerAllowedToSeeEventRequest + var response api.QueryServerAllowedToSeeEventResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryServerAllowedToSeeEvent(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryMissingEventsPath, + common.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { + var request api.QueryMissingEventsRequest + var response api.QueryMissingEventsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryMissingEvents(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryStateAndAuthChainPath, + common.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { + var request api.QueryStateAndAuthChainRequest + var response api.QueryStateAndAuthChainResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryStateAndAuthChain(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryBackfillPath, + common.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse { + var request api.QueryBackfillRequest + var response api.QueryBackfillResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryBackfill(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryRoomVersionCapabilitiesPath, + common.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { + var request api.QueryRoomVersionCapabilitiesRequest + var response api.QueryRoomVersionCapabilitiesResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryRoomVersionCapabilities(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryRoomVersionForRoomPath, + common.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryRoomVersionForRoomRequest + var response api.QueryRoomVersionForRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryRoomVersionForRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverSetRoomAliasPath, + common.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { + var request api.SetRoomAliasRequest + var response api.SetRoomAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.SetRoomAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverGetRoomIDForAliasPath, + common.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { + var request api.GetRoomIDForAliasRequest + var response api.GetRoomIDForAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetRoomIDForAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverGetCreatorIDForAliasPath, + common.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { + var request api.GetCreatorIDForAliasRequest + var response api.GetCreatorIDForAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetCreatorIDForAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverGetAliasesForRoomIDPath, + common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { + var request api.GetAliasesForRoomIDRequest + var response api.GetAliasesForRoomIDResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverRemoveRoomAliasPath, + common.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { + var request api.RemoveRoomAliasRequest + var response api.RemoveRoomAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.RemoveRoomAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/roomserver/input/input.go b/roomserver/internal/input.go similarity index 55% rename from roomserver/input/input.go rename to roomserver/internal/input.go index cb588380a..19ebea660 100644 --- a/roomserver/input/input.go +++ b/roomserver/internal/input.go @@ -13,34 +13,27 @@ // limitations under the License. // Package input contains the code processes new room events -package input +package internal import ( "context" "encoding/json" - "net/http" - "sync" "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/roomserver/storage" - "github.com/matrix-org/util" + + fsAPI "github.com/matrix-org/dendrite/federationsender/api" ) -// RoomserverInputAPI implements api.RoomserverInputAPI -type RoomserverInputAPI struct { - DB storage.Database - Producer sarama.SyncProducer - // The kafkaesque topic to output new room events to. - // This is the name used in kafka to identify the stream to write events to. - OutputRoomEventTopic string - // Protects calls to processRoomEvent - mutex sync.Mutex +// SetFederationSenderInputAPI passes in a federation sender input API reference +// so that we can avoid the chicken-and-egg problem of both the roomserver input API +// and the federation sender input API being interdependent. +func (r *RoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) { + r.fsAPI = fsAPI } // WriteOutputEvents implements OutputRoomEventWriter -func (r *RoomserverInputAPI) WriteOutputEvents(roomID string, updates []api.OutputEvent) error { +func (r *RoomserverInternalAPI) WriteOutputEvents(roomID string, updates []api.OutputEvent) error { messages := make([]*sarama.ProducerMessage, len(updates)) for i := range updates { value, err := json.Marshal(updates[i]) @@ -56,8 +49,8 @@ func (r *RoomserverInputAPI) WriteOutputEvents(roomID string, updates []api.Outp return r.Producer.SendMessages(messages) } -// InputRoomEvents implements api.RoomserverInputAPI -func (r *RoomserverInputAPI) InputRoomEvents( +// InputRoomEvents implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) InputRoomEvents( ctx context.Context, request *api.InputRoomEventsRequest, response *api.InputRoomEventsResponse, @@ -77,20 +70,3 @@ func (r *RoomserverInputAPI) InputRoomEvents( } return nil } - -// SetupHTTP adds the RoomserverInputAPI handlers to the http.ServeMux. -func (r *RoomserverInputAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle(api.RoomserverInputRoomEventsPath, - common.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { - var request api.InputRoomEventsRequest - var response api.InputRoomEventsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := r.InputRoomEvents(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/roomserver/input/authevents.go b/roomserver/internal/input_authevents.go similarity index 99% rename from roomserver/input/authevents.go rename to roomserver/internal/input_authevents.go index 2c2e14b3a..e3828f566 100644 --- a/roomserver/input/authevents.go +++ b/roomserver/internal/input_authevents.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "context" diff --git a/roomserver/input/authevents_test.go b/roomserver/internal/input_authevents_test.go similarity index 99% rename from roomserver/input/authevents_test.go rename to roomserver/internal/input_authevents_test.go index 0621a0842..6b981571b 100644 --- a/roomserver/input/authevents_test.go +++ b/roomserver/internal/input_authevents_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "testing" diff --git a/roomserver/input/events.go b/roomserver/internal/input_events.go similarity index 99% rename from roomserver/input/events.go rename to roomserver/internal/input_events.go index 69828d9f9..6da63716c 100644 --- a/roomserver/input/events.go +++ b/roomserver/internal/input_events.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "context" diff --git a/roomserver/input/latest_events.go b/roomserver/internal/input_latest_events.go similarity index 99% rename from roomserver/input/latest_events.go rename to roomserver/internal/input_latest_events.go index cac3968d9..42be0f408 100644 --- a/roomserver/input/latest_events.go +++ b/roomserver/internal/input_latest_events.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "bytes" diff --git a/roomserver/input/membership.go b/roomserver/internal/input_membership.go similarity index 99% rename from roomserver/input/membership.go rename to roomserver/internal/input_membership.go index 351e63d61..cba75b4fc 100644 --- a/roomserver/input/membership.go +++ b/roomserver/internal/input_membership.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "context" diff --git a/roomserver/query/query.go b/roomserver/internal/query.go similarity index 70% rename from roomserver/query/query.go rename to roomserver/internal/query.go index a54fa58d9..98adc24b3 100644 --- a/roomserver/query/query.go +++ b/roomserver/internal/query.go @@ -14,16 +14,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package query +package internal import ( "context" - "encoding/json" "fmt" - "net/http" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/caching" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/auth" "github.com/matrix-org/dendrite/roomserver/state" @@ -35,17 +31,8 @@ import ( "github.com/sirupsen/logrus" ) -// RoomserverQueryAPI is an implementation of api.RoomserverQueryAPI -type RoomserverQueryAPI struct { - DB storage.Database - ImmutableCache caching.ImmutableCache - ServerName gomatrixserverlib.ServerName - KeyRing gomatrixserverlib.JSONVerifier - FedClient *gomatrixserverlib.FederationClient -} - -// QueryLatestEventsAndState implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryLatestEventsAndState( +// QueryLatestEventsAndState implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryLatestEventsAndState( ctx context.Context, request *api.QueryLatestEventsAndStateRequest, response *api.QueryLatestEventsAndStateResponse, @@ -104,8 +91,8 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( return nil } -// QueryStateAfterEvents implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryStateAfterEvents( +// QueryStateAfterEvents implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryStateAfterEvents( ctx context.Context, request *api.QueryStateAfterEventsRequest, response *api.QueryStateAfterEventsResponse, @@ -160,8 +147,8 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( return nil } -// QueryEventsByID implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryEventsByID( +// QueryEventsByID implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryEventsByID( ctx context.Context, request *api.QueryEventsByIDRequest, response *api.QueryEventsByIDResponse, @@ -195,7 +182,7 @@ func (r *RoomserverQueryAPI) QueryEventsByID( return nil } -func (r *RoomserverQueryAPI) loadStateEvents( +func (r *RoomserverInternalAPI) loadStateEvents( ctx context.Context, stateEntries []types.StateEntry, ) ([]gomatrixserverlib.Event, error) { eventNIDs := make([]types.EventNID, len(stateEntries)) @@ -205,7 +192,7 @@ func (r *RoomserverQueryAPI) loadStateEvents( return r.loadEvents(ctx, eventNIDs) } -func (r *RoomserverQueryAPI) loadEvents( +func (r *RoomserverInternalAPI) loadEvents( ctx context.Context, eventNIDs []types.EventNID, ) ([]gomatrixserverlib.Event, error) { stateEvents, err := r.DB.Events(ctx, eventNIDs) @@ -220,8 +207,8 @@ func (r *RoomserverQueryAPI) loadEvents( return result, nil } -// QueryMembershipForUser implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryMembershipForUser( +// QueryMembershipForUser implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryMembershipForUser( ctx context.Context, request *api.QueryMembershipForUserRequest, response *api.QueryMembershipForUserResponse, @@ -251,8 +238,8 @@ func (r *RoomserverQueryAPI) QueryMembershipForUser( return nil } -// QueryMembershipsForRoom implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryMembershipsForRoom( +// QueryMembershipsForRoom implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryMembershipsForRoom( ctx context.Context, request *api.QueryMembershipsForRoomRequest, response *api.QueryMembershipsForRoomResponse, @@ -277,6 +264,7 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( response.JoinEvents = []gomatrixserverlib.ClientEvent{} var events []types.Event + var stateEntries []types.StateEntry if stillInRoom { var eventNIDs []types.EventNID eventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, roomNID, request.JoinedOnly) @@ -286,7 +274,12 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( events, err = r.DB.Events(ctx, eventNIDs) } else { - events, err = getMembershipsBeforeEventNID(ctx, r.DB, membershipEventNID, request.JoinedOnly) + stateEntries, err = stateBeforeEvent(ctx, r.DB, membershipEventNID) + if err != nil { + logrus.WithField("membership_event_nid", membershipEventNID).WithError(err).Error("failed to load state before event") + return err + } + events, err = getMembershipsAtState(ctx, r.DB, stateEntries, request.JoinedOnly) } if err != nil { @@ -301,15 +294,8 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( return nil } -// getMembershipsBeforeEventNID takes the numeric ID of an event and fetches the state -// of the event's room as it was when this event was fired, then filters the state events to -// only keep the "m.room.member" events with a "join" membership. These events are returned. -// Returns an error if there was an issue fetching the events. -func getMembershipsBeforeEventNID( - ctx context.Context, db storage.Database, eventNID types.EventNID, joinedOnly bool, -) ([]types.Event, error) { +func stateBeforeEvent(ctx context.Context, db storage.Database, eventNID types.EventNID) ([]types.StateEntry, error) { roomState := state.NewStateResolution(db) - events := []types.Event{} // Lookup the event NID eIDs, err := db.EventIDs(ctx, []types.EventNID{eventNID}) if err != nil { @@ -323,10 +309,15 @@ func getMembershipsBeforeEventNID( } // Fetch the state as it was when this event was fired - stateEntries, err := roomState.LoadCombinedStateAfterEvents(ctx, prevState) - if err != nil { - return nil, err - } + return roomState.LoadCombinedStateAfterEvents(ctx, prevState) +} + +// getMembershipsAtState filters the state events to +// only keep the "m.room.member" events with a "join" membership. These events are returned. +// Returns an error if there was an issue fetching the events. +func getMembershipsAtState( + ctx context.Context, db storage.Database, stateEntries []types.StateEntry, joinedOnly bool, +) ([]types.Event, error) { var eventNIDs []types.EventNID for _, entry := range stateEntries { @@ -347,6 +338,7 @@ func getMembershipsBeforeEventNID( } // Filter the events to only keep the "join" membership events + var events []types.Event for _, event := range stateEvents { membership, err := event.Membership() if err != nil { @@ -361,8 +353,8 @@ func getMembershipsBeforeEventNID( return events, nil } -// QueryInvitesForUser implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryInvitesForUser( +// QueryInvitesForUser implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryInvitesForUser( ctx context.Context, request *api.QueryInvitesForUserRequest, response *api.QueryInvitesForUserResponse, @@ -395,8 +387,8 @@ func (r *RoomserverQueryAPI) QueryInvitesForUser( return nil } -// QueryServerAllowedToSeeEvent implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryServerAllowedToSeeEvent( +// QueryServerAllowedToSeeEvent implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryServerAllowedToSeeEvent( ctx context.Context, request *api.QueryServerAllowedToSeeEventRequest, response *api.QueryServerAllowedToSeeEventResponse, @@ -419,7 +411,7 @@ func (r *RoomserverQueryAPI) QueryServerAllowedToSeeEvent( return } -func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent( +func (r *RoomserverInternalAPI) checkServerAllowedToSeeEvent( ctx context.Context, eventID string, serverName gomatrixserverlib.ServerName, isServerInRoom bool, ) (bool, error) { roomState := state.NewStateResolution(r.DB) @@ -438,8 +430,8 @@ func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent( return auth.IsServerAllowed(serverName, isServerInRoom, stateAtEvent), nil } -// QueryMissingEvents implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryMissingEvents( +// QueryMissingEvents implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryMissingEvents( ctx context.Context, request *api.QueryMissingEventsRequest, response *api.QueryMissingEventsResponse, @@ -484,7 +476,7 @@ func (r *RoomserverQueryAPI) QueryMissingEvents( } // QueryBackfill implements api.RoomServerQueryAPI -func (r *RoomserverQueryAPI) QueryBackfill( +func (r *RoomserverInternalAPI) QueryBackfill( ctx context.Context, request *api.QueryBackfillRequest, response *api.QueryBackfillResponse, @@ -537,7 +529,7 @@ func (r *RoomserverQueryAPI) QueryBackfill( return err } -func (r *RoomserverQueryAPI) backfillViaFederation(ctx context.Context, req *api.QueryBackfillRequest, res *api.QueryBackfillResponse) error { +func (r *RoomserverInternalAPI) backfillViaFederation(ctx context.Context, req *api.QueryBackfillRequest, res *api.QueryBackfillResponse) error { roomVer, err := r.DB.GetRoomVersionForRoom(ctx, req.RoomID) if err != nil { return fmt.Errorf("backfillViaFederation: unknown room version for room %s : %w", req.RoomID, err) @@ -563,20 +555,29 @@ func (r *RoomserverQueryAPI) backfillViaFederation(ctx context.Context, req *api if !ok { // this should be impossible as all events returned must have pass Step 5 of the PDU checks // which requires a list of state IDs. - logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to find state IDs for event which passed auth checks") + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to find state IDs for event which passed auth checks") continue } var entries []types.StateEntry if entries, err = r.DB.StateEntriesForEventIDs(ctx, stateIDs); err != nil { - return err + // attempt to fetch the missing events + r.fetchAndStoreMissingEvents(ctx, roomVer, requester, stateIDs) + // try again + entries, err = r.DB.StateEntriesForEventIDs(ctx, stateIDs) + if err != nil { + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to get state entries for event") + return err + } } var beforeStateSnapshotNID types.StateSnapshotNID if beforeStateSnapshotNID, err = r.DB.AddState(ctx, roomNID, nil, entries); err != nil { + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to persist state entries to get snapshot nid") return err } + util.GetLogger(ctx).Infof("Backfilled event %s (nid=%d) getting snapshot %v with entries %+v", ev.EventID(), ev.EventNID, beforeStateSnapshotNID, entries) if err = r.DB.SetState(ctx, ev.EventNID, beforeStateSnapshotNID); err != nil { - logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to set state before event") + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to persist snapshot nid") } } @@ -586,7 +587,7 @@ func (r *RoomserverQueryAPI) backfillViaFederation(ctx context.Context, req *api return nil } -func (r *RoomserverQueryAPI) isServerCurrentlyInRoom(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID string) (bool, error) { +func (r *RoomserverInternalAPI) isServerCurrentlyInRoom(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID string) (bool, error) { roomNID, err := r.DB.RoomNID(ctx, roomID) if err != nil { return false, err @@ -608,9 +609,69 @@ func (r *RoomserverQueryAPI) isServerCurrentlyInRoom(ctx context.Context, server return auth.IsAnyUserOnServerWithMembership(serverName, gmslEvents, gomatrixserverlib.Join), nil } +// fetchAndStoreMissingEvents does a best-effort fetch and store of missing events specified in stateIDs. Returns no error as it is just +// best effort. +func (r *RoomserverInternalAPI) fetchAndStoreMissingEvents(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, + backfillRequester *backfillRequester, stateIDs []string) { + + servers := backfillRequester.servers + + // work out which are missing + nidMap, err := r.DB.EventNIDs(ctx, stateIDs) + if err != nil { + util.GetLogger(ctx).WithError(err).Warn("cannot query missing events") + return + } + missingMap := make(map[string]*gomatrixserverlib.HeaderedEvent) // id -> event + for _, id := range stateIDs { + if _, ok := nidMap[id]; !ok { + missingMap[id] = nil + } + } + util.GetLogger(ctx).Infof("Fetching %d missing state events (from %d possible servers)", len(missingMap), len(servers)) + + // fetch the events from federation. Loop the servers first so if we find one that works we stick with them + for _, srv := range servers { + for id, ev := range missingMap { + if ev != nil { + continue // already found + } + logger := util.GetLogger(ctx).WithField("server", srv).WithField("event_id", id) + res, err := r.FedClient.GetEvent(ctx, srv, id) + if err != nil { + logger.WithError(err).Warn("failed to get event from server") + continue + } + loader := gomatrixserverlib.NewEventsLoader(roomVer, r.KeyRing, backfillRequester, backfillRequester.ProvideEvents, false) + result, err := loader.LoadAndVerify(ctx, res.PDUs, gomatrixserverlib.TopologicalOrderByPrevEvents) + if err != nil { + logger.WithError(err).Warn("failed to load and verify event") + continue + } + logger.Infof("returned %d PDUs which made events %+v", len(res.PDUs), result) + for _, res := range result { + if res.Error != nil { + logger.WithError(err).Warn("event failed PDU checks") + continue + } + missingMap[id] = res.Event + } + } + } + + var newEvents []gomatrixserverlib.HeaderedEvent + for _, ev := range missingMap { + if ev != nil { + newEvents = append(newEvents, *ev) + } + } + util.GetLogger(ctx).Infof("Persisting %d new events", len(newEvents)) + persistEvents(ctx, r.DB, newEvents) +} + // TODO: Remove this when we have tests to assert correctness of this function // nolint:gocyclo -func (r *RoomserverQueryAPI) scanEventTree( +func (r *RoomserverInternalAPI) scanEventTree( ctx context.Context, front []string, visited map[string]bool, limit int, serverName gomatrixserverlib.ServerName, ) ([]types.EventNID, error) { @@ -703,8 +764,8 @@ BFSLoop: return resultNIDs, err } -// QueryStateAndAuthChain implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryStateAndAuthChain( +// QueryStateAndAuthChain implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryStateAndAuthChain( ctx context.Context, request *api.QueryStateAndAuthChainRequest, response *api.QueryStateAndAuthChainResponse, @@ -763,7 +824,7 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( return err } -func (r *RoomserverQueryAPI) loadStateAtEventIDs(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { +func (r *RoomserverInternalAPI) loadStateAtEventIDs(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { roomState := state.NewStateResolution(r.DB) prevStates, err := r.DB.StateAtEventIDs(ctx, eventIDs) if err != nil { @@ -857,7 +918,7 @@ func persistEvents(ctx context.Context, db storage.Database, events []gomatrixse var stateAtEvent types.StateAtEvent roomNID, stateAtEvent, err = db.StoreEvent(ctx, ev.Unwrap(), nil, authNids) if err != nil { - logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to store backfilled event") + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to persist event") continue } backfilledEventMap[ev.EventID()] = types.Event{ @@ -868,8 +929,8 @@ func persistEvents(ctx context.Context, db storage.Database, events []gomatrixse return roomNID, backfilledEventMap } -// QueryRoomVersionCapabilities implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities( +// QueryRoomVersionCapabilities implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryRoomVersionCapabilities( ctx context.Context, request *api.QueryRoomVersionCapabilitiesRequest, response *api.QueryRoomVersionCapabilitiesResponse, @@ -886,8 +947,8 @@ func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities( return nil } -// QueryRoomVersionCapabilities implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryRoomVersionForRoom( +// QueryRoomVersionCapabilities implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryRoomVersionForRoom( ctx context.Context, request *api.QueryRoomVersionForRoomRequest, response *api.QueryRoomVersionForRoomResponse, @@ -905,176 +966,3 @@ func (r *RoomserverQueryAPI) QueryRoomVersionForRoom( r.ImmutableCache.StoreRoomVersion(request.RoomID, response.RoomVersion) return nil } - -// SetupHTTP adds the RoomserverQueryAPI handlers to the http.ServeMux. -// nolint: gocyclo -func (r *RoomserverQueryAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle( - api.RoomserverQueryLatestEventsAndStatePath, - common.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { - var request api.QueryLatestEventsAndStateRequest - var response api.QueryLatestEventsAndStateResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryLatestEventsAndState(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryStateAfterEventsPath, - common.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { - var request api.QueryStateAfterEventsRequest - var response api.QueryStateAfterEventsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryStateAfterEvents(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryEventsByIDPath, - common.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { - var request api.QueryEventsByIDRequest - var response api.QueryEventsByIDResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryEventsByID(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryMembershipForUserPath, - common.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { - var request api.QueryMembershipForUserRequest - var response api.QueryMembershipForUserResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryMembershipForUser(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryMembershipsForRoomPath, - common.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryMembershipsForRoomRequest - var response api.QueryMembershipsForRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryMembershipsForRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryInvitesForUserPath, - common.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse { - var request api.QueryInvitesForUserRequest - var response api.QueryInvitesForUserResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryInvitesForUser(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryServerAllowedToSeeEventPath, - common.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { - var request api.QueryServerAllowedToSeeEventRequest - var response api.QueryServerAllowedToSeeEventResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryServerAllowedToSeeEvent(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryMissingEventsPath, - common.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { - var request api.QueryMissingEventsRequest - var response api.QueryMissingEventsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryMissingEvents(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryStateAndAuthChainPath, - common.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { - var request api.QueryStateAndAuthChainRequest - var response api.QueryStateAndAuthChainResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryStateAndAuthChain(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryBackfillPath, - common.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse { - var request api.QueryBackfillRequest - var response api.QueryBackfillResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryBackfill(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryRoomVersionCapabilitiesPath, - common.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { - var request api.QueryRoomVersionCapabilitiesRequest - var response api.QueryRoomVersionCapabilitiesResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryRoomVersionCapabilities(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryRoomVersionForRoomPath, - common.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryRoomVersionForRoomRequest - var response api.QueryRoomVersionForRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryRoomVersionForRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/roomserver/query/backfill.go b/roomserver/internal/query_backfill.go similarity index 72% rename from roomserver/query/backfill.go rename to roomserver/internal/query_backfill.go index 09a515e99..d42038e74 100644 --- a/roomserver/query/backfill.go +++ b/roomserver/internal/query_backfill.go @@ -1,8 +1,9 @@ -package query +package internal import ( "context" + "github.com/matrix-org/dendrite/roomserver/auth" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -62,9 +63,9 @@ FederationHit: logrus.WithField("event_id", targetEvent.EventID()).Info("Requesting /state_ids at event") for _, srv := range b.servers { // hit any valid server c := gomatrixserverlib.FederatedStateProvider{ - FedClient: b.fedClient, - AuthEventsOnly: false, - Server: srv, + FedClient: b.fedClient, + RememberAuthEvents: false, + Server: srv, } res, err := c.StateIDsBeforeEvent(ctx, targetEvent) if err != nil { @@ -114,7 +115,9 @@ func (b *backfillRequester) calculateNewStateIDs(targetEvent, prevEvent gomatrix return nil } -func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, event gomatrixserverlib.HeaderedEvent, eventIDs []string) (map[string]*gomatrixserverlib.Event, error) { +func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, + event gomatrixserverlib.HeaderedEvent, eventIDs []string) (map[string]*gomatrixserverlib.Event, error) { + // try to fetch the events from the database first events, err := b.ProvideEvents(roomVer, eventIDs) if err != nil { @@ -133,9 +136,9 @@ func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatr } c := gomatrixserverlib.FederatedStateProvider{ - FedClient: b.fedClient, - AuthEventsOnly: false, - Server: b.servers[0], + FedClient: b.fedClient, + RememberAuthEvents: false, + Server: b.servers[0], } result, err := c.StateBeforeEvent(ctx, roomVer, event, eventIDs) if err != nil { @@ -160,18 +163,33 @@ func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID return } + stateEntries, err := stateBeforeEvent(ctx, b.db, NIDs[eventID]) + if err != nil { + logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to load state before event") + return + } + + // possibly return all joined servers depending on history visiblity + memberEventsFromVis, err := joinEventsFromHistoryVisibility(ctx, b.db, roomID, stateEntries) + if err != nil { + logrus.WithError(err).Error("ServersAtEvent: failed calculate servers from history visibility rules") + return + } + logrus.Infof("ServersAtEvent including %d current events from history visibility", len(memberEventsFromVis)) + // Retrieve all "m.room.member" state events of "join" membership, which // contains the list of users in the room before the event, therefore all // the servers in it at that moment. - events, err := getMembershipsBeforeEventNID(ctx, b.db, NIDs[eventID], true) + memberEvents, err := getMembershipsAtState(ctx, b.db, stateEntries, true) if err != nil { logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to get memberships before event") return } + memberEvents = append(memberEvents, memberEventsFromVis...) // Store the server names in a temporary map to avoid duplicates. serverSet := make(map[gomatrixserverlib.ServerName]bool) - for _, event := range events { + for _, event := range memberEvents { serverSet[event.Origin()] = true } for server := range serverSet { @@ -186,7 +204,9 @@ func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID // Backfill performs a backfill request to the given server. // https://matrix.org/docs/spec/server_server/latest#get-matrix-federation-v1-backfill-roomid -func (b *backfillRequester) Backfill(ctx context.Context, server gomatrixserverlib.ServerName, roomID string, fromEventIDs []string, limit int) (*gomatrixserverlib.Transaction, error) { +func (b *backfillRequester) Backfill(ctx context.Context, server gomatrixserverlib.ServerName, roomID string, + fromEventIDs []string, limit int) (*gomatrixserverlib.Transaction, error) { + tx, err := b.fedClient.Backfill(ctx, server, roomID, limit, fromEventIDs) return &tx, err } @@ -215,3 +235,44 @@ func (b *backfillRequester) ProvideEvents(roomVer gomatrixserverlib.RoomVersion, } return events, nil } + +// joinEventsFromHistoryVisibility returns all CURRENTLY joined members if the provided state indicated a 'shared' history visibility. +// TODO: Long term we probably want a history_visibility table which stores eventNID | visibility_enum so we can just +// pull all events and then filter by that table. +func joinEventsFromHistoryVisibility( + ctx context.Context, db storage.Database, roomID string, stateEntries []types.StateEntry) ([]types.Event, error) { + + var eventNIDs []types.EventNID + for _, entry := range stateEntries { + // Filter the events to retrieve to only keep the membership events + if entry.EventTypeNID == types.MRoomHistoryVisibilityNID && entry.EventStateKeyNID == types.EmptyStateKeyNID { + eventNIDs = append(eventNIDs, entry.EventNID) + break + } + } + + // Get all of the events in this state + stateEvents, err := db.Events(ctx, eventNIDs) + if err != nil { + return nil, err + } + events := make([]gomatrixserverlib.Event, len(stateEvents)) + for i := range stateEvents { + events[i] = stateEvents[i].Event + } + visibility := auth.HistoryVisibilityForRoom(events) + if visibility != "shared" { + logrus.Infof("ServersAtEvent history visibility not shared: %s", visibility) + return nil, nil + } + // get joined members + roomNID, err := db.RoomNID(ctx, roomID) + if err != nil { + return nil, err + } + joinEventNIDs, err := db.GetMembershipEventNIDsForRoom(ctx, roomNID, true) + if err != nil { + return nil, err + } + return db.Events(ctx, joinEventNIDs) +} diff --git a/roomserver/query/query_test.go b/roomserver/internal/query_test.go similarity index 96% rename from roomserver/query/query_test.go rename to roomserver/internal/query_test.go index 8fb6a082e..211ab5083 100644 --- a/roomserver/query/query_test.go +++ b/roomserver/internal/query_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package query +package internal import ( "context" @@ -24,7 +24,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// used to implement RoomserverQueryAPIEventDB to test getAuthChain +// used to implement RoomserverInternalAPIEventDB to test getAuthChain type getEventDB struct { eventMap map[string]gomatrixserverlib.Event } @@ -79,7 +79,7 @@ func (db *getEventDB) addFakeEvents(graph map[string][]string) error { return nil } -// EventsFromIDs implements RoomserverQueryAPIEventDB +// EventsFromIDs implements RoomserverInternalAPIEventDB func (db *getEventDB) EventsFromIDs(ctx context.Context, eventIDs []string) (res []types.Event, err error) { for _, evID := range eventIDs { res = append(res, types.Event{ diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 0bc7c1ca7..916e25fbd 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -20,11 +20,8 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" - asQuery "github.com/matrix-org/dendrite/appservice/query" "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/roomserver/alias" - "github.com/matrix-org/dendrite/roomserver/input" - "github.com/matrix-org/dendrite/roomserver/query" + "github.com/matrix-org/dendrite/roomserver/internal" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/sirupsen/logrus" ) @@ -34,44 +31,27 @@ import ( // allowing other components running in the same process to hit the query the // APIs directly instead of having to use HTTP. func SetupRoomServerComponent( - base *basecomponent.BaseDendrite, keyRing gomatrixserverlib.JSONVerifier, -) (api.RoomserverAliasAPI, api.RoomserverInputAPI, api.RoomserverQueryAPI) { - roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer), base.Cfg.DbProperties()) + base *basecomponent.BaseDendrite, + keyRing gomatrixserverlib.JSONVerifier, + fedClient *gomatrixserverlib.FederationClient, +) api.RoomserverInternalAPI { + roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer)) if err != nil { logrus.WithError(err).Panicf("failed to connect to room server db") } - inputAPI := input.RoomserverInputAPI{ + internalAPI := internal.RoomserverInternalAPI{ DB: roomserverDB, + Cfg: base.Cfg, Producer: base.KafkaProducer, OutputRoomEventTopic: string(base.Cfg.Kafka.Topics.OutputRoomEvent), + ImmutableCache: base.ImmutableCache, + ServerName: base.Cfg.Matrix.ServerName, + FedClient: fedClient, + KeyRing: keyRing, } - inputAPI.SetupHTTP(http.DefaultServeMux) + internalAPI.SetupHTTP(http.DefaultServeMux) - queryAPI := query.RoomserverQueryAPI{ - DB: roomserverDB, - ImmutableCache: base.ImmutableCache, - ServerName: base.Cfg.Matrix.ServerName, - FedClient: base.CreateFederationClient(), - // TODO: We should have a key server so we don't keep adding components - // which talk to the same DB. - KeyRing: keyRing, - } - - queryAPI.SetupHTTP(http.DefaultServeMux) - - asAPI := asQuery.AppServiceQueryAPI{Cfg: base.Cfg} - - aliasAPI := alias.RoomserverAliasAPI{ - DB: roomserverDB, - Cfg: base.Cfg, - InputAPI: &inputAPI, - QueryAPI: &queryAPI, - AppserviceAPI: &asAPI, - } - - aliasAPI.SetupHTTP(http.DefaultServeMux) - - return &aliasAPI, &inputAPI, &queryAPI + return &internalAPI } diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 389c94400..9b005ee6a 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -86,7 +86,10 @@ func (v StateResolution) LoadStateAtEvent( ) ([]types.StateEntry, error) { snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID) if err != nil { - return nil, err + return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID failed for event %s : %s", eventID, err) + } + if snapshotNID == 0 { + return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID(%s) returned 0 NID, was this event stored?", eventID) } stateEntries, err := v.LoadStateAtSnapshot(ctx, snapshotNID) diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index d881fa91f..a63596aeb 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -48,11 +48,6 @@ const insertEventSQL = ` ON CONFLICT DO NOTHING; ` -const insertEventResultSQL = ` - SELECT event_nid, state_snapshot_nid FROM roomserver_events - WHERE rowid = last_insert_rowid(); -` - const selectEventSQL = "" + "SELECT event_nid, state_snapshot_nid FROM roomserver_events WHERE event_id = $1" @@ -102,7 +97,6 @@ const selectRoomNIDForEventNIDSQL = "" + type eventStatements struct { db *sql.DB insertEventStmt *sql.Stmt - insertEventResultStmt *sql.Stmt selectEventStmt *sql.Stmt bulkSelectStateEventByIDStmt *sql.Stmt bulkSelectStateAtEventByIDStmt *sql.Stmt @@ -126,7 +120,6 @@ func (s *eventStatements) prepare(db *sql.DB) (err error) { return statementList{ {&s.insertEventStmt, insertEventSQL}, - {&s.insertEventResultStmt, insertEventResultSQL}, {&s.selectEventStmt, selectEventSQL}, {&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL}, {&s.bulkSelectStateAtEventByIDStmt, bulkSelectStateAtEventByIDSQL}, @@ -152,19 +145,22 @@ func (s *eventStatements) insertEvent( referenceSHA256 []byte, authEventNIDs []types.EventNID, depth int64, -) (types.EventNID, types.StateSnapshotNID, error) { - var eventNID int64 - var stateNID int64 - var err error +) (types.EventNID, error) { + // attempt to insert: the last_row_id is the event NID insertStmt := common.TxStmt(txn, s.insertEventStmt) - resultStmt := common.TxStmt(txn, s.insertEventResultStmt) - if _, err = insertStmt.ExecContext( + result, err := insertStmt.ExecContext( ctx, int64(roomNID), int64(eventTypeNID), int64(eventStateKeyNID), eventID, referenceSHA256, eventNIDsAsArray(authEventNIDs), depth, - ); err == nil { - err = resultStmt.QueryRowContext(ctx).Scan(&eventNID, &stateNID) + ) + if err != nil { + return 0, err } - return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err + modified, err := result.RowsAffected() + if modified == 0 && err == nil { + return 0, sql.ErrNoRows + } + eventNID, err := result.LastInsertId() + return types.EventNID(eventNID), err } func (s *eventStatements) selectEvent( diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 6e5a1fbf2..e77fea9cf 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -124,7 +124,7 @@ func (d *Database) StoreEvent( } } - if eventNID, stateNID, err = d.statements.insertEvent( + if eventNID, err = d.statements.insertEvent( ctx, txn, roomNID, diff --git a/roomserver/version/version.go b/roomserver/version/version.go index f2a67e74d..ddd0f23a6 100644 --- a/roomserver/version/version.go +++ b/roomserver/version/version.go @@ -51,15 +51,15 @@ var roomVersions = map[gomatrixserverlib.RoomVersion]RoomVersionDescription{ Stable: true, }, gomatrixserverlib.RoomVersionV5: RoomVersionDescription{ - Supported: false, - Stable: false, + Supported: true, + Stable: true, }, } // DefaultRoomVersion contains the room version that will, by // default, be used to create new rooms on this server. func DefaultRoomVersion() gomatrixserverlib.RoomVersion { - return gomatrixserverlib.RoomVersionV4 + return gomatrixserverlib.RoomVersionV5 } // RoomVersions returns a map of all known room versions to this diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 1d512972d..987cc5df6 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -32,10 +32,10 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - roomServerConsumer *common.ContinualConsumer - db storage.Database - notifier *sync.Notifier - query api.RoomserverQueryAPI + rsAPI api.RoomserverInternalAPI + rsConsumer *common.ContinualConsumer + db storage.Database + notifier *sync.Notifier } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -44,7 +44,7 @@ func NewOutputRoomEventConsumer( kafkaConsumer sarama.Consumer, n *sync.Notifier, store storage.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ @@ -53,10 +53,10 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ - roomServerConsumer: &consumer, - db: store, - notifier: n, - query: queryAPI, + rsConsumer: &consumer, + db: store, + notifier: n, + rsAPI: rsAPI, } consumer.ProcessMessage = s.onMessage @@ -65,7 +65,7 @@ func NewOutputRoomEventConsumer( // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return s.roomServerConsumer.Start() + return s.rsConsumer.Start() } // onMessage is called when the sync server receives a new event from the room server output log. @@ -226,7 +226,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // from the roomserver using the query API. eventReq := api.QueryEventsByIDRequest{EventIDs: missing} var eventResp api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { return nil, err } diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index c48414ab6..d7e39704d 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -34,7 +34,7 @@ import ( type messagesReq struct { ctx context.Context db storage.Database - queryAPI api.RoomserverQueryAPI + rsAPI api.RoomserverInternalAPI federation *gomatrixserverlib.FederationClient cfg *config.Dendrite roomID string @@ -59,7 +59,7 @@ const defaultMessagesLimit = 10 func OnIncomingMessagesRequest( req *http.Request, db storage.Database, roomID string, federation *gomatrixserverlib.FederationClient, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, cfg *config.Dendrite, ) util.JSONResponse { var err error @@ -135,7 +135,7 @@ func OnIncomingMessagesRequest( mReq := messagesReq{ ctx: req.Context(), db: db, - queryAPI: queryAPI, + rsAPI: rsAPI, federation: federation, cfg: cfg, roomID: roomID, @@ -229,14 +229,14 @@ func (r *messagesReq) retrieveEvents() ( // change the way topological positions are defined (as depth isn't the most // reliable way to define it), it would be easier and less troublesome to // only have to change it in one place, i.e. the database. - startPos, err := r.db.EventPositionInTopology( + startPos, startStreamPos, err := r.db.EventPositionInTopology( r.ctx, events[0].EventID(), ) if err != nil { err = fmt.Errorf("EventPositionInTopology: for start event %s: %w", events[0].EventID(), err) return } - endPos, err := r.db.EventPositionInTopology( + endPos, endStreamPos, err := r.db.EventPositionInTopology( r.ctx, events[len(events)-1].EventID(), ) if err != nil { @@ -246,10 +246,10 @@ func (r *messagesReq) retrieveEvents() ( // Generate pagination tokens to send to the client using the positions // retrieved previously. start = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, startPos, 0, + types.PaginationTokenTypeTopology, startPos, startStreamPos, ) end = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, endPos, 0, + types.PaginationTokenTypeTopology, endPos, endStreamPos, ) if r.backwardOrdering { @@ -360,7 +360,7 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent // the room or sending the request. func (r *messagesReq) backfill(roomID string, fromEventIDs []string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { var res api.QueryBackfillResponse - err := r.queryAPI.QueryBackfill(context.Background(), &api.QueryBackfillRequest{ + err := r.rsAPI.QueryBackfill(context.Background(), &api.QueryBackfillRequest{ RoomID: roomID, EarliestEventsIDs: fromEventIDs, Limit: limit, @@ -407,13 +407,13 @@ func setToDefault( // go 1 earlier than the first event so we correctly fetch the earliest event to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) } else { - var pos types.StreamPosition - pos, err = db.MaxTopologicalPosition(ctx, roomID) + var pos, stream types.StreamPosition + pos, stream, err = db.MaxTopologicalPosition(ctx, roomID) if err != nil { return } - to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, pos, 0) + to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, pos, stream) } return diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 9078b87ff..5a36a279f 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -40,7 +40,7 @@ const pathPrefixR0 = "/_matrix/client/r0" func Setup( apiMux *mux.Router, srp *sync.RequestPool, syncDB storage.Database, deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, cfg *config.Dendrite, ) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() @@ -61,6 +61,6 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], federation, queryAPI, cfg) + return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], federation, rsAPI, cfg) })).Methods(http.MethodGet, http.MethodOptions) } diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index a3efd8d58..7d6376438 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -28,26 +28,83 @@ import ( type Database interface { common.PartitionStorer + // AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) + // Events lookups a list of event by their event ID. + // Returns a list of events matching the requested IDs found in the database. + // If an event is not found in the database then it will be omitted from the list. + // Returns an error if there was a problem talking with the database. + // Does not include any transaction IDs in the returned events. Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) - WriteEvent(context.Context, *gomatrixserverlib.HeaderedEvent, []gomatrixserverlib.HeaderedEvent, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) + // WriteEvent into the database. It is not safe to call this function from multiple goroutines, as it would create races + // when generating the sync stream position for this event. Returns the sync stream position for the inserted event. + // Returns an error if there was a problem inserting this event. + WriteEvent(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent, addStateEvents []gomatrixserverlib.HeaderedEvent, + addStateEventIDs []string, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool) (types.StreamPosition, error) + // GetStateEvent returns the Matrix state event of a given type for a given room with a given state key + // If no event could be found, returns nil + // If there was an issue during the retrieval, returns an error GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) + // GetStateEventsForRoom fetches the state events for a given room. + // Returns an empty slice if no state events could be found for this room. + // Returns an error if there was an issue with the retrieval. GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) + // SyncPosition returns the latest positions for syncing. SyncPosition(ctx context.Context) (types.PaginationToken, error) + // IncrementalSync returns all the data needed in order to create an incremental + // sync response for the given user. Events returned will include any client + // transaction IDs associated with the given device. These transaction IDs come + // from when the device sent the event via an API that included a transaction + // ID. IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) + // CompleteSync returns a complete /sync API response for the given user. CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error) + // GetAccountDataInRange returns all account data for a given user inserted or + // updated between two given positions + // Returns a map following the format data[roomID] = []dataTypes + // If no data is retrieved, returns an empty map + // If there was an issue with the retrieval, returns an error GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) + // UpsertAccountData keeps track of new or updated account data, by saving the type + // of the new/updated data, and the user ID and room ID the data is related to (empty) + // room ID means the data isn't specific to any room) + // If no data with the given type, user ID and room ID exists in the database, + // creates a new row, else update the existing one + // Returns an error if there was an issue with the upsert UpsertAccountData(ctx context.Context, userID, roomID, dataType string) (types.StreamPosition, error) + // AddInviteEvent stores a new invite event for a user. + // If the invite was successfully stored this returns the stream ID it was stored at. + // Returns an error if there was a problem communicating with the database. AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent) (types.StreamPosition, error) + // RetireInviteEvent removes an old invite event from the database. + // Returns an error if there was a problem communicating with the database. RetireInviteEvent(ctx context.Context, inviteEventID string) error + // SetTypingTimeoutCallback sets a callback function that is called right after + // a user is removed from the typing user list due to timeout. SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) + // AddTypingUser adds a typing user to the typing cache. + // Returns the newly calculated sync position for typing notifications. AddTypingUser(userID, roomID string, expireTime *time.Time) types.StreamPosition + // RemoveTypingUser removes a typing user from the typing cache. + // Returns the newly calculated sync position for typing notifications. RemoveTypingUser(userID, roomID string) types.StreamPosition + // GetEventsInRange retrieves all of the events on a given ordering using the + // given extremities and limit. GetEventsInRange(ctx context.Context, from, to *types.PaginationToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) - EventPositionInTopology(ctx context.Context, eventID string) (types.StreamPosition, error) + // EventPositionInTopology returns the depth and stream position of the given event. + EventPositionInTopology(ctx context.Context, eventID string) (depth types.StreamPosition, stream types.StreamPosition, err error) + // EventsAtTopologicalPosition returns all of the events matching a given + // position in the topology of a given room. EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error) + // BackwardExtremitiesForRoom returns the event IDs of all of the backward + // extremities we know of for a given room. BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) - MaxTopologicalPosition(ctx context.Context, roomID string) (types.StreamPosition, error) + // MaxTopologicalPosition returns the highest topological position for a given room. + MaxTopologicalPosition(ctx context.Context, roomID string) (depth types.StreamPosition, stream types.StreamPosition, err error) + // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and + // matches the streamevent.transactionID device then the transaction ID gets + // added to the unsigned section of the output event. StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent + // SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) } diff --git a/syncapi/storage/postgres/backward_extremities_table.go b/syncapi/storage/postgres/backward_extremities_table.go deleted file mode 100644 index cb3629644..000000000 --- a/syncapi/storage/postgres/backward_extremities_table.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2018 New Vector Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "context" - "database/sql" - - "github.com/matrix-org/dendrite/common" -) - -// The purpose of this table is to keep track of backwards extremities for a room. -// Backwards extremities are the earliest (DAG-wise) known events which we have -// the entire event JSON. These event IDs are used in federation requests to fetch -// even earlier events. -// -// We persist the previous event IDs as well, one per row, so when we do fetch even -// earlier events we can simply delete rows which referenced it. Consider the graph: -// A -// | Event C has 1 prev_event ID: A. -// B C -// |___| Event D has 2 prev_event IDs: B and C. -// | -// D -// The earliest known event we have is D, so this table has 2 rows. -// A backfill request gives us C but not B. We delete rows where prev_event=C. This -// still means that D is a backwards extremity as we do not have event B. However, event -// C is *also* a backwards extremity at this point as we do not have event A. Later, -// when we fetch event B, we delete rows where prev_event=B. This then removes D as -// a backwards extremity because there are no more rows with event_id=B. -const backwardExtremitiesSchema = ` --- Stores output room events received from the roomserver. -CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( - -- The 'room_id' key for the event. - room_id TEXT NOT NULL, - -- The event ID for the last known event. This is the backwards extremity. - event_id TEXT NOT NULL, - -- The prev_events for the last known event. This is used to update extremities. - prev_event_id TEXT NOT NULL, - - PRIMARY KEY(room_id, event_id, prev_event_id) -); -` - -const insertBackwardExtremitySQL = "" + - "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT DO NOTHING" - -const selectBackwardExtremitiesForRoomSQL = "" + - "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" - -const deleteBackwardExtremitySQL = "" + - "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" - -type backwardExtremitiesStatements struct { - insertBackwardExtremityStmt *sql.Stmt - selectBackwardExtremitiesForRoomStmt *sql.Stmt - deleteBackwardExtremityStmt *sql.Stmt -} - -func (s *backwardExtremitiesStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(backwardExtremitiesSchema) - if err != nil { - return - } - if s.insertBackwardExtremityStmt, err = db.Prepare(insertBackwardExtremitySQL); err != nil { - return - } - if s.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(selectBackwardExtremitiesForRoomSQL); err != nil { - return - } - if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil { - return - } - return -} - -func (s *backwardExtremitiesStatements) insertsBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, -) (err error) { - _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) - return -} - -func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( - ctx context.Context, roomID string, -) (eventIDs []string, err error) { - rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) - if err != nil { - return - } - defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") - - for rows.Next() { - var eID string - if err = rows.Scan(&eID); err != nil { - return - } - - eventIDs = append(eventIDs, eID) - } - - return eventIDs, rows.Err() -} - -func (s *backwardExtremitiesStatements) deleteBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, knownEventID string, -) (err error) { - _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) - return -} diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 0b53dfa9e..769823e92 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -94,6 +94,9 @@ const selectEarlyEventsSQL = "" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id ASC LIMIT $4" +const selectStreamPositionForEventIDSQL = "" + + "SELECT id FROM syncapi_output_room_events WHERE event_id = $1" + const selectMaxEventIDSQL = "" + "SELECT MAX(id) FROM syncapi_output_room_events" @@ -111,13 +114,14 @@ const selectStateInRangeSQL = "" + " LIMIT $8" type outputRoomEventsStatements struct { - insertEventStmt *sql.Stmt - selectEventsStmt *sql.Stmt - selectMaxEventIDStmt *sql.Stmt - selectRecentEventsStmt *sql.Stmt - selectRecentEventsForSyncStmt *sql.Stmt - selectEarlyEventsStmt *sql.Stmt - selectStateInRangeStmt *sql.Stmt + insertEventStmt *sql.Stmt + selectEventsStmt *sql.Stmt + selectMaxEventIDStmt *sql.Stmt + selectRecentEventsStmt *sql.Stmt + selectRecentEventsForSyncStmt *sql.Stmt + selectEarlyEventsStmt *sql.Stmt + selectStateInRangeStmt *sql.Stmt + selectStreamPositionForEventIDStmt *sql.Stmt } func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) { @@ -146,9 +150,18 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) { if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil { return } + if s.selectStreamPositionForEventIDStmt, err = db.Prepare(selectStreamPositionForEventIDSQL); err != nil { + return + } return } +func (s *outputRoomEventsStatements) selectStreamPositionForEventID(ctx context.Context, eventID string) (types.StreamPosition, error) { + var id int64 + err := s.selectStreamPositionForEventIDStmt.QueryRowContext(ctx, eventID).Scan(&id) + return types.StreamPosition(id), err +} + // selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos. // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the // two positions, only the most recent state is returned. diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index 280d4ec39..f77365c8d 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -32,35 +32,44 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events_topology ( -- The place of the event in the room's topology. This can usually be determined -- from the event's depth. topological_position BIGINT NOT NULL, + stream_position BIGINT NOT NULL, -- The 'room_id' key for the event. room_id TEXT NOT NULL ); -- The topological order will be used in events selection and ordering -CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, room_id); +CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, stream_position, room_id); ` const insertEventInTopologySQL = "" + - "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT (topological_position, room_id) DO UPDATE SET event_id = $1" + "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id, stream_position)" + + " VALUES ($1, $2, $3, $4)" + + " ON CONFLICT (topological_position, stream_position, room_id) DO UPDATE SET event_id = $1" const selectEventIDsInRangeASCSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + - " ORDER BY topological_position ASC LIMIT $4" + " WHERE room_id = $1 AND" + + "(topological_position > $2 AND topological_position < $3) OR" + + "(topological_position = $4 AND stream_position <= $5)" + + " ORDER BY topological_position ASC, stream_position ASC LIMIT $6" const selectEventIDsInRangeDESCSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + - " ORDER BY topological_position DESC LIMIT $4" + " WHERE room_id = $1 AND" + + "(topological_position > $2 AND topological_position < $3) OR" + + "(topological_position = $4 AND stream_position <= $5)" + + " ORDER BY topological_position DESC, stream_position DESC LIMIT $6" const selectPositionInTopologySQL = "" + "SELECT topological_position FROM syncapi_output_room_events_topology" + " WHERE event_id = $1" + // Select the max topological position for the room, then sort by stream position and take the highest, + // returning both topological and stream positions. const selectMaxPositionInTopologySQL = "" + - "SELECT MAX(topological_position) FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1" + "SELECT topological_position, stream_position FROM syncapi_output_room_events_topology" + + " WHERE topological_position=(" + + "SELECT MAX(topological_position) FROM syncapi_output_room_events_topology WHERE room_id=$1" + + ") ORDER BY stream_position DESC LIMIT 1" const selectEventIDsFromPositionSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + @@ -104,10 +113,10 @@ func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { // insertEventInTopology inserts the given event in the room's topology, based // on the event's depth. func (s *outputRoomEventsTopologyStatements) insertEventInTopology( - ctx context.Context, event *gomatrixserverlib.HeaderedEvent, + ctx context.Context, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition, ) (err error) { _, err = s.insertEventInTopologyStmt.ExecContext( - ctx, event.EventID(), event.Depth(), event.RoomID(), + ctx, event.EventID(), event.Depth(), event.RoomID(), pos, ) return } @@ -116,7 +125,7 @@ func (s *outputRoomEventsTopologyStatements) insertEventInTopology( // given range in a given room's topological order. // Returns an empty slice if no events match the given range. func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( - ctx context.Context, roomID string, fromPos, toPos types.StreamPosition, + ctx context.Context, roomID string, fromPos, toPos, toMicroPos types.StreamPosition, limit int, chronologicalOrder bool, ) (eventIDs []string, err error) { // Decide on the selection's order according to whether chronological order @@ -129,7 +138,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( } // Query the event IDs. - rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) + rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, toPos, toMicroPos, limit) if err == sql.ErrNoRows { // If no event matched the request, return an empty slice. return []string{}, nil @@ -161,8 +170,8 @@ func (s *outputRoomEventsTopologyStatements) selectPositionInTopology( func (s *outputRoomEventsTopologyStatements) selectMaxPositionInTopology( ctx context.Context, roomID string, -) (pos types.StreamPosition, err error) { - err = s.selectMaxPositionInTopologyStmt.QueryRowContext(ctx, roomID).Scan(&pos) +) (pos types.StreamPosition, spos types.StreamPosition, err error) { + err = s.selectMaxPositionInTopologyStmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos) return } diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index cbcbdbd2f..1ea82ea15 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -32,6 +32,7 @@ import ( _ "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -56,7 +57,7 @@ type SyncServerDatasource struct { invites inviteEventsStatements eduCache *cache.EDUCache topology outputRoomEventsTopologyStatements - backwardExtremities backwardExtremitiesStatements + backwardExtremities tables.BackwardsExtremities } // NewSyncServerDatasource creates a new sync server database @@ -75,32 +76,27 @@ func NewSyncServerDatasource(dbDataSourceName string, dbProperties common.DbProp if err = d.events.prepare(d.db); err != nil { return nil, err } - if err := d.roomstate.prepare(d.db); err != nil { + if err = d.roomstate.prepare(d.db); err != nil { return nil, err } - if err := d.invites.prepare(d.db); err != nil { + if err = d.invites.prepare(d.db); err != nil { return nil, err } - if err := d.topology.prepare(d.db); err != nil { + if err = d.topology.prepare(d.db); err != nil { return nil, err } - if err := d.backwardExtremities.prepare(d.db); err != nil { + d.backwardExtremities, err = tables.NewBackwardsExtremities(d.db, &tables.PostgresBackwardsExtremitiesStatements{}) + if err != nil { return nil, err } d.eduCache = cache.New() return &d, nil } -// AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { return d.roomstate.selectJoinedUsers(ctx) } -// Events lookups a list of event by their event ID. -// Returns a list of events matching the requested IDs found in the database. -// If an event is not found in the database then it will be omitted from the list. -// Returns an error if there was a problem talking with the database. -// Does not include any transaction IDs in the returned events. func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) { streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs) if err != nil { @@ -116,7 +112,7 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([ // the events listed in the event's 'prev_events'. This function also updates the backwards extremities table // to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { - if err := d.backwardExtremities.deleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { + if err := d.backwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { return err } @@ -137,7 +133,7 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx // If the event is missing, consider it a backward extremity. if !found { - if err = d.backwardExtremities.insertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { + if err = d.backwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { return err } } @@ -146,9 +142,6 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx return nil } -// WriteEvent into the database. It is not safe to call this function from multiple goroutines, as it would create races -// when generating the sync stream position for this event. Returns the sync stream position for the inserted event. -// Returns an error if there was a problem inserting this event. func (d *SyncServerDatasource) WriteEvent( ctx context.Context, ev *gomatrixserverlib.HeaderedEvent, @@ -166,7 +159,7 @@ func (d *SyncServerDatasource) WriteEvent( } pduPosition = pos - if err = d.topology.insertEventInTopology(ctx, ev); err != nil { + if err = d.topology.insertEventInTopology(ctx, ev, pos); err != nil { return err } @@ -219,18 +212,12 @@ func (d *SyncServerDatasource) updateRoomState( return nil } -// GetStateEvent returns the Matrix state event of a given type for a given room with a given state key -// If no event could be found, returns nil -// If there was an issue during the retrieval, returns an error func (d *SyncServerDatasource) GetStateEvent( ctx context.Context, roomID, evType, stateKey string, ) (*gomatrixserverlib.HeaderedEvent, error) { return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey) } -// GetStateEventsForRoom fetches the state events for a given room. -// Returns an empty slice if no state events could be found for this room. -// Returns an error if there was an issue with the retrieval. func (d *SyncServerDatasource) GetStateEventsForRoom( ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter, ) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) { @@ -241,8 +228,6 @@ func (d *SyncServerDatasource) GetStateEventsForRoom( return } -// GetEventsInRange retrieves all of the events on a given ordering using the -// given extremities and limit. func (d *SyncServerDatasource) GetEventsInRange( ctx context.Context, from, to *types.PaginationToken, @@ -255,12 +240,13 @@ func (d *SyncServerDatasource) GetEventsInRange( if from.Type == types.PaginationTokenTypeTopology { // Determine the backward and forward limit, i.e. the upper and lower // limits to the selection in the room's topology, from the direction. - var backwardLimit, forwardLimit types.StreamPosition + var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition if backwardOrdering { // Backward ordering is antichronological (latest event to oldest // one). backwardLimit = to.PDUPosition forwardLimit = from.PDUPosition + forwardMicroLimit = from.EDUTypingPosition } else { // Forward ordering is chronological (oldest event to latest one). backwardLimit = from.PDUPosition @@ -270,7 +256,7 @@ func (d *SyncServerDatasource) GetEventsInRange( // Select the event IDs from the defined range. var eIDs []string eIDs, err = d.topology.selectEventIDsInRange( - ctx, roomID, backwardLimit, forwardLimit, limit, !backwardOrdering, + ctx, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering, ) if err != nil { return @@ -304,29 +290,22 @@ func (d *SyncServerDatasource) GetEventsInRange( return } -// SyncPosition returns the latest positions for syncing. func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.PaginationToken, error) { return d.syncPositionTx(ctx, nil) } -// BackwardExtremitiesForRoom returns the event IDs of all of the backward -// extremities we know of for a given room. func (d *SyncServerDatasource) BackwardExtremitiesForRoom( ctx context.Context, roomID string, ) (backwardExtremities []string, err error) { - return d.backwardExtremities.selectBackwardExtremitiesForRoom(ctx, roomID) + return d.backwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID) } -// MaxTopologicalPosition returns the highest topological position for a given -// room. func (d *SyncServerDatasource) MaxTopologicalPosition( ctx context.Context, roomID string, -) (types.StreamPosition, error) { +) (depth types.StreamPosition, stream types.StreamPosition, err error) { return d.topology.selectMaxPositionInTopology(ctx, roomID) } -// EventsAtTopologicalPosition returns all of the events matching a given -// position in the topology of a given room. func (d *SyncServerDatasource) EventsAtTopologicalPosition( ctx context.Context, roomID string, pos types.StreamPosition, ) ([]types.StreamEvent, error) { @@ -340,11 +319,15 @@ func (d *SyncServerDatasource) EventsAtTopologicalPosition( func (d *SyncServerDatasource) EventPositionInTopology( ctx context.Context, eventID string, -) (types.StreamPosition, error) { - return d.topology.selectPositionInTopology(ctx, eventID) +) (depth types.StreamPosition, stream types.StreamPosition, err error) { + depth, err = d.topology.selectPositionInTopology(ctx, eventID) + if err != nil { + return + } + stream, err = d.events.selectStreamPositionForEventID(ctx, eventID) + return } -// SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. func (d *SyncServerDatasource) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { return d.syncStreamPositionTx(ctx, nil) } @@ -509,11 +492,6 @@ func (d *SyncServerDatasource) addEDUDeltaToResponse( return } -// IncrementalSync returns all the data needed in order to create an incremental -// sync response for the given user. Events returned will include any client -// transaction IDs associated with the given device. These transaction IDs come -// from when the device sent the event via an API that included a transaction -// ID. func (d *SyncServerDatasource) IncrementalSync( ctx context.Context, device authtypes.Device, @@ -643,7 +621,6 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( return res, toPos, joinedRoomIDs, err } -// CompleteSync returns a complete /sync API response for the given user. func (d *SyncServerDatasource) CompleteSync( ctx context.Context, userID string, numRecentEventsPerRoom int, ) (*types.Response, error) { @@ -675,11 +652,6 @@ var txReadOnlySnapshot = sql.TxOptions{ ReadOnly: true, } -// GetAccountDataInRange returns all account data for a given user inserted or -// updated between two given positions -// Returns a map following the format data[roomID] = []dataTypes -// If no data is retrieved, returns an empty map -// If there was an issue with the retrieval, returns an error func (d *SyncServerDatasource) GetAccountDataInRange( ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter, @@ -687,29 +659,18 @@ func (d *SyncServerDatasource) GetAccountDataInRange( return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos, accountDataFilterPart) } -// UpsertAccountData keeps track of new or updated account data, by saving the type -// of the new/updated data, and the user ID and room ID the data is related to (empty) -// room ID means the data isn't specific to any room) -// If no data with the given type, user ID and room ID exists in the database, -// creates a new row, else update the existing one -// Returns an error if there was an issue with the upsert func (d *SyncServerDatasource) UpsertAccountData( ctx context.Context, userID, roomID, dataType string, ) (types.StreamPosition, error) { return d.accountData.insertAccountData(ctx, userID, roomID, dataType) } -// AddInviteEvent stores a new invite event for a user. -// If the invite was successfully stored this returns the stream ID it was stored at. -// Returns an error if there was a problem communicating with the database. func (d *SyncServerDatasource) AddInviteEvent( ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (types.StreamPosition, error) { return d.invites.insertInviteEvent(ctx, inviteEvent) } -// RetireInviteEvent removes an old invite event from the database. -// Returns an error if there was a problem communicating with the database. func (d *SyncServerDatasource) RetireInviteEvent( ctx context.Context, inviteEventID string, ) error { @@ -723,16 +684,12 @@ func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallback d.eduCache.SetTimeoutCallback(fn) } -// AddTypingUser adds a typing user to the typing cache. -// Returns the newly calculated sync position for typing notifications. func (d *SyncServerDatasource) AddTypingUser( userID, roomID string, expireTime *time.Time, ) types.StreamPosition { return types.StreamPosition(d.eduCache.AddTypingUser(userID, roomID, expireTime)) } -// RemoveTypingUser removes a typing user from the typing cache. -// Returns the newly calculated sync position for typing notifications. func (d *SyncServerDatasource) RemoveTypingUser( userID, roomID string, ) types.StreamPosition { @@ -1071,9 +1028,6 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( return s, nil } -// StreamEventsToEvents converts streamEvent to Event. If device is non-nil and -// matches the streamevent.transactionID device then the transaction ID gets -// added to the unsigned section of the output event. func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent { out := make([]gomatrixserverlib.HeaderedEvent, len(in)) for i := 0; i < len(in); i++ { diff --git a/syncapi/storage/sqlite3/backward_extremities_table.go b/syncapi/storage/sqlite3/backward_extremities_table.go deleted file mode 100644 index 3d8cb91fc..000000000 --- a/syncapi/storage/sqlite3/backward_extremities_table.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2018 New Vector Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "context" - "database/sql" - - "github.com/matrix-org/dendrite/common" -) - -// The purpose of this table is to keep track of backwards extremities for a room. -// Backwards extremities are the earliest (DAG-wise) known events which we have -// the entire event JSON. These event IDs are used in federation requests to fetch -// even earlier events. -// -// We persist the previous event IDs as well, one per row, so when we do fetch even -// earlier events we can simply delete rows which referenced it. Consider the graph: -// A -// | Event C has 1 prev_event ID: A. -// B C -// |___| Event D has 2 prev_event IDs: B and C. -// | -// D -// The earliest known event we have is D, so this table has 2 rows. -// A backfill request gives us C but not B. We delete rows where prev_event=C. This -// still means that D is a backwards extremity as we do not have event B. However, event -// C is *also* a backwards extremity at this point as we do not have event A. Later, -// when we fetch event B, we delete rows where prev_event=B. This then removes D as -// a backwards extremity because there are no more rows with event_id=B. -const backwardExtremitiesSchema = ` --- Stores output room events received from the roomserver. -CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( - -- The 'room_id' key for the event. - room_id TEXT NOT NULL, - -- The event ID for the last known event. This is the backwards extremity. - event_id TEXT NOT NULL, - -- The prev_events for the last known event. This is used to update extremities. - prev_event_id TEXT NOT NULL, - - PRIMARY KEY(room_id, event_id, prev_event_id) -); -` - -const insertBackwardExtremitySQL = "" + - "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING" - -const selectBackwardExtremitiesForRoomSQL = "" + - "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" - -const deleteBackwardExtremitySQL = "" + - "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" - -type backwardExtremitiesStatements struct { - insertBackwardExtremityStmt *sql.Stmt - selectBackwardExtremitiesForRoomStmt *sql.Stmt - deleteBackwardExtremityStmt *sql.Stmt -} - -func (s *backwardExtremitiesStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(backwardExtremitiesSchema) - if err != nil { - return - } - if s.insertBackwardExtremityStmt, err = db.Prepare(insertBackwardExtremitySQL); err != nil { - return - } - if s.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(selectBackwardExtremitiesForRoomSQL); err != nil { - return - } - if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil { - return - } - return -} - -func (s *backwardExtremitiesStatements) insertsBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, -) (err error) { - _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) - return -} - -func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( - ctx context.Context, roomID string, -) (eventIDs []string, err error) { - rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) - if err != nil { - return - } - defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") - - for rows.Next() { - var eID string - if err = rows.Scan(&eID); err != nil { - return - } - - eventIDs = append(eventIDs, eID) - } - - return eventIDs, rows.Err() -} - -func (s *backwardExtremitiesStatements) deleteBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, knownEventID string, -) (err error) { - _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) - return -} diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 08299f64b..83d7940ad 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -74,6 +74,9 @@ const selectEarlyEventsSQL = "" + const selectMaxEventIDSQL = "" + "SELECT MAX(id) FROM syncapi_output_room_events" +const selectStreamPositionForEventIDSQL = "" + + "SELECT id FROM syncapi_output_room_events WHERE event_id = $1" + // In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). /* $1 = oldPos, @@ -99,14 +102,15 @@ const selectStateInRangeSQL = "" + " LIMIT $8" // limit type outputRoomEventsStatements struct { - streamIDStatements *streamIDStatements - insertEventStmt *sql.Stmt - selectEventsStmt *sql.Stmt - selectMaxEventIDStmt *sql.Stmt - selectRecentEventsStmt *sql.Stmt - selectRecentEventsForSyncStmt *sql.Stmt - selectEarlyEventsStmt *sql.Stmt - selectStateInRangeStmt *sql.Stmt + streamIDStatements *streamIDStatements + insertEventStmt *sql.Stmt + selectEventsStmt *sql.Stmt + selectMaxEventIDStmt *sql.Stmt + selectRecentEventsStmt *sql.Stmt + selectRecentEventsForSyncStmt *sql.Stmt + selectEarlyEventsStmt *sql.Stmt + selectStateInRangeStmt *sql.Stmt + selectStreamPositionForEventIDStmt *sql.Stmt } func (s *outputRoomEventsStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) { @@ -136,9 +140,18 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB, streamID *streamIDState if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil { return } + if s.selectStreamPositionForEventIDStmt, err = db.Prepare(selectStreamPositionForEventIDSQL); err != nil { + return + } return } +func (s *outputRoomEventsStatements) selectStreamPositionForEventID(ctx context.Context, eventID string) (types.StreamPosition, error) { + var id int64 + err := s.selectStreamPositionForEventIDStmt.QueryRowContext(ctx, eventID).Scan(&id) + return types.StreamPosition(id), err +} + // selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos. // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the // two positions, only the most recent state is returned. diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index a2944c2f9..d64894663 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -27,37 +27,42 @@ const outputRoomEventsTopologySchema = ` -- Stores output room events received from the roomserver. CREATE TABLE IF NOT EXISTS syncapi_output_room_events_topology ( event_id TEXT PRIMARY KEY, - topological_position BIGINT NOT NULL, + topological_position BIGINT NOT NULL, + stream_position BIGINT NOT NULL, room_id TEXT NOT NULL, - UNIQUE(topological_position, room_id) + UNIQUE(topological_position, room_id, stream_position) ); -- The topological order will be used in events selection and ordering --- CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, room_id); +-- CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, stream_position, room_id); ` const insertEventInTopologySQL = "" + - "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT (topological_position, room_id) DO UPDATE SET event_id = $1" + "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id, stream_position)" + + " VALUES ($1, $2, $3, $4)" + + " ON CONFLICT DO NOTHING" const selectEventIDsInRangeASCSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + - " ORDER BY topological_position ASC LIMIT $4" + " WHERE room_id = $1 AND" + + "(topological_position > $2 AND topological_position < $3) OR" + + "(topological_position = $4 AND stream_position <= $5)" + + " ORDER BY topological_position ASC, stream_position ASC LIMIT $6" const selectEventIDsInRangeDESCSQL = "" + - "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + - " ORDER BY topological_position DESC LIMIT $4" + "SELECT event_id FROM syncapi_output_room_events_topology" + + " WHERE room_id = $1 AND" + + "(topological_position > $2 AND topological_position < $3) OR" + + "(topological_position = $4 AND stream_position <= $5)" + + " ORDER BY topological_position DESC, stream_position DESC LIMIT $6" const selectPositionInTopologySQL = "" + "SELECT topological_position FROM syncapi_output_room_events_topology" + " WHERE event_id = $1" const selectMaxPositionInTopologySQL = "" + - "SELECT MAX(topological_position) FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1" + "SELECT MAX(topological_position), stream_position FROM syncapi_output_room_events_topology" + + " WHERE room_id = $1 ORDER BY stream_position DESC" const selectEventIDsFromPositionSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + @@ -101,11 +106,11 @@ func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { // insertEventInTopology inserts the given event in the room's topology, based // on the event's depth. func (s *outputRoomEventsTopologyStatements) insertEventInTopology( - ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, + ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition, ) (err error) { stmt := common.TxStmt(txn, s.insertEventInTopologyStmt) _, err = stmt.ExecContext( - ctx, event.EventID(), event.Depth(), event.RoomID(), + ctx, event.EventID(), event.Depth(), event.RoomID(), pos, ) return } @@ -115,7 +120,7 @@ func (s *outputRoomEventsTopologyStatements) insertEventInTopology( // Returns an empty slice if no events match the given range. func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( ctx context.Context, txn *sql.Tx, roomID string, - fromPos, toPos types.StreamPosition, + fromPos, toPos, toMicroPos types.StreamPosition, limit int, chronologicalOrder bool, ) (eventIDs []string, err error) { // Decide on the selection's order according to whether chronological order @@ -128,7 +133,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( } // Query the event IDs. - rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) + rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, toPos, toMicroPos, limit) if err == sql.ErrNoRows { // If no event matched the request, return an empty slice. return []string{}, nil @@ -160,9 +165,9 @@ func (s *outputRoomEventsTopologyStatements) selectPositionInTopology( func (s *outputRoomEventsTopologyStatements) selectMaxPositionInTopology( ctx context.Context, txn *sql.Tx, roomID string, -) (pos types.StreamPosition, err error) { +) (pos types.StreamPosition, spos types.StreamPosition, err error) { stmt := common.TxStmt(txn, s.selectMaxPositionInTopologyStmt) - err = stmt.QueryRowContext(ctx, roomID).Scan(&pos) + err = stmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos) return } diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 94e36d99b..7eccc7b5e 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -35,6 +35,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -60,7 +61,7 @@ type SyncServerDatasource struct { invites inviteEventsStatements eduCache *cache.EDUCache topology outputRoomEventsTopologyStatements - backwardExtremities backwardExtremitiesStatements + backwardExtremities tables.BackwardsExtremities } // NewSyncServerDatasource creates a new sync server database @@ -102,16 +103,17 @@ func (d *SyncServerDatasource) prepare() (err error) { if err = d.events.prepare(d.db, &d.streamID); err != nil { return err } - if err := d.roomstate.prepare(d.db, &d.streamID); err != nil { + if err = d.roomstate.prepare(d.db, &d.streamID); err != nil { return err } - if err := d.invites.prepare(d.db, &d.streamID); err != nil { + if err = d.invites.prepare(d.db, &d.streamID); err != nil { return err } - if err := d.topology.prepare(d.db); err != nil { + if err = d.topology.prepare(d.db); err != nil { return err } - if err := d.backwardExtremities.prepare(d.db); err != nil { + d.backwardExtremities, err = tables.NewBackwardsExtremities(d.db, &tables.SqliteBackwardsExtremitiesStatements{}) + if err != nil { return err } return nil @@ -142,7 +144,7 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([ // the events listed in the event's 'prev_events'. This function also updates the backwards extremities table // to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { - if err := d.backwardExtremities.deleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { + if err := d.backwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { return err } @@ -163,7 +165,7 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx // If the event is missing, consider it a backward extremity. if !found { - if err = d.backwardExtremities.insertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { + if err = d.backwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { return err } } @@ -192,7 +194,7 @@ func (d *SyncServerDatasource) WriteEvent( } pduPosition = pos - if err = d.topology.insertEventInTopology(ctx, txn, ev); err != nil { + if err = d.topology.insertEventInTopology(ctx, txn, ev, pos); err != nil { return err } @@ -279,14 +281,16 @@ func (d *SyncServerDatasource) GetEventsInRange( // events must be retrieved from the rooms' topology table rather than the // table contaning the syncapi server's whole stream of events. if from.Type == types.PaginationTokenTypeTopology { + // TODO: ARGH CONFUSING // Determine the backward and forward limit, i.e. the upper and lower // limits to the selection in the room's topology, from the direction. - var backwardLimit, forwardLimit types.StreamPosition + var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition if backwardOrdering { // Backward ordering is antichronological (latest event to oldest // one). backwardLimit = to.PDUPosition forwardLimit = from.PDUPosition + forwardMicroLimit = from.EDUTypingPosition } else { // Forward ordering is chronological (oldest event to latest one). backwardLimit = from.PDUPosition @@ -296,7 +300,7 @@ func (d *SyncServerDatasource) GetEventsInRange( // Select the event IDs from the defined range. var eIDs []string eIDs, err = d.topology.selectEventIDsInRange( - ctx, nil, roomID, backwardLimit, forwardLimit, limit, !backwardOrdering, + ctx, nil, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering, ) if err != nil { return @@ -326,8 +330,7 @@ func (d *SyncServerDatasource) GetEventsInRange( return } } - - return + return events, err } // SyncPosition returns the latest positions for syncing. @@ -344,14 +347,14 @@ func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (tok types.Pagi func (d *SyncServerDatasource) BackwardExtremitiesForRoom( ctx context.Context, roomID string, ) (backwardExtremities []string, err error) { - return d.backwardExtremities.selectBackwardExtremitiesForRoom(ctx, roomID) + return d.backwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID) } // MaxTopologicalPosition returns the highest topological position for a given // room. func (d *SyncServerDatasource) MaxTopologicalPosition( ctx context.Context, roomID string, -) (types.StreamPosition, error) { +) (types.StreamPosition, types.StreamPosition, error) { return d.topology.selectMaxPositionInTopology(ctx, nil, roomID) } @@ -370,8 +373,13 @@ func (d *SyncServerDatasource) EventsAtTopologicalPosition( func (d *SyncServerDatasource) EventPositionInTopology( ctx context.Context, eventID string, -) (types.StreamPosition, error) { - return d.topology.selectPositionInTopology(ctx, nil, eventID) +) (depth types.StreamPosition, stream types.StreamPosition, err error) { + depth, err = d.topology.selectPositionInTopology(ctx, nil, eventID) + if err != nil { + return + } + stream, err = d.events.selectStreamPositionForEventID(ctx, eventID) + return } // SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go new file mode 100644 index 000000000..a57d59176 --- /dev/null +++ b/syncapi/storage/storage_test.go @@ -0,0 +1,413 @@ +package storage_test + +import ( + "context" + "crypto/ed25519" + "fmt" + "testing" + "time" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +var ( + ctx = context.Background() + emptyStateKey = "" + testOrigin = gomatrixserverlib.ServerName("hollow.knight") + testRoomID = fmt.Sprintf("!hallownest:%s", testOrigin) + testUserIDA = fmt.Sprintf("@hornet:%s", testOrigin) + testUserIDB = fmt.Sprintf("@paleking:%s", testOrigin) + testUserDeviceA = authtypes.Device{ + UserID: testUserIDA, + ID: "device_id_A", + DisplayName: "Device A", + } + testRoomVersion = gomatrixserverlib.RoomVersionV4 + testKeyID = gomatrixserverlib.KeyID("ed25519:storage_test") + testPrivateKey = ed25519.NewKeyFromSeed([]byte{ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + }) +) + +func MustCreateEvent(t *testing.T, roomID string, prevs []gomatrixserverlib.HeaderedEvent, b *gomatrixserverlib.EventBuilder) gomatrixserverlib.HeaderedEvent { + b.RoomID = roomID + if prevs != nil { + prevIDs := make([]string, len(prevs)) + for i := range prevs { + prevIDs[i] = prevs[i].EventID() + } + b.PrevEvents = prevIDs + } + e, err := b.Build(time.Now(), testOrigin, testKeyID, testPrivateKey, testRoomVersion) + if err != nil { + t.Fatalf("failed to build event: %s", err) + } + return e.Headered(testRoomVersion) +} + +func MustCreateDatabase(t *testing.T) storage.Database { + db, err := sqlite3.NewSyncServerDatasource("file::memory:") + if err != nil { + t.Fatalf("NewSyncServerDatasource returned %s", err) + } + return db +} + +// Create a list of events which include a create event, join event and some messages. +func SimpleRoom(t *testing.T, roomID, userA, userB string) (msgs []gomatrixserverlib.HeaderedEvent, state []gomatrixserverlib.HeaderedEvent) { + var events []gomatrixserverlib.HeaderedEvent + events = append(events, MustCreateEvent(t, roomID, nil, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"room_version":"4","creator":"%s"}`, userA)), + Type: "m.room.create", + StateKey: &emptyStateKey, + Sender: userA, + Depth: int64(len(events) + 1), + })) + state = append(state, events[len(events)-1]) + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Type: "m.room.member", + StateKey: &userA, + Sender: userA, + Depth: int64(len(events) + 1), + })) + state = append(state, events[len(events)-1]) + for i := 0; i < 10; i++ { + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"body":"Message A %d"}`, i+1)), + Type: "m.room.message", + Sender: userA, + Depth: int64(len(events) + 1), + })) + } + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Type: "m.room.member", + StateKey: &userB, + Sender: userB, + Depth: int64(len(events) + 1), + })) + state = append(state, events[len(events)-1]) + for i := 0; i < 10; i++ { + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"body":"Message B %d"}`, i+1)), + Type: "m.room.message", + Sender: userB, + Depth: int64(len(events) + 1), + })) + } + + return events, state +} + +func MustWriteEvents(t *testing.T, db storage.Database, events []gomatrixserverlib.HeaderedEvent) (positions []types.StreamPosition) { + for _, ev := range events { + var addStateEvents []gomatrixserverlib.HeaderedEvent + var addStateEventIDs []string + var removeStateEventIDs []string + if ev.StateKey() != nil { + addStateEvents = append(addStateEvents, ev) + addStateEventIDs = append(addStateEventIDs, ev.EventID()) + } + pos, err := db.WriteEvent(ctx, &ev, addStateEvents, addStateEventIDs, removeStateEventIDs, nil, false) + if err != nil { + t.Fatalf("WriteEvent failed: %s", err) + } + positions = append(positions, pos) + } + return +} + +func TestWriteEvents(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + MustWriteEvents(t, db, events) +} + +// These tests assert basic functionality of the IncrementalSync and CompleteSync functions. +func TestSyncResponse(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, state := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + positions := MustWriteEvents(t, db, events) + latest, err := db.SyncPosition(ctx) + if err != nil { + t.Fatalf("failed to get SyncPosition: %s", err) + } + + testCases := []struct { + Name string + DoSync func() (*types.Response, error) + WantTimeline []gomatrixserverlib.HeaderedEvent + WantState []gomatrixserverlib.HeaderedEvent + }{ + // The purpose of this test is to make sure that incremental syncs are including up to the latest events. + // It's a basic sanity test that sync works. It creates a `since` token that is on the penultimate event. + // It makes sure the response includes the final event. + { + Name: "IncrementalSync penultimate", + DoSync: func() (*types.Response, error) { + from := types.NewPaginationTokenFromTypeAndPosition( // pretend we are at the penultimate event + types.PaginationTokenTypeStream, positions[len(positions)-2], types.StreamPosition(0), + ) + return db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false) + }, + WantTimeline: events[len(events)-1:], + }, + // The purpose of this test is to check that passing a `numRecentEventsPerRoom` correctly limits the + // number of returned events. This is critical for big rooms hence the test here. + { + Name: "IncrementalSync limited", + DoSync: func() (*types.Response, error) { + from := types.NewPaginationTokenFromTypeAndPosition( // pretend we are 10 events behind + types.PaginationTokenTypeStream, positions[len(positions)-11], types.StreamPosition(0), + ) + // limit is set to 5 + return db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false) + }, + // want the last 5 events, NOT the last 10. + WantTimeline: events[len(events)-5:], + }, + // The purpose of this test is to check that CompleteSync returns all the current state as well as + // honouring the `numRecentEventsPerRoom` value + { + Name: "CompleteSync limited", + DoSync: func() (*types.Response, error) { + // limit set to 5 + return db.CompleteSync(ctx, testUserIDA, 5) + }, + // want the last 5 events + WantTimeline: events[len(events)-5:], + // want all state for the room + WantState: state, + }, + // The purpose of this test is to check that CompleteSync can return everything with a high enough + // `numRecentEventsPerRoom`. + { + Name: "CompleteSync", + DoSync: func() (*types.Response, error) { + return db.CompleteSync(ctx, testUserIDA, len(events)+1) + }, + WantTimeline: events, + // We want no state at all as that field in /sync is the delta between the token (beginning of time) + // and the START of the timeline. + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(st *testing.T) { + res, err := tc.DoSync() + if err != nil { + st.Fatalf("failed to do sync: %s", err) + } + next := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeStream, latest.PDUPosition, latest.EDUTypingPosition) + if res.NextBatch != next.String() { + st.Errorf("NextBatch got %s want %s", res.NextBatch, next.String()) + } + roomRes, ok := res.Rooms.Join[testRoomID] + if !ok { + st.Fatalf("IncrementalSync response missing room %s - response: %+v", testRoomID, res) + } + assertEventsEqual(st, "state for "+testRoomID, false, roomRes.State.Events, tc.WantState) + assertEventsEqual(st, "timeline for "+testRoomID, false, roomRes.Timeline.Events, tc.WantTimeline) + }) + } +} + +// The purpose of this test is to ensure that backfill does indeed go backwards, using a stream token. +func TestGetEventsInRangeWithStreamToken(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + MustWriteEvents(t, db, events) + latest, err := db.SyncPosition(ctx) + if err != nil { + t.Fatalf("failed to get SyncPosition: %s", err) + } + // head towards the beginning of time + to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + + // backpaginate 5 messages starting at the latest position. + paginatedEvents, err := db.GetEventsInRange(ctx, &latest, to, testRoomID, 5, true) + if err != nil { + t.Fatalf("GetEventsInRange returned an error: %s", err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + assertEventsEqual(t, "", true, gots, reversed(events[len(events)-5:])) +} + +// The purpose of this test is to ensure that backfill does indeed go backwards, using a topology token +func TestGetEventsInRangeWithTopologyToken(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + MustWriteEvents(t, db, events) + latest, latestStream, err := db.MaxTopologicalPosition(ctx, testRoomID) + if err != nil { + t.Fatalf("failed to get MaxTopologicalPosition: %s", err) + } + from := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latest, latestStream) + // head towards the beginning of time + to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + + // backpaginate 5 messages starting at the latest position. + paginatedEvents, err := db.GetEventsInRange(ctx, from, to, testRoomID, 5, true) + if err != nil { + t.Fatalf("GetEventsInRange returned an error: %s", err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + assertEventsEqual(t, "", true, gots, reversed(events[len(events)-5:])) +} + +// The purpose of this test is to make sure that backpagination returns all events, even if some events have the same depth. +// For cases where events have the same depth, the streaming token should be used to tie break so events written via WriteEvent +// will appear FIRST when going backwards. This test creates a DAG like: +// .-----> Message ---. +// Create -> Membership --------> Message -------> Message +// `-----> Message ---` +// depth 1 2 3 4 +// +// With a total depth of 4. It tests that: +// - Backpagination over the whole fork should include all messages and not leave any out. +// - Backpagination from the middle of the fork should not return duplicates (things later than the token). +func TestGetEventsInRangeWithEventsSameDepth(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + + var events []gomatrixserverlib.HeaderedEvent + events = append(events, MustCreateEvent(t, testRoomID, nil, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"room_version":"4","creator":"%s"}`, testUserIDA)), + Type: "m.room.create", + StateKey: &emptyStateKey, + Sender: testUserIDA, + Depth: int64(len(events) + 1), + })) + events = append(events, MustCreateEvent(t, testRoomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Type: "m.room.member", + StateKey: &testUserIDA, + Sender: testUserIDA, + Depth: int64(len(events) + 1), + })) + // fork the dag into three, same prev_events and depth + parent := []gomatrixserverlib.HeaderedEvent{events[len(events)-1]} + depth := int64(len(events) + 1) + for i := 0; i < 3; i++ { + events = append(events, MustCreateEvent(t, testRoomID, parent, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"body":"Message A %d"}`, i+1)), + Type: "m.room.message", + Sender: testUserIDA, + Depth: depth, + })) + } + // merge the fork, prev_events are all 3 messages, depth is increased by 1. + events = append(events, MustCreateEvent(t, testRoomID, events[len(events)-3:], &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"body":"Message merge"}`)), + Type: "m.room.message", + Sender: testUserIDA, + Depth: depth + 1, + })) + MustWriteEvents(t, db, events) + latestPos, latestStreamPos, err := db.EventPositionInTopology(ctx, events[len(events)-1].EventID()) + if err != nil { + t.Fatalf("failed to get EventPositionInTopology: %s", err) + } + topoPos, streamPos, err := db.EventPositionInTopology(ctx, events[len(events)-3].EventID()) // Message 2 + if err != nil { + t.Fatalf("failed to get EventPositionInTopology for event: %s", err) + } + fromLatest := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latestPos, latestStreamPos) + fromFork := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, topoPos, streamPos) + // head towards the beginning of time + to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + + testCases := []struct { + Name string + From *types.PaginationToken + Limit int + Wants []gomatrixserverlib.HeaderedEvent + }{ + { + Name: "Pagination over the whole fork", + From: fromLatest, + Limit: 5, + Wants: reversed(events[len(events)-5:]), + }, + { + Name: "Paginating to the middle of the fork", + From: fromLatest, + Limit: 2, + Wants: reversed(events[len(events)-2:]), + }, + { + Name: "Pagination FROM the middle of the fork", + From: fromFork, + Limit: 3, + Wants: reversed(events[len(events)-5 : len(events)-2]), + }, + } + + for _, tc := range testCases { + // backpaginate messages starting at the latest position. + paginatedEvents, err := db.GetEventsInRange(ctx, tc.From, to, testRoomID, tc.Limit, true) + if err != nil { + t.Fatalf("%s GetEventsInRange returned an error: %s", tc.Name, err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + assertEventsEqual(t, tc.Name, true, gots, tc.Wants) + } +} + +func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatrixserverlib.ClientEvent, wants []gomatrixserverlib.HeaderedEvent) { + if len(gots) != len(wants) { + t.Fatalf("%s response returned %d events, want %d", msg, len(gots), len(wants)) + } + for i := range gots { + g := gots[i] + w := wants[i] + if g.EventID != w.EventID() { + t.Errorf("%s event[%d] event_id mismatch: got %s want %s", msg, i, g.EventID, w.EventID()) + } + if g.Sender != w.Sender() { + t.Errorf("%s event[%d] sender mismatch: got %s want %s", msg, i, g.Sender, w.Sender()) + } + if checkRoomID && g.RoomID != w.RoomID() { + t.Errorf("%s event[%d] room_id mismatch: got %s want %s", msg, i, g.RoomID, w.RoomID()) + } + if g.Type != w.Type() { + t.Errorf("%s event[%d] event type mismatch: got %s want %s", msg, i, g.Type, w.Type()) + } + if g.OriginServerTS != w.OriginServerTS() { + t.Errorf("%s event[%d] origin_server_ts mismatch: got %v want %v", msg, i, g.OriginServerTS, w.OriginServerTS()) + } + if string(g.Content) != string(w.Content()) { + t.Errorf("%s event[%d] content mismatch: got %s want %s", msg, i, string(g.Content), string(w.Content())) + } + if string(g.Unsigned) != string(w.Unsigned()) { + t.Errorf("%s event[%d] unsigned mismatch: got %s want %s", msg, i, string(g.Unsigned), string(w.Unsigned())) + } + if (g.StateKey == nil && w.StateKey() != nil) || (g.StateKey != nil && w.StateKey() == nil) { + t.Errorf("%s event[%d] state_key [not] missing: got %v want %v", msg, i, g.StateKey, w.StateKey()) + continue + } + if g.StateKey != nil { + if !w.StateKeyEquals(*g.StateKey) { + t.Errorf("%s event[%d] state_key mismatch: got %s want %s", msg, i, *g.StateKey, *w.StateKey()) + } + } + } +} + +func reversed(in []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { + out := make([]gomatrixserverlib.HeaderedEvent, len(in)) + for i := 0; i < len(in); i++ { + out[i] = in[len(in)-i-1] + } + return out +} diff --git a/syncapi/storage/tables/backward_extremities.go b/syncapi/storage/tables/backward_extremities.go new file mode 100644 index 000000000..babd9aaa1 --- /dev/null +++ b/syncapi/storage/tables/backward_extremities.go @@ -0,0 +1,175 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tables + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" +) + +// BackwardsExtremitiesStatements contains the SQL statements to implement. +// See BackwardsExtremities to see the parameter and response types. +type BackwardsExtremitiesStatements interface { + Schema() string + InsertBackwardExtremity() string + SelectBackwardExtremitiesForRoom() string + DeleteBackwardExtremity() string +} + +type PostgresBackwardsExtremitiesStatements struct{} + +func (s *PostgresBackwardsExtremitiesStatements) Schema() string { + return `-- Stores output room events received from the roomserver. + CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( + -- The 'room_id' key for the event. + room_id TEXT NOT NULL, + -- The event ID for the last known event. This is the backwards extremity. + event_id TEXT NOT NULL, + -- The prev_events for the last known event. This is used to update extremities. + prev_event_id TEXT NOT NULL, + + PRIMARY KEY(room_id, event_id, prev_event_id) + ); + ` +} +func (s *PostgresBackwardsExtremitiesStatements) InsertBackwardExtremity() string { + return "" + + "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + + " VALUES ($1, $2, $3)" + + " ON CONFLICT DO NOTHING" +} +func (s *PostgresBackwardsExtremitiesStatements) SelectBackwardExtremitiesForRoom() string { + return "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" +} +func (s *PostgresBackwardsExtremitiesStatements) DeleteBackwardExtremity() string { + return "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" +} + +type SqliteBackwardsExtremitiesStatements struct{} + +func (s *SqliteBackwardsExtremitiesStatements) Schema() string { + return `-- Stores output room events received from the roomserver. +CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( + -- The 'room_id' key for the event. + room_id TEXT NOT NULL, + -- The event ID for the last known event. This is the backwards extremity. + event_id TEXT NOT NULL, + -- The prev_events for the last known event. This is used to update extremities. + prev_event_id TEXT NOT NULL, + + PRIMARY KEY(room_id, event_id, prev_event_id) +); +` +} + +func (s *SqliteBackwardsExtremitiesStatements) InsertBackwardExtremity() string { + return "" + + "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + + " VALUES ($1, $2, $3)" + + " ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING" +} + +func (s *SqliteBackwardsExtremitiesStatements) SelectBackwardExtremitiesForRoom() string { + return "" + + "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" +} + +func (s *SqliteBackwardsExtremitiesStatements) DeleteBackwardExtremity() string { + return "" + + "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" +} + +// BackwardsExtremities keeps track of backwards extremities for a room. +// Backwards extremities are the earliest (DAG-wise) known events which we have +// the entire event JSON. These event IDs are used in federation requests to fetch +// even earlier events. +// +// We persist the previous event IDs as well, one per row, so when we do fetch even +// earlier events we can simply delete rows which referenced it. Consider the graph: +// A +// | Event C has 1 prev_event ID: A. +// B C +// |___| Event D has 2 prev_event IDs: B and C. +// | +// D +// The earliest known event we have is D, so this table has 2 rows. +// A backfill request gives us C but not B. We delete rows where prev_event=C. This +// still means that D is a backwards extremity as we do not have event B. However, event +// C is *also* a backwards extremity at this point as we do not have event A. Later, +// when we fetch event B, we delete rows where prev_event=B. This then removes D as +// a backwards extremity because there are no more rows with event_id=B. +type BackwardsExtremities struct { + insertBackwardExtremityStmt *sql.Stmt + selectBackwardExtremitiesForRoomStmt *sql.Stmt + deleteBackwardExtremityStmt *sql.Stmt +} + +// NewBackwardsExtremities prepares the table +func NewBackwardsExtremities(db *sql.DB, stmts BackwardsExtremitiesStatements) (table BackwardsExtremities, err error) { + _, err = db.Exec(stmts.Schema()) + if err != nil { + return + } + if table.insertBackwardExtremityStmt, err = db.Prepare(stmts.InsertBackwardExtremity()); err != nil { + return + } + if table.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(stmts.SelectBackwardExtremitiesForRoom()); err != nil { + return + } + if table.deleteBackwardExtremityStmt, err = db.Prepare(stmts.DeleteBackwardExtremity()); err != nil { + return + } + return +} + +// InsertsBackwardExtremity inserts a new backwards extremity. +func (s *BackwardsExtremities) InsertsBackwardExtremity( + ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, +) (err error) { + _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) + return +} + +// SelectBackwardExtremitiesForRoom retrieves all backwards extremities for the room. +func (s *BackwardsExtremities) SelectBackwardExtremitiesForRoom( + ctx context.Context, roomID string, +) (eventIDs []string, err error) { + rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) + if err != nil { + return + } + defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") + + for rows.Next() { + var eID string + if err = rows.Scan(&eID); err != nil { + return + } + + eventIDs = append(eventIDs, eID) + } + + return eventIDs, rows.Err() +} + +// DeleteBackwardExtremity removes a backwards extremity for a room, if one existed. +func (s *BackwardsExtremities) DeleteBackwardExtremity( + ctx context.Context, txn *sql.Tx, roomID, knownEventID string, +) (err error) { + _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) + return +} diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index b6f773f04..4219d5603 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -38,7 +38,7 @@ func SetupSyncAPIComponent( base *basecomponent.BaseDendrite, deviceDB devices.Database, accountsDB accounts.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, ) { @@ -61,7 +61,7 @@ func SetupSyncAPIComponent( requestPool := sync.NewRequestPool(syncDB, notifier, accountsDB) roomConsumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, notifier, syncDB, queryAPI, + base.Cfg, base.KafkaConsumer, notifier, syncDB, rsAPI, ) if err = roomConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start room server consumer") @@ -81,5 +81,5 @@ func SetupSyncAPIComponent( logrus.WithError(err).Panicf("failed to start typing server consumer") } - routing.Setup(base.APIMux, requestPool, syncDB, deviceDB, federation, queryAPI, cfg) + routing.Setup(base.APIMux, requestPool, syncDB, deviceDB, federation, rsAPI, cfg) } diff --git a/syncapi/types/types.go b/syncapi/types/types.go index cfd49ff1f..c04fe5219 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -64,8 +64,14 @@ const ( // /sync or /messages, for example. type PaginationToken struct { //Position StreamPosition - Type PaginationTokenType - PDUPosition StreamPosition + Type PaginationTokenType + // For /sync, this is the PDU position. For /messages, this is the topological position (depth). + // TODO: Given how different the positions are depending on the token type, they should probably be renamed + // or use different structs altogether. + PDUPosition StreamPosition + // For /sync, this is the EDU position. For /messages, this is the stream (PDU) position. + // TODO: Given how different the positions are depending on the token type, they should probably be renamed + // or use different structs altogether. EDUTypingPosition StreamPosition } diff --git a/sytest-whitelist b/sytest-whitelist index 439c306c7..c957021cb 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -258,3 +258,9 @@ User can invite remote user to room with version 1 User can invite remote user to room with version 2 User can invite remote user to room with version 3 User can invite remote user to room with version 4 +User can create and send/receive messages in a room with version 5 +local user can join room with version 5 +User can invite local user to room with version 5 +remote user can join room with version 5 +User can invite remote user to room with version 5 +Remote user can backfill in a room with version 5