From d8eb0e1b4fc536c763cf7a3dfa5fc3d4e6692b7e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 28 Jul 2020 17:43:32 +0100 Subject: [PATCH] Initial pass at refactoring config (not finished) --- appservice/appservice.go | 2 +- appservice/consumers/roomserver.go | 4 +- clientapi/auth/password.go | 2 +- clientapi/auth/user_interactive_test.go | 7 +- clientapi/clientapi.go | 4 +- clientapi/routing/auth_fallback.go | 8 +- clientapi/routing/createroom.go | 6 +- clientapi/routing/directory.go | 30 +- clientapi/routing/getevent.go | 4 +- clientapi/routing/login.go | 2 +- clientapi/routing/membership.go | 18 +- clientapi/routing/memberships.go | 2 +- clientapi/routing/profile.go | 16 +- clientapi/routing/redaction.go | 4 +- clientapi/routing/register.go | 42 +- clientapi/routing/register_test.go | 2 +- clientapi/routing/routing.go | 2 +- clientapi/routing/sendevent.go | 6 +- clientapi/routing/threepid.go | 4 +- clientapi/routing/voip.go | 2 +- clientapi/threepid/invites.go | 10 +- clientapi/threepid/threepid.go | 8 +- cmd/dendrite-appservice-server/main.go | 2 +- cmd/dendrite-client-api-server/main.go | 4 +- cmd/dendrite-current-state-server/main.go | 4 +- cmd/dendrite-edu-server/main.go | 2 +- cmd/dendrite-federation-api-server/main.go | 4 +- cmd/dendrite-federation-sender-server/main.go | 2 +- cmd/dendrite-key-server/main.go | 4 +- cmd/dendrite-media-api-server/main.go | 4 +- cmd/dendrite-monolith-server/main.go | 22 +- cmd/dendrite-room-server/main.go | 2 +- cmd/dendrite-server-key-api-server/main.go | 4 +- cmd/dendrite-sync-api-server/main.go | 4 +- cmd/dendrite-user-api-server/main.go | 4 +- cmd/generate-config/main.go | 20 + cmd/mediaapi-integration-tests/main.go | 6 +- cmd/roomserver-integration-tests/main.go | 2 +- cmd/syncserver-integration-tests/main.go | 6 +- federationapi/federationapi.go | 2 +- federationapi/federationapi_test.go | 26 +- federationapi/routing/backfill.go | 2 +- federationapi/routing/invite.go | 6 +- federationapi/routing/join.go | 6 +- federationapi/routing/keys.go | 6 +- federationapi/routing/leave.go | 6 +- federationapi/routing/profile.go | 2 +- federationapi/routing/query.go | 2 +- federationapi/routing/routing.go | 2 +- federationapi/routing/send.go | 2 +- federationapi/routing/threepid.go | 10 +- federationsender/consumers/eduserver.go | 10 +- federationsender/consumers/roomserver.go | 6 +- federationsender/federationsender.go | 20 +- federationsender/internal/api.go | 4 +- internal/config/config.go | 534 ++++-------------- .../{appservice.go => config_appservice.go} | 50 +- internal/config/config_clientapi.go | 87 +++ internal/config/config_currentstate.go | 26 + internal/config/config_eduserver.go | 18 + internal/config/config_federationapi.go | 32 ++ internal/config/config_federationsender.go | 55 ++ internal/config/config_global.go | 174 ++++++ internal/config/config_keyserver.go | 24 + internal/config/config_mediaapi.go | 63 +++ internal/config/config_roomserver.go | 24 + internal/config/config_serverkey.go | 30 + internal/config/config_syncapi.go | 24 + internal/config/config_userapi.go | 33 ++ internal/eventutil/events.go | 6 +- internal/httputil/httpapi.go | 2 +- internal/setup/base.go | 36 +- internal/setup/monolith.go | 8 +- keyserver/keyserver.go | 9 +- mediaapi/mediaapi.go | 4 +- mediaapi/routing/download.go | 18 +- mediaapi/routing/routing.go | 4 +- mediaapi/routing/upload.go | 16 +- roomserver/internal/api.go | 2 +- roomserver/internal/perform_join.go | 12 +- roomserver/internal/perform_leave.go | 12 +- roomserver/roomserver.go | 10 +- roomserver/roomserver_test.go | 18 +- serverkeyapi/serverkeyapi.go | 8 +- serverkeyapi/serverkeyapi_test.go | 18 +- syncapi/consumers/clientapi.go | 4 +- syncapi/consumers/eduserver_sendtodevice.go | 4 +- syncapi/consumers/eduserver_typing.go | 4 +- syncapi/consumers/roomserver.go | 4 +- syncapi/routing/messages.go | 4 +- syncapi/routing/routing.go | 2 +- syncapi/syncapi.go | 4 +- 92 files changed, 1066 insertions(+), 710 deletions(-) create mode 100644 cmd/generate-config/main.go rename internal/config/{appservice.go => config_appservice.go} (87%) create mode 100644 internal/config/config_clientapi.go create mode 100644 internal/config/config_currentstate.go create mode 100644 internal/config/config_eduserver.go create mode 100644 internal/config/config_federationapi.go create mode 100644 internal/config/config_federationsender.go create mode 100644 internal/config/config_global.go create mode 100644 internal/config/config_keyserver.go create mode 100644 internal/config/config_mediaapi.go create mode 100644 internal/config/config_roomserver.go create mode 100644 internal/config/config_serverkey.go create mode 100644 internal/config/config_syncapi.go create mode 100644 internal/config/config_userapi.go diff --git a/appservice/appservice.go b/appservice/appservice.go index 728690414..594b537c4 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -48,7 +48,7 @@ func NewInternalAPI( rsAPI roomserverAPI.RoomserverInternalAPI, ) appserviceAPI.AppServiceQueryAPI { // Create a connection to the appservice postgres DB - appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService), base.Cfg.DbProperties()) + appserviceDB, err := storage.NewDatabase(string(base.Cfg.AppServiceAPI.Database), base.Cfg.AppServiceAPI.DatabaseOptions) if err != nil { logrus.WithError(err).Panicf("failed to connect to appservice db") } diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 4c0156b2c..84f2b9ab8 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -48,7 +48,7 @@ func NewOutputRoomEventConsumer( workerStates []types.ApplicationServiceWorkerState, ) *OutputRoomEventConsumer { consumer := internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputRoomEvent), + Topic: string(cfg.Global.Kafka.Topics.OutputRoomEvent), Consumer: kafkaConsumer, PartitionStore: appserviceDB, } @@ -56,7 +56,7 @@ func NewOutputRoomEventConsumer( roomServerConsumer: &consumer, asDB: appserviceDB, rsAPI: rsAPI, - serverName: string(cfg.Matrix.ServerName), + serverName: string(cfg.Global.ServerName), workerStates: workerStates, } consumer.ProcessMessage = s.onMessage diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index f48149252..d98019550 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -35,7 +35,7 @@ type PasswordRequest struct { // LoginTypePassword implements https://matrix.org/docs/spec/client_server/r0.6.1#password-based type LoginTypePassword struct { GetAccountByPassword GetAccountByPassword - Config *config.Dendrite + Config *config.ClientAPI } func (t *LoginTypePassword) Name() string { diff --git a/clientapi/auth/user_interactive_test.go b/clientapi/auth/user_interactive_test.go index d12652c08..47d1cad36 100644 --- a/clientapi/auth/user_interactive_test.go +++ b/clientapi/auth/user_interactive_test.go @@ -33,8 +33,11 @@ func getAccountByPassword(ctx context.Context, localpart, plaintextPassword stri } func setup() *UserInteractive { - cfg := &config.Dendrite{} - cfg.Matrix.ServerName = serverName + cfg := &config.ClientAPI{ + Matrix: &config.Global{ + ServerName: serverName, + }, + } return NewUserInteractive(getAccountByPassword, cfg) } diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 9ed285a86..f3789521c 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -37,7 +37,7 @@ import ( // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component. func AddPublicRoutes( router *mux.Router, - cfg *config.Dendrite, + cfg *config.ClientAPI, producer sarama.SyncProducer, deviceDB devices.Database, accountsDB accounts.Database, @@ -54,7 +54,7 @@ func AddPublicRoutes( ) { syncProducer := &producers.SyncAPIProducer{ Producer: producer, - Topic: string(cfg.Kafka.Topics.OutputClientData), + Topic: string(cfg.Matrix.Kafka.Topics.OutputClientData), } routing.Setup( diff --git a/clientapi/routing/auth_fallback.go b/clientapi/routing/auth_fallback.go index b7f2cd6d3..e639b1015 100644 --- a/clientapi/routing/auth_fallback.go +++ b/clientapi/routing/auth_fallback.go @@ -101,7 +101,7 @@ func serveTemplate(w http.ResponseWriter, templateHTML string, data map[string]s // AuthFallback implements GET and POST /auth/{authType}/fallback/web?session={sessionID} func AuthFallback( w http.ResponseWriter, req *http.Request, authType string, - cfg *config.Dendrite, + cfg *config.ClientAPI, ) *util.JSONResponse { sessionID := req.URL.Query().Get("session") @@ -116,7 +116,7 @@ func AuthFallback( data := map[string]string{ "myUrl": req.URL.String(), "session": sessionID, - "siteKey": cfg.Matrix.RecaptchaPublicKey, + "siteKey": cfg.RecaptchaPublicKey, } serveTemplate(w, recaptchaTemplate, data) } @@ -181,11 +181,11 @@ func AuthFallback( // checkRecaptchaEnabled creates an error response if recaptcha is not usable on homeserver. func checkRecaptchaEnabled( - cfg *config.Dendrite, + cfg *config.ClientAPI, w http.ResponseWriter, req *http.Request, ) *util.JSONResponse { - if !cfg.Matrix.RecaptchaEnabled { + if !cfg.RecaptchaEnabled { return writeHTTPMessage(w, req, "Recaptcha login is disabled on this Homeserver", http.StatusBadRequest, diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 027a21e76..5412c2223 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -135,7 +135,7 @@ type fledglingEvent struct { // CreateRoom implements /createRoom func CreateRoom( req *http.Request, device *api.Device, - cfg *config.Dendrite, + cfg *config.ClientAPI, accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { @@ -149,7 +149,7 @@ func CreateRoom( // nolint: gocyclo func createRoom( req *http.Request, device *api.Device, - cfg *config.Dendrite, roomID string, + cfg *config.ClientAPI, roomID string, accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { @@ -438,7 +438,7 @@ func createRoom( func buildEvent( builder *gomatrixserverlib.EventBuilder, provider gomatrixserverlib.AuthEventProvider, - cfg *config.Dendrite, + cfg *config.ClientAPI, evTime time.Time, roomVersion gomatrixserverlib.RoomVersion, ) (*gomatrixserverlib.Event, error) { diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 0f78f4a24..11831c1f6 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -47,7 +47,7 @@ func DirectoryRoom( req *http.Request, roomAlias string, federation *gomatrixserverlib.FederationClient, - cfg *config.Dendrite, + cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, fedSenderAPI federationSenderAPI.FederationSenderInternalAPI, ) util.JSONResponse { @@ -116,7 +116,7 @@ func SetLocalAlias( req *http.Request, device *api.Device, alias string, - cfg *config.Dendrite, + cfg *config.ClientAPI, aliasAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('#', alias) @@ -139,22 +139,26 @@ func SetLocalAlias( // TODO: This code should eventually be refactored with: // 1. The new method for checking for things matching an AS's namespace // 2. Using an overall Regex object for all AS's just like we did for usernames - for _, appservice := range cfg.Derived.ApplicationServices { - // Don't prevent AS from creating aliases in its own namespace - // Note that Dendrite uses SenderLocalpart as UserID for AS users - if device.UserID != appservice.SenderLocalpart { - if aliasNamespaces, ok := appservice.NamespaceMap["aliases"]; ok { - for _, namespace := range aliasNamespaces { - if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.ASExclusive("Alias is reserved by an application service"), + + // TODO: What to do with derived? + /* + for _, appservice := range cfg.Derived.ApplicationServices { + // Don't prevent AS from creating aliases in its own namespace + // Note that Dendrite uses SenderLocalpart as UserID for AS users + if device.UserID != appservice.SenderLocalpart { + if aliasNamespaces, ok := appservice.NamespaceMap["aliases"]; ok { + for _, namespace := range aliasNamespaces { + if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.ASExclusive("Alias is reserved by an application service"), + } } } } } } - } + */ var r struct { RoomID string `json:"room_id"` diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go index 2a51db730..c74509f07 100644 --- a/clientapi/routing/getevent.go +++ b/clientapi/routing/getevent.go @@ -30,7 +30,7 @@ type getEventRequest struct { device *userapi.Device roomID string eventID string - cfg *config.Dendrite + cfg *config.ClientAPI federation *gomatrixserverlib.FederationClient requestedEvent gomatrixserverlib.Event } @@ -42,7 +42,7 @@ func GetEvent( device *userapi.Device, roomID string, eventID string, - cfg *config.Dendrite, + cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index 7f47aafff..dae593bf0 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -58,7 +58,7 @@ func passwordLogin() flows { // Login implements GET and POST /login func Login( req *http.Request, accountDB accounts.Database, deviceDB devices.Database, - cfg *config.Dendrite, + cfg *config.ClientAPI, ) util.JSONResponse { if req.Method == http.MethodGet { // TODO: support other forms of login other than password, depending on config options diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 90ddb6995..8303a68ef 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -41,7 +41,7 @@ var errMissingUserID = errors.New("'user_id' must be supplied") func SendBan( req *http.Request, accountDB accounts.Database, device *userapi.Device, - roomID string, cfg *config.Dendrite, + roomID string, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI) @@ -52,7 +52,7 @@ func SendBan( } func sendMembership(ctx context.Context, accountDB accounts.Database, device *userapi.Device, - roomID, membership, reason string, cfg *config.Dendrite, targetUserID string, evTime time.Time, + roomID, membership, reason string, cfg *config.ClientAPI, targetUserID string, evTime time.Time, roomVer gomatrixserverlib.RoomVersion, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI) util.JSONResponse { @@ -94,7 +94,7 @@ func sendMembership(ctx context.Context, accountDB accounts.Database, device *us func SendKick( req *http.Request, accountDB accounts.Database, device *userapi.Device, - roomID string, cfg *config.Dendrite, + roomID string, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, stateAPI currentstateAPI.CurrentStateInternalAPI, ) util.JSONResponse { @@ -135,7 +135,7 @@ func SendKick( func SendUnban( req *http.Request, accountDB accounts.Database, device *userapi.Device, - roomID string, cfg *config.Dendrite, + roomID string, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI) @@ -170,7 +170,7 @@ func SendUnban( func SendInvite( req *http.Request, accountDB accounts.Database, device *userapi.Device, - roomID string, cfg *config.Dendrite, + roomID string, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI) @@ -236,7 +236,7 @@ func buildMembershipEvent( targetUserID, reason string, accountDB accounts.Database, device *userapi.Device, membership, roomID string, isDirect bool, - cfg *config.Dendrite, evTime time.Time, + cfg *config.ClientAPI, evTime time.Time, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) (*gomatrixserverlib.HeaderedEvent, error) { profile, err := loadProfile(ctx, targetUserID, cfg, accountDB, asAPI) @@ -263,7 +263,7 @@ func buildMembershipEvent( return nil, err } - return eventutil.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) + return eventutil.BuildEvent(ctx, &builder, cfg.Matrix, evTime, rsAPI, nil) } // loadProfile lookups the profile of a given user from the database and returns @@ -273,7 +273,7 @@ func buildMembershipEvent( func loadProfile( ctx context.Context, userID string, - cfg *config.Dendrite, + cfg *config.ClientAPI, accountDB accounts.Database, asAPI appserviceAPI.AppServiceQueryAPI, ) (*authtypes.Profile, error) { @@ -326,7 +326,7 @@ func checkAndProcessThreepid( req *http.Request, device *userapi.Device, body *threepid.MembershipRequest, - cfg *config.Dendrite, + cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, accountDB accounts.Database, roomID string, diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index 9c4cf7497..560593500 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -48,7 +48,7 @@ type joinedMember struct { // GetMemberships implements GET /rooms/{roomId}/members func GetMemberships( req *http.Request, device *userapi.Device, roomID string, joinedOnly bool, - _ *config.Dendrite, + _ *config.ClientAPI, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { queryReq := api.QueryMembershipsForRoomRequest{ diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 1df4c9b33..faf92451e 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -37,7 +37,7 @@ import ( // GetProfile implements GET /profile/{userID} func GetProfile( - req *http.Request, accountDB accounts.Database, cfg *config.Dendrite, + req *http.Request, accountDB accounts.Database, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceQueryAPI, federation *gomatrixserverlib.FederationClient, @@ -66,7 +66,7 @@ func GetProfile( // GetAvatarURL implements GET /profile/{userID}/avatar_url func GetAvatarURL( - req *http.Request, accountDB accounts.Database, cfg *config.Dendrite, + req *http.Request, accountDB accounts.Database, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceQueryAPI, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { @@ -95,7 +95,7 @@ func GetAvatarURL( // nolint:gocyclo func SetAvatarURL( req *http.Request, accountDB accounts.Database, stateAPI currentstateAPI.CurrentStateInternalAPI, - device *userapi.Device, userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, + device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -184,7 +184,7 @@ func SetAvatarURL( // GetDisplayName implements GET /profile/{userID}/displayname func GetDisplayName( - req *http.Request, accountDB accounts.Database, cfg *config.Dendrite, + req *http.Request, accountDB accounts.Database, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceQueryAPI, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { @@ -213,7 +213,7 @@ func GetDisplayName( // nolint:gocyclo func SetDisplayName( req *http.Request, accountDB accounts.Database, stateAPI currentstateAPI.CurrentStateInternalAPI, - device *userapi.Device, userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, + device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -305,7 +305,7 @@ func SetDisplayName( // Returns an error when something goes wrong or specifically // eventutil.ErrProfileNoExists when the profile doesn't exist. func getProfile( - ctx context.Context, accountDB accounts.Database, cfg *config.Dendrite, + ctx context.Context, accountDB accounts.Database, cfg *config.ClientAPI, userID string, asAPI appserviceAPI.AppServiceQueryAPI, federation *gomatrixserverlib.FederationClient, @@ -345,7 +345,7 @@ func getProfile( func buildMembershipEvents( ctx context.Context, roomIDs []string, - newProfile authtypes.Profile, userID string, cfg *config.Dendrite, + newProfile authtypes.Profile, userID string, cfg *config.ClientAPI, evTime time.Time, rsAPI api.RoomserverInternalAPI, ) ([]gomatrixserverlib.HeaderedEvent, error) { evs := []gomatrixserverlib.HeaderedEvent{} @@ -375,7 +375,7 @@ func buildMembershipEvents( return nil, err } - event, err := eventutil.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) + event, err := eventutil.BuildEvent(ctx, &builder, cfg.Matrix, evTime, rsAPI, nil) if err != nil { return nil, err } diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go index fd80e0ab4..bb5265135 100644 --- a/clientapi/routing/redaction.go +++ b/clientapi/routing/redaction.go @@ -40,7 +40,7 @@ type redactionResponse struct { } func SendRedaction( - req *http.Request, device *userapi.Device, roomID, eventID string, cfg *config.Dendrite, + req *http.Request, device *userapi.Device, roomID, eventID string, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI, ) util.JSONResponse { resErr := checkMemberInRoom(req.Context(), stateAPI, device.UserID, roomID) @@ -115,7 +115,7 @@ func SendRedaction( } var queryRes api.QueryLatestEventsAndStateResponse - e, err := eventutil.BuildEvent(req.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) + e, err := eventutil.BuildEvent(req.Context(), &builder, cfg.Matrix, time.Now(), rsAPI, &queryRes) if err == eventutil.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 69ebdfd70..937abc83d 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -255,11 +255,11 @@ func validatePassword(password string) *util.JSONResponse { // validateRecaptcha returns an error response if the captcha response is invalid func validateRecaptcha( - cfg *config.Dendrite, + cfg *config.ClientAPI, response string, clientip string, ) *util.JSONResponse { - if !cfg.Matrix.RecaptchaEnabled { + if !cfg.RecaptchaEnabled { return &util.JSONResponse{ Code: http.StatusConflict, JSON: jsonerror.Unknown("Captcha registration is disabled"), @@ -274,9 +274,9 @@ func validateRecaptcha( } // Make a POST request to Google's API to check the captcha response - resp, err := http.PostForm(cfg.Matrix.RecaptchaSiteVerifyAPI, + resp, err := http.PostForm(cfg.RecaptchaSiteVerifyAPI, url.Values{ - "secret": {cfg.Matrix.RecaptchaPrivateKey}, + "secret": {cfg.RecaptchaPrivateKey}, "response": {response}, "remoteip": {clientip}, }, @@ -324,7 +324,7 @@ func validateRecaptcha( // Application Service is given, it will check to see if it matches any // Application Service's namespace. func UserIDIsWithinApplicationServiceNamespace( - cfg *config.Dendrite, + cfg *config.ClientAPI, userID string, appservice *config.ApplicationService, ) bool { @@ -354,7 +354,7 @@ func UserIDIsWithinApplicationServiceNamespace( // UsernameMatchesMultipleExclusiveNamespaces will check if a given username matches // more than one exclusive namespace. More than one is not allowed func UsernameMatchesMultipleExclusiveNamespaces( - cfg *config.Dendrite, + cfg *config.ClientAPI, username string, ) bool { userID := userutil.MakeUserID(username, cfg.Matrix.ServerName) @@ -374,7 +374,7 @@ func UsernameMatchesMultipleExclusiveNamespaces( // UsernameMatchesExclusiveNamespaces will check if a given username matches any // application service's exclusive users namespace func UsernameMatchesExclusiveNamespaces( - cfg *config.Dendrite, + cfg *config.ClientAPI, username string, ) bool { userID := userutil.MakeUserID(username, cfg.Matrix.ServerName) @@ -386,7 +386,7 @@ func UsernameMatchesExclusiveNamespaces( // username is within that application service's namespace. As long as these // two requirements are met, no error will be returned. func validateApplicationService( - cfg *config.Dendrite, + cfg *config.ClientAPI, username string, accessToken string, ) (string, *util.JSONResponse) { @@ -442,7 +442,7 @@ func Register( req *http.Request, userAPI userapi.UserInternalAPI, accountDB accounts.Database, - cfg *config.Dendrite, + cfg *config.ClientAPI, ) util.JSONResponse { var r registerRequest resErr := httputil.UnmarshalJSONRequest(req, &r) @@ -512,7 +512,7 @@ func Register( func handleGuestRegistration( req *http.Request, r registerRequest, - cfg *config.Dendrite, + cfg *config.ClientAPI, userAPI userapi.UserInternalAPI, ) util.JSONResponse { var res userapi.PerformAccountCreationResponse @@ -568,7 +568,7 @@ func handleRegistrationFlow( req *http.Request, r registerRequest, sessionID string, - cfg *config.Dendrite, + cfg *config.ClientAPI, userAPI userapi.UserInternalAPI, ) util.JSONResponse { // TODO: Shared secret registration (create new user scripts) @@ -580,7 +580,7 @@ func handleRegistrationFlow( // TODO: email / msisdn auth types. - if cfg.Matrix.RegistrationDisabled && r.Auth.Type != authtypes.LoginTypeSharedSecret { + if cfg.RegistrationDisabled && r.Auth.Type != authtypes.LoginTypeSharedSecret { return util.MessageResponse(http.StatusForbidden, "Registration has been disabled") } @@ -666,7 +666,7 @@ func handleApplicationServiceRegistration( tokenErr error, req *http.Request, r registerRequest, - cfg *config.Dendrite, + cfg *config.ClientAPI, userAPI userapi.UserInternalAPI, ) util.JSONResponse { // Check if we previously had issues extracting the access token from the @@ -704,7 +704,7 @@ func checkAndCompleteFlow( req *http.Request, r registerRequest, sessionID string, - cfg *config.Dendrite, + cfg *config.ClientAPI, userAPI userapi.UserInternalAPI, ) util.JSONResponse { if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) { @@ -728,7 +728,7 @@ func checkAndCompleteFlow( func LegacyRegister( req *http.Request, userAPI userapi.UserInternalAPI, - cfg *config.Dendrite, + cfg *config.ClientAPI, ) util.JSONResponse { var r legacyRegisterRequest resErr := parseAndValidateLegacyLogin(req, &r) @@ -742,13 +742,13 @@ func LegacyRegister( "auth.type": r.Type, }).Info("Processing registration request") - if cfg.Matrix.RegistrationDisabled && r.Type != authtypes.LoginTypeSharedSecret { + if cfg.RegistrationDisabled && r.Type != authtypes.LoginTypeSharedSecret { return util.MessageResponse(http.StatusForbidden, "Registration has been disabled") } switch r.Type { case authtypes.LoginTypeSharedSecret: - if cfg.Matrix.RegistrationSharedSecret == "" { + if cfg.RegistrationSharedSecret == "" { return util.MessageResponse(http.StatusBadRequest, "Shared secret registration is disabled") } @@ -902,15 +902,15 @@ func completeRegistration( // Used for shared secret registration. // Checks if the username, password and isAdmin flag matches the given mac. func isValidMacLogin( - cfg *config.Dendrite, + cfg *config.ClientAPI, username, password string, isAdmin bool, givenMac []byte, ) (bool, error) { - sharedSecret := cfg.Matrix.RegistrationSharedSecret + sharedSecret := cfg.RegistrationSharedSecret // Check that shared secret registration isn't disabled. - if cfg.Matrix.RegistrationSharedSecret == "" { + if cfg.RegistrationSharedSecret == "" { return false, errors.New("Shared secret registration is disabled") } @@ -1001,7 +1001,7 @@ type availableResponse struct { // RegisterAvailable checks if the username is already taken or invalid. func RegisterAvailable( req *http.Request, - cfg *config.Dendrite, + cfg *config.ClientAPI, accountDB accounts.Database, ) util.JSONResponse { username := req.URL.Query().Get("username") diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index a44389f94..e10583b1c 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -179,7 +179,7 @@ func TestValidationOfApplicationServices(t *testing.T) { } // Set up a config - fakeConfig := config.Dendrite{} + fakeConfig := config.ClientAPI{} fakeConfig.Matrix.ServerName = "localhost" fakeConfig.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index ebb141ef8..57aefd0ad 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -51,7 +51,7 @@ const pathPrefixUnstable = "/client/unstable" // applied: // nolint: gocyclo func Setup( - publicAPIMux *mux.Router, cfg *config.Dendrite, + publicAPIMux *mux.Router, cfg *config.ClientAPI, eduAPI eduServerAPI.EDUServerInputAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index bf32992f5..e0cd7eb5d 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -43,7 +43,7 @@ func SendEvent( req *http.Request, device *userapi.Device, roomID, eventType string, txnID, stateKey *string, - cfg *config.Dendrite, + cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI, txnCache *transactions.Cache, ) util.JSONResponse { @@ -112,7 +112,7 @@ func generateSendEvent( req *http.Request, device *userapi.Device, roomID, eventType string, stateKey *string, - cfg *config.Dendrite, + cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI, ) (*gomatrixserverlib.Event, *util.JSONResponse) { // parse the incoming http request @@ -146,7 +146,7 @@ func generateSendEvent( } var queryRes api.QueryLatestEventsAndStateResponse - e, err := eventutil.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes) + e, err := eventutil.BuildEvent(req.Context(), &builder, cfg.Matrix, evTime, rsAPI, &queryRes) if err == eventutil.ErrRoomNoExists { return nil, &util.JSONResponse{ Code: http.StatusNotFound, diff --git a/clientapi/routing/threepid.go b/clientapi/routing/threepid.go index e7aaadf54..54ffa53f6 100644 --- a/clientapi/routing/threepid.go +++ b/clientapi/routing/threepid.go @@ -40,7 +40,7 @@ type threePIDsResponse struct { // RequestEmailToken implements: // POST /account/3pid/email/requestToken // POST /register/email/requestToken -func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *config.Dendrite) util.JSONResponse { +func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *config.ClientAPI) util.JSONResponse { var body threepid.EmailAssociationRequest if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { return *reqErr @@ -86,7 +86,7 @@ func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *conf // CheckAndSave3PIDAssociation implements POST /account/3pid func CheckAndSave3PIDAssociation( req *http.Request, accountDB accounts.Database, device *api.Device, - cfg *config.Dendrite, + cfg *config.ClientAPI, ) util.JSONResponse { var body threepid.EmailAssociationCheckRequest if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { diff --git a/clientapi/routing/voip.go b/clientapi/routing/voip.go index 046e87811..536c69fba 100644 --- a/clientapi/routing/voip.go +++ b/clientapi/routing/voip.go @@ -31,7 +31,7 @@ import ( // RequestTurnServer implements: // GET /voip/turnServer -func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.Dendrite) util.JSONResponse { +func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.ClientAPI) util.JSONResponse { turnConfig := cfg.TURN // TODO Guest Support diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 89bc86064..f1d54a47b 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -86,7 +86,7 @@ var ( // can be emitted. func CheckAndProcessInvite( ctx context.Context, - device *userapi.Device, body *MembershipRequest, cfg *config.Dendrite, + device *userapi.Device, body *MembershipRequest, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI, db accounts.Database, roomID string, evTime time.Time, @@ -137,7 +137,7 @@ func CheckAndProcessInvite( // Returns an error if a check or a request failed. func queryIDServer( ctx context.Context, - db accounts.Database, cfg *config.Dendrite, device *userapi.Device, + db accounts.Database, cfg *config.ClientAPI, device *userapi.Device, body *MembershipRequest, roomID string, ) (lookupRes *idServerLookupResponse, storeInviteRes *idServerStoreInviteResponse, err error) { if err = isTrusted(body.IDServer, cfg); err != nil { @@ -206,7 +206,7 @@ func queryIDServerLookup(ctx context.Context, body *MembershipRequest) (*idServe // Returns an error if the request failed to send or if the response couldn't be parsed. func queryIDServerStoreInvite( ctx context.Context, - db accounts.Database, cfg *config.Dendrite, device *userapi.Device, + db accounts.Database, cfg *config.ClientAPI, device *userapi.Device, body *MembershipRequest, roomID string, ) (*idServerStoreInviteResponse, error) { // Retrieve the sender's profile to get their display name @@ -330,7 +330,7 @@ func checkIDServerSignatures( func emit3PIDInviteEvent( ctx context.Context, body *MembershipRequest, res *idServerStoreInviteResponse, - device *userapi.Device, roomID string, cfg *config.Dendrite, + device *userapi.Device, roomID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI, evTime time.Time, ) error { @@ -354,7 +354,7 @@ func emit3PIDInviteEvent( } queryRes := api.QueryLatestEventsAndStateResponse{} - event, err := eventutil.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes) + event, err := eventutil.BuildEvent(ctx, builder, cfg.Matrix, evTime, rsAPI, &queryRes) if err != nil { return err } diff --git a/clientapi/threepid/threepid.go b/clientapi/threepid/threepid.go index bffe31adc..40fd161d6 100644 --- a/clientapi/threepid/threepid.go +++ b/clientapi/threepid/threepid.go @@ -53,7 +53,7 @@ type Credentials struct { // Returns an error if there was a problem sending the request or decoding the // response, or if the identity server responded with a non-OK status. func CreateSession( - ctx context.Context, req EmailAssociationRequest, cfg *config.Dendrite, + ctx context.Context, req EmailAssociationRequest, cfg *config.ClientAPI, ) (string, error) { if err := isTrusted(req.IDServer, cfg); err != nil { return "", err @@ -101,7 +101,7 @@ func CreateSession( // Returns an error if there was a problem sending the request or decoding the // response, or if the identity server responded with a non-OK status. func CheckAssociation( - ctx context.Context, creds Credentials, cfg *config.Dendrite, + ctx context.Context, creds Credentials, cfg *config.ClientAPI, ) (bool, string, string, error) { if err := isTrusted(creds.IDServer, cfg); err != nil { return false, "", "", err @@ -142,7 +142,7 @@ func CheckAssociation( // identifier and a Matrix ID. // Returns an error if there was a problem sending the request or decoding the // response, or if the identity server responded with a non-OK status. -func PublishAssociation(creds Credentials, userID string, cfg *config.Dendrite) error { +func PublishAssociation(creds Credentials, userID string, cfg *config.ClientAPI) error { if err := isTrusted(creds.IDServer, cfg); err != nil { return err } @@ -177,7 +177,7 @@ func PublishAssociation(creds Credentials, userID string, cfg *config.Dendrite) // isTrusted checks if a given identity server is part of the list of trusted // identity servers in the configuration file. // Returns an error if the server isn't trusted. -func isTrusted(idServer string, cfg *config.Dendrite) error { +func isTrusted(idServer string, cfg *config.ClientAPI) error { for _, server := range cfg.Matrix.TrustedIDServers { if idServer == server { return nil diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index 6719d0471..632b45e65 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -30,6 +30,6 @@ func main() { intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) appservice.AddInternalRoutes(base.InternalAPIMux, intAPI) - base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI)) + base.SetupAndServeHTTP(string(base.Cfg.AppServiceAPI.Bind), string(base.Cfg.AppServiceAPI.Listen)) } diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 367f27d15..2b6c39b62 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -39,10 +39,10 @@ func main() { keyAPI := base.KeyServerHTTPClient() clientapi.AddPublicRoutes( - base.PublicAPIMux, base.Cfg, base.KafkaProducer, deviceDB, accountDB, federation, + base.PublicAPIMux, &base.Cfg.ClientAPI, base.KafkaProducer, deviceDB, accountDB, federation, rsAPI, eduInputAPI, asQuery, stateAPI, transactions.New(), fsAPI, userAPI, keyAPI, nil, ) - base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI)) + base.SetupAndServeHTTP(string(base.Cfg.ClientAPI.Bind), string(base.Cfg.ClientAPI.Listen)) } diff --git a/cmd/dendrite-current-state-server/main.go b/cmd/dendrite-current-state-server/main.go index 0d4eae7b5..a8c0813f0 100644 --- a/cmd/dendrite-current-state-server/main.go +++ b/cmd/dendrite-current-state-server/main.go @@ -24,10 +24,10 @@ func main() { base := setup.NewBaseDendrite(cfg, "CurrentStateServer", true) defer base.Close() // nolint: errcheck - stateAPI := currentstateserver.NewInternalAPI(cfg, base.KafkaConsumer) + stateAPI := currentstateserver.NewInternalAPI(&cfg.CurrentStateServer, base.KafkaConsumer) currentstateserver.AddInternalRoutes(base.InternalAPIMux, stateAPI) - base.SetupAndServeHTTP(string(base.Cfg.Bind.CurrentState), string(base.Cfg.Listen.CurrentState)) + base.SetupAndServeHTTP(string(base.Cfg.CurrentStateServer.Bind), string(base.Cfg.CurrentStateServer.Listen)) } diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-edu-server/main.go index 6704ebd09..d3e4e0a0d 100644 --- a/cmd/dendrite-edu-server/main.go +++ b/cmd/dendrite-edu-server/main.go @@ -33,6 +33,6 @@ func main() { intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient()) eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI) - base.SetupAndServeHTTP(string(base.Cfg.Bind.EDUServer), string(base.Cfg.Listen.EDUServer)) + base.SetupAndServeHTTP(string(base.Cfg.EDUServer.Bind), string(base.Cfg.EDUServer.Listen)) } diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index 70d8394f5..3b12a295f 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -33,10 +33,10 @@ func main() { keyAPI := base.KeyServerHTTPClient() federationapi.AddPublicRoutes( - base.PublicAPIMux, base.Cfg, userAPI, federation, keyRing, + base.PublicAPIMux, &base.Cfg.FederationAPI, userAPI, federation, keyRing, rsAPI, fsAPI, base.EDUServerClient(), base.CurrentStateAPIClient(), keyAPI, ) - base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) + base.SetupAndServeHTTP(string(base.Cfg.FederationAPI.Bind), string(base.Cfg.FederationAPI.Listen)) } diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 20bc1070f..f75bc3143 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -35,6 +35,6 @@ func main() { ) federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI) - base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender)) + base.SetupAndServeHTTP(string(base.Cfg.FederationSender.Bind), string(base.Cfg.FederationSender.Listen)) } diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-key-server/main.go index d58d475a2..669808b2a 100644 --- a/cmd/dendrite-key-server/main.go +++ b/cmd/dendrite-key-server/main.go @@ -24,10 +24,10 @@ func main() { base := setup.NewBaseDendrite(cfg, "KeyServer", true) defer base.Close() // nolint: errcheck - intAPI := keyserver.NewInternalAPI(base.Cfg, base.CreateFederationClient(), base.UserAPIClient(), base.KafkaProducer) + intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient(), base.UserAPIClient(), base.KafkaProducer) keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI) - base.SetupAndServeHTTP(string(base.Cfg.Bind.KeyServer), string(base.Cfg.Listen.KeyServer)) + base.SetupAndServeHTTP(string(base.Cfg.KeyServer.Bind), string(base.Cfg.KeyServer.Listen)) } diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go index 1582a33a8..cada8b742 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-media-api-server/main.go @@ -28,8 +28,8 @@ func main() { userAPI := base.UserAPIClient() client := gomatrixserverlib.NewClient() - mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, userAPI, client) + mediaapi.AddPublicRoutes(base.PublicAPIMux, &base.Cfg.MediaAPI, userAPI, client) - base.SetupAndServeHTTP(string(base.Cfg.Bind.MediaAPI), string(base.Cfg.Listen.MediaAPI)) + base.SetupAndServeHTTP(string(base.Cfg.MediaAPI.Bind), string(base.Cfg.MediaAPI.Listen)) } diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index b312579cb..289aa7754 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -54,11 +54,13 @@ func main() { // the API endpoints. They'll listen on the same port as the monolith // itself. addr := config.Address(*httpBindAddr) - cfg.Listen.RoomServer = addr - cfg.Listen.EDUServer = addr - cfg.Listen.AppServiceAPI = addr - cfg.Listen.FederationSender = addr - cfg.Listen.ServerKeyAPI = addr + cfg.RoomServer.Listen = addr + cfg.EDUServer.Listen = addr + cfg.AppServiceAPI.Listen = addr + cfg.FederationSender.Listen = addr + cfg.ServerKeyAPI.Listen = addr + cfg.CurrentStateServer.Listen = addr + cfg.KeyServer.Listen = addr } base := setup.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs) @@ -69,14 +71,14 @@ func main() { federation := base.CreateFederationClient() serverKeyAPI := serverkeyapi.NewInternalAPI( - base.Cfg, federation, base.Caches, + &base.Cfg.ServerKeyAPI, federation, base.Caches, ) if base.UseHTTPAPIs { serverkeyapi.AddInternalRoutes(base.InternalAPIMux, serverKeyAPI, base.Caches) serverKeyAPI = base.ServerKeyAPIClient() } keyRing := serverKeyAPI.KeyRing() - userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices) + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, cfg.Derived.ApplicationServices) rsImpl := roomserver.NewInternalAPI( base, keyRing, federation, @@ -118,8 +120,8 @@ func main() { // This is different to rsAPI which can be the http client which doesn't need this dependency rsImpl.SetFederationSenderAPI(fsAPI) - stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer) - keyAPI := keyserver.NewInternalAPI(base.Cfg, federation, userAPI, base.KafkaProducer) + stateAPI := currentstateserver.NewInternalAPI(&base.Cfg.CurrentStateServer, base.KafkaConsumer) + keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation, userAPI, base.KafkaProducer) monolith := setup.Monolith{ Config: base.Cfg, @@ -146,7 +148,7 @@ func main() { base.BaseMux, base.PublicAPIMux, base.InternalAPIMux, - cfg, + &cfg.Global, base.UseHTTPAPIs, ) diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index 627a68677..e1d4b16ba 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -33,6 +33,6 @@ func main() { rsAPI.SetFederationSenderAPI(fsAPI) roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI) - base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer)) + base.SetupAndServeHTTP(string(base.Cfg.RoomServer.Bind), string(base.Cfg.RoomServer.Listen)) } diff --git a/cmd/dendrite-server-key-api-server/main.go b/cmd/dendrite-server-key-api-server/main.go index 9ffaeee31..36f34890b 100644 --- a/cmd/dendrite-server-key-api-server/main.go +++ b/cmd/dendrite-server-key-api-server/main.go @@ -26,8 +26,8 @@ func main() { federation := base.CreateFederationClient() - intAPI := serverkeyapi.NewInternalAPI(base.Cfg, federation, base.Caches) + intAPI := serverkeyapi.NewInternalAPI(&base.Cfg.ServerKeyAPI, federation, base.Caches) serverkeyapi.AddInternalRoutes(base.InternalAPIMux, intAPI, base.Caches) - base.SetupAndServeHTTP(string(base.Cfg.Bind.ServerKeyAPI), string(base.Cfg.Listen.ServerKeyAPI)) + base.SetupAndServeHTTP(string(base.Cfg.ServerKeyAPI.Bind), string(base.Cfg.ServerKeyAPI.Listen)) } diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index d67395fb3..39d48a6bd 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -29,8 +29,8 @@ func main() { rsAPI := base.RoomserverHTTPClient() - syncapi.AddPublicRoutes(base.PublicAPIMux, base.KafkaConsumer, userAPI, rsAPI, federation, cfg) + syncapi.AddPublicRoutes(base.PublicAPIMux, base.KafkaConsumer, userAPI, rsAPI, federation, &cfg.SyncAPI) - base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI)) + base.SetupAndServeHTTP(string(base.Cfg.SyncAPI.Bind), string(base.Cfg.SyncAPI.Listen)) } diff --git a/cmd/dendrite-user-api-server/main.go b/cmd/dendrite-user-api-server/main.go index 4257da3f3..4655cd09a 100644 --- a/cmd/dendrite-user-api-server/main.go +++ b/cmd/dendrite-user-api-server/main.go @@ -27,9 +27,9 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() - userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices) + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, cfg.Derived.ApplicationServices) userapi.AddInternalRoutes(base.InternalAPIMux, userAPI) - base.SetupAndServeHTTP(string(base.Cfg.Bind.UserAPI), string(base.Cfg.Listen.UserAPI)) + base.SetupAndServeHTTP(string(base.Cfg.UserAPI.Bind), string(base.Cfg.UserAPI.Listen)) } diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go new file mode 100644 index 000000000..024fafaee --- /dev/null +++ b/cmd/generate-config/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "fmt" + + "github.com/matrix-org/dendrite/internal/config" + "gopkg.in/yaml.v2" +) + +func main() { + config := &config.Dendrite{} + config.Defaults() + + j, err := yaml.Marshal(config) + if err != nil { + panic(err) + } + + fmt.Println(string(j)) +} diff --git a/cmd/mediaapi-integration-tests/main.go b/cmd/mediaapi-integration-tests/main.go index e6ce14d28..5cfb3cb07 100644 --- a/cmd/mediaapi-integration-tests/main.go +++ b/cmd/mediaapi-integration-tests/main.go @@ -88,9 +88,9 @@ func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error if err != nil { panic(err) } - cfg.Matrix.ServerName = gomatrixserverlib.ServerName(proxyAddr) - cfg.Media.DynamicThumbnails = dynamicThumbnails - if err = yaml.Unmarshal([]byte(thumbnailSizes), &cfg.Media.ThumbnailSizes); err != nil { + cfg.Global.ServerName = gomatrixserverlib.ServerName(proxyAddr) + cfg.MediaAPI.DynamicThumbnails = dynamicThumbnails + if err = yaml.Unmarshal([]byte(thumbnailSizes), &cfg.MediaAPI.ThumbnailSizes); err != nil { panic(err) } diff --git a/cmd/roomserver-integration-tests/main.go b/cmd/roomserver-integration-tests/main.go index 3860ca1f7..0ff3cd139 100644 --- a/cmd/roomserver-integration-tests/main.go +++ b/cmd/roomserver-integration-tests/main.go @@ -240,7 +240,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R panic(err) } - outputTopic := string(cfg.Kafka.Topics.OutputRoomEvent) + outputTopic := string(cfg.Global.Kafka.Topics.OutputRoomEvent) err = exe.DeleteTopic(outputTopic) if err != nil { diff --git a/cmd/syncserver-integration-tests/main.go b/cmd/syncserver-integration-tests/main.go index cfe8cc165..62c15e364 100644 --- a/cmd/syncserver-integration-tests/main.go +++ b/cmd/syncserver-integration-tests/main.go @@ -132,10 +132,10 @@ func startSyncServer() (*exec.Cmd, chan error) { panic(err) } // TODO use the address assigned by the config generator rather than clobbering. - cfg.Matrix.ServerName = "localhost" + cfg.Global.ServerName = "localhost" cfg.Listen.SyncAPI = config.Address(syncserverAddr) - cfg.Kafka.Topics.OutputRoomEvent = config.Topic(inputTopic) - cfg.Kafka.Topics.OutputClientData = config.Topic(clientTopic) + cfg.Global.Kafka.Topics.OutputRoomEvent = config.Topic(inputTopic) + cfg.Global.Kafka.Topics.OutputClientData = config.Topic(clientTopic) if err := test.WriteConfig(cfg, dir); err != nil { panic(err) diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index 079f333a4..e9a8e40a9 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -31,7 +31,7 @@ import ( // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component. func AddPublicRoutes( router *mux.Router, - cfg *config.Dendrite, + cfg *config.FederationAPI, userAPI userapi.UserInternalAPI, federation *gomatrixserverlib.FederationClient, keyRing gomatrixserverlib.JSONVerifier, diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index 8bc4277eb..a9d42ecdf 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -19,31 +19,37 @@ import ( // Relevant for v3 rooms and a cause of flakey sytests as the IDs are randomly generated. func TestRoomsV3URLEscapeDoNot404(t *testing.T) { _, privKey, _ := ed25519.GenerateKey(nil) - cfg := &config.Dendrite{} - cfg.Matrix.KeyID = gomatrixserverlib.KeyID("ed25519:auto") - cfg.Matrix.ServerName = gomatrixserverlib.ServerName("localhost") - cfg.Matrix.PrivateKey = privKey - cfg.Kafka.UseNaffka = true - cfg.Database.Naffka = "file::memory:" - cfg.SetDefaults() + cfg := &config.Dendrite{ + Global: config.Global{ + KeyID: gomatrixserverlib.KeyID("ed25519:auto"), + ServerName: gomatrixserverlib.ServerName("localhost"), + PrivateKey: privKey, + }, + FederationSender: config.FederationSender{ + Database: "file::memory:", + }, + } + cfg.FederationSender.Matrix = &cfg.Global + cfg.Global.Kafka.UseNaffka = true + // TODO: cfg.SetDefaults() base := setup.NewBaseDendrite(cfg, "Test", false) keyRing := &test.NopJSONVerifier{} fsAPI := base.FederationSenderHTTPClient() // TODO: This is pretty fragile, as if anything calls anything on these nils this test will break. // Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing. - federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil, nil, nil) + federationapi.AddPublicRoutes(base.PublicAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, nil, nil) httputil.SetupHTTPAPI( base.BaseMux, base.PublicAPIMux, base.InternalAPIMux, - cfg, + &cfg.Global, base.UseHTTPAPIs, ) baseURL, cancel := test.ListenAndServe(t, base.BaseMux, true) defer cancel() serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://")) - fedCli := gomatrixserverlib.NewFederationClient(serverName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey) + fedCli := gomatrixserverlib.NewFederationClient(serverName, cfg.Global.KeyID, cfg.Global.PrivateKey) testCases := []struct { roomVer gomatrixserverlib.RoomVersion diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index f906c73c9..ea77c947f 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -35,7 +35,7 @@ func Backfill( request *gomatrixserverlib.FederationRequest, rsAPI api.RoomserverInternalAPI, roomID string, - cfg *config.Dendrite, + cfg *config.FederationAPI, ) util.JSONResponse { var res api.PerformBackfillResponse var eIDs []string diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 4a49463a2..3f9661eef 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -34,7 +34,7 @@ func InviteV2( request *gomatrixserverlib.FederationRequest, roomID string, eventID string, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, keys gomatrixserverlib.JSONVerifier, ) util.JSONResponse { @@ -56,7 +56,7 @@ func InviteV1( request *gomatrixserverlib.FederationRequest, roomID string, eventID string, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, keys gomatrixserverlib.JSONVerifier, ) util.JSONResponse { @@ -86,7 +86,7 @@ func processInvite( strippedState []gomatrixserverlib.InviteV2StrippedState, roomID string, eventID string, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, keys gomatrixserverlib.JSONVerifier, ) util.JSONResponse { diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 17981f532..4874f4d1c 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -32,7 +32,7 @@ import ( func MakeJoin( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, roomID, userID string, remoteVersions []gomatrixserverlib.RoomVersion, @@ -95,7 +95,7 @@ func MakeJoin( queryRes := api.QueryLatestEventsAndStateResponse{ RoomVersion: verRes.RoomVersion, } - event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) + event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg.Matrix, time.Now(), rsAPI, &queryRes) if err == eventutil.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, @@ -141,7 +141,7 @@ func MakeJoin( func SendJoin( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, keys gomatrixserverlib.JSONVerifier, roomID, eventID string, diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go index 90eec9e0e..f1ed4176b 100644 --- a/federationapi/routing/keys.go +++ b/federationapi/routing/keys.go @@ -121,7 +121,7 @@ func ClaimOneTimeKeys( // LocalKeys returns the local keys for the server. // See https://matrix.org/docs/spec/server_server/unstable.html#publishing-keys -func LocalKeys(cfg *config.Dendrite) util.JSONResponse { +func LocalKeys(cfg *config.FederationAPI) util.JSONResponse { keys, err := localKeys(cfg, time.Now().Add(cfg.Matrix.KeyValidityPeriod)) if err != nil { return util.ErrorResponse(err) @@ -129,7 +129,7 @@ func LocalKeys(cfg *config.Dendrite) util.JSONResponse { return util.JSONResponse{Code: http.StatusOK, JSON: keys} } -func localKeys(cfg *config.Dendrite, validUntil time.Time) (*gomatrixserverlib.ServerKeys, error) { +func localKeys(cfg *config.FederationAPI, validUntil time.Time) (*gomatrixserverlib.ServerKeys, error) { var keys gomatrixserverlib.ServerKeys keys.ServerName = cfg.Matrix.ServerName @@ -142,7 +142,7 @@ func localKeys(cfg *config.Dendrite, validUntil time.Time) (*gomatrixserverlib.S }, } - keys.TLSFingerprints = cfg.Matrix.TLSFingerPrints + keys.TLSFingerprints = cfg.TLSFingerPrints keys.OldVerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.OldVerifyKey{} keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(validUntil) diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 56f1b05af..314b4c729 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -28,7 +28,7 @@ import ( func MakeLeave( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, roomID, userID string, ) util.JSONResponse { @@ -60,7 +60,7 @@ func MakeLeave( } var queryRes api.QueryLatestEventsAndStateResponse - event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) + event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg.Matrix, time.Now(), rsAPI, &queryRes) if err == eventutil.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, @@ -102,7 +102,7 @@ func MakeLeave( func SendLeave( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, keys gomatrixserverlib.JSONVerifier, roomID, eventID string, diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go index a6180ae6d..f1d90bbf4 100644 --- a/federationapi/routing/profile.go +++ b/federationapi/routing/profile.go @@ -30,7 +30,7 @@ import ( func GetProfile( httpReq *http.Request, userAPI userapi.UserInternalAPI, - cfg *config.Dendrite, + cfg *config.FederationAPI, ) util.JSONResponse { userID, field := httpReq.FormValue("user_id"), httpReq.FormValue("field") diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 39fd6d2ee..99b5460bc 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -31,7 +31,7 @@ import ( func RoomAliasToID( httpReq *http.Request, federation *gomatrixserverlib.FederationClient, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI roomserverAPI.RoomserverInternalAPI, senderAPI federationSenderAPI.FederationSenderInternalAPI, ) util.JSONResponse { diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 50b7bdd28..c7aa20e55 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -47,7 +47,7 @@ const ( // nolint: gocyclo func Setup( publicAPIMux *mux.Router, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI roomserverAPI.RoomserverInternalAPI, eduAPI eduserverAPI.EDUServerInputAPI, fsAPI federationSenderAPI.FederationSenderInternalAPI, diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 680eaccd3..62fe4da5f 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -34,7 +34,7 @@ func Send( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, txnID gomatrixserverlib.TransactionID, - cfg *config.Dendrite, + cfg *config.FederationAPI, rsAPI api.RoomserverInternalAPI, eduAPI eduserverAPI.EDUServerInputAPI, keys gomatrixserverlib.JSONVerifier, diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 61788010b..e8d9a9397 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -56,7 +56,7 @@ var ( // CreateInvitesFrom3PIDInvites implements POST /_matrix/federation/v1/3pid/onbind func CreateInvitesFrom3PIDInvites( req *http.Request, rsAPI api.RoomserverInternalAPI, - cfg *config.Dendrite, + cfg *config.FederationAPI, federation *gomatrixserverlib.FederationClient, userAPI userapi.UserInternalAPI, ) util.JSONResponse { @@ -106,7 +106,7 @@ func ExchangeThirdPartyInvite( request *gomatrixserverlib.FederationRequest, roomID string, rsAPI api.RoomserverInternalAPI, - cfg *config.Dendrite, + cfg *config.FederationAPI, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { var builder gomatrixserverlib.EventBuilder @@ -196,7 +196,7 @@ func ExchangeThirdPartyInvite( // necessary data to do so. func createInviteFrom3PIDInvite( ctx context.Context, rsAPI api.RoomserverInternalAPI, - cfg *config.Dendrite, + cfg *config.FederationAPI, inv invite, federation *gomatrixserverlib.FederationClient, userAPI userapi.UserInternalAPI, ) (*gomatrixserverlib.Event, error) { @@ -263,7 +263,7 @@ func createInviteFrom3PIDInvite( func buildMembershipEvent( ctx context.Context, builder *gomatrixserverlib.EventBuilder, rsAPI api.RoomserverInternalAPI, - cfg *config.Dendrite, + cfg *config.FederationAPI, ) (*gomatrixserverlib.Event, error) { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) if err != nil { @@ -327,7 +327,7 @@ func buildMembershipEvent( // them responded with an error. func sendToRemoteServer( ctx context.Context, inv invite, - federation *gomatrixserverlib.FederationClient, _ *config.Dendrite, + federation *gomatrixserverlib.FederationClient, _ *config.FederationAPI, builder gomatrixserverlib.EventBuilder, ) (err error) { remoteServers := make([]gomatrixserverlib.ServerName, 2) diff --git a/federationsender/consumers/eduserver.go b/federationsender/consumers/eduserver.go index 161de8c68..dd72f5f20 100644 --- a/federationsender/consumers/eduserver.go +++ b/federationsender/consumers/eduserver.go @@ -41,27 +41,27 @@ type OutputEDUConsumer struct { // NewOutputEDUConsumer creates a new OutputEDUConsumer. Call Start() to begin consuming from EDU servers. func NewOutputEDUConsumer( - cfg *config.Dendrite, + cfg *config.FederationSender, kafkaConsumer sarama.Consumer, queues *queue.OutgoingQueues, store storage.Database, ) *OutputEDUConsumer { c := &OutputEDUConsumer{ typingConsumer: &internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputTypingEvent), + Topic: string(cfg.Matrix.Kafka.Topics.OutputTypingEvent), Consumer: kafkaConsumer, PartitionStore: store, }, sendToDeviceConsumer: &internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputSendToDeviceEvent), + Topic: string(cfg.Matrix.Kafka.Topics.OutputSendToDeviceEvent), Consumer: kafkaConsumer, PartitionStore: store, }, queues: queues, db: store, ServerName: cfg.Matrix.ServerName, - TypingTopic: string(cfg.Kafka.Topics.OutputTypingEvent), - SendToDeviceTopic: string(cfg.Kafka.Topics.OutputSendToDeviceEvent), + TypingTopic: string(cfg.Matrix.Kafka.Topics.OutputTypingEvent), + SendToDeviceTopic: string(cfg.Matrix.Kafka.Topics.OutputSendToDeviceEvent), } c.typingConsumer.ProcessMessage = c.onTypingEvent c.sendToDeviceConsumer.ProcessMessage = c.onSendToDeviceEvent diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 299c7b37a..b3a4cde34 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -33,7 +33,7 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - cfg *config.Dendrite + cfg *config.FederationSender rsAPI api.RoomserverInternalAPI rsConsumer *internal.ContinualConsumer db storage.Database @@ -42,14 +42,14 @@ type OutputRoomEventConsumer struct { // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. func NewOutputRoomEventConsumer( - cfg *config.Dendrite, + cfg *config.FederationSender, kafkaConsumer sarama.Consumer, queues *queue.OutgoingQueues, store storage.Database, rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputRoomEvent), + Topic: string(cfg.Matrix.Kafka.Topics.OutputRoomEvent), Consumer: kafkaConsumer, PartitionStore: store, } diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 9e14f6ec5..4f69c793a 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -43,27 +43,29 @@ func NewInternalAPI( rsAPI roomserverAPI.RoomserverInternalAPI, keyRing *gomatrixserverlib.KeyRing, ) api.FederationSenderInternalAPI { - federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender), base.Cfg.DbProperties()) + cfg := &base.Cfg.FederationSender + + federationSenderDB, err := storage.NewDatabase(string(cfg.Database), cfg.DatabaseOptions) if err != nil { logrus.WithError(err).Panic("failed to connect to federation sender db") } stats := &statistics.Statistics{ DB: federationSenderDB, - FailuresUntilBlacklist: base.Cfg.Matrix.FederationMaxRetries, + FailuresUntilBlacklist: cfg.FederationMaxRetries, } queues := queue.NewOutgoingQueues( - federationSenderDB, base.Cfg.Matrix.ServerName, federation, rsAPI, stats, + federationSenderDB, cfg.Matrix.ServerName, federation, rsAPI, stats, &queue.SigningInfo{ - KeyID: base.Cfg.Matrix.KeyID, - PrivateKey: base.Cfg.Matrix.PrivateKey, - ServerName: base.Cfg.Matrix.ServerName, + KeyID: cfg.Matrix.KeyID, + PrivateKey: cfg.Matrix.PrivateKey, + ServerName: cfg.Matrix.ServerName, }, ) rsConsumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, queues, + cfg, base.KafkaConsumer, queues, federationSenderDB, rsAPI, ) if err = rsConsumer.Start(); err != nil { @@ -71,11 +73,11 @@ func NewInternalAPI( } tsConsumer := consumers.NewOutputEDUConsumer( - base.Cfg, base.KafkaConsumer, queues, federationSenderDB, + cfg, base.KafkaConsumer, queues, federationSenderDB, ) if err := tsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start typing server consumer") } - return internal.NewFederationSenderInternalAPI(federationSenderDB, base.Cfg, rsAPI, federation, keyRing, stats, queues) + return internal.NewFederationSenderInternalAPI(federationSenderDB, cfg, rsAPI, federation, keyRing, stats, queues) } diff --git a/federationsender/internal/api.go b/federationsender/internal/api.go index 9a9880ce1..647e3fcb8 100644 --- a/federationsender/internal/api.go +++ b/federationsender/internal/api.go @@ -12,7 +12,7 @@ import ( // FederationSenderInternalAPI is an implementation of api.FederationSenderInternalAPI type FederationSenderInternalAPI struct { db storage.Database - cfg *config.Dendrite + cfg *config.FederationSender statistics *statistics.Statistics rsAPI api.RoomserverInternalAPI federation *gomatrixserverlib.FederationClient @@ -21,7 +21,7 @@ type FederationSenderInternalAPI struct { } func NewFederationSenderInternalAPI( - db storage.Database, cfg *config.Dendrite, + db storage.Database, cfg *config.FederationSender, rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, keyRing *gomatrixserverlib.KeyRing, diff --git a/internal/config/config.go b/internal/config/config.go index 657e804f6..4b7a92fdd 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -24,7 +24,6 @@ import ( "path/filepath" "regexp" "strings" - "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/gomatrixserverlib" @@ -38,7 +37,7 @@ import ( // Version is the current version of the config format. // This will change whenever we make breaking changes to the config format. -const Version = 0 +const Version = 1 // Dendrite contains all the config used by a dendrite process. // Relative paths are resolved relative to the current working directory @@ -51,214 +50,19 @@ type Dendrite struct { // been a breaking change to the config file format. Version int `yaml:"version"` - // The configuration required for a matrix server. - Matrix struct { - // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. - ServerName gomatrixserverlib.ServerName `yaml:"server_name"` - // Path to the private key which will be used to sign requests and events. - PrivateKeyPath Path `yaml:"private_key"` - // The private key which will be used to sign requests and events. - PrivateKey ed25519.PrivateKey `yaml:"-"` - // An arbitrary string used to uniquely identify the PrivateKey. Must start with the - // prefix "ed25519:". - KeyID gomatrixserverlib.KeyID `yaml:"-"` - // List of paths to X509 certificates used by the external federation listeners. - // These are used to calculate the TLS fingerprints to publish for this server. - // Other matrix servers talking to this server will expect the x509 certificate - // to match one of these certificates. - // The certificates should be in PEM format. - FederationCertificatePaths []Path `yaml:"federation_certificates"` - // A list of SHA256 TLS fingerprints for the X509 certificates used by the - // federation listener for this server. - TLSFingerPrints []gomatrixserverlib.TLSFingerprint `yaml:"-"` - // How long a remote server can cache our server key for before requesting it again. - // Increasing this number will reduce the number of requests made by remote servers - // for our key, but increases the period a compromised key will be considered valid - // by remote servers. - // Defaults to 24 hours. - KeyValidityPeriod time.Duration `yaml:"key_validity_period"` - // List of domains that the server will trust as identity servers to - // verify third-party identifiers. - // Defaults to an empty array. - TrustedIDServers []string `yaml:"trusted_third_party_id_servers"` - // If set, allows registration by anyone who also has the shared - // secret, even if registration is otherwise disabled. - RegistrationSharedSecret string `yaml:"registration_shared_secret"` - // This Home Server's ReCAPTCHA public key. - RecaptchaPublicKey string `yaml:"recaptcha_public_key"` - // This Home Server's ReCAPTCHA private key. - RecaptchaPrivateKey string `yaml:"recaptcha_private_key"` - // Boolean stating whether catpcha registration is enabled - // and required - RecaptchaEnabled bool `yaml:"enable_registration_captcha"` - // Secret used to bypass the captcha registration entirely - RecaptchaBypassSecret string `yaml:"captcha_bypass_secret"` - // HTTP API endpoint used to verify whether the captcha response - // was successful - RecaptchaSiteVerifyAPI string `yaml:"recaptcha_siteverify_api"` - // If set disables new users from registering (except via shared - // secrets) - RegistrationDisabled bool `yaml:"registration_disabled"` - // Perspective keyservers, to use as a backup when direct key fetch - // requests don't succeed - KeyPerspectives KeyPerspectives `yaml:"key_perspectives"` - // Federation failure threshold. How many consecutive failures that we should - // tolerate when sending federation requests to a specific server. The backoff - // is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. - // The default value is 16 if not specified, which is circa 18 hours. - FederationMaxRetries uint32 `yaml:"federation_max_retries"` - } `yaml:"matrix"` - - // The configuration specific to the media repostitory. - Media struct { - // The base path to where the media files will be stored. May be relative or absolute. - BasePath Path `yaml:"base_path"` - // The absolute base path to where media files will be stored. - AbsBasePath Path `yaml:"-"` - // The maximum file size in bytes that is allowed to be stored on this server. - // Note: if max_file_size_bytes is set to 0, the size is unlimited. - // Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) - MaxFileSizeBytes *FileSizeBytes `yaml:"max_file_size_bytes,omitempty"` - // Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated - DynamicThumbnails bool `yaml:"dynamic_thumbnails"` - // The maximum number of simultaneous thumbnail generators. default: 10 - MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"` - // A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content - ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"` - } `yaml:"media"` - - // The configuration to use for Prometheus metrics - Metrics struct { - // Whether or not the metrics are enabled - Enabled bool `yaml:"enabled"` - // Use BasicAuth for Authorization - BasicAuth struct { - // Authorization via Static Username & Password - // Hardcoded Username and Password - Username string `yaml:"username"` - Password string `yaml:"password"` - } `yaml:"basic_auth"` - } `yaml:"metrics"` - - // The configuration for talking to kafka. - Kafka struct { - // A list of kafka addresses to connect to. - Addresses []string `yaml:"addresses"` - // Whether to use naffka instead of kafka. - // Naffka can only be used when running dendrite as a single monolithic server. - // Kafka can be used both with a monolithic server and when running the - // components as separate servers. - UseNaffka bool `yaml:"use_naffka,omitempty"` - // The names of the topics to use when reading and writing from kafka. - Topics struct { - // Topic for roomserver/api.OutputRoomEvent events. - OutputRoomEvent Topic `yaml:"output_room_event"` - // Topic for sending account data from client API to sync API - OutputClientData Topic `yaml:"output_client_data"` - // Topic for eduserver/api.OutputTypingEvent events. - OutputTypingEvent Topic `yaml:"output_typing_event"` - // Topic for eduserver/api.OutputSendToDeviceEvent events. - OutputSendToDeviceEvent Topic `yaml:"output_send_to_device_event"` - // Topic for keyserver when new device keys are added. - OutputKeyChangeEvent Topic `yaml:"output_key_change_event"` - } - } `yaml:"kafka"` - - // Postgres Config - Database struct { - // The Account database stores the login details and account information - // for local users. It is accessed by the UserAPI. - Account DataSource `yaml:"account"` - // The CurrentState database stores the current state of all rooms. - // It is accessed by the CurrentStateServer. - CurrentState DataSource `yaml:"current_state"` - // The Device database stores session information for the devices of logged - // in local users. It is accessed by the UserAPI. - Device DataSource `yaml:"device"` - // The MediaAPI database stores information about files uploaded and downloaded - // by local users. It is only accessed by the MediaAPI. - MediaAPI DataSource `yaml:"media_api"` - // The ServerKey database caches the public keys of remote servers. - // It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI. - ServerKey DataSource `yaml:"server_key"` - // The E2EKey database stores one-time public keys for devices in addition to - // signed device keys. Used for E2E. - E2EKey DataSource `yaml:"e2e_key"` - // The SyncAPI stores information used by the SyncAPI server. - // It is only accessed by the SyncAPI server. - SyncAPI DataSource `yaml:"sync_api"` - // The RoomServer database stores information about matrix rooms. - // It is only accessed by the RoomServer. - RoomServer DataSource `yaml:"room_server"` - // The FederationSender database stores information used by the FederationSender - // It is only accessed by the FederationSender. - FederationSender DataSource `yaml:"federation_sender"` - // The AppServices database stores information used by the AppService component. - // It is only accessed by the AppService component. - AppService DataSource `yaml:"appservice"` - // The Naffka database is used internally by the naffka library, if used. - Naffka DataSource `yaml:"naffka,omitempty"` - // Maximum open connections to the DB (0 = use default, negative means unlimited) - MaxOpenConns int `yaml:"max_open_conns"` - // Maximum idle connections to the DB (0 = use default, negative means unlimited) - MaxIdleConns int `yaml:"max_idle_conns"` - // maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited) - ConnMaxLifetimeSec int `yaml:"conn_max_lifetime"` - } `yaml:"database"` - - // TURN Server Config - TURN struct { - // TODO Guest Support - // Whether or not guests can request TURN credentials - //AllowGuests bool `yaml:"turn_allow_guests"` - // How long the authorization should last - UserLifetime string `yaml:"turn_user_lifetime"` - // The list of TURN URIs to pass to clients - URIs []string `yaml:"turn_uris"` - - // Authorization via Shared Secret - // The shared secret from coturn - SharedSecret string `yaml:"turn_shared_secret"` - - // Authorization via Static Username & Password - // Hardcoded Username and Password - Username string `yaml:"turn_username"` - Password string `yaml:"turn_password"` - } `yaml:"turn"` - - // The internal addresses the components will listen on. - // These should not be exposed externally as they expose metrics and debugging APIs. - // Falls back to addresses listed in Listen if not specified - Bind struct { - MediaAPI Address `yaml:"media_api"` - ClientAPI Address `yaml:"client_api"` - CurrentState Address `yaml:"current_state_server"` - FederationAPI Address `yaml:"federation_api"` - ServerKeyAPI Address `yaml:"server_key_api"` - AppServiceAPI Address `yaml:"appservice_api"` - SyncAPI Address `yaml:"sync_api"` - UserAPI Address `yaml:"user_api"` - RoomServer Address `yaml:"room_server"` - FederationSender Address `yaml:"federation_sender"` - EDUServer Address `yaml:"edu_server"` - KeyServer Address `yaml:"key_server"` - } `yaml:"bind"` - - // The addresses for talking to other microservices. - Listen struct { - MediaAPI Address `yaml:"media_api"` - ClientAPI Address `yaml:"client_api"` - CurrentState Address `yaml:"current_state_server"` - FederationAPI Address `yaml:"federation_api"` - ServerKeyAPI Address `yaml:"server_key_api"` - AppServiceAPI Address `yaml:"appservice_api"` - SyncAPI Address `yaml:"sync_api"` - UserAPI Address `yaml:"user_api"` - RoomServer Address `yaml:"room_server"` - FederationSender Address `yaml:"federation_sender"` - EDUServer Address `yaml:"edu_server"` - KeyServer Address `yaml:"key_server"` - } `yaml:"listen"` + Global Global `yaml:"global"` + AppServiceAPI AppServiceAPI `yaml:"app_service_api"` + ClientAPI ClientAPI `yaml:"client_api"` + CurrentStateServer CurrentStateServer `yaml:"current_state_server"` + EDUServer EDUServer `yaml:"edu_server"` + FederationAPI FederationAPI `yaml:"federation_api"` + FederationSender FederationSender `yaml:"federation_sender"` + KeyServer KeyServer `yaml:"key_server"` + MediaAPI MediaAPI `yaml:"media_api"` + RoomServer RoomServer `yaml:"room_server"` + ServerKeyAPI ServerKeyAPI `yaml:"server_key_api"` + SyncAPI SyncAPI `yaml:"sync_api"` + UserAPI UserAPI `yaml:"user_api"` // The config for tracing the dendrite servers. Tracing struct { @@ -268,56 +72,42 @@ type Dendrite struct { Jaeger jaegerconfig.Configuration `yaml:"jaeger"` } `yaml:"tracing"` - // Application Services - // https://matrix.org/docs/spec/application_service/unstable.html - ApplicationServices struct { - // Configuration files for various application services - ConfigFiles []string `yaml:"config_files"` - } `yaml:"application_services"` - // The config for logging informations. Each hook will be added to logrus. Logging []LogrusHook `yaml:"logging"` - // The config for setting a proxy to use for server->server requests - Proxy *struct { - // The protocol for the proxy (http / https / socks5) - Protocol string `yaml:"protocol"` - // The host where the proxy is listening - Host string `yaml:"host"` - // The port on which the proxy is listening - Port uint16 `yaml:"port"` - } `yaml:"proxy"` - // Any information derived from the configuration options for later use. - Derived struct { - Registration struct { - // Flows is a slice of flows, which represent one possible way that the client can authenticate a request. - // http://matrix.org/docs/spec/HEAD/client_server/r0.3.0.html#user-interactive-authentication-api - // As long as the generated flows only rely on config file options, - // we can generate them on startup and store them until needed - Flows []authtypes.Flow `json:"flows"` + Derived Derived `yaml:"-"` +} - // Params that need to be returned to the client during - // registration in order to complete registration stages. - Params map[string]interface{} `json:"params"` - } +// TODO: Kill Derived +type Derived struct { + Registration struct { + // Flows is a slice of flows, which represent one possible way that the client can authenticate a request. + // http://matrix.org/docs/spec/HEAD/client_server/r0.3.0.html#user-interactive-authentication-api + // As long as the generated flows only rely on config file options, + // we can generate them on startup and store them until needed + Flows []authtypes.Flow `json:"flows"` - // Application services parsed from their config files - // The paths of which were given above in the main config file - ApplicationServices []ApplicationService + // Params that need to be returned to the client during + // registration in order to complete registration stages. + Params map[string]interface{} `json:"params"` + } - // Meta-regexes compiled from all exclusive application service - // Regexes. - // - // When a user registers, we check that their username does not match any - // exclusive application service namespaces - ExclusiveApplicationServicesUsernameRegexp *regexp.Regexp - // When a user creates a room alias, we check that it isn't already - // reserved by an application service - ExclusiveApplicationServicesAliasRegexp *regexp.Regexp - // Note: An Exclusive Regex for room ID isn't necessary as we aren't blocking - // servers from creating RoomIDs in exclusive application service namespaces - } `yaml:"-"` + // Application services parsed from their config files + // The paths of which were given above in the main config file + ApplicationServices []ApplicationService + + // Meta-regexes compiled from all exclusive application service + // Regexes. + // + // When a user registers, we check that their username does not match any + // exclusive application service namespaces + ExclusiveApplicationServicesUsernameRegexp *regexp.Regexp + // When a user creates a room alias, we check that it isn't already + // reserved by an application service + ExclusiveApplicationServicesAliasRegexp *regexp.Regexp + // Note: An Exclusive Regex for room ID isn't necessary as we aren't blocking + // servers from creating RoomIDs in exclusive application service namespaces } // KeyPerspectives are used to configure perspective key servers for @@ -402,29 +192,29 @@ func loadConfig( readFile func(string) ([]byte, error), monolithic bool, ) (*Dendrite, error) { - var config Dendrite + var c Dendrite + c.Defaults() + var err error - if err = yaml.Unmarshal(configData, &config); err != nil { + if err = yaml.Unmarshal(configData, &c); err != nil { return nil, err } - config.SetDefaults() - - if err = config.check(monolithic); err != nil { + if err = c.check(monolithic); err != nil { return nil, err } - privateKeyPath := absPath(basePath, config.Matrix.PrivateKeyPath) + privateKeyPath := absPath(basePath, c.Global.PrivateKeyPath) privateKeyData, err := readFile(privateKeyPath) if err != nil { return nil, err } - if config.Matrix.KeyID, config.Matrix.PrivateKey, err = readKeyPEM(privateKeyPath, privateKeyData); err != nil { + if c.Global.KeyID, c.Global.PrivateKey, err = readKeyPEM(privateKeyPath, privateKeyData); err != nil { return nil, err } - for _, certPath := range config.Matrix.FederationCertificatePaths { + for _, certPath := range c.FederationAPI.FederationCertificatePaths { absCertPath := absPath(basePath, certPath) var pemData []byte pemData, err = readFile(absCertPath) @@ -435,18 +225,19 @@ func loadConfig( if fingerprint == nil { return nil, fmt.Errorf("no certificate PEM data in %q", absCertPath) } - config.Matrix.TLSFingerPrints = append(config.Matrix.TLSFingerPrints, *fingerprint) + c.FederationAPI.TLSFingerPrints = append(c.FederationAPI.TLSFingerPrints, *fingerprint) } - config.Media.AbsBasePath = Path(absPath(basePath, config.Media.BasePath)) + c.MediaAPI.AbsBasePath = Path(absPath(basePath, c.MediaAPI.BasePath)) // Generate data from config options - err = config.Derive() + err = c.Derive() if err != nil { return nil, err } - return &config, nil + c.Wiring() + return &c, nil } // Derive generates data that is derived from various values provided in @@ -459,8 +250,8 @@ func (config *Dendrite) Derive() error { // TODO: Add email auth type // TODO: Add MSISDN auth type - if config.Matrix.RecaptchaEnabled { - config.Derived.Registration.Params[authtypes.LoginTypeRecaptcha] = map[string]string{"public_key": config.Matrix.RecaptchaPublicKey} + if config.ClientAPI.RecaptchaEnabled { + config.Derived.Registration.Params[authtypes.LoginTypeRecaptcha] = map[string]string{"public_key": config.ClientAPI.RecaptchaPublicKey} config.Derived.Registration.Flows = append(config.Derived.Registration.Flows, authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeRecaptcha}}) } else { @@ -469,44 +260,65 @@ func (config *Dendrite) Derive() error { } // Load application service configuration files - if err := loadAppServices(config); err != nil { - return err - } + //if err := loadAppServices(config); err != nil { + // return err + //} return nil } // SetDefaults sets default config values if they are not explicitly set. -func (config *Dendrite) SetDefaults() { - if config.Matrix.KeyValidityPeriod == 0 { - config.Matrix.KeyValidityPeriod = 24 * time.Hour - } +func (c *Dendrite) Defaults() { + c.Version = 1 - if config.Matrix.TrustedIDServers == nil { - config.Matrix.TrustedIDServers = []string{} - } + c.Global.Defaults() + c.ClientAPI.Defaults() + c.CurrentStateServer.Defaults() + c.EDUServer.Defaults() + c.FederationAPI.Defaults() + c.FederationSender.Defaults() + c.KeyServer.Defaults() + c.MediaAPI.Defaults() + c.RoomServer.Defaults() + c.ServerKeyAPI.Defaults() + c.SyncAPI.Defaults() + c.UserAPI.Defaults() + c.AppServiceAPI.Defaults() - if config.Matrix.FederationMaxRetries == 0 { - config.Matrix.FederationMaxRetries = 16 - } + c.Wiring() +} - if config.Media.MaxThumbnailGenerators == 0 { - config.Media.MaxThumbnailGenerators = 10 +func (c *Dendrite) Verify(configErrs *configErrors) { + type verifiable interface { + Verify(configErrs *configErrors) } - - if config.Media.MaxFileSizeBytes == nil { - defaultMaxFileSizeBytes := FileSizeBytes(10485760) - config.Media.MaxFileSizeBytes = &defaultMaxFileSizeBytes + for _, c := range []verifiable{ + &c.Global, &c.ClientAPI, &c.CurrentStateServer, + &c.EDUServer, &c.FederationAPI, &c.FederationSender, + &c.KeyServer, &c.MediaAPI, &c.RoomServer, + &c.ServerKeyAPI, &c.SyncAPI, &c.UserAPI, + &c.AppServiceAPI, + } { + c.Verify(configErrs) } +} - if config.Database.MaxIdleConns == 0 { - config.Database.MaxIdleConns = 2 - } - - if config.Database.MaxOpenConns == 0 { - config.Database.MaxOpenConns = 100 - } +func (c *Dendrite) Wiring() { + c.ClientAPI.Matrix = &c.Global + c.CurrentStateServer.Matrix = &c.Global + c.EDUServer.Matrix = &c.Global + c.FederationAPI.Matrix = &c.Global + c.FederationSender.Matrix = &c.Global + c.KeyServer.Matrix = &c.Global + c.MediaAPI.Matrix = &c.Global + c.RoomServer.Matrix = &c.Global + c.ServerKeyAPI.Matrix = &c.Global + c.SyncAPI.Matrix = &c.Global + c.UserAPI.Matrix = &c.Global + c.AppServiceAPI.Matrix = &c.Global + c.ClientAPI.Derived = &c.Derived + c.AppServiceAPI.Derived = &c.Derived } // Error returns a string detailing how many errors were contained within a @@ -553,88 +365,6 @@ func checkPositive(configErrs *configErrors, key string, value int64) { } } -// checkTurn verifies the parameters turn.* are valid. -func (config *Dendrite) checkTurn(configErrs *configErrors) { - value := config.TURN.UserLifetime - if value != "" { - if _, err := time.ParseDuration(value); err != nil { - configErrs.Add(fmt.Sprintf("invalid duration for config key %q: %s", "turn.turn_user_lifetime", value)) - } - } -} - -// checkMatrix verifies the parameters matrix.* are valid. -func (config *Dendrite) checkMatrix(configErrs *configErrors) { - checkNotEmpty(configErrs, "matrix.server_name", string(config.Matrix.ServerName)) - checkNotEmpty(configErrs, "matrix.private_key", string(config.Matrix.PrivateKeyPath)) - checkNotZero(configErrs, "matrix.federation_certificates", int64(len(config.Matrix.FederationCertificatePaths))) - if config.Matrix.RecaptchaEnabled { - checkNotEmpty(configErrs, "matrix.recaptcha_public_key", string(config.Matrix.RecaptchaPublicKey)) - checkNotEmpty(configErrs, "matrix.recaptcha_private_key", string(config.Matrix.RecaptchaPrivateKey)) - checkNotEmpty(configErrs, "matrix.recaptcha_siteverify_api", string(config.Matrix.RecaptchaSiteVerifyAPI)) - } -} - -// checkMedia verifies the parameters media.* are valid. -func (config *Dendrite) checkMedia(configErrs *configErrors) { - checkNotEmpty(configErrs, "media.base_path", string(config.Media.BasePath)) - checkPositive(configErrs, "media.max_file_size_bytes", int64(*config.Media.MaxFileSizeBytes)) - checkPositive(configErrs, "media.max_thumbnail_generators", int64(config.Media.MaxThumbnailGenerators)) - - for i, size := range config.Media.ThumbnailSizes { - checkPositive(configErrs, fmt.Sprintf("media.thumbnail_sizes[%d].width", i), int64(size.Width)) - checkPositive(configErrs, fmt.Sprintf("media.thumbnail_sizes[%d].height", i), int64(size.Height)) - } -} - -// checkKafka verifies the parameters kafka.* and the related -// database.naffka are valid. -func (config *Dendrite) checkKafka(configErrs *configErrors, monolithic bool) { - - if config.Kafka.UseNaffka { - if !monolithic { - configErrs.Add(fmt.Sprintf("naffka can only be used in a monolithic server")) - } - - checkNotEmpty(configErrs, "database.naffka", string(config.Database.Naffka)) - } else { - // If we aren't using naffka then we need to have at least one kafka - // server to talk to. - checkNotZero(configErrs, "kafka.addresses", int64(len(config.Kafka.Addresses))) - } - checkNotEmpty(configErrs, "kafka.topics.output_room_event", string(config.Kafka.Topics.OutputRoomEvent)) - checkNotEmpty(configErrs, "kafka.topics.output_client_data", string(config.Kafka.Topics.OutputClientData)) - checkNotEmpty(configErrs, "kafka.topics.output_typing_event", string(config.Kafka.Topics.OutputTypingEvent)) - checkNotEmpty(configErrs, "kafka.topics.output_send_to_device_event", string(config.Kafka.Topics.OutputSendToDeviceEvent)) - checkNotEmpty(configErrs, "kafka.topics.output_key_change_event", string(config.Kafka.Topics.OutputKeyChangeEvent)) -} - -// checkDatabase verifies the parameters database.* are valid. -func (config *Dendrite) checkDatabase(configErrs *configErrors) { - checkNotEmpty(configErrs, "database.account", string(config.Database.Account)) - checkNotEmpty(configErrs, "database.device", string(config.Database.Device)) - checkNotEmpty(configErrs, "database.server_key", string(config.Database.ServerKey)) - checkNotEmpty(configErrs, "database.media_api", string(config.Database.MediaAPI)) - checkNotEmpty(configErrs, "database.sync_api", string(config.Database.SyncAPI)) - checkNotEmpty(configErrs, "database.room_server", string(config.Database.RoomServer)) - checkNotEmpty(configErrs, "database.current_state", string(config.Database.CurrentState)) - checkNotEmpty(configErrs, "database.e2e_key", string(config.Database.E2EKey)) -} - -// checkListen verifies the parameters listen.* are valid. -func (config *Dendrite) checkListen(configErrs *configErrors) { - checkNotEmpty(configErrs, "listen.media_api", string(config.Listen.MediaAPI)) - checkNotEmpty(configErrs, "listen.client_api", string(config.Listen.ClientAPI)) - checkNotEmpty(configErrs, "listen.federation_api", string(config.Listen.FederationAPI)) - checkNotEmpty(configErrs, "listen.sync_api", string(config.Listen.SyncAPI)) - checkNotEmpty(configErrs, "listen.room_server", string(config.Listen.RoomServer)) - checkNotEmpty(configErrs, "listen.edu_server", string(config.Listen.EDUServer)) - checkNotEmpty(configErrs, "listen.server_key_api", string(config.Listen.EDUServer)) - checkNotEmpty(configErrs, "listen.user_api", string(config.Listen.UserAPI)) - checkNotEmpty(configErrs, "listen.current_state_server", string(config.Listen.CurrentState)) - checkNotEmpty(configErrs, "listen.key_server", string(config.Listen.KeyServer)) -} - // checkLogging verifies the parameters logging.* are valid. func (config *Dendrite) checkLogging(configErrs *configErrors) { for _, logrusHook := range config.Logging { @@ -645,7 +375,7 @@ func (config *Dendrite) checkLogging(configErrs *configErrors) { // check returns an error type containing all errors found within the config // file. -func (config *Dendrite) check(monolithic bool) error { +func (config *Dendrite) check(_ bool) error { // monolithic var configErrs configErrors if config.Version != Version { @@ -655,17 +385,8 @@ func (config *Dendrite) check(monolithic bool) error { return configErrs } - config.checkMatrix(&configErrs) - config.checkMedia(&configErrs) - config.checkTurn(&configErrs) - config.checkKafka(&configErrs, monolithic) - config.checkDatabase(&configErrs) config.checkLogging(&configErrs) - if !monolithic { - config.checkListen(&configErrs) - } - // Due to how Golang manages its interface types, this condition is not redundant. // In order to get the proper behaviour, it is necessary to return an explicit nil // and not a nil configErrors. @@ -734,7 +455,7 @@ func (config *Dendrite) AppServiceURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.AppServiceAPI) + return "http://" + string(config.AppServiceAPI.Listen) } // RoomServerURL returns an HTTP URL for where the roomserver is listening. @@ -743,7 +464,7 @@ func (config *Dendrite) RoomServerURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.RoomServer) + return "http://" + string(config.RoomServer.Listen) } // UserAPIURL returns an HTTP URL for where the userapi is listening. @@ -752,7 +473,7 @@ func (config *Dendrite) UserAPIURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.UserAPI) + return "http://" + string(config.UserAPI.Listen) } // CurrentStateAPIURL returns an HTTP URL for where the currentstateserver is listening. @@ -761,7 +482,7 @@ func (config *Dendrite) CurrentStateAPIURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.CurrentState) + return "http://" + string(config.CurrentStateServer.Listen) } // EDUServerURL returns an HTTP URL for where the EDU server is listening. @@ -770,7 +491,7 @@ func (config *Dendrite) EDUServerURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.EDUServer) + return "http://" + string(config.EDUServer.Listen) } // FederationSenderURL returns an HTTP URL for where the federation sender is listening. @@ -779,7 +500,7 @@ func (config *Dendrite) FederationSenderURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.FederationSender) + return "http://" + string(config.FederationSender.Listen) } // ServerKeyAPIURL returns an HTTP URL for where the server key API is listening. @@ -788,7 +509,7 @@ func (config *Dendrite) ServerKeyAPIURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.ServerKeyAPI) + return "http://" + string(config.ServerKeyAPI.Listen) } // KeyServerURL returns an HTTP URL for where the key server is listening. @@ -797,7 +518,7 @@ func (config *Dendrite) KeyServerURL() string { // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.KeyServer) + return "http://" + string(config.KeyServer.Listen) } // SetupTracing configures the opentracing using the supplied configuration. @@ -812,33 +533,6 @@ func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err ) } -// MaxIdleConns returns maximum idle connections to the DB -func (config Dendrite) MaxIdleConns() int { - return config.Database.MaxIdleConns -} - -// MaxOpenConns returns maximum open connections to the DB -func (config Dendrite) MaxOpenConns() int { - return config.Database.MaxOpenConns -} - -// ConnMaxLifetime returns maximum amount of time a connection may be reused -func (config Dendrite) ConnMaxLifetime() time.Duration { - return time.Duration(config.Database.ConnMaxLifetimeSec) * time.Second -} - -// DbProperties functions return properties used by database/sql/DB -type DbProperties interface { - MaxIdleConns() int - MaxOpenConns() int - ConnMaxLifetime() time.Duration -} - -// DbProperties returns cfg as a DbProperties interface -func (config Dendrite) DbProperties() DbProperties { - return config -} - // logrusLogger is a small wrapper that implements jaeger.Logger using logrus. type logrusLogger struct { l *logrus.Logger diff --git a/internal/config/appservice.go b/internal/config/config_appservice.go similarity index 87% rename from internal/config/appservice.go rename to internal/config/config_appservice.go index bf5f089b7..51a40c3cd 100644 --- a/internal/config/appservice.go +++ b/internal/config/config_appservice.go @@ -25,6 +25,32 @@ import ( yaml "gopkg.in/yaml.v2" ) +type AppServiceAPI struct { + Matrix *Global `yaml:"-"` + Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + Database DataSource `yaml:"database"` + DatabaseOptions DatabaseOptions `yaml:"database_options"` + + ConfigFiles []string `yaml:"config_files"` +} + +func (c *AppServiceAPI) Defaults() { + c.Listen = "localhost:7777" + c.Bind = "localhost:7777" + c.Database = "file:appservice.db" + c.DatabaseOptions.Defaults() +} + +func (c *AppServiceAPI) Verify(configErrs *configErrors) { + checkNotEmpty(configErrs, "app_service_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "app_service_api.bind", string(c.Bind)) + checkNotEmpty(configErrs, "app_service_api.database", string(c.Database)) +} + // ApplicationServiceNamespace is the namespace that a specific application // service has management over. type ApplicationServiceNamespace struct { @@ -132,8 +158,8 @@ func (a *ApplicationService) IsInterestedInRoomAlias( // loadAppServices iterates through all application service config files // and loads their data into the config object for later access. -func loadAppServices(config *Dendrite) error { - for _, configPath := range config.ApplicationServices.ConfigFiles { +func loadAppServices(config *AppServiceAPI, derived *Derived) error { + for _, configPath := range config.ConfigFiles { // Create a new application service with default options appservice := ApplicationService{ RateLimited: true, @@ -157,26 +183,26 @@ func loadAppServices(config *Dendrite) error { } // Append the parsed application service to the global config - config.Derived.ApplicationServices = append( - config.Derived.ApplicationServices, appservice, + derived.ApplicationServices = append( + derived.ApplicationServices, appservice, ) } // Check for any errors in the loaded application services - return checkErrors(config) + return checkErrors(config, derived) } // setupRegexps will create regex objects for exclusive and non-exclusive // usernames, aliases and rooms of all application services, so that other // methods can quickly check if a particular string matches any of them. -func setupRegexps(cfg *Dendrite) (err error) { +func setupRegexps(cfg *AppServiceAPI, derived *Derived) (err error) { // Combine all exclusive namespaces for later string checking var exclusiveUsernameStrings, exclusiveAliasStrings []string // If an application service's regex is marked as exclusive, add // its contents to the overall exlusive regex string. Room regex // not necessary as we aren't denying exclusive room ID creation - for _, appservice := range cfg.Derived.ApplicationServices { + for _, appservice := range derived.ApplicationServices { for key, namespaceSlice := range appservice.NamespaceMap { switch key { case "users": @@ -204,10 +230,10 @@ func setupRegexps(cfg *Dendrite) (err error) { } // Store compiled Regex - if cfg.Derived.ExclusiveApplicationServicesUsernameRegexp, err = regexp.Compile(exclusiveUsernames); err != nil { + if derived.ExclusiveApplicationServicesUsernameRegexp, err = regexp.Compile(exclusiveUsernames); err != nil { return err } - if cfg.Derived.ExclusiveApplicationServicesAliasRegexp, err = regexp.Compile(exclusiveAliases); err != nil { + if derived.ExclusiveApplicationServicesAliasRegexp, err = regexp.Compile(exclusiveAliases); err != nil { return err } @@ -234,7 +260,7 @@ func appendExclusiveNamespaceRegexs( // checkErrors checks for any configuration errors amongst the loaded // application services according to the application service spec. -func checkErrors(config *Dendrite) (err error) { +func checkErrors(config *AppServiceAPI, derived *Derived) (err error) { var idMap = make(map[string]bool) var tokenMap = make(map[string]bool) @@ -242,7 +268,7 @@ func checkErrors(config *Dendrite) (err error) { groupIDRegexp := regexp.MustCompile(`\+.*:.*`) // Check each application service for any config errors - for _, appservice := range config.Derived.ApplicationServices { + for _, appservice := range derived.ApplicationServices { // Namespace-related checks for key, namespaceSlice := range appservice.NamespaceMap { for _, namespace := range namespaceSlice { @@ -284,7 +310,7 @@ func checkErrors(config *Dendrite) (err error) { } } - return setupRegexps(config) + return setupRegexps(config, derived) } // validateNamespace returns nil or an error based on whether a given diff --git a/internal/config/config_clientapi.go b/internal/config/config_clientapi.go new file mode 100644 index 000000000..d52b5dedc --- /dev/null +++ b/internal/config/config_clientapi.go @@ -0,0 +1,87 @@ +package config + +import ( + "fmt" + "time" +) + +type ClientAPI struct { + Matrix *Global `yaml:"-"` + Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // If set, allows registration by anyone who also has the shared + // secret, even if registration is otherwise disabled. + RegistrationSharedSecret string `yaml:"registration_shared_secret"` + // This Home Server's ReCAPTCHA public key. + RecaptchaPublicKey string `yaml:"recaptcha_public_key"` + // This Home Server's ReCAPTCHA private key. + RecaptchaPrivateKey string `yaml:"recaptcha_private_key"` + // Boolean stating whether catpcha registration is enabled + // and required + RecaptchaEnabled bool `yaml:"enable_registration_captcha"` + // Secret used to bypass the captcha registration entirely + RecaptchaBypassSecret string `yaml:"captcha_bypass_secret"` + // HTTP API endpoint used to verify whether the captcha response + // was successful + RecaptchaSiteVerifyAPI string `yaml:"recaptcha_siteverify_api"` + // If set disables new users from registering (except via shared + // secrets) + RegistrationDisabled bool `yaml:"registration_disabled"` + + // TURN options + TURN TURN `yaml:"turn"` +} + +func (c *ClientAPI) Defaults() { + c.Listen = "localhost:7771" + c.Bind = "localhost:7771" + c.RegistrationSharedSecret = "" + c.RecaptchaPublicKey = "" + c.RecaptchaPrivateKey = "" + c.RecaptchaEnabled = false + c.RecaptchaBypassSecret = "" + c.RecaptchaSiteVerifyAPI = "" + c.RegistrationDisabled = false +} + +func (c *ClientAPI) Verify(configErrs *configErrors) { + checkNotEmpty(configErrs, "client_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "client_api.bind", string(c.Bind)) + if c.RecaptchaEnabled { + checkNotEmpty(configErrs, "client_api.recaptcha_public_key", string(c.RecaptchaPublicKey)) + checkNotEmpty(configErrs, "client_api.recaptcha_private_key", string(c.RecaptchaPrivateKey)) + checkNotEmpty(configErrs, "client_api.recaptcha_siteverify_api", string(c.RecaptchaSiteVerifyAPI)) + } + c.TURN.Verify(configErrs) +} + +type TURN struct { + // TODO Guest Support + // Whether or not guests can request TURN credentials + // AllowGuests bool `yaml:"turn_allow_guests"` + // How long the authorization should last + UserLifetime string `yaml:"turn_user_lifetime"` + // The list of TURN URIs to pass to clients + URIs []string `yaml:"turn_uris"` + + // Authorization via Shared Secret + // The shared secret from coturn + SharedSecret string `yaml:"turn_shared_secret"` + + // Authorization via Static Username & Password + // Hardcoded Username and Password + Username string `yaml:"turn_username"` + Password string `yaml:"turn_password"` +} + +func (c *TURN) Verify(configErrs *configErrors) { + value := c.UserLifetime + if value != "" { + if _, err := time.ParseDuration(value); err != nil { + configErrs.Add(fmt.Sprintf("invalid duration for config key %q: %s", "client_api.turn.turn_user_lifetime", value)) + } + } +} diff --git a/internal/config/config_currentstate.go b/internal/config/config_currentstate.go new file mode 100644 index 000000000..2f32848ae --- /dev/null +++ b/internal/config/config_currentstate.go @@ -0,0 +1,26 @@ +package config + +type CurrentStateServer struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // The CurrentState database stores the current state of all rooms. + // It is accessed by the CurrentStateServer. + Database DataSource `yaml:"database"` + DatabaseOptions DatabaseOptions `yaml:"database_options"` +} + +func (c *CurrentStateServer) Defaults() { + c.Listen = "localhost:7782" + c.Bind = "localhost:7782" + c.Database = "file:currentstate.db" + c.DatabaseOptions.Defaults() +} + +func (c *CurrentStateServer) Verify(configErrs *configErrors) { + checkNotEmpty(configErrs, "current_state_server.listen", string(c.Listen)) + checkNotEmpty(configErrs, "current_state_server.bind", string(c.Bind)) + checkNotEmpty(configErrs, "current_state_server.database", string(c.Database)) +} diff --git a/internal/config/config_eduserver.go b/internal/config/config_eduserver.go new file mode 100644 index 000000000..a76612b5c --- /dev/null +++ b/internal/config/config_eduserver.go @@ -0,0 +1,18 @@ +package config + +type EDUServer struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` +} + +func (c *EDUServer) Defaults() { + c.Listen = "localhost:7778" + c.Bind = "localhost:7778" +} + +func (c *EDUServer) Verify(configErrs *configErrors) { + checkNotEmpty(configErrs, "edu_server.listen", string(c.Listen)) + checkNotEmpty(configErrs, "edu_server.bind", string(c.Bind)) +} diff --git a/internal/config/config_federationapi.go b/internal/config/config_federationapi.go new file mode 100644 index 000000000..8f7bc9396 --- /dev/null +++ b/internal/config/config_federationapi.go @@ -0,0 +1,32 @@ +package config + +import "github.com/matrix-org/gomatrixserverlib" + +type FederationAPI struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // List of paths to X509 certificates used by the external federation listeners. + // These are used to calculate the TLS fingerprints to publish for this server. + // Other matrix servers talking to this server will expect the x509 certificate + // to match one of these certificates. + // The certificates should be in PEM format. + FederationCertificatePaths []Path `yaml:"federation_certificates"` + + // A list of SHA256 TLS fingerprints for the X509 certificates used by the + // federation listener for this server. + TLSFingerPrints []gomatrixserverlib.TLSFingerprint `yaml:"-"` +} + +func (c *FederationAPI) Defaults() { + c.Listen = "localhost:7772" + c.Bind = "localhost:7772" +} + +func (c *FederationAPI) Verify(configErrs *configErrors) { + checkNotEmpty(configErrs, "federation_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "federation_api.bind", string(c.Bind)) + checkNotZero(configErrs, "federation_api.federation_certificates", int64(len(c.FederationCertificatePaths))) +} diff --git a/internal/config/config_federationsender.go b/internal/config/config_federationsender.go new file mode 100644 index 000000000..693fcdf10 --- /dev/null +++ b/internal/config/config_federationsender.go @@ -0,0 +1,55 @@ +package config + +type FederationSender struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // The FederationSender database stores information used by the FederationSender + // It is only accessed by the FederationSender. + Database DataSource `yaml:"database"` + DatabaseOptions DatabaseOptions `yaml:"database_options"` + + // Federation failure threshold. How many consecutive failures that we should + // tolerate when sending federation requests to a specific server. The backoff + // is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. + // The default value is 16 if not specified, which is circa 18 hours. + FederationMaxRetries uint32 `yaml:"federation_max_retries"` + + Proxy Proxy `yaml:"proxy"` +} + +func (c *FederationSender) Defaults() { + c.Listen = "localhost:7775" + c.Bind = "localhost:7775" + c.Database = "file:federationsender.db" + c.DatabaseOptions.Defaults() + + c.FederationMaxRetries = 16 +} + +func (c *FederationSender) Verify(configErrs *configErrors) { + checkNotEmpty(configErrs, "federation_sender.listen", string(c.Listen)) + checkNotEmpty(configErrs, "federation_sender.bind", string(c.Bind)) + checkNotEmpty(configErrs, "federation_sender.database", string(c.Database)) +} + +// The config for setting a proxy to use for server->server requests +type Proxy struct { + // The protocol for the proxy (http / https / socks5) + Protocol string `yaml:"protocol"` + // The host where the proxy is listening + Host string `yaml:"host"` + // The port on which the proxy is listening + Port uint16 `yaml:"port"` +} + +func (c *Proxy) Defaults() { + c.Protocol = "" + c.Host = "" + c.Port = 0 +} + +func (c *Proxy) Verify(configErrs *configErrors) { +} diff --git a/internal/config/config_global.go b/internal/config/config_global.go new file mode 100644 index 000000000..83a31faa4 --- /dev/null +++ b/internal/config/config_global.go @@ -0,0 +1,174 @@ +package config + +import ( + "math/rand" + "time" + + "github.com/matrix-org/gomatrixserverlib" + "golang.org/x/crypto/ed25519" +) + +type Global struct { + // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. + ServerName gomatrixserverlib.ServerName `yaml:"server_name"` + + // Path to the private key which will be used to sign requests and events. + PrivateKeyPath Path `yaml:"private_key"` + + // The private key which will be used to sign requests and events. + PrivateKey ed25519.PrivateKey `yaml:"-"` + + // An arbitrary string used to uniquely identify the PrivateKey. Must start with the + // prefix "ed25519:". + KeyID gomatrixserverlib.KeyID `yaml:"-"` + + // How long a remote server can cache our server key for before requesting it again. + // Increasing this number will reduce the number of requests made by remote servers + // for our key, but increases the period a compromised key will be considered valid + // by remote servers. + // Defaults to 24 hours. + KeyValidityPeriod time.Duration `yaml:"key_validity_period"` + + // List of domains that the server will trust as identity servers to + // verify third-party identifiers. + // Defaults to an empty array. + TrustedIDServers []string `yaml:"trusted_third_party_id_servers"` + + // Kafka/Naffka configuration + Kafka Kafka `yaml:"kafka"` + + // Metrics configuration + Metrics Metrics `yaml:"metrics"` +} + +func (c *Global) Defaults() { + c.ServerName = "localhost" + c.PrivateKeyPath = "matrix.pem" + _, c.PrivateKey, _ = ed25519.GenerateKey(rand.New(rand.NewSource(0))) + c.KeyID = "ed25519:auto" + c.KeyValidityPeriod = time.Hour * 24 * 7 + + c.Kafka.Defaults() + c.Metrics.Defaults() +} + +func (c *Global) Verify(configErrs *configErrors) { + checkNotEmpty(configErrs, "global.server_name", string(c.ServerName)) + checkNotEmpty(configErrs, "global.private_key", string(c.PrivateKeyPath)) + + c.Kafka.Verify(configErrs) + c.Metrics.Verify(configErrs) +} + +type Kafka struct { + // A list of kafka addresses to connect to. + Addresses []string `yaml:"addresses"` + // Whether to use naffka instead of kafka. + // Naffka can only be used when running dendrite as a single monolithic server. + // Kafka can be used both with a monolithic server and when running the + // components as separate servers. + UseNaffka bool `yaml:"use_naffka"` + // The Naffka database is used internally by the naffka library, if used. + Database DataSource `yaml:"naffka_database"` + DatabaseOptions DatabaseOptions `yaml:"naffka_database_options"` + // The names of the topics to use when reading and writing from kafka. + Topics struct { + // Topic for roomserver/api.OutputRoomEvent events. + OutputRoomEvent Topic `yaml:"output_room_event"` + // Topic for sending account data from client API to sync API + OutputClientData Topic `yaml:"output_client_data"` + // Topic for eduserver/api.OutputTypingEvent events. + OutputTypingEvent Topic `yaml:"output_typing_event"` + // Topic for eduserver/api.OutputSendToDeviceEvent events. + OutputSendToDeviceEvent Topic `yaml:"output_send_to_device_event"` + // Topic for keyserver when new device keys are added. + OutputKeyChangeEvent Topic `yaml:"output_key_change_event"` + } +} + +func (c *Kafka) Defaults() { + c.UseNaffka = true + c.Database = DataSource("file:naffka.db") + c.DatabaseOptions.Defaults() + c.Topics.OutputRoomEvent = "OutputRoomEventTopic" + c.Topics.OutputClientData = "OutputClientDataTopic" + c.Topics.OutputTypingEvent = "OutputTypingEventTopic" + c.Topics.OutputSendToDeviceEvent = "OutputSendToDeviceEventTopic" + c.Topics.OutputKeyChangeEvent = "OutputKeyChangeEventTopic" +} + +func (c *Kafka) Verify(configErrs *configErrors) { + if c.UseNaffka { + // TODO: monolithic check + /* + if !monolithic { + configErrs.Add(fmt.Sprintf("naffka can only be used in a monolithic server")) + } + */ + checkNotEmpty(configErrs, "global.kafka.database", string(c.Database)) + } else { + // If we aren't using naffka then we need to have at least one kafka + // server to talk to. + checkNotZero(configErrs, "global.kafka.addresses", int64(len(c.Addresses))) + } + checkNotEmpty(configErrs, "global.kafka.topics.output_room_event", string(c.Topics.OutputRoomEvent)) + checkNotEmpty(configErrs, "global.kafka.topics.output_client_data", string(c.Topics.OutputClientData)) + checkNotEmpty(configErrs, "global.kafka.topics.output_typing_event", string(c.Topics.OutputTypingEvent)) + checkNotEmpty(configErrs, "global.kafka.topics.output_send_to_device_event", string(c.Topics.OutputSendToDeviceEvent)) + checkNotEmpty(configErrs, "global.kafka.topics.output_key_change_event", string(c.Topics.OutputKeyChangeEvent)) +} + +// The configuration to use for Prometheus metrics +type Metrics struct { + // Whether or not the metrics are enabled + Enabled bool `yaml:"enabled"` + // Use BasicAuth for Authorization + BasicAuth struct { + // Authorization via Static Username & Password + // Hardcoded Username and Password + Username string `yaml:"username"` + Password string `yaml:"password"` + } `yaml:"basic_auth"` +} + +func (c *Metrics) Defaults() { + c.Enabled = false + c.BasicAuth.Username = "metrics" + c.BasicAuth.Password = "metrics" +} + +func (c *Metrics) Verify(configErrs *configErrors) { +} + +type DatabaseOptions struct { + // Maximum open connections to the DB (0 = use default, negative means unlimited) + MaxOpenConnections int `yaml:"database_max_open_conns"` + // Maximum idle connections to the DB (0 = use default, negative means unlimited) + MaxIdleConnections int `yaml:"database_max_idle_conns"` + // maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited) + ConnMaxLifetimeSeconds int `yaml:"database_conn_max_lifetime"` +} + +func (c *DatabaseOptions) Defaults() { + c.MaxOpenConnections = 0 + c.MaxIdleConnections = 0 + c.ConnMaxLifetimeSeconds = -1 +} + +func (c *DatabaseOptions) Verify(configErrs *configErrors) { +} + +// MaxIdleConns returns maximum idle connections to the DB +func (c DatabaseOptions) MaxIdleConns() int { + return c.MaxIdleConnections +} + +// MaxOpenConns returns maximum open connections to the DB +func (c DatabaseOptions) MaxOpenConns() int { + return c.MaxOpenConnections +} + +// ConnMaxLifetime returns maximum amount of time a connection may be reused +func (c DatabaseOptions) ConnMaxLifetime() time.Duration { + return time.Duration(c.ConnMaxLifetimeSeconds) * time.Second +} diff --git a/internal/config/config_keyserver.go b/internal/config/config_keyserver.go new file mode 100644 index 000000000..1920e77c5 --- /dev/null +++ b/internal/config/config_keyserver.go @@ -0,0 +1,24 @@ +package config + +type KeyServer struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + Database DataSource `yaml:"database"` + DatabaseOptions DatabaseOptions `yaml:"database_options"` +} + +func (c *KeyServer) Defaults() { + c.Listen = "localhost:7779" + c.Bind = "localhost:7779" + c.Database = "file:keyserver.db" + c.DatabaseOptions.Defaults() +} + +func (c *KeyServer) Verify(configErrs *configErrors) { + checkNotEmpty(configErrs, "key_server.listen", string(c.Listen)) + checkNotEmpty(configErrs, "key_server.bind", string(c.Bind)) + checkNotEmpty(configErrs, "key_server.database", string(c.Database)) +} diff --git a/internal/config/config_mediaapi.go b/internal/config/config_mediaapi.go new file mode 100644 index 000000000..0d35e7001 --- /dev/null +++ b/internal/config/config_mediaapi.go @@ -0,0 +1,63 @@ +package config + +import ( + "fmt" +) + +type MediaAPI struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // The MediaAPI database stores information about files uploaded and downloaded + // by local users. It is only accessed by the MediaAPI. + Database DataSource `yaml:"database"` + DatabaseOptions DatabaseOptions `yaml:"database_options"` + + // The base path to where the media files will be stored. May be relative or absolute. + BasePath Path `yaml:"base_path"` + + // The absolute base path to where media files will be stored. + AbsBasePath Path `yaml:"-"` + + // The maximum file size in bytes that is allowed to be stored on this server. + // Note: if max_file_size_bytes is set to 0, the size is unlimited. + // Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) + MaxFileSizeBytes *FileSizeBytes `yaml:"max_file_size_bytes,omitempty"` + + // Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated + DynamicThumbnails bool `yaml:"dynamic_thumbnails"` + + // The maximum number of simultaneous thumbnail generators. default: 10 + MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"` + + // A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content + ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"` +} + +func (c *MediaAPI) Defaults() { + c.Listen = "localhost:7774" + c.Bind = "localhost:7774" + c.Database = "file:mediaapi.db" + c.DatabaseOptions.Defaults() + + defaultMaxFileSizeBytes := FileSizeBytes(10485760) + c.MaxFileSizeBytes = &defaultMaxFileSizeBytes + c.MaxThumbnailGenerators = 10 +} + +func (c *MediaAPI) Verify(configErrs *configErrors) { + checkNotEmpty(configErrs, "media_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "media_api.bind", string(c.Bind)) + checkNotEmpty(configErrs, "media_api.database", string(c.Database)) + + checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath)) + checkPositive(configErrs, "media_api.max_file_size_bytes", int64(*c.MaxFileSizeBytes)) + checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators)) + + for i, size := range c.ThumbnailSizes { + checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].width", i), int64(size.Width)) + checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].height", i), int64(size.Height)) + } +} diff --git a/internal/config/config_roomserver.go b/internal/config/config_roomserver.go new file mode 100644 index 000000000..33a212554 --- /dev/null +++ b/internal/config/config_roomserver.go @@ -0,0 +1,24 @@ +package config + +type RoomServer struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + Database DataSource `yaml:"database"` + DatabaseOptions DatabaseOptions `yaml:"database_options"` +} + +func (c *RoomServer) Defaults() { + c.Listen = "localhost:7770" + c.Bind = "localhost:7770" + c.Database = "file:roomserver.db" + c.DatabaseOptions.Defaults() +} + +func (c *RoomServer) Verify(configErrs *configErrors) { + checkNotEmpty(configErrs, "room_server.listen", string(c.Listen)) + checkNotEmpty(configErrs, "room_server.bind", string(c.Bind)) + checkNotEmpty(configErrs, "room_server.database", string(c.Database)) +} diff --git a/internal/config/config_serverkey.go b/internal/config/config_serverkey.go new file mode 100644 index 000000000..e22660d4a --- /dev/null +++ b/internal/config/config_serverkey.go @@ -0,0 +1,30 @@ +package config + +type ServerKeyAPI struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // The ServerKey database caches the public keys of remote servers. + // It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI. + Database DataSource `yaml:"database"` + DatabaseOptions DatabaseOptions `yaml:"database_options"` + + // Perspective keyservers, to use as a backup when direct key fetch + // requests don't succeed + KeyPerspectives KeyPerspectives `yaml:"key_perspectives"` +} + +func (c *ServerKeyAPI) Defaults() { + c.Listen = "localhost:7780" + c.Bind = "localhost:7780" + c.Database = "file:serverkeyapi.db" + c.DatabaseOptions.Defaults() +} + +func (c *ServerKeyAPI) Verify(configErrs *configErrors) { + checkNotEmpty(configErrs, "server_key_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "server_key_api.bind", string(c.Bind)) + checkNotEmpty(configErrs, "server_key_api.database", string(c.Database)) +} diff --git a/internal/config/config_syncapi.go b/internal/config/config_syncapi.go new file mode 100644 index 000000000..687bf6043 --- /dev/null +++ b/internal/config/config_syncapi.go @@ -0,0 +1,24 @@ +package config + +type SyncAPI struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + Database DataSource `yaml:"database"` + DatabaseOptions DatabaseOptions `yaml:"database_options"` +} + +func (c *SyncAPI) Defaults() { + c.Listen = "localhost:7773" + c.Bind = "localhost:7773" + c.Database = "file:syncapi.db" + c.DatabaseOptions.Defaults() +} + +func (c *SyncAPI) Verify(configErrs *configErrors) { + checkNotEmpty(configErrs, "sync_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "sync_api.bind", string(c.Bind)) + checkNotEmpty(configErrs, "sync_api.database", string(c.Database)) +} diff --git a/internal/config/config_userapi.go b/internal/config/config_userapi.go new file mode 100644 index 000000000..2c1c4ff8e --- /dev/null +++ b/internal/config/config_userapi.go @@ -0,0 +1,33 @@ +package config + +type UserAPI struct { + Matrix *Global `yaml:"-"` + + Listen Address `yaml:"listen"` + Bind Address `yaml:"bind"` + + // The Account database stores the login details and account information + // for local users. It is accessed by the UserAPI. + AccountDatabase DataSource `yaml:"account_database"` + AccountDatabaseOptions DatabaseOptions `yaml:"account_database_options"` + // The Device database stores session information for the devices of logged + // in local users. It is accessed by the UserAPI. + DeviceDatabase DataSource `yaml:"device_database"` + DeviceDatabaseOptions DatabaseOptions `yaml:"device_database_options"` +} + +func (c *UserAPI) Defaults() { + c.Listen = "localhost:7781" + c.Bind = "localhost:7781" + c.AccountDatabase = "file:userapi_accounts.db" + c.DeviceDatabase = "file:userapi_devices.db" + c.AccountDatabaseOptions.Defaults() + c.DeviceDatabaseOptions.Defaults() +} + +func (c *UserAPI) Verify(configErrs *configErrors) { + checkNotEmpty(configErrs, "user_api.listen", string(c.Listen)) + checkNotEmpty(configErrs, "user_api.bind", string(c.Bind)) + checkNotEmpty(configErrs, "user_api.account_database", string(c.AccountDatabase)) + checkNotEmpty(configErrs, "user_api.device_database", string(c.DeviceDatabase)) +} diff --git a/internal/eventutil/events.go b/internal/eventutil/events.go index 1e3afac8a..35c7f33d8 100644 --- a/internal/eventutil/events.go +++ b/internal/eventutil/events.go @@ -38,7 +38,7 @@ var ErrRoomNoExists = errors.New("Room does not exist") // Returns an error if something else went wrong func BuildEvent( ctx context.Context, - builder *gomatrixserverlib.EventBuilder, cfg *config.Dendrite, evTime time.Time, + builder *gomatrixserverlib.EventBuilder, cfg *config.Global, evTime time.Time, rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse, ) (*gomatrixserverlib.HeaderedEvent, error) { if queryRes == nil { @@ -52,8 +52,8 @@ func BuildEvent( } event, err := builder.Build( - evTime, cfg.Matrix.ServerName, cfg.Matrix.KeyID, - cfg.Matrix.PrivateKey, queryRes.RoomVersion, + evTime, cfg.ServerName, cfg.KeyID, + cfg.PrivateKey, queryRes.RoomVersion, ) if err != nil { return nil, err diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go index d371d1728..8f7723efa 100644 --- a/internal/httputil/httpapi.go +++ b/internal/httputil/httpapi.go @@ -234,7 +234,7 @@ func (f *FederationWakeups) Wakeup(ctx context.Context, origin gomatrixserverlib } // SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics listener -func SetupHTTPAPI(servMux, publicApiMux, internalApiMux *mux.Router, cfg *config.Dendrite, enableHTTPAPIs bool) { +func SetupHTTPAPI(servMux, publicApiMux, internalApiMux *mux.Router, cfg *config.Global, enableHTTPAPIs bool) { if cfg.Metrics.Enabled { servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth)) } diff --git a/internal/setup/base.go b/internal/setup/base.go index 333c01736..0999803b7 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -16,7 +16,6 @@ package setup import ( "database/sql" - "fmt" "io" "net/http" "net/url" @@ -96,7 +95,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo var kafkaConsumer sarama.Consumer var kafkaProducer sarama.SyncProducer - if cfg.Kafka.UseNaffka { + if cfg.Global.Kafka.UseNaffka { kafkaConsumer, kafkaProducer = setupNaffka(cfg) } else { kafkaConsumer, kafkaProducer = setupKafka(cfg) @@ -108,12 +107,15 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo } client := http.Client{Timeout: HTTPClientTimeout} - if cfg.Proxy != nil { - client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{ - Scheme: cfg.Proxy.Protocol, - Host: fmt.Sprintf("%s:%d", cfg.Proxy.Host, cfg.Proxy.Port), - })} - } + // TODO: fix this + /* + if cfg.Proxy != nil { + client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{ + Scheme: cfg.Proxy.Protocol, + Host: fmt.Sprintf("%s:%d", cfg.Proxy.Host, cfg.Proxy.Port), + })} + } + */ // Ideally we would only use SkipClean on routes which we know can allow '/' but due to // https://github.com/gorilla/mux/issues/460 we have to attach this at the top router. @@ -228,7 +230,7 @@ func (b *BaseDendrite) KeyServerHTTPClient() keyserverAPI.KeyInternalAPI { // CreateDeviceDB creates a new instance of the device database. Should only be // called once per component. func (b *BaseDendrite) CreateDeviceDB() devices.Database { - db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName) + db, err := devices.NewDatabase(string(b.Cfg.UserAPI.DeviceDatabase), &b.Cfg.UserAPI.DeviceDatabaseOptions, b.Cfg.Global.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to devices db") } @@ -239,7 +241,7 @@ func (b *BaseDendrite) CreateDeviceDB() devices.Database { // CreateAccountsDB creates a new instance of the accounts database. Should only // be called once per component. func (b *BaseDendrite) CreateAccountsDB() accounts.Database { - db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName) + db, err := accounts.NewDatabase(string(b.Cfg.UserAPI.AccountDatabase), &b.Cfg.UserAPI.AccountDatabaseOptions, b.Cfg.Global.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to accounts db") } @@ -251,7 +253,7 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database { // once per component. func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationClient { return gomatrixserverlib.NewFederationClient( - b.Cfg.Matrix.ServerName, b.Cfg.Matrix.KeyID, b.Cfg.Matrix.PrivateKey, + b.Cfg.Global.ServerName, b.Cfg.Global.KeyID, b.Cfg.Global.PrivateKey, ) } @@ -276,7 +278,7 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { b.BaseMux, b.PublicAPIMux, b.InternalAPIMux, - b.Cfg, + &b.Cfg.Global, b.UseHTTPAPIs, ) serv.Handler = b.BaseMux @@ -292,12 +294,12 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { // setupKafka creates kafka consumer/producer pair from the config. func setupKafka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { - consumer, err := sarama.NewConsumer(cfg.Kafka.Addresses, nil) + consumer, err := sarama.NewConsumer(cfg.Global.Kafka.Addresses, nil) if err != nil { logrus.WithError(err).Panic("failed to start kafka consumer") } - producer, err := sarama.NewSyncProducer(cfg.Kafka.Addresses, nil) + producer, err := sarama.NewSyncProducer(cfg.Global.Kafka.Addresses, nil) if err != nil { logrus.WithError(err).Panic("failed to setup kafka producers") } @@ -311,10 +313,10 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { var db *sql.DB var naffkaDB *naffka.DatabaseImpl - uri, err := url.Parse(string(cfg.Database.Naffka)) + uri, err := url.Parse(string(cfg.Global.Kafka.Database)) if err != nil || uri.Scheme == "file" { var cs string - cs, err = sqlutil.ParseFileURI(string(cfg.Database.Naffka)) + cs, err = sqlutil.ParseFileURI(string(cfg.Global.Kafka.Database)) if err != nil { logrus.WithError(err).Panic("Failed to parse naffka database file URI") } @@ -328,7 +330,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { logrus.WithError(err).Panic("Failed to setup naffka database") } } else { - db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka), nil) + db, err = sqlutil.Open("postgres", string(cfg.Global.Kafka.Database), nil) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index 1f6d9a761..a2cd9be23 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -65,18 +65,18 @@ type Monolith struct { // AddAllPublicRoutes attaches all public paths to the given router func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) { clientapi.AddPublicRoutes( - publicMux, m.Config, m.KafkaProducer, m.DeviceDB, m.AccountDB, + publicMux, &m.Config.ClientAPI, m.KafkaProducer, m.DeviceDB, m.AccountDB, m.FedClient, m.RoomserverAPI, m.EDUInternalAPI, m.AppserviceAPI, m.StateAPI, transactions.New(), m.FederationSenderAPI, m.UserAPI, m.KeyAPI, m.ExtPublicRoomsProvider, ) federationapi.AddPublicRoutes( - publicMux, m.Config, m.UserAPI, m.FedClient, + publicMux, &m.Config.FederationAPI, m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationSenderAPI, m.EDUInternalAPI, m.StateAPI, m.KeyAPI, ) - mediaapi.AddPublicRoutes(publicMux, m.Config, m.UserAPI, m.Client) + mediaapi.AddPublicRoutes(publicMux, &m.Config.MediaAPI, m.UserAPI, m.Client) syncapi.AddPublicRoutes( - publicMux, m.KafkaConsumer, m.UserAPI, m.RoomserverAPI, m.FedClient, m.Config, + publicMux, m.KafkaConsumer, m.UserAPI, m.RoomserverAPI, m.FedClient, &m.Config.SyncAPI, ) } diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index 47c6a8c37..2f1bfb016 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -37,17 +37,14 @@ func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI) { // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - cfg *config.Dendrite, fedClient *gomatrixserverlib.FederationClient, userAPI userapi.UserInternalAPI, producer sarama.SyncProducer, + cfg *config.KeyServer, fedClient *gomatrixserverlib.FederationClient, userAPI userapi.UserInternalAPI, producer sarama.SyncProducer, ) api.KeyInternalAPI { - db, err := storage.NewDatabase( - string(cfg.Database.E2EKey), - cfg.DbProperties(), - ) + db, err := storage.NewDatabase(string(cfg.Database), cfg.DatabaseOptions) if err != nil { logrus.WithError(err).Panicf("failed to connect to key server database") } keyChangeProducer := &producers.KeyChange{ - Topic: string(cfg.Kafka.Topics.OutputKeyChangeEvent), + Topic: string(cfg.Matrix.Kafka.Topics.OutputKeyChangeEvent), Producer: producer, } return &internal.KeyInternalAPI{ diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index 290ef46e1..79f5e8ab7 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -26,11 +26,11 @@ import ( // AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component. func AddPublicRoutes( - router *mux.Router, cfg *config.Dendrite, + router *mux.Router, cfg *config.MediaAPI, userAPI userapi.UserInternalAPI, client *gomatrixserverlib.Client, ) { - mediaDB, err := storage.Open(string(cfg.Database.MediaAPI), cfg.DbProperties()) + mediaDB, err := storage.Open(string(cfg.Database), cfg.DatabaseOptions) if err != nil { logrus.WithError(err).Panicf("failed to connect to media db") } diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 7e121de3e..e61fa82b0 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -73,7 +73,7 @@ func Download( req *http.Request, origin gomatrixserverlib.ServerName, mediaID types.MediaID, - cfg *config.Dendrite, + cfg *config.MediaAPI, db storage.Database, client *gomatrixserverlib.Client, activeRemoteRequests *types.ActiveRemoteRequests, @@ -203,7 +203,7 @@ func (r *downloadRequest) Validate() *util.JSONResponse { func (r *downloadRequest) doDownload( ctx context.Context, w http.ResponseWriter, - cfg *config.Dendrite, + cfg *config.MediaAPI, db storage.Database, client *gomatrixserverlib.Client, activeRemoteRequests *types.ActiveRemoteRequests, @@ -233,9 +233,9 @@ func (r *downloadRequest) doDownload( r.MediaMetadata = mediaMetadata } return r.respondFromLocalFile( - ctx, w, cfg.Media.AbsBasePath, activeThumbnailGeneration, - cfg.Media.MaxThumbnailGenerators, db, - cfg.Media.DynamicThumbnails, cfg.Media.ThumbnailSizes, + ctx, w, cfg.AbsBasePath, activeThumbnailGeneration, + cfg.MaxThumbnailGenerators, db, + cfg.DynamicThumbnails, cfg.ThumbnailSizes, ) } @@ -514,7 +514,7 @@ func (r *downloadRequest) generateThumbnail( func (r *downloadRequest) getRemoteFile( ctx context.Context, client *gomatrixserverlib.Client, - cfg *config.Dendrite, + cfg *config.MediaAPI, db storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, @@ -550,9 +550,9 @@ func (r *downloadRequest) getRemoteFile( // If we do not have a record, we need to fetch the remote file first and then respond from the local file err := r.fetchRemoteFileAndStoreMetadata( ctx, client, - cfg.Media.AbsBasePath, *cfg.Media.MaxFileSizeBytes, db, - cfg.Media.ThumbnailSizes, activeThumbnailGeneration, - cfg.Media.MaxThumbnailGenerators, + cfg.AbsBasePath, *cfg.MaxFileSizeBytes, db, + cfg.ThumbnailSizes, activeThumbnailGeneration, + cfg.MaxThumbnailGenerators, ) if err != nil { return errors.Wrap(err, "error querying the database.") diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index f4a8b1575..ee5f28f93 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -42,7 +42,7 @@ const pathPrefixV1 = "/media/v1" // TODO: remove when synapse is fixed // nolint: gocyclo func Setup( publicAPIMux *mux.Router, - cfg *config.Dendrite, + cfg *config.MediaAPI, db storage.Database, userAPI userapi.UserInternalAPI, client *gomatrixserverlib.Client, @@ -81,7 +81,7 @@ func Setup( func makeDownloadAPI( name string, - cfg *config.Dendrite, + cfg *config.MediaAPI, db storage.Database, client *gomatrixserverlib.Client, activeRemoteRequests *types.ActiveRemoteRequests, diff --git a/mediaapi/routing/upload.go b/mediaapi/routing/upload.go index 9b5dc3df8..9f35e90c0 100644 --- a/mediaapi/routing/upload.go +++ b/mediaapi/routing/upload.go @@ -54,7 +54,7 @@ type uploadResponse struct { // This implementation supports a configurable maximum file size limit in bytes. If a user tries to upload more than this, they will receive an error that their upload is too large. // Uploaded files are processed piece-wise to avoid DoS attacks which would starve the server of memory. // TODO: We should time out requests if they have not received any data within a configured timeout period. -func Upload(req *http.Request, cfg *config.Dendrite, db storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse { +func Upload(req *http.Request, cfg *config.MediaAPI, db storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse { r, resErr := parseAndValidateRequest(req, cfg) if resErr != nil { return *resErr @@ -75,7 +75,7 @@ func Upload(req *http.Request, cfg *config.Dendrite, db storage.Database, active // parseAndValidateRequest parses the incoming upload request to validate and extract // all the metadata about the media being uploaded. // Returns either an uploadRequest or an error formatted as a util.JSONResponse -func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRequest, *util.JSONResponse) { +func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRequest, *util.JSONResponse) { r := &uploadRequest{ MediaMetadata: &types.MediaMetadata{ Origin: cfg.Matrix.ServerName, @@ -86,7 +86,7 @@ func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRe Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.Matrix.ServerName), } - if resErr := r.Validate(*cfg.Media.MaxFileSizeBytes); resErr != nil { + if resErr := r.Validate(*cfg.MaxFileSizeBytes); resErr != nil { return nil, resErr } @@ -96,7 +96,7 @@ func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRe func (r *uploadRequest) doUpload( ctx context.Context, reqReader io.Reader, - cfg *config.Dendrite, + cfg *config.MediaAPI, db storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration, ) *util.JSONResponse { @@ -110,10 +110,10 @@ func (r *uploadRequest) doUpload( // method of deduplicating files to save storage, as well as a way to conduct // integrity checks on the file data in the repository. // Data is truncated to maxFileSizeBytes. Content-Length was reported as 0 < Content-Length <= maxFileSizeBytes so this is OK. - hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.Media.MaxFileSizeBytes, cfg.Media.AbsBasePath) + hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.MaxFileSizeBytes, cfg.AbsBasePath) if err != nil { r.Logger.WithError(err).WithFields(log.Fields{ - "MaxFileSizeBytes": *cfg.Media.MaxFileSizeBytes, + "MaxFileSizeBytes": *cfg.MaxFileSizeBytes, }).Warn("Error while transferring file") fileutils.RemoveDir(tmpDir, r.Logger) return &util.JSONResponse{ @@ -159,8 +159,8 @@ func (r *uploadRequest) doUpload( } return r.storeFileAndMetadata( - ctx, tmpDir, cfg.Media.AbsBasePath, db, cfg.Media.ThumbnailSizes, - activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators, + ctx, tmpDir, cfg.AbsBasePath, db, cfg.ThumbnailSizes, + activeThumbnailGeneration, cfg.MaxThumbnailGenerators, ) } diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index 37a8a39bf..6d7c8fcfe 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -14,7 +14,7 @@ import ( // RoomserverInternalAPI is an implementation of api.RoomserverInternalAPI type RoomserverInternalAPI struct { DB storage.Database - Cfg *config.Dendrite + Cfg *config.RoomServer Producer sarama.SyncProducer Cache caching.RoomVersionCache ServerName gomatrixserverlib.ServerName diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go index c5480a5bd..73ea008d9 100644 --- a/roomserver/internal/perform_join.go +++ b/roomserver/internal/perform_join.go @@ -189,12 +189,12 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( // but everyone has since left. I suspect it does the wrong thing. buildRes := api.QueryLatestEventsAndStateResponse{} event, err := eventutil.BuildEvent( - ctx, // the request context - &eb, // the template join event - r.Cfg, // the server configuration - time.Now(), // the event timestamp to use - r, // the roomserver API to use - &buildRes, // the query response + ctx, // the request context + &eb, // the template join event + r.Cfg.Matrix, // the server configuration + time.Now(), // the event timestamp to use + r, // the roomserver API to use + &buildRes, // the query response ) switch err { diff --git a/roomserver/internal/perform_leave.go b/roomserver/internal/perform_leave.go index a19d0da9f..79676530f 100644 --- a/roomserver/internal/perform_leave.go +++ b/roomserver/internal/perform_leave.go @@ -99,12 +99,12 @@ func (r *RoomserverInternalAPI) performLeaveRoomByID( // but everyone has since left. I suspect it does the wrong thing. buildRes := api.QueryLatestEventsAndStateResponse{} event, err := eventutil.BuildEvent( - ctx, // the request context - &eb, // the template leave event - r.Cfg, // the server configuration - time.Now(), // the event timestamp to use - r, // the roomserver API to use - &buildRes, // the query response + ctx, // the request context + &eb, // the template leave event + r.Cfg.Matrix, // the server configuration + time.Now(), // the event timestamp to use + r, // the roomserver API to use + &buildRes, // the query response ) if err != nil { return fmt.Errorf("eventutil.BuildEvent: %w", err) diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 427d5ff36..250b46993 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -39,18 +39,20 @@ func NewInternalAPI( keyRing gomatrixserverlib.JSONVerifier, fedClient *gomatrixserverlib.FederationClient, ) api.RoomserverInternalAPI { - roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer), base.Cfg.DbProperties()) + cfg := &base.Cfg.RoomServer + + roomserverDB, err := storage.Open(string(cfg.Database), cfg.DatabaseOptions) if err != nil { logrus.WithError(err).Panicf("failed to connect to room server db") } return &internal.RoomserverInternalAPI{ DB: roomserverDB, - Cfg: base.Cfg, + Cfg: cfg, Producer: base.KafkaProducer, - OutputRoomEventTopic: string(base.Cfg.Kafka.Topics.OutputRoomEvent), + OutputRoomEventTopic: string(cfg.Matrix.Kafka.Topics.OutputRoomEvent), Cache: base.Caches, - ServerName: base.Cfg.Matrix.ServerName, + ServerName: cfg.Matrix.ServerName, FedClient: fedClient, KeyRing: keyRing, } diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index d553c5b79..2c172981d 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -94,13 +94,19 @@ func mustLoadEvents(t *testing.T, ver gomatrixserverlib.RoomVersion, events []js } func mustSendEvents(t *testing.T, ver gomatrixserverlib.RoomVersion, events []json.RawMessage) (api.RoomserverInternalAPI, *dummyProducer, []gomatrixserverlib.HeaderedEvent) { - cfg := &config.Dendrite{} - cfg.Database.RoomServer = roomserverDBFileURI - cfg.Kafka.Topics.OutputRoomEvent = "output_room_event" - cfg.Matrix.ServerName = testOrigin - cfg.Kafka.UseNaffka = true + cfg := &config.Dendrite{ + Global: config.Global{ + ServerName: testOrigin, + Kafka: config.Kafka{ + UseNaffka: true, + }, + }, + RoomServer: config.RoomServer{}, + } + cfg.RoomServer.Database = roomserverDBFileURI + cfg.Global.Kafka.Topics.OutputRoomEvent = "output_room_event" dp := &dummyProducer{ - topic: string(cfg.Kafka.Topics.OutputRoomEvent), + topic: string(cfg.Global.Kafka.Topics.OutputRoomEvent), } cache, err := caching.NewInMemoryLRUCache(true) if err != nil { diff --git a/serverkeyapi/serverkeyapi.go b/serverkeyapi/serverkeyapi.go index cddd392ed..2919bc75f 100644 --- a/serverkeyapi/serverkeyapi.go +++ b/serverkeyapi/serverkeyapi.go @@ -25,13 +25,13 @@ func AddInternalRoutes(router *mux.Router, intAPI api.ServerKeyInternalAPI, cach // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - cfg *config.Dendrite, + cfg *config.ServerKeyAPI, fedClient *gomatrixserverlib.FederationClient, caches *caching.Caches, ) api.ServerKeyInternalAPI { innerDB, err := storage.NewDatabase( - string(cfg.Database.ServerKey), - cfg.DbProperties(), + string(cfg.Database), + cfg.DatabaseOptions, cfg.Matrix.ServerName, cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), cfg.Matrix.KeyID, @@ -62,7 +62,7 @@ func NewInternalAPI( } var b64e = base64.StdEncoding.WithPadding(base64.NoPadding) - for _, ps := range cfg.Matrix.KeyPerspectives { + for _, ps := range cfg.KeyPerspectives { perspective := &gomatrixserverlib.PerspectiveKeyFetcher{ PerspectiveServerName: ps.ServerName, PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{}, diff --git a/serverkeyapi/serverkeyapi_test.go b/serverkeyapi/serverkeyapi_test.go index 3368f5b2a..c8f9c3a4a 100644 --- a/serverkeyapi/serverkeyapi_test.go +++ b/serverkeyapi/serverkeyapi_test.go @@ -23,7 +23,7 @@ import ( type server struct { name gomatrixserverlib.ServerName // server name validity time.Duration // key validity duration from now - config *config.Dendrite // skeleton config, from TestMain + config *config.ServerKeyAPI // skeleton config, from TestMain fedclient *gomatrixserverlib.FederationClient // uses MockRoundTripper cache *caching.Caches // server-specific cache api api.ServerKeyInternalAPI // server-specific server key API @@ -69,13 +69,15 @@ func TestMain(m *testing.M) { // Draw up just enough Dendrite config for the server key // API to work. - s.config = &config.Dendrite{} - s.config.SetDefaults() - s.config.Matrix.ServerName = gomatrixserverlib.ServerName(s.name) - s.config.Matrix.PrivateKey = testPriv - s.config.Matrix.KeyID = serverKeyID - s.config.Matrix.KeyValidityPeriod = s.validity - s.config.Database.ServerKey = config.DataSource("file::memory:") + s.config = &config.ServerKeyAPI{ + Matrix: &config.Global{ + ServerName: gomatrixserverlib.ServerName(s.name), + PrivateKey: testPriv, + KeyID: serverKeyID, + KeyValidityPeriod: s.validity, + }, + Database: config.DataSource("file::memory:"), + } // Create a transport which redirects federation requests to // the mock round tripper. Since we're not *really* listening for diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index ad6290e3f..813feee15 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -37,14 +37,14 @@ type OutputClientDataConsumer struct { // NewOutputClientDataConsumer creates a new OutputClientData consumer. Call Start() to begin consuming from room servers. func NewOutputClientDataConsumer( - cfg *config.Dendrite, + cfg *config.SyncAPI, kafkaConsumer sarama.Consumer, n *sync.Notifier, store storage.Database, ) *OutputClientDataConsumer { consumer := internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputClientData), + Topic: string(cfg.Matrix.Kafka.Topics.OutputClientData), Consumer: kafkaConsumer, PartitionStore: store, } diff --git a/syncapi/consumers/eduserver_sendtodevice.go b/syncapi/consumers/eduserver_sendtodevice.go index 487018031..de7b61db9 100644 --- a/syncapi/consumers/eduserver_sendtodevice.go +++ b/syncapi/consumers/eduserver_sendtodevice.go @@ -41,14 +41,14 @@ type OutputSendToDeviceEventConsumer struct { // NewOutputSendToDeviceEventConsumer creates a new OutputSendToDeviceEventConsumer. // Call Start() to begin consuming from the EDU server. func NewOutputSendToDeviceEventConsumer( - cfg *config.Dendrite, + cfg *config.SyncAPI, kafkaConsumer sarama.Consumer, n *sync.Notifier, store storage.Database, ) *OutputSendToDeviceEventConsumer { consumer := internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputSendToDeviceEvent), + Topic: string(cfg.Matrix.Kafka.Topics.OutputSendToDeviceEvent), Consumer: kafkaConsumer, PartitionStore: store, } diff --git a/syncapi/consumers/eduserver_typing.go b/syncapi/consumers/eduserver_typing.go index 12b1efbc0..72e0deaa3 100644 --- a/syncapi/consumers/eduserver_typing.go +++ b/syncapi/consumers/eduserver_typing.go @@ -37,14 +37,14 @@ type OutputTypingEventConsumer struct { // NewOutputTypingEventConsumer creates a new OutputTypingEventConsumer. // Call Start() to begin consuming from the EDU server. func NewOutputTypingEventConsumer( - cfg *config.Dendrite, + cfg *config.SyncAPI, kafkaConsumer sarama.Consumer, n *sync.Notifier, store storage.Database, ) *OutputTypingEventConsumer { consumer := internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputTypingEvent), + Topic: string(cfg.Matrix.Kafka.Topics.OutputTypingEvent), Consumer: kafkaConsumer, PartitionStore: store, } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index c65027168..91f215f84 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -39,7 +39,7 @@ type OutputRoomEventConsumer struct { // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. func NewOutputRoomEventConsumer( - cfg *config.Dendrite, + cfg *config.SyncAPI, kafkaConsumer sarama.Consumer, n *sync.Notifier, store storage.Database, @@ -47,7 +47,7 @@ func NewOutputRoomEventConsumer( ) *OutputRoomEventConsumer { consumer := internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputRoomEvent), + Topic: string(cfg.Matrix.Kafka.Topics.OutputRoomEvent), Consumer: kafkaConsumer, PartitionStore: store, } diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 15add1b45..0999d3e8c 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -36,7 +36,7 @@ type messagesReq struct { db storage.Database rsAPI api.RoomserverInternalAPI federation *gomatrixserverlib.FederationClient - cfg *config.Dendrite + cfg *config.SyncAPI roomID string from *types.TopologyToken to *types.TopologyToken @@ -61,7 +61,7 @@ func OnIncomingMessagesRequest( req *http.Request, db storage.Database, roomID string, federation *gomatrixserverlib.FederationClient, rsAPI api.RoomserverInternalAPI, - cfg *config.Dendrite, + cfg *config.SyncAPI, ) util.JSONResponse { var err error diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index a98955c57..ab0a35090 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -39,7 +39,7 @@ func Setup( publicAPIMux *mux.Router, srp *sync.RequestPool, syncDB storage.Database, userAPI userapi.UserInternalAPI, federation *gomatrixserverlib.FederationClient, rsAPI api.RoomserverInternalAPI, - cfg *config.Dendrite, + cfg *config.SyncAPI, ) { r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index caf91e27e..a7a4add02 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -40,9 +40,9 @@ func AddPublicRoutes( userAPI userapi.UserInternalAPI, rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, - cfg *config.Dendrite, + cfg *config.SyncAPI, ) { - syncDB, err := storage.NewSyncServerDatasource(string(cfg.Database.SyncAPI), cfg.DbProperties()) + syncDB, err := storage.NewSyncServerDatasource(string(cfg.Database), cfg.DatabaseOptions) if err != nil { logrus.WithError(err).Panicf("failed to connect to sync db") }