Configuration format v1 (#1230)

* Initial pass at refactoring config (not finished)

* Don't forget current state and EDU servers

* More shifting around

* Update server key API tests

* Fix roomserver test

* Fix more tests

* Further tweaks

* Fix current state server test (sort of)

* Maybe fix appservices

* Fix client API test

* Include database connection string in database options

* Fix sync API build

* Update config test

* Fix unit tests

* Fix federation sender build

* Fix gobind build

* Set Listen address for all services in HTTP monolith mode

* Validate config, reinstate appservice derived in directory, tweaks

* Tweak federation API test

* Set MaxOpenConnections/MaxIdleConnections to previous values

* Update generate-config

appservice/appservice.go

@@ -48,7 +48,7 @@ func NewInternalAPI(
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) appserviceAPI.AppServiceQueryAPI {
 	// Create a connection to the appservice postgres DB
-	appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService), base.Cfg.DbProperties())
+	appserviceDB, err := storage.NewDatabase(&base.Cfg.AppServiceAPI.Database)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to appservice db")
 	}

appservice/consumers/roomserver.go

@@ -48,7 +48,7 @@ func NewOutputRoomEventConsumer(
 	workerStates []types.ApplicationServiceWorkerState,
 ) *OutputRoomEventConsumer {
 	consumer := internal.ContinualConsumer{
-		Topic:          string(cfg.Kafka.Topics.OutputRoomEvent),
+		Topic:          string(cfg.Global.Kafka.Topics.OutputRoomEvent),
 		Consumer:       kafkaConsumer,
 		PartitionStore: appserviceDB,
 	}
@@ -56,7 +56,7 @@ func NewOutputRoomEventConsumer(
 		roomServerConsumer: &consumer,
 		asDB:               appserviceDB,
 		rsAPI:              rsAPI,
-		serverName:         string(cfg.Matrix.ServerName),
+		serverName:         string(cfg.Global.ServerName),
 		workerStates:       workerStates,
 	}
 	consumer.ProcessMessage = s.onMessage

appservice/storage/postgres/storage.go

@@ -21,6 +21,7 @@ import (
 
 	// Import postgres database driver
 	_ "github.com/lib/pq"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -34,10 +35,10 @@ type Database struct {
 }
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) {
+func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
+	if result.db, err = sqlutil.Open(dbProperties); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {

appservice/storage/sqlite3/storage.go

@@ -20,6 +20,7 @@ import (
 	"database/sql"
 
 	// Import SQLite database driver
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 	_ "github.com/mattn/go-sqlite3"
@@ -34,14 +35,10 @@ type Database struct {
 }
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string) (*Database, error) {
+func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) {
 	var result Database
 	var err error
-	cs, err := sqlutil.ParseFileURI(dataSourceName)
-	if err != nil {
-		return nil, err
-	}
-	if result.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil {
+	if result.db, err = sqlutil.Open(dbProperties); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {

appservice/storage/storage.go

@@ -17,26 +17,22 @@
 package storage
 
 import (
-	"net/url"
+	"fmt"
 
 	"github.com/matrix-org/dendrite/appservice/storage/postgres"
 	"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 )
 
 // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
 // and sets DB connection parameters
-func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return postgres.NewDatabase(dataSourceName, dbProperties)
-	}
-	switch uri.Scheme {
-	case "postgres":
-		return postgres.NewDatabase(dataSourceName, dbProperties)
-	case "file":
-		return sqlite3.NewDatabase(dataSourceName)
+func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.NewDatabase(dbProperties)
+	case dbProperties.ConnectionString.IsPostgres():
+		return postgres.NewDatabase(dbProperties)
 	default:
-		return postgres.NewDatabase(dataSourceName, dbProperties)
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }

appservice/storage/storage_wasm.go

@@ -16,26 +16,18 @@ package storage
 
 import (
 	"fmt"
-	"net/url"
 
 	"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 )
 
-func NewDatabase(
-	dataSourceName string,
-	dbProperties sqlutil.DbProperties, // nolint:unparam
-) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	}
-	switch uri.Scheme {
-	case "postgres":
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	case "file":
-		return sqlite3.NewDatabase(dataSourceName)
+func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.NewDatabase(dbProperties)
+	case dbProperties.ConnectionString.IsPostgres():
+		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
-		return nil, fmt.Errorf("Cannot use postgres implementation")
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }

build/gobind/monolith.go

@@ -83,29 +83,29 @@ func (m *DendriteMonolith) Start() {
 	m.YggdrasilNode = ygg
 
 	cfg := &config.Dendrite{}
-	cfg.SetDefaults()
-	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
-	cfg.Matrix.PrivateKey = ygg.SigningPrivateKey()
-	cfg.Matrix.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
-	cfg.Matrix.FederationMaxRetries = 8
-	cfg.Kafka.UseNaffka = true
-	cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput"
-	cfg.Kafka.Topics.OutputClientData = "clientapiOutput"
-	cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput"
-	cfg.Kafka.Topics.OutputSendToDeviceEvent = "sendToDeviceOutput"
-	cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s/dendrite-account.db", m.StorageDirectory))
-	cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s/dendrite-device.db", m.StorageDirectory))
-	cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-mediaapi.db", m.StorageDirectory))
-	cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-syncapi.db", m.StorageDirectory))
-	cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s/dendrite-roomserver.db", m.StorageDirectory))
-	cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s/dendrite-serverkey.db", m.StorageDirectory))
-	cfg.Database.E2EKey = config.DataSource(fmt.Sprintf("file:%s/dendrite-keyserver.db", m.StorageDirectory))
-	cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s/dendrite-federationsender.db", m.StorageDirectory))
-	cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s/dendrite-appservice.db", m.StorageDirectory))
-	cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s/dendrite-currentstate.db", m.StorageDirectory))
-	cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s/dendrite-naffka.db", m.StorageDirectory))
-	cfg.Media.BasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory))
-	cfg.Media.AbsBasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory))
+	cfg.Defaults()
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
+	cfg.Global.PrivateKey = ygg.SigningPrivateKey()
+	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
+	cfg.Global.Kafka.UseNaffka = true
+	cfg.Global.Kafka.Topics.OutputRoomEvent = "roomserverOutput"
+	cfg.Global.Kafka.Topics.OutputClientData = "clientapiOutput"
+	cfg.Global.Kafka.Topics.OutputTypingEvent = "typingServerOutput"
+	cfg.Global.Kafka.Topics.OutputSendToDeviceEvent = "sendToDeviceOutput"
+	cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-naffka.db", m.StorageDirectory))
+	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-account.db", m.StorageDirectory))
+	cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-device.db", m.StorageDirectory))
+	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-mediaapi.db", m.StorageDirectory))
+	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-syncapi.db", m.StorageDirectory))
+	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-roomserver.db", m.StorageDirectory))
+	cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-serverkey.db", m.StorageDirectory))
+	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-keyserver.db", m.StorageDirectory))
+	cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-federationsender.db", m.StorageDirectory))
+	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-appservice.db", m.StorageDirectory))
+	cfg.CurrentStateServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-currentstate.db", m.StorageDirectory))
+	cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory))
+	cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory))
+	cfg.FederationSender.FederationMaxRetries = 8
 	if err = cfg.Derive(); err != nil {
 		panic(err)
 	}
@@ -119,8 +119,8 @@ func (m *DendriteMonolith) Start() {
 
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
-	keyAPI := keyserver.NewInternalAPI(base.Cfg, federation, base.KafkaProducer)
-	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices, keyAPI)
+	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation, base.KafkaProducer)
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, cfg.Derived.ApplicationServices, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
 	rsAPI := roomserver.NewInternalAPI(
@@ -132,7 +132,7 @@ func (m *DendriteMonolith) Start() {
 	)
 
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
-	stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
+	stateAPI := currentstateserver.NewInternalAPI(&base.Cfg.CurrentStateServer, base.KafkaConsumer)
 	fsAPI := federationsender.NewInternalAPI(
 		base, federation, rsAPI, stateAPI, keyRing,
 	)
@@ -180,7 +180,7 @@ func (m *DendriteMonolith) Start() {
 		base.BaseMux,
 		base.PublicAPIMux,
 		base.InternalAPIMux,
-		cfg,
+		&cfg.Global,
 		base.UseHTTPAPIs,
 	)
 

clientapi/auth/password.go

@@ -35,7 +35,7 @@ type PasswordRequest struct {
 // LoginTypePassword implements https://matrix.org/docs/spec/client_server/r0.6.1#password-based
 type LoginTypePassword struct {
 	GetAccountByPassword GetAccountByPassword
-	Config               *config.Dendrite
+	Config               *config.ClientAPI
 }
 
 func (t *LoginTypePassword) Name() string {

clientapi/auth/user_interactive.go

@@ -110,7 +110,7 @@ type UserInteractive struct {
 	Sessions map[string][]string
 }
 
-func NewUserInteractive(getAccByPass GetAccountByPassword, cfg *config.Dendrite) *UserInteractive {
+func NewUserInteractive(getAccByPass GetAccountByPassword, cfg *config.ClientAPI) *UserInteractive {
 	typePassword := &LoginTypePassword{
 		GetAccountByPassword: getAccByPass,
 		Config:               cfg,

clientapi/auth/user_interactive_test.go

@@ -33,8 +33,11 @@ func getAccountByPassword(ctx context.Context, localpart, plaintextPassword stri
 }
 
 func setup() *UserInteractive {
-	cfg := &config.Dendrite{}
-	cfg.Matrix.ServerName = serverName
+	cfg := &config.ClientAPI{
+		Matrix: &config.Global{
+			ServerName: serverName,
+		},
+	}
 	return NewUserInteractive(getAccountByPassword, cfg)
 }
 

clientapi/clientapi.go

@@ -37,7 +37,7 @@ import (
 // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component.
 func AddPublicRoutes(
 	router *mux.Router,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	producer sarama.SyncProducer,
 	deviceDB devices.Database,
 	accountsDB accounts.Database,
@@ -54,7 +54,7 @@ func AddPublicRoutes(
 ) {
 	syncProducer := &producers.SyncAPIProducer{
 		Producer: producer,
-		Topic:    string(cfg.Kafka.Topics.OutputClientData),
+		Topic:    string(cfg.Matrix.Kafka.Topics.OutputClientData),
 	}
 
 	routing.Setup(

clientapi/routing/auth_fallback.go

@@ -101,7 +101,7 @@ func serveTemplate(w http.ResponseWriter, templateHTML string, data map[string]s
 // AuthFallback implements GET and POST /auth/{authType}/fallback/web?session={sessionID}
 func AuthFallback(
 	w http.ResponseWriter, req *http.Request, authType string,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 ) *util.JSONResponse {
 	sessionID := req.URL.Query().Get("session")
 
@@ -116,7 +116,7 @@ func AuthFallback(
 		data := map[string]string{
 			"myUrl":   req.URL.String(),
 			"session": sessionID,
-			"siteKey": cfg.Matrix.RecaptchaPublicKey,
+			"siteKey": cfg.RecaptchaPublicKey,
 		}
 		serveTemplate(w, recaptchaTemplate, data)
 	}
@@ -181,11 +181,11 @@ func AuthFallback(
 
 // checkRecaptchaEnabled creates an error response if recaptcha is not usable on homeserver.
 func checkRecaptchaEnabled(
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	w http.ResponseWriter,
 	req *http.Request,
 ) *util.JSONResponse {
-	if !cfg.Matrix.RecaptchaEnabled {
+	if !cfg.RecaptchaEnabled {
 		return writeHTTPMessage(w, req,
 			"Recaptcha login is disabled on this Homeserver",
 			http.StatusBadRequest,

clientapi/routing/createroom.go

@@ -135,7 +135,7 @@ type fledglingEvent struct {
 // CreateRoom implements /createRoom
 func CreateRoom(
 	req *http.Request, device *api.Device,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
@@ -149,7 +149,7 @@ func CreateRoom(
 // nolint: gocyclo
 func createRoom(
 	req *http.Request, device *api.Device,
-	cfg *config.Dendrite, roomID string,
+	cfg *config.ClientAPI, roomID string,
 	accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
@@ -438,7 +438,7 @@ func createRoom(
 func buildEvent(
 	builder *gomatrixserverlib.EventBuilder,
 	provider gomatrixserverlib.AuthEventProvider,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	evTime time.Time,
 	roomVersion gomatrixserverlib.RoomVersion,
 ) (*gomatrixserverlib.Event, error) {

clientapi/routing/directory.go

@@ -47,7 +47,7 @@ func DirectoryRoom(
 	req *http.Request,
 	roomAlias string,
 	federation *gomatrixserverlib.FederationClient,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	fedSenderAPI federationSenderAPI.FederationSenderInternalAPI,
 ) util.JSONResponse {
@@ -116,7 +116,7 @@ func SetLocalAlias(
 	req *http.Request,
 	device *api.Device,
 	alias string,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	aliasAPI roomserverAPI.RoomserverInternalAPI,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('#', alias)
@@ -139,6 +139,7 @@ func SetLocalAlias(
 	// TODO: This code should eventually be refactored with:
 	// 1. The new method for checking for things matching an AS's namespace
 	// 2. Using an overall Regex object for all AS's just like we did for usernames
+
 	for _, appservice := range cfg.Derived.ApplicationServices {
 		// Don't prevent AS from creating aliases in its own namespace
 		// Note that Dendrite uses SenderLocalpart as UserID for AS users

clientapi/routing/getevent.go

@@ -30,7 +30,7 @@ type getEventRequest struct {
 	device         *userapi.Device
 	roomID         string
 	eventID        string
-	cfg            *config.Dendrite
+	cfg            *config.ClientAPI
 	federation     *gomatrixserverlib.FederationClient
 	requestedEvent gomatrixserverlib.Event
 }
@@ -42,7 +42,7 @@ func GetEvent(
 	device *userapi.Device,
 	roomID string,
 	eventID string,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	rsAPI api.RoomserverInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {

clientapi/routing/login.go

@@ -58,7 +58,7 @@ func passwordLogin() flows {
 // Login implements GET and POST /login
 func Login(
 	req *http.Request, accountDB accounts.Database, userAPI userapi.UserInternalAPI,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	if req.Method == http.MethodGet {
 		// TODO: support other forms of login other than password, depending on config options

clientapi/routing/membership.go

@@ -41,7 +41,7 @@ var errMissingUserID = errors.New("'user_id' must be supplied")
 
 func SendBan(
 	req *http.Request, accountDB accounts.Database, device *userapi.Device,
-	roomID string, cfg *config.Dendrite,
+	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
@@ -52,7 +52,7 @@ func SendBan(
 }
 
 func sendMembership(ctx context.Context, accountDB accounts.Database, device *userapi.Device,
-	roomID, membership, reason string, cfg *config.Dendrite, targetUserID string, evTime time.Time,
+	roomID, membership, reason string, cfg *config.ClientAPI, targetUserID string, evTime time.Time,
 	roomVer gomatrixserverlib.RoomVersion,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI) util.JSONResponse {
 
@@ -94,7 +94,7 @@ func sendMembership(ctx context.Context, accountDB accounts.Database, device *us
 
 func SendKick(
 	req *http.Request, accountDB accounts.Database, device *userapi.Device,
-	roomID string, cfg *config.Dendrite,
+	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 	stateAPI currentstateAPI.CurrentStateInternalAPI,
 ) util.JSONResponse {
@@ -135,7 +135,7 @@ func SendKick(
 
 func SendUnban(
 	req *http.Request, accountDB accounts.Database, device *userapi.Device,
-	roomID string, cfg *config.Dendrite,
+	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
@@ -170,7 +170,7 @@ func SendUnban(
 
 func SendInvite(
 	req *http.Request, accountDB accounts.Database, device *userapi.Device,
-	roomID string, cfg *config.Dendrite,
+	roomID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
@@ -236,7 +236,7 @@ func buildMembershipEvent(
 	targetUserID, reason string, accountDB accounts.Database,
 	device *userapi.Device,
 	membership, roomID string, isDirect bool,
-	cfg *config.Dendrite, evTime time.Time,
+	cfg *config.ClientAPI, evTime time.Time,
 	rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) (*gomatrixserverlib.HeaderedEvent, error) {
 	profile, err := loadProfile(ctx, targetUserID, cfg, accountDB, asAPI)
@@ -263,7 +263,7 @@ func buildMembershipEvent(
 		return nil, err
 	}
 
-	return eventutil.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil)
+	return eventutil.BuildEvent(ctx, &builder, cfg.Matrix, evTime, rsAPI, nil)
 }
 
 // loadProfile lookups the profile of a given user from the database and returns
@@ -273,7 +273,7 @@ func buildMembershipEvent(
 func loadProfile(
 	ctx context.Context,
 	userID string,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	accountDB accounts.Database,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 ) (*authtypes.Profile, error) {
@@ -326,7 +326,7 @@ func checkAndProcessThreepid(
 	req *http.Request,
 	device *userapi.Device,
 	body *threepid.MembershipRequest,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	accountDB accounts.Database,
 	roomID string,

clientapi/routing/memberships.go

@@ -48,7 +48,7 @@ type joinedMember struct {
 // GetMemberships implements GET /rooms/{roomId}/members
 func GetMemberships(
 	req *http.Request, device *userapi.Device, roomID string, joinedOnly bool,
-	_ *config.Dendrite,
+	_ *config.ClientAPI,
 	rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	queryReq := api.QueryMembershipsForRoomRequest{

clientapi/routing/profile.go

@@ -37,7 +37,7 @@ import (
 
 // GetProfile implements GET /profile/{userID}
 func GetProfile(
-	req *http.Request, accountDB accounts.Database, cfg *config.Dendrite,
+	req *http.Request, accountDB accounts.Database, cfg *config.ClientAPI,
 	userID string,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	federation *gomatrixserverlib.FederationClient,
@@ -66,7 +66,7 @@ func GetProfile(
 
 // GetAvatarURL implements GET /profile/{userID}/avatar_url
 func GetAvatarURL(
-	req *http.Request, accountDB accounts.Database, cfg *config.Dendrite,
+	req *http.Request, accountDB accounts.Database, cfg *config.ClientAPI,
 	userID string, asAPI appserviceAPI.AppServiceQueryAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
@@ -95,7 +95,7 @@ func GetAvatarURL(
 // nolint:gocyclo
 func SetAvatarURL(
 	req *http.Request, accountDB accounts.Database, stateAPI currentstateAPI.CurrentStateInternalAPI,
-	device *userapi.Device, userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI,
+	device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{
@@ -184,7 +184,7 @@ func SetAvatarURL(
 
 // GetDisplayName implements GET /profile/{userID}/displayname
 func GetDisplayName(
-	req *http.Request, accountDB accounts.Database, cfg *config.Dendrite,
+	req *http.Request, accountDB accounts.Database, cfg *config.ClientAPI,
 	userID string, asAPI appserviceAPI.AppServiceQueryAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
@@ -213,7 +213,7 @@ func GetDisplayName(
 // nolint:gocyclo
 func SetDisplayName(
 	req *http.Request, accountDB accounts.Database, stateAPI currentstateAPI.CurrentStateInternalAPI,
-	device *userapi.Device, userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI,
+	device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI,
 ) util.JSONResponse {
 	if userID != device.UserID {
 		return util.JSONResponse{
@@ -305,7 +305,7 @@ func SetDisplayName(
 // Returns an error when something goes wrong or specifically
 // eventutil.ErrProfileNoExists when the profile doesn't exist.
 func getProfile(
-	ctx context.Context, accountDB accounts.Database, cfg *config.Dendrite,
+	ctx context.Context, accountDB accounts.Database, cfg *config.ClientAPI,
 	userID string,
 	asAPI appserviceAPI.AppServiceQueryAPI,
 	federation *gomatrixserverlib.FederationClient,
@@ -345,7 +345,7 @@ func getProfile(
 func buildMembershipEvents(
 	ctx context.Context,
 	roomIDs []string,
-	newProfile authtypes.Profile, userID string, cfg *config.Dendrite,
+	newProfile authtypes.Profile, userID string, cfg *config.ClientAPI,
 	evTime time.Time, rsAPI api.RoomserverInternalAPI,
 ) ([]gomatrixserverlib.HeaderedEvent, error) {
 	evs := []gomatrixserverlib.HeaderedEvent{}
@@ -375,7 +375,7 @@ func buildMembershipEvents(
 			return nil, err
 		}
 
-		event, err := eventutil.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil)
+		event, err := eventutil.BuildEvent(ctx, &builder, cfg.Matrix, evTime, rsAPI, nil)
 		if err != nil {
 			return nil, err
 		}

clientapi/routing/redaction.go

@@ -40,7 +40,7 @@ type redactionResponse struct {
 }
 
 func SendRedaction(
-	req *http.Request, device *userapi.Device, roomID, eventID string, cfg *config.Dendrite,
+	req *http.Request, device *userapi.Device, roomID, eventID string, cfg *config.ClientAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI,
 ) util.JSONResponse {
 	resErr := checkMemberInRoom(req.Context(), stateAPI, device.UserID, roomID)
@@ -115,7 +115,7 @@ func SendRedaction(
 	}
 
 	var queryRes api.QueryLatestEventsAndStateResponse
-	e, err := eventutil.BuildEvent(req.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes)
+	e, err := eventutil.BuildEvent(req.Context(), &builder, cfg.Matrix, time.Now(), rsAPI, &queryRes)
 	if err == eventutil.ErrRoomNoExists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,

clientapi/routing/register.go

@@ -255,11 +255,11 @@ func validatePassword(password string) *util.JSONResponse {
 
 // validateRecaptcha returns an error response if the captcha response is invalid
 func validateRecaptcha(
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	response string,
 	clientip string,
 ) *util.JSONResponse {
-	if !cfg.Matrix.RecaptchaEnabled {
+	if !cfg.RecaptchaEnabled {
 		return &util.JSONResponse{
 			Code: http.StatusConflict,
 			JSON: jsonerror.Unknown("Captcha registration is disabled"),
@@ -274,9 +274,9 @@ func validateRecaptcha(
 	}
 
 	// Make a POST request to Google's API to check the captcha response
-	resp, err := http.PostForm(cfg.Matrix.RecaptchaSiteVerifyAPI,
+	resp, err := http.PostForm(cfg.RecaptchaSiteVerifyAPI,
 		url.Values{
-			"secret":   {cfg.Matrix.RecaptchaPrivateKey},
+			"secret":   {cfg.RecaptchaPrivateKey},
 			"response": {response},
 			"remoteip": {clientip},
 		},
@@ -324,7 +324,7 @@ func validateRecaptcha(
 // Application Service is given, it will check to see if it matches any
 // Application Service's namespace.
 func UserIDIsWithinApplicationServiceNamespace(
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	userID string,
 	appservice *config.ApplicationService,
 ) bool {
@@ -354,7 +354,7 @@ func UserIDIsWithinApplicationServiceNamespace(
 // UsernameMatchesMultipleExclusiveNamespaces will check if a given username matches
 // more than one exclusive namespace. More than one is not allowed
 func UsernameMatchesMultipleExclusiveNamespaces(
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	username string,
 ) bool {
 	userID := userutil.MakeUserID(username, cfg.Matrix.ServerName)
@@ -374,7 +374,7 @@ func UsernameMatchesMultipleExclusiveNamespaces(
 // UsernameMatchesExclusiveNamespaces will check if a given username matches any
 // application service's exclusive users namespace
 func UsernameMatchesExclusiveNamespaces(
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	username string,
 ) bool {
 	userID := userutil.MakeUserID(username, cfg.Matrix.ServerName)
@@ -386,7 +386,7 @@ func UsernameMatchesExclusiveNamespaces(
 // username is within that application service's namespace. As long as these
 // two requirements are met, no error will be returned.
 func validateApplicationService(
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	username string,
 	accessToken string,
 ) (string, *util.JSONResponse) {
@@ -442,7 +442,7 @@ func Register(
 	req *http.Request,
 	userAPI userapi.UserInternalAPI,
 	accountDB accounts.Database,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	var r registerRequest
 	resErr := httputil.UnmarshalJSONRequest(req, &r)
@@ -512,7 +512,7 @@ func Register(
 func handleGuestRegistration(
 	req *http.Request,
 	r registerRequest,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	userAPI userapi.UserInternalAPI,
 ) util.JSONResponse {
 	var res userapi.PerformAccountCreationResponse
@@ -568,7 +568,7 @@ func handleRegistrationFlow(
 	req *http.Request,
 	r registerRequest,
 	sessionID string,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	userAPI userapi.UserInternalAPI,
 ) util.JSONResponse {
 	// TODO: Shared secret registration (create new user scripts)
@@ -580,7 +580,7 @@ func handleRegistrationFlow(
 
 	// TODO: email / msisdn auth types.
 
-	if cfg.Matrix.RegistrationDisabled && r.Auth.Type != authtypes.LoginTypeSharedSecret {
+	if cfg.RegistrationDisabled && r.Auth.Type != authtypes.LoginTypeSharedSecret {
 		return util.MessageResponse(http.StatusForbidden, "Registration has been disabled")
 	}
 
@@ -666,7 +666,7 @@ func handleApplicationServiceRegistration(
 	tokenErr error,
 	req *http.Request,
 	r registerRequest,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	userAPI userapi.UserInternalAPI,
 ) util.JSONResponse {
 	// Check if we previously had issues extracting the access token from the
@@ -704,7 +704,7 @@ func checkAndCompleteFlow(
 	req *http.Request,
 	r registerRequest,
 	sessionID string,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	userAPI userapi.UserInternalAPI,
 ) util.JSONResponse {
 	if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
@@ -728,7 +728,7 @@ func checkAndCompleteFlow(
 func LegacyRegister(
 	req *http.Request,
 	userAPI userapi.UserInternalAPI,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	var r legacyRegisterRequest
 	resErr := parseAndValidateLegacyLogin(req, &r)
@@ -742,13 +742,13 @@ func LegacyRegister(
 		"auth.type": r.Type,
 	}).Info("Processing registration request")
 
-	if cfg.Matrix.RegistrationDisabled && r.Type != authtypes.LoginTypeSharedSecret {
+	if cfg.RegistrationDisabled && r.Type != authtypes.LoginTypeSharedSecret {
 		return util.MessageResponse(http.StatusForbidden, "Registration has been disabled")
 	}
 
 	switch r.Type {
 	case authtypes.LoginTypeSharedSecret:
-		if cfg.Matrix.RegistrationSharedSecret == "" {
+		if cfg.RegistrationSharedSecret == "" {
 			return util.MessageResponse(http.StatusBadRequest, "Shared secret registration is disabled")
 		}
 
@@ -902,15 +902,15 @@ func completeRegistration(
 // Used for shared secret registration.
 // Checks if the username, password and isAdmin flag matches the given mac.
 func isValidMacLogin(
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	username, password string,
 	isAdmin bool,
 	givenMac []byte,
 ) (bool, error) {
-	sharedSecret := cfg.Matrix.RegistrationSharedSecret
+	sharedSecret := cfg.RegistrationSharedSecret
 
 	// Check that shared secret registration isn't disabled.
-	if cfg.Matrix.RegistrationSharedSecret == "" {
+	if cfg.RegistrationSharedSecret == "" {
 		return false, errors.New("Shared secret registration is disabled")
 	}
 
@@ -1001,7 +1001,7 @@ type availableResponse struct {
 // RegisterAvailable checks if the username is already taken or invalid.
 func RegisterAvailable(
 	req *http.Request,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	accountDB accounts.Database,
 ) util.JSONResponse {
 	username := req.URL.Query().Get("username")

clientapi/routing/register_test.go

@@ -179,30 +179,31 @@ func TestValidationOfApplicationServices(t *testing.T) {
 	}
 
 	// Set up a config
-	fakeConfig := config.Dendrite{}
-	fakeConfig.Matrix.ServerName = "localhost"
-	fakeConfig.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService}
+	fakeConfig := &config.Dendrite{}
+	fakeConfig.Defaults()
+	fakeConfig.Global.ServerName = "localhost"
+	fakeConfig.ClientAPI.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService}
 
 	// Access token is correct, user_id omitted so we are acting as SenderLocalpart
-	asID, resp := validateApplicationService(&fakeConfig, fakeSenderLocalpart, "1234")
+	asID, resp := validateApplicationService(&fakeConfig.ClientAPI, fakeSenderLocalpart, "1234")
 	if resp != nil || asID != fakeID {
 		t.Errorf("appservice should have validated and returned correct ID: %s", resp.JSON)
 	}
 
 	// Access token is incorrect, user_id omitted so we are acting as SenderLocalpart
-	asID, resp = validateApplicationService(&fakeConfig, fakeSenderLocalpart, "xxxx")
+	asID, resp = validateApplicationService(&fakeConfig.ClientAPI, fakeSenderLocalpart, "xxxx")
 	if resp == nil || asID == fakeID {
 		t.Errorf("access_token should have been marked as invalid")
 	}
 
 	// Access token is correct, acting as valid user_id
-	asID, resp = validateApplicationService(&fakeConfig, "_appservice_bob", "1234")
+	asID, resp = validateApplicationService(&fakeConfig.ClientAPI, "_appservice_bob", "1234")
 	if resp != nil || asID != fakeID {
 		t.Errorf("access_token and user_id should've been valid: %s", resp.JSON)
 	}
 
 	// Access token is correct, acting as invalid user_id
-	asID, resp = validateApplicationService(&fakeConfig, "_something_else", "1234")
+	asID, resp = validateApplicationService(&fakeConfig.ClientAPI, "_something_else", "1234")
 	if resp == nil || asID == fakeID {
 		t.Errorf("user_id should not have been valid: @_something_else:localhost")
 	}

clientapi/routing/routing.go

@@ -51,7 +51,7 @@ const pathPrefixUnstable = "/client/unstable"
 // applied:
 // nolint: gocyclo
 func Setup(
-	publicAPIMux *mux.Router, cfg *config.Dendrite,
+	publicAPIMux *mux.Router, cfg *config.ClientAPI,
 	eduAPI eduServerAPI.EDUServerInputAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	asAPI appserviceAPI.AppServiceQueryAPI,

clientapi/routing/sendevent.go

@@ -43,7 +43,7 @@ func SendEvent(
 	req *http.Request,
 	device *userapi.Device,
 	roomID, eventType string, txnID, stateKey *string,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	rsAPI api.RoomserverInternalAPI,
 	txnCache *transactions.Cache,
 ) util.JSONResponse {
@@ -112,7 +112,7 @@ func generateSendEvent(
 	req *http.Request,
 	device *userapi.Device,
 	roomID, eventType string, stateKey *string,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 	rsAPI api.RoomserverInternalAPI,
 ) (*gomatrixserverlib.Event, *util.JSONResponse) {
 	// parse the incoming http request
@@ -146,7 +146,7 @@ func generateSendEvent(
 	}
 
 	var queryRes api.QueryLatestEventsAndStateResponse
-	e, err := eventutil.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes)
+	e, err := eventutil.BuildEvent(req.Context(), &builder, cfg.Matrix, evTime, rsAPI, &queryRes)
 	if err == eventutil.ErrRoomNoExists {
 		return nil, &util.JSONResponse{
 			Code: http.StatusNotFound,

clientapi/routing/threepid.go

@@ -40,7 +40,7 @@ type threePIDsResponse struct {
 // RequestEmailToken implements:
 //     POST /account/3pid/email/requestToken
 //     POST /register/email/requestToken
-func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *config.Dendrite) util.JSONResponse {
+func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *config.ClientAPI) util.JSONResponse {
 	var body threepid.EmailAssociationRequest
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
 		return *reqErr
@@ -86,7 +86,7 @@ func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *conf
 // CheckAndSave3PIDAssociation implements POST /account/3pid
 func CheckAndSave3PIDAssociation(
 	req *http.Request, accountDB accounts.Database, device *api.Device,
-	cfg *config.Dendrite,
+	cfg *config.ClientAPI,
 ) util.JSONResponse {
 	var body threepid.EmailAssociationCheckRequest
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {

clientapi/routing/voip.go

@@ -31,7 +31,7 @@ import (
 
 // RequestTurnServer implements:
 //     GET /voip/turnServer
-func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.Dendrite) util.JSONResponse {
+func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.ClientAPI) util.JSONResponse {
 	turnConfig := cfg.TURN
 
 	// TODO Guest Support

clientapi/threepid/invites.go

@@ -86,7 +86,7 @@ var (
 // can be emitted.
 func CheckAndProcessInvite(
 	ctx context.Context,
-	device *userapi.Device, body *MembershipRequest, cfg *config.Dendrite,
+	device *userapi.Device, body *MembershipRequest, cfg *config.ClientAPI,
 	rsAPI api.RoomserverInternalAPI, db accounts.Database,
 	roomID string,
 	evTime time.Time,
@@ -137,7 +137,7 @@ func CheckAndProcessInvite(
 // Returns an error if a check or a request failed.
 func queryIDServer(
 	ctx context.Context,
-	db accounts.Database, cfg *config.Dendrite, device *userapi.Device,
+	db accounts.Database, cfg *config.ClientAPI, device *userapi.Device,
 	body *MembershipRequest, roomID string,
 ) (lookupRes *idServerLookupResponse, storeInviteRes *idServerStoreInviteResponse, err error) {
 	if err = isTrusted(body.IDServer, cfg); err != nil {
@@ -206,7 +206,7 @@ func queryIDServerLookup(ctx context.Context, body *MembershipRequest) (*idServe
 // Returns an error if the request failed to send or if the response couldn't be parsed.
 func queryIDServerStoreInvite(
 	ctx context.Context,
-	db accounts.Database, cfg *config.Dendrite, device *userapi.Device,
+	db accounts.Database, cfg *config.ClientAPI, device *userapi.Device,
 	body *MembershipRequest, roomID string,
 ) (*idServerStoreInviteResponse, error) {
 	// Retrieve the sender's profile to get their display name
@@ -330,7 +330,7 @@ func checkIDServerSignatures(
 func emit3PIDInviteEvent(
 	ctx context.Context,
 	body *MembershipRequest, res *idServerStoreInviteResponse,
-	device *userapi.Device, roomID string, cfg *config.Dendrite,
+	device *userapi.Device, roomID string, cfg *config.ClientAPI,
 	rsAPI api.RoomserverInternalAPI,
 	evTime time.Time,
 ) error {
@@ -354,7 +354,7 @@ func emit3PIDInviteEvent(
 	}
 
 	queryRes := api.QueryLatestEventsAndStateResponse{}
-	event, err := eventutil.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes)
+	event, err := eventutil.BuildEvent(ctx, builder, cfg.Matrix, evTime, rsAPI, &queryRes)
 	if err != nil {
 		return err
 	}

clientapi/threepid/threepid.go

@@ -53,7 +53,7 @@ type Credentials struct {
 // Returns an error if there was a problem sending the request or decoding the
 // response, or if the identity server responded with a non-OK status.
 func CreateSession(
-	ctx context.Context, req EmailAssociationRequest, cfg *config.Dendrite,
+	ctx context.Context, req EmailAssociationRequest, cfg *config.ClientAPI,
 ) (string, error) {
 	if err := isTrusted(req.IDServer, cfg); err != nil {
 		return "", err
@@ -101,7 +101,7 @@ func CreateSession(
 // Returns an error if there was a problem sending the request or decoding the
 // response, or if the identity server responded with a non-OK status.
 func CheckAssociation(
-	ctx context.Context, creds Credentials, cfg *config.Dendrite,
+	ctx context.Context, creds Credentials, cfg *config.ClientAPI,
 ) (bool, string, string, error) {
 	if err := isTrusted(creds.IDServer, cfg); err != nil {
 		return false, "", "", err
@@ -142,7 +142,7 @@ func CheckAssociation(
 // identifier and a Matrix ID.
 // Returns an error if there was a problem sending the request or decoding the
 // response, or if the identity server responded with a non-OK status.
-func PublishAssociation(creds Credentials, userID string, cfg *config.Dendrite) error {
+func PublishAssociation(creds Credentials, userID string, cfg *config.ClientAPI) error {
 	if err := isTrusted(creds.IDServer, cfg); err != nil {
 		return err
 	}
@@ -177,7 +177,7 @@ func PublishAssociation(creds Credentials, userID string, cfg *config.Dendrite)
 // isTrusted checks if a given identity server is part of the list of trusted
 // identity servers in the configuration file.
 // Returns an error if the server isn't trusted.
-func isTrusted(idServer string, cfg *config.Dendrite) error {
+func isTrusted(idServer string, cfg *config.ClientAPI) error {
 	for _, server := range cfg.Matrix.TrustedIDServers {
 		if idServer == server {
 			return nil

cmd/create-account/main.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
 	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -63,7 +64,9 @@ func main() {
 
 	serverName := gomatrixserverlib.ServerName(*serverNameStr)
 
-	accountDB, err := accounts.NewDatabase(*database, nil, serverName)
+	accountDB, err := accounts.NewDatabase(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(*database),
+	}, serverName)
 	if err != nil {
 		fmt.Println(err.Error())
 		os.Exit(1)
@@ -75,7 +78,9 @@ func main() {
 		os.Exit(1)
 	}
 
-	deviceDB, err := devices.NewDatabase(*database, nil, serverName)
+	deviceDB, err := devices.NewDatabase(&config.DatabaseOptions{
+		ConnectionString: config.DataSource(*database),
+	}, serverName)
 	if err != nil {
 		fmt.Println(err.Error())
 		os.Exit(1)

cmd/dendrite-appservice-server/main.go

@@ -30,6 +30,6 @@ func main() {
 	intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
 	appservice.AddInternalRoutes(base.InternalAPIMux, intAPI)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI))
+	base.SetupAndServeHTTP(string(base.Cfg.AppServiceAPI.Bind), string(base.Cfg.AppServiceAPI.Listen))
 
 }

cmd/dendrite-client-api-server/main.go

@@ -39,10 +39,10 @@ func main() {
 	keyAPI := base.KeyServerHTTPClient()
 
 	clientapi.AddPublicRoutes(
-		base.PublicAPIMux, base.Cfg, base.KafkaProducer, deviceDB, accountDB, federation,
+		base.PublicAPIMux, &base.Cfg.ClientAPI, base.KafkaProducer, deviceDB, accountDB, federation,
 		rsAPI, eduInputAPI, asQuery, stateAPI, transactions.New(), fsAPI, userAPI, keyAPI, nil,
 	)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI))
+	base.SetupAndServeHTTP(string(base.Cfg.ClientAPI.Bind), string(base.Cfg.ClientAPI.Listen))
 
 }

cmd/dendrite-current-state-server/main.go

@@ -24,10 +24,10 @@ func main() {
 	base := setup.NewBaseDendrite(cfg, "CurrentStateServer", true)
 	defer base.Close() // nolint: errcheck
 
-	stateAPI := currentstateserver.NewInternalAPI(cfg, base.KafkaConsumer)
+	stateAPI := currentstateserver.NewInternalAPI(&cfg.CurrentStateServer, base.KafkaConsumer)
 
 	currentstateserver.AddInternalRoutes(base.InternalAPIMux, stateAPI)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.CurrentState), string(base.Cfg.Listen.CurrentState))
+	base.SetupAndServeHTTP(string(base.Cfg.CurrentStateServer.Bind), string(base.Cfg.CurrentStateServer.Listen))
 
 }

cmd/dendrite-demo-libp2p/main.go

@@ -75,8 +75,8 @@ func createFederationClient(
 		p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")),
 	)
 	return gomatrixserverlib.NewFederationClientWithTransport(
-		base.Base.Cfg.Matrix.ServerName, base.Base.Cfg.Matrix.KeyID,
-		base.Base.Cfg.Matrix.PrivateKey, true, tr,
+		base.Base.Cfg.Global.ServerName, base.Base.Cfg.Global.KeyID,
+		base.Base.Cfg.Global.PrivateKey, true, tr,
 	)
 }
 
@@ -113,25 +113,28 @@ func main() {
 	}
 
 	cfg := config.Dendrite{}
-	cfg.SetDefaults()
-	cfg.Matrix.ServerName = "p2p"
-	cfg.Matrix.PrivateKey = privKey
-	cfg.Matrix.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName))
-	cfg.Kafka.UseNaffka = true
-	cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput"
-	cfg.Kafka.Topics.OutputClientData = "clientapiOutput"
-	cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput"
-	cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-	cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
-	cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
-	cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
-	cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
-	cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName))
-	cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
-	cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
-	cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
-	cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName))
-	cfg.Database.E2EKey = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName))
+	cfg.Defaults()
+	cfg.Global.ServerName = "p2p"
+	cfg.Global.PrivateKey = privKey
+	cfg.Global.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName))
+	cfg.Global.Kafka.UseNaffka = true
+	cfg.Global.Kafka.Topics.OutputRoomEvent = "roomserverOutput"
+	cfg.Global.Kafka.Topics.OutputClientData = "clientapiOutput"
+	cfg.Global.Kafka.Topics.OutputTypingEvent = "typingServerOutput"
+	cfg.Global.Kafka.Topics.OutputSendToDeviceEvent = "sendToDeviceOutput"
+	cfg.Global.Kafka.Topics.OutputKeyChangeEvent = "keyChangeOutput"
+	cfg.FederationSender.FederationMaxRetries = 6
+	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
+	cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
+	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
+	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
+	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
+	cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName))
+	cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
+	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
+	cfg.CurrentStateServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName))
+	cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
+	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName))
 	if err = cfg.Derive(); err != nil {
 		panic(err)
 	}
@@ -142,19 +145,19 @@ func main() {
 	accountDB := base.Base.CreateAccountsDB()
 	deviceDB := base.Base.CreateDeviceDB()
 	federation := createFederationClient(base)
-	keyAPI := keyserver.NewInternalAPI(base.Base.Cfg, federation, base.Base.KafkaProducer)
-	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil, keyAPI)
+	keyAPI := keyserver.NewInternalAPI(&base.Base.Cfg.KeyServer, federation, base.Base.KafkaProducer)
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, nil, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
 	serverKeyAPI := serverkeyapi.NewInternalAPI(
-		base.Base.Cfg, federation, base.Base.Caches,
+		&base.Base.Cfg.ServerKeyAPI, federation, base.Base.Caches,
 	)
 	keyRing := serverKeyAPI.KeyRing()
 	createKeyDB(
 		base, serverKeyAPI,
 	)
 
-	stateAPI := currentstateserver.NewInternalAPI(base.Base.Cfg, base.Base.KafkaConsumer)
+	stateAPI := currentstateserver.NewInternalAPI(&base.Base.Cfg.CurrentStateServer, base.Base.KafkaConsumer)
 	rsAPI := roomserver.NewInternalAPI(
 		&base.Base, keyRing, federation,
 	)
@@ -198,7 +201,7 @@ func main() {
 		base.Base.BaseMux,
 		base.Base.PublicAPIMux,
 		base.Base.InternalAPIMux,
-		&cfg,
+		&cfg.Global,
 		base.Base.UseHTTPAPIs,
 	)
 

cmd/dendrite-demo-libp2p/p2pdendrite.go

@@ -58,7 +58,7 @@ func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite {
 
 	ctx, cancel := context.WithCancel(context.Background())
 
-	privKey, err := crypto.UnmarshalEd25519PrivateKey(cfg.Matrix.PrivateKey[:])
+	privKey, err := crypto.UnmarshalEd25519PrivateKey(cfg.Global.PrivateKey[:])
 	if err != nil {
 		panic(err)
 	}
@@ -97,7 +97,7 @@ func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite {
 	fmt.Println("Our node ID:", libp2p.ID())
 	fmt.Println("Our addresses:", libp2p.Addrs())
 
-	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(libp2p.ID().String())
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(libp2p.ID().String())
 
 	return &P2PDendrite{
 		Base:          *baseDendrite,

cmd/dendrite-demo-yggdrasil/main.go

@@ -68,27 +68,28 @@ func main() {
 	}
 
 	cfg := &config.Dendrite{}
-	cfg.SetDefaults()
-	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
-	cfg.Matrix.PrivateKey = ygg.SigningPrivateKey()
-	cfg.Matrix.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
-	cfg.Matrix.FederationMaxRetries = 8
-	cfg.Kafka.UseNaffka = true
-	cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput"
-	cfg.Kafka.Topics.OutputClientData = "clientapiOutput"
-	cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput"
-	cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-	cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
-	cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
-	cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
-	cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
-	cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName))
-	cfg.Database.E2EKey = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
-	cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
-	cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
-	cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName))
-	cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
-	cfg.Database.E2EKey = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName))
+	cfg.Defaults()
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
+	cfg.Global.PrivateKey = ygg.SigningPrivateKey()
+	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
+	cfg.Global.Kafka.UseNaffka = true
+	cfg.Global.Kafka.Topics.OutputRoomEvent = "roomserverOutput"
+	cfg.Global.Kafka.Topics.OutputClientData = "clientapiOutput"
+	cfg.Global.Kafka.Topics.OutputTypingEvent = "typingServerOutput"
+	cfg.Global.Kafka.Topics.OutputSendToDeviceEvent = "sendToDeviceOutput"
+	cfg.Global.Kafka.Topics.OutputKeyChangeEvent = "keyChangeOutput"
+	cfg.FederationSender.FederationMaxRetries = 8
+	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
+	cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
+	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
+	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
+	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
+	cfg.ServerKeyAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName))
+	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
+	cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
+	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
+	cfg.CurrentStateServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName))
+	cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
 	if err = cfg.Derive(); err != nil {
 		panic(err)
 	}
@@ -103,8 +104,8 @@ func main() {
 	serverKeyAPI := &signing.YggdrasilKeys{}
 	keyRing := serverKeyAPI.KeyRing()
 
-	keyAPI := keyserver.NewInternalAPI(base.Cfg, federation, base.KafkaProducer)
-	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil, keyAPI)
+	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation, base.KafkaProducer)
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, nil, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
 	rsComponent := roomserver.NewInternalAPI(
@@ -117,7 +118,7 @@ func main() {
 	)
 
 	asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
-	stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
+	stateAPI := currentstateserver.NewInternalAPI(&base.Cfg.CurrentStateServer, base.KafkaConsumer)
 	fsAPI := federationsender.NewInternalAPI(
 		base, federation, rsAPI, stateAPI, keyRing,
 	)
@@ -155,7 +156,6 @@ func main() {
 		UserAPI:             userAPI,
 		StateAPI:            stateAPI,
 		KeyAPI:              keyAPI,
-		//ServerKeyAPI:        serverKeyAPI,
 		ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider(
 			ygg, fsAPI, federation,
 		),
@@ -166,7 +166,7 @@ func main() {
 		base.BaseMux,
 		base.PublicAPIMux,
 		base.InternalAPIMux,
-		cfg,
+		&cfg.Global,
 		base.UseHTTPAPIs,
 	)
 

cmd/dendrite-demo-yggdrasil/yggconn/client.go

@@ -54,7 +54,7 @@ func (n *Node) CreateFederationClient(
 		},
 	)
 	return gomatrixserverlib.NewFederationClientWithTransport(
-		base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID,
-		base.Cfg.Matrix.PrivateKey, true, tr,
+		base.Cfg.Global.ServerName, base.Cfg.Global.KeyID,
+		base.Cfg.Global.PrivateKey, true, tr,
 	)
 }

cmd/dendrite-edu-server/main.go

@@ -33,6 +33,6 @@ func main() {
 	intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient())
 	eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.EDUServer), string(base.Cfg.Listen.EDUServer))
+	base.SetupAndServeHTTP(string(base.Cfg.EDUServer.Bind), string(base.Cfg.EDUServer.Listen))
 
 }

cmd/dendrite-federation-api-server/main.go

@@ -33,10 +33,10 @@ func main() {
 	keyAPI := base.KeyServerHTTPClient()
 
 	federationapi.AddPublicRoutes(
-		base.PublicAPIMux, base.Cfg, userAPI, federation, keyRing,
+		base.PublicAPIMux, &base.Cfg.FederationAPI, userAPI, federation, keyRing,
 		rsAPI, fsAPI, base.EDUServerClient(), base.CurrentStateAPIClient(), keyAPI,
 	)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI))
+	base.SetupAndServeHTTP(string(base.Cfg.FederationAPI.Bind), string(base.Cfg.FederationAPI.Listen))
 
 }

cmd/dendrite-federation-sender-server/main.go

@@ -35,6 +35,6 @@ func main() {
 	)
 	federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender))
+	base.SetupAndServeHTTP(string(base.Cfg.FederationSender.Bind), string(base.Cfg.FederationSender.Listen))
 
 }

cmd/dendrite-key-server/main.go

@@ -24,11 +24,11 @@ func main() {
 	base := setup.NewBaseDendrite(cfg, "KeyServer", true)
 	defer base.Close() // nolint: errcheck
 
-	intAPI := keyserver.NewInternalAPI(base.Cfg, base.CreateFederationClient(), base.KafkaProducer)
+	intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient(), base.KafkaProducer)
 	intAPI.SetUserAPI(base.UserAPIClient())
 
 	keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.KeyServer), string(base.Cfg.Listen.KeyServer))
+	base.SetupAndServeHTTP(string(base.Cfg.KeyServer.Bind), string(base.Cfg.KeyServer.Listen))
 
 }

cmd/dendrite-media-api-server/main.go

@@ -26,10 +26,10 @@ func main() {
 	defer base.Close() // nolint: errcheck
 
 	userAPI := base.UserAPIClient()
-	client := gomatrixserverlib.NewClient(cfg.Matrix.FederationDisableTLSValidation)
+	client := gomatrixserverlib.NewClient(cfg.FederationSender.DisableTLSValidation)
 
-	mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, userAPI, client)
+	mediaapi.AddPublicRoutes(base.PublicAPIMux, &base.Cfg.MediaAPI, userAPI, client)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.MediaAPI), string(base.Cfg.Listen.MediaAPI))
+	base.SetupAndServeHTTP(string(base.Cfg.MediaAPI.Bind), string(base.Cfg.MediaAPI.Listen))
 
 }

cmd/dendrite-monolith-server/main.go

@@ -54,11 +54,17 @@ func main() {
 		// the API endpoints. They'll listen on the same port as the monolith
 		// itself.
 		addr := config.Address(*httpBindAddr)
-		cfg.Listen.RoomServer = addr
-		cfg.Listen.EDUServer = addr
-		cfg.Listen.AppServiceAPI = addr
-		cfg.Listen.FederationSender = addr
-		cfg.Listen.ServerKeyAPI = addr
+		cfg.AppServiceAPI.Listen = addr
+		cfg.ClientAPI.Listen = addr
+		cfg.CurrentStateServer.Listen = addr
+		cfg.EDUServer.Listen = addr
+		cfg.FederationAPI.Listen = addr
+		cfg.FederationSender.Listen = addr
+		cfg.KeyServer.Listen = addr
+		cfg.MediaAPI.Listen = addr
+		cfg.RoomServer.Listen = addr
+		cfg.ServerKeyAPI.Listen = addr
+		cfg.SyncAPI.Listen = addr
 	}
 
 	base := setup.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs)
@@ -69,15 +75,15 @@ func main() {
 	federation := base.CreateFederationClient()
 
 	serverKeyAPI := serverkeyapi.NewInternalAPI(
-		base.Cfg, federation, base.Caches,
+		&base.Cfg.ServerKeyAPI, federation, base.Caches,
 	)
 	if base.UseHTTPAPIs {
 		serverkeyapi.AddInternalRoutes(base.InternalAPIMux, serverKeyAPI, base.Caches)
 		serverKeyAPI = base.ServerKeyAPIClient()
 	}
 	keyRing := serverKeyAPI.KeyRing()
-	keyAPI := keyserver.NewInternalAPI(base.Cfg, federation, base.KafkaProducer)
-	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices, keyAPI)
+	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation, base.KafkaProducer)
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, cfg.Derived.ApplicationServices, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
 	rsImpl := roomserver.NewInternalAPI(
@@ -109,7 +115,7 @@ func main() {
 		asAPI = base.AppserviceHTTPClient()
 	}
 
-	stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
+	stateAPI := currentstateserver.NewInternalAPI(&base.Cfg.CurrentStateServer, base.KafkaConsumer)
 
 	fsAPI := federationsender.NewInternalAPI(
 		base, federation, rsAPI, stateAPI, keyRing,
@@ -126,7 +132,7 @@ func main() {
 		Config:        base.Cfg,
 		AccountDB:     accountDB,
 		DeviceDB:      deviceDB,
-		Client:        gomatrixserverlib.NewClient(cfg.Matrix.FederationDisableTLSValidation),
+		Client:        gomatrixserverlib.NewClient(cfg.FederationSender.DisableTLSValidation),
 		FedClient:     federation,
 		KeyRing:       keyRing,
 		KafkaConsumer: base.KafkaConsumer,
@@ -147,7 +153,7 @@ func main() {
 		base.BaseMux,
 		base.PublicAPIMux,
 		base.InternalAPIMux,
-		cfg,
+		&cfg.Global,
 		base.UseHTTPAPIs,
 	)
 

cmd/dendrite-room-server/main.go

@@ -33,6 +33,6 @@ func main() {
 	rsAPI.SetFederationSenderAPI(fsAPI)
 	roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer))
+	base.SetupAndServeHTTP(string(base.Cfg.RoomServer.Bind), string(base.Cfg.RoomServer.Listen))
 
 }

cmd/dendrite-server-key-api-server/main.go

@@ -26,8 +26,8 @@ func main() {
 
 	federation := base.CreateFederationClient()
 
-	intAPI := serverkeyapi.NewInternalAPI(base.Cfg, federation, base.Caches)
+	intAPI := serverkeyapi.NewInternalAPI(&base.Cfg.ServerKeyAPI, federation, base.Caches)
 	serverkeyapi.AddInternalRoutes(base.InternalAPIMux, intAPI, base.Caches)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.ServerKeyAPI), string(base.Cfg.Listen.ServerKeyAPI))
+	base.SetupAndServeHTTP(string(base.Cfg.ServerKeyAPI.Bind), string(base.Cfg.ServerKeyAPI.Listen))
 }

cmd/dendrite-sync-api-server/main.go

@@ -31,8 +31,8 @@ func main() {
 
 	syncapi.AddPublicRoutes(
 		base.PublicAPIMux, base.KafkaConsumer, userAPI, rsAPI, base.KeyServerHTTPClient(), base.CurrentStateAPIClient(),
-		federation, cfg)
+		federation, &cfg.SyncAPI)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI))
+	base.SetupAndServeHTTP(string(base.Cfg.SyncAPI.Bind), string(base.Cfg.SyncAPI.Listen))
 
 }

cmd/dendrite-user-api-server/main.go

@@ -27,9 +27,9 @@ func main() {
 	accountDB := base.CreateAccountsDB()
 	deviceDB := base.CreateDeviceDB()
 
-	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient())
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient())
 
 	userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
 
-	base.SetupAndServeHTTP(string(base.Cfg.Bind.UserAPI), string(base.Cfg.Listen.UserAPI))
+	base.SetupAndServeHTTP(string(base.Cfg.UserAPI.Bind), string(base.Cfg.UserAPI.Listen))
 }

cmd/dendritejs/main.go

@@ -139,7 +139,7 @@ func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLoc
 	tr := go_http_js_libp2p.NewP2pTransport(node)
 
 	fed := gomatrixserverlib.NewFederationClient(
-		cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, true,
+		cfg.Global.ServerName, cfg.Global.KeyID, cfg.Global.PrivateKey, true,
 	)
 	fed.Client = *gomatrixserverlib.NewClientWithTransport(true, tr)
 
@@ -161,31 +161,31 @@ func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http
 
 func main() {
 	cfg := &config.Dendrite{}
-	cfg.SetDefaults()
-	cfg.Kafka.UseNaffka = true
-	cfg.Database.Account = "file:/idb/dendritejs_account.db"
-	cfg.Database.AppService = "file:/idb/dendritejs_appservice.db"
-	cfg.Database.Device = "file:/idb/dendritejs_device.db"
-	cfg.Database.FederationSender = "file:/idb/dendritejs_fedsender.db"
-	cfg.Database.MediaAPI = "file:/idb/dendritejs_mediaapi.db"
-	cfg.Database.Naffka = "file:/idb/dendritejs_naffka.db"
-	cfg.Database.RoomServer = "file:/idb/dendritejs_roomserver.db"
-	cfg.Database.ServerKey = "file:/idb/dendritejs_serverkey.db"
-	cfg.Database.SyncAPI = "file:/idb/dendritejs_syncapi.db"
-	cfg.Database.CurrentState = "file:/idb/dendritejs_currentstate.db"
-	cfg.Database.E2EKey = "file:/idb/dendritejs_e2ekey.db"
-	cfg.Kafka.Topics.OutputTypingEvent = "output_typing_event"
-	cfg.Kafka.Topics.OutputSendToDeviceEvent = "output_send_to_device_event"
-	cfg.Kafka.Topics.OutputClientData = "output_client_data"
-	cfg.Kafka.Topics.OutputRoomEvent = "output_room_event"
-	cfg.Matrix.TrustedIDServers = []string{
+	cfg.Defaults()
+	cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db"
+	cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db"
+	cfg.UserAPI.DeviceDatabase.ConnectionString = "file:/idb/dendritejs_device.db"
+	cfg.FederationSender.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db"
+	cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db"
+	cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db"
+	cfg.ServerKeyAPI.Database.ConnectionString = "file:/idb/dendritejs_serverkey.db"
+	cfg.SyncAPI.Database.ConnectionString = "file:/idb/dendritejs_syncapi.db"
+	cfg.CurrentStateServer.Database.ConnectionString = "file:/idb/dendritejs_currentstate.db"
+	cfg.KeyServer.Database.ConnectionString = "file:/idb/dendritejs_e2ekey.db"
+	cfg.Global.Kafka.UseNaffka = true
+	cfg.Global.Kafka.Database.ConnectionString = "file:/idb/dendritejs_naffka.db"
+	cfg.Global.Kafka.Topics.OutputTypingEvent = "output_typing_event"
+	cfg.Global.Kafka.Topics.OutputSendToDeviceEvent = "output_send_to_device_event"
+	cfg.Global.Kafka.Topics.OutputClientData = "output_client_data"
+	cfg.Global.Kafka.Topics.OutputRoomEvent = "output_room_event"
+	cfg.Global.TrustedIDServers = []string{
 		"matrix.org", "vector.im",
 	}
-	cfg.Matrix.KeyID = libp2pMatrixKeyID
-	cfg.Matrix.PrivateKey = generateKey()
+	cfg.Global.KeyID = libp2pMatrixKeyID
+	cfg.Global.PrivateKey = generateKey()
 
-	serverName, node := createP2PNode(cfg.Matrix.PrivateKey)
-	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(serverName)
+	serverName, node := createP2PNode(cfg.Global.PrivateKey)
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(serverName)
 
 	if err := cfg.Derive(); err != nil {
 		logrus.Fatalf("Failed to derive values from config: %s", err)
@@ -196,8 +196,8 @@ func main() {
 	accountDB := base.CreateAccountsDB()
 	deviceDB := base.CreateDeviceDB()
 	federation := createFederationClient(cfg, node)
-	keyAPI := keyserver.NewInternalAPI(base.Cfg, federation, base.KafkaProducer)
-	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil, keyAPI)
+	keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation, base.KafkaProducer)
+	userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Global.ServerName, nil, keyAPI)
 	keyAPI.SetUserAPI(userAPI)
 
 	fetcher := &libp2pKeyFetcher{}
@@ -208,7 +208,7 @@ func main() {
 		KeyDatabase: fetcher,
 	}
 
-	stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
+	stateAPI := currentstateserver.NewInternalAPI(&base.Cfg.CurrentStateServer, base.KafkaConsumer)
 	rsAPI := roomserver.NewInternalAPI(base, keyRing, federation)
 	eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), userAPI)
 	asQuery := appservice.NewInternalAPI(
@@ -244,7 +244,7 @@ func main() {
 		base.BaseMux,
 		base.PublicAPIMux,
 		base.InternalAPIMux,
-		cfg,
+		&cfg.Global,
 		base.UseHTTPAPIs,
 	)
 

cmd/generate-config/main.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/config"
+	"gopkg.in/yaml.v2"
+)
+
+func main() {
+	cfg := &config.Dendrite{}
+	cfg.Defaults()
+	cfg.Logging = []config.LogrusHook{
+		{
+			Type:  "file",
+			Level: "info",
+			Params: map[string]interface{}{
+				"path": "/var/log/dendrite",
+			},
+		},
+	}
+
+	j, err := yaml.Marshal(cfg)
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Println(string(j))
+}

cmd/mediaapi-integration-tests/main.go

@@ -88,9 +88,9 @@ func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error
 	if err != nil {
 		panic(err)
 	}
-	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(proxyAddr)
-	cfg.Media.DynamicThumbnails = dynamicThumbnails
-	if err = yaml.Unmarshal([]byte(thumbnailSizes), &cfg.Media.ThumbnailSizes); err != nil {
+	cfg.Global.ServerName = gomatrixserverlib.ServerName(proxyAddr)
+	cfg.MediaAPI.DynamicThumbnails = dynamicThumbnails
+	if err = yaml.Unmarshal([]byte(thumbnailSizes), &cfg.MediaAPI.ThumbnailSizes); err != nil {
 		panic(err)
 	}
 
@@ -120,7 +120,7 @@ func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error
 		serverArgs,
 	)
 
-	fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, cfg.Listen.MediaAPI, dir)
+	fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, cfg.MediaAPI.Listen, dir)
 	return cmd, cmdChan, proxyCmd, proxyAddr, dir
 }
 

cmd/roomserver-integration-tests/main.go

@@ -240,7 +240,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R
 		panic(err)
 	}
 
-	outputTopic := string(cfg.Kafka.Topics.OutputRoomEvent)
+	outputTopic := string(cfg.Global.Kafka.Topics.OutputRoomEvent)
 
 	err = exe.DeleteTopic(outputTopic)
 	if err != nil {
@@ -277,7 +277,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R
 	cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)}
 
 	gotOutput, err := runAndReadFromTopic(cmd, cfg.RoomServerURL()+"/metrics", doInput, outputTopic, len(wantOutput), func() {
-		queryAPI, _ := inthttp.NewRoomserverClient("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache)
+		queryAPI, _ := inthttp.NewRoomserverClient("http://"+string(cfg.RoomServer.Listen), &http.Client{Timeout: timeoutHTTP}, cache)
 		checkQueries(queryAPI)
 	})
 	if err != nil {

cmd/syncserver-integration-tests/main.go

@@ -132,10 +132,10 @@ func startSyncServer() (*exec.Cmd, chan error) {
 		panic(err)
 	}
 	// TODO use the address assigned by the config generator rather than clobbering.
-	cfg.Matrix.ServerName = "localhost"
-	cfg.Listen.SyncAPI = config.Address(syncserverAddr)
-	cfg.Kafka.Topics.OutputRoomEvent = config.Topic(inputTopic)
-	cfg.Kafka.Topics.OutputClientData = config.Topic(clientTopic)
+	cfg.Global.ServerName = "localhost"
+	cfg.SyncAPI.Listen = config.Address(syncserverAddr)
+	cfg.Global.Kafka.Topics.OutputRoomEvent = config.Topic(inputTopic)
+	cfg.Global.Kafka.Topics.OutputClientData = config.Topic(clientTopic)
 
 	if err := test.WriteConfig(cfg, dir); err != nil {
 		panic(err)

currentstateserver/currentstateserver.go

@@ -34,13 +34,13 @@ func AddInternalRoutes(router *mux.Router, intAPI api.CurrentStateInternalAPI) {
 
 // NewInternalAPI returns a concrete implementation of the internal API. Callers
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
-func NewInternalAPI(cfg *config.Dendrite, consumer sarama.Consumer) api.CurrentStateInternalAPI {
-	csDB, err := storage.NewDatabase(string(cfg.Database.CurrentState), cfg.DbProperties())
+func NewInternalAPI(cfg *config.CurrentStateServer, consumer sarama.Consumer) api.CurrentStateInternalAPI {
+	csDB, err := storage.NewDatabase(&cfg.Database)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to open database")
 	}
 	roomConsumer := consumers.NewOutputRoomEventConsumer(
-		string(cfg.Kafka.Topics.OutputRoomEvent), consumer, csDB,
+		string(cfg.Matrix.Kafka.Topics.OutputRoomEvent), consumer, csDB,
 	)
 	if err = roomConsumer.Start(); err != nil {
 		logrus.WithError(err).Panicf("failed to start room server consumer")

currentstateserver/currentstateserver_test.go

@@ -15,6 +15,7 @@
 package currentstateserver
 
 import (
+	"bytes"
 	"context"
 	"crypto/ed25519"
 	"encoding/json"
@@ -94,11 +95,15 @@ func MustWriteOutputEvent(t *testing.T, producer sarama.SyncProducer, out *rooms
 
 func MustMakeInternalAPI(t *testing.T) (api.CurrentStateInternalAPI, sarama.SyncProducer, func()) {
 	cfg := &config.Dendrite{}
+	cfg.Defaults()
 	stateDBName := "test_state.db"
 	naffkaDBName := "test_naffka.db"
-	cfg.Kafka.Topics.OutputRoomEvent = config.Topic(kafkaTopic)
-	cfg.Database.CurrentState = config.DataSource("file:" + stateDBName)
-	db, err := sqlutil.Open(sqlutil.SQLiteDriverName(), "file:"+naffkaDBName, nil)
+	cfg.Global.ServerName = "kaer.morhen"
+	cfg.Global.Kafka.Topics.OutputRoomEvent = config.Topic(kafkaTopic)
+	cfg.CurrentStateServer.Database.ConnectionString = config.DataSource("file:" + stateDBName)
+	db, err := sqlutil.Open(&config.DatabaseOptions{
+		ConnectionString: config.DataSource("file:" + naffkaDBName),
+	})
 	if err != nil {
 		t.Fatalf("Failed to open naffka database: %s", err)
 	}
@@ -110,7 +115,7 @@ func MustMakeInternalAPI(t *testing.T) (api.CurrentStateInternalAPI, sarama.Sync
 	if err != nil {
 		t.Fatalf("Failed to create naffka consumer: %s", err)
 	}
-	return NewInternalAPI(cfg, naff), naff, func() {
+	return NewInternalAPI(&cfg.CurrentStateServer, naff), naff, func() {
 		os.Remove(naffkaDBName)
 		os.Remove(stateDBName)
 	}
@@ -165,8 +170,13 @@ func TestQueryCurrentState(t *testing.T) {
 					t.Errorf("QueryCurrentState want tuple %+v but it is missing from the response", tuple)
 					continue
 				}
-				if !reflect.DeepEqual(gotEvent.JSON(), wantEvent.JSON()) {
-					t.Errorf("QueryCurrentState tuple %+v got event JSON %s want %s", tuple, string(gotEvent.JSON()), string(wantEvent.JSON()))
+				gotCanon, err := gomatrixserverlib.CanonicalJSON(gotEvent.JSON())
+				if err != nil {
+					t.Errorf("CanonicalJSON failed: %w", err)
+					continue
+				}
+				if !bytes.Equal(gotCanon, wantEvent.JSON()) {
+					t.Errorf("QueryCurrentState tuple %+v got event JSON %s want %s", tuple, string(gotCanon), string(wantEvent.JSON()))
 				}
 			}
 		}

currentstateserver/storage/postgres/storage.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 
 	"github.com/matrix-org/dendrite/currentstateserver/storage/shared"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
@@ -14,10 +15,10 @@ type Database struct {
 }
 
 // NewDatabase creates a new sync server database
-func NewDatabase(dbDataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) {
+func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) {
 	var d Database
 	var err error
-	if d.db, err = sqlutil.Open("postgres", dbDataSourceName, dbProperties); err != nil {
+	if d.db, err = sqlutil.Open(dbProperties); err != nil {
 		return nil, err
 	}
 	if err = d.PartitionOffsetStatements.Prepare(d.db, "currentstate"); err != nil {

currentstateserver/storage/sqlite3/storage.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 
 	"github.com/matrix-org/dendrite/currentstateserver/storage/shared"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
@@ -15,13 +16,10 @@ type Database struct {
 
 // NewDatabase creates a new sync server database
 // nolint: gocyclo
-func NewDatabase(dataSourceName string) (*Database, error) {
+func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) {
 	var d Database
-	cs, err := sqlutil.ParseFileURI(dataSourceName)
-	if err != nil {
-		return nil, err
-	}
-	if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil {
+	var err error
+	if d.db, err = sqlutil.Open(dbProperties); err != nil {
 		return nil, err
 	}
 	if err = d.PartitionOffsetStatements.Prepare(d.db, "currentstate"); err != nil {

currentstateserver/storage/storage.go

@@ -17,25 +17,21 @@
 package storage
 
 import (
-	"net/url"
+	"fmt"
 
 	"github.com/matrix-org/dendrite/currentstateserver/storage/postgres"
 	"github.com/matrix-org/dendrite/currentstateserver/storage/sqlite3"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 )
 
 // NewDatabase opens a database connection.
-func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return postgres.NewDatabase(dataSourceName, dbProperties)
-	}
-	switch uri.Scheme {
-	case "postgres":
-		return postgres.NewDatabase(dataSourceName, dbProperties)
-	case "file":
-		return sqlite3.NewDatabase(dataSourceName)
+func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.NewDatabase(dbProperties)
+	case dbProperties.ConnectionString.IsPostgres():
+		return postgres.NewDatabase(dbProperties)
 	default:
-		return postgres.NewDatabase(dataSourceName, dbProperties)
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }

currentstateserver/storage/storage_wasm.go

@@ -16,27 +16,19 @@ package storage
 
 import (
 	"fmt"
-	"net/url"
 
 	"github.com/matrix-org/dendrite/currentstateserver/storage/sqlite3"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 )
 
 // NewDatabase opens a database connection.
-func NewDatabase(
-	dataSourceName string,
-	dbProperties sqlutil.DbProperties, // nolint:unparam
-) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	}
-	switch uri.Scheme {
-	case "postgres":
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	case "file":
-		return sqlite3.NewDatabase(dataSourceName)
+func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.NewDatabase(dbProperties)
+	case dbProperties.ConnectionString.IsPostgres():
+		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
-		return nil, fmt.Errorf("Cannot use postgres implementation")
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }

dendrite-config.yaml

@@ -1,178 +0,0 @@
-# The config file format version
-# This is used by dendrite to tell if it understands the config format.
-# This will change if the structure of the config file changes or if the meaning
-# of an existing config key changes.
-version: 0
-
-# The matrix specific config
-matrix:
-    # The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
-    server_name: "example.com"
-    # The path to the PEM formatted matrix private key.
-    private_key: "/etc/dendrite/matrix_key.pem"
-    # The x509 certificates used by the federation listeners for this server
-    federation_certificates: ["/etc/dendrite/server.crt"]
-    # The list of identity servers trusted to verify third party identifiers by this server.
-    # Defaults to no trusted servers.
-    trusted_third_party_id_servers:
-      - vector.im
-      - matrix.org
-    # Perspective key servers which are used when direct key requests fail
-    #key_perspectives:
-    #  - server_name: matrix.org
-    #    keys:
-    #      - key_id: ed25519:auto
-    #        public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw
-    #      - key_id: ed25519:a_RXGa
-    #        public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ
-    # Disables new users from registering (except via shared secrets)
-    registration_disabled: false
-    # Whether to disable TLS certificate validation. Warning: this reduces federation
-    # security and should not be enabled in production!
-    federation_disable_tls_validation: false
-
-# The media repository config
-media:
-    # The base path to where the media files will be stored. May be relative or absolute.
-    base_path: /var/dendrite/media
-
-    # The maximum file size in bytes that is allowed to be stored on this server.
-    # Note: if max_file_size_bytes is set to 0, the size is unlimited.
-    # Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB)
-    max_file_size_bytes: 10485760
-
-    # Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated
-    # NOTE: This is a possible denial-of-service attack vector - use at your own risk
-    dynamic_thumbnails: false
-
-    # A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content
-    # method is one of crop or scale. If omitted, it will default to scale.
-    # crop scales to fill the requested dimensions and crops the excess.
-    # scale scales to fit the requested dimensions and one dimension may be smaller than requested.
-    thumbnail_sizes:
-      - width: 32
-        height: 32
-        method: crop
-      - width: 96
-        height: 96
-        method: crop
-      - width: 320
-        height: 240
-        method: scale
-      - width: 640
-        height: 480
-        method: scale
-      - width: 800
-        height: 600
-        method: scale
-
-# Metrics config for Prometheus
-metrics:
-    # Whether or not metrics are enabled
-    enabled: false
-    # Use basic auth to protect the metrics. Uncomment to the complete block to enable.
-    #basic_auth:
-    #  username: prometheusUser
-    #  password: y0ursecr3tPa$$w0rd
-
-# The config for the TURN server
-turn:
-    # Whether or not guests can request TURN credentials
-    turn_allow_guests: true
-    # How long the authorization should last
-    turn_user_lifetime: "1h"
-    # The list of TURN URIs to pass to clients
-    turn_uris: []
-
-    # Authorization via Shared Secret
-    # The shared secret from coturn
-    turn_shared_secret: "<SECRET STRING GOES HERE>"
-
-    # Authorization via Static Username & Password
-    # Hardcoded Username and Password
-    turn_username: ""
-    turn_password: ""
-
-# The config for communicating with kafka
-kafka:
-    # Where the kafka servers are running.
-    addresses: ["localhost:9092"]
-    # Whether to use naffka instead of kafka.
-    # Naffka can only be used when running dendrite as a single monolithic server.
-    # Kafka can be used both with a monolithic server and when running the
-    # components as separate servers.
-    # If enabled database.naffka must also be specified.
-    use_naffka: false
-    # The names of the kafka topics to use.
-    topics:
-        output_room_event: roomserverOutput
-        output_client_data: clientapiOutput
-        output_typing_event: eduServerTypingOutput
-        output_send_to_device_event: eduServerSendToDeviceOutput
-        user_updates: userUpdates
-
-# The postgres connection configs for connecting to the databases, e.g.
-# for Postgres: postgres://username:password@hostname/database
-# for SQLite: file:filename.db or file:///path/to/filename.db
-database:
-    account: "postgres://dendrite:itsasecret@localhost/dendrite_account?sslmode=disable"
-    device: "postgres://dendrite:itsasecret@localhost/dendrite_device?sslmode=disable"
-    media_api: "postgres://dendrite:itsasecret@localhost/dendrite_mediaapi?sslmode=disable"
-    sync_api: "postgres://dendrite:itsasecret@localhost/dendrite_syncapi?sslmode=disable"
-    room_server: "postgres://dendrite:itsasecret@localhost/dendrite_roomserver?sslmode=disable"
-    server_key: "postgres://dendrite:itsasecret@localhost/dendrite_serverkey?sslmode=disable"
-    federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable"
-    appservice: "postgres://dendrite:itsasecret@localhost/dendrite_appservice?sslmode=disable"
-    current_state: "postgres://dendrite:itsasecret@localhost/dendrite_currentstate?sslmode=disable"
-    e2e_key: "postgres://dendrite:itsasecret@localhost/dendrite_e2ekey?sslmode=disable"
-    max_open_conns: 100
-    max_idle_conns: 2
-    conn_max_lifetime: -1
-    # If 'use_naffka: true' set above then you need to specify a naffka database
-    # naffka: "postgres://dendrite:itsasecret@localhost/dendrite_naffka?sslmode=disable"
-
-# The TCP host:port pairs to bind the internal HTTP APIs to.
-# These shouldn't be exposed to the public internet.
-# These aren't needed when running dendrite as a monolithic server.
-listen:
-    room_server: "localhost:7770"
-    client_api: "localhost:7771"
-    federation_api: "localhost:7772"
-    sync_api: "localhost:7773"
-    media_api: "localhost:7774"
-    federation_sender: "localhost:7776"
-    appservice_api: "localhost:7777"
-    edu_server: "localhost:7778"
-    key_server: "localhost:7779"
-    server_key_api: "localhost:7780"
-    user_api: "localhost:7781"
-    current_state_server: "localhost:7782"
-
-# The configuration for tracing the dendrite components.
-tracing:
-    # Config for the jaeger opentracing reporter.
-    # See https://godoc.org/github.com/uber/jaeger-client-go/config#Configuration
-    # for documentation.
-    jaeger:
-        disabled: true
-
-# A list of application service config files to use
-application_services:
-    config_files: []
-
-# The configuration for dendrite logs
-logging:
-    # The logging type, only "file" is supported at the moment
-    - type: "file"
-      # The logging level, must be one of debug, info, warn, error, fatal, panic.
-      level: "info"
-      # Parameters for this type of log
-      params:
-        # File logging must be given a path to a directory. Each component will write to a different file. Logs are rotated each day and gzipped
-        path: "/var/log/dendrite"
-    # It is possible to have multiple logging hooks at the same time.
-    # To save only errors in a different directory, uncomment the following.
-    # - type: "file"
-    #   level: "error"
-    #   params:
-    #     path: "/var/log/dendrite/errors"

eduserver/eduserver.go

@@ -39,12 +39,13 @@ func NewInternalAPI(
 	eduCache *cache.EDUCache,
 	userAPI userapi.UserInternalAPI,
 ) api.EDUServerInputAPI {
+	cfg := &base.Cfg.EDUServer
 	return &input.EDUServerInputAPI{
 		Cache:                        eduCache,
 		UserAPI:                      userAPI,
 		Producer:                     base.KafkaProducer,
-		OutputTypingEventTopic:       string(base.Cfg.Kafka.Topics.OutputTypingEvent),
-		OutputSendToDeviceEventTopic: string(base.Cfg.Kafka.Topics.OutputSendToDeviceEvent),
-		ServerName:                   base.Cfg.Matrix.ServerName,
+		OutputTypingEventTopic:       string(cfg.Matrix.Kafka.Topics.OutputTypingEvent),
+		OutputSendToDeviceEventTopic: string(cfg.Matrix.Kafka.Topics.OutputSendToDeviceEvent),
+		ServerName:                   cfg.Matrix.ServerName,
 	}
 }

federationapi/federationapi.go

@@ -31,7 +31,7 @@ import (
 // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component.
 func AddPublicRoutes(
 	router *mux.Router,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	userAPI userapi.UserInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
 	keyRing gomatrixserverlib.JSONVerifier,

federationapi/federationapi_test.go

@@ -20,30 +20,31 @@ import (
 func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
 	_, privKey, _ := ed25519.GenerateKey(nil)
 	cfg := &config.Dendrite{}
-	cfg.Matrix.KeyID = gomatrixserverlib.KeyID("ed25519:auto")
-	cfg.Matrix.ServerName = gomatrixserverlib.ServerName("localhost")
-	cfg.Matrix.PrivateKey = privKey
-	cfg.Kafka.UseNaffka = true
-	cfg.Database.Naffka = "file::memory:"
-	cfg.SetDefaults()
-	base := setup.NewBaseDendrite(cfg, "Test", false)
+	cfg.Defaults()
+	cfg.Global.KeyID = gomatrixserverlib.KeyID("ed25519:auto")
+	cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost")
+	cfg.Global.PrivateKey = privKey
+	cfg.Global.Kafka.UseNaffka = true
+	cfg.Global.Kafka.Database.ConnectionString = config.DataSource("file::memory:")
+	cfg.FederationSender.Database.ConnectionString = config.DataSource("file::memory:")
+	base := setup.NewBaseDendrite(cfg, "Monolith", false)
 	keyRing := &test.NopJSONVerifier{}
 	fsAPI := base.FederationSenderHTTPClient()
 	// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
 	// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
-	federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil, nil, nil)
+	federationapi.AddPublicRoutes(base.PublicAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, nil, nil)
 	httputil.SetupHTTPAPI(
 		base.BaseMux,
 		base.PublicAPIMux,
 		base.InternalAPIMux,
-		cfg,
+		&cfg.Global,
 		base.UseHTTPAPIs,
 	)
 	baseURL, cancel := test.ListenAndServe(t, base.BaseMux, true)
 	defer cancel()
 	serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://"))
 
-	fedCli := gomatrixserverlib.NewFederationClient(serverName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, true)
+	fedCli := gomatrixserverlib.NewFederationClient(serverName, cfg.Global.KeyID, cfg.Global.PrivateKey, true)
 
 	testCases := []struct {
 		roomVer   gomatrixserverlib.RoomVersion

federationapi/routing/backfill.go

@@ -35,7 +35,7 @@ func Backfill(
 	request *gomatrixserverlib.FederationRequest,
 	rsAPI api.RoomserverInternalAPI,
 	roomID string,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 ) util.JSONResponse {
 	var res api.PerformBackfillResponse
 	var eIDs []string

federationapi/routing/invite.go

@@ -34,7 +34,7 @@ func InviteV2(
 	request *gomatrixserverlib.FederationRequest,
 	roomID string,
 	eventID string,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
 ) util.JSONResponse {
@@ -56,7 +56,7 @@ func InviteV1(
 	request *gomatrixserverlib.FederationRequest,
 	roomID string,
 	eventID string,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
 ) util.JSONResponse {
@@ -86,7 +86,7 @@ func processInvite(
 	strippedState []gomatrixserverlib.InviteV2StrippedState,
 	roomID string,
 	eventID string,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
 ) util.JSONResponse {

federationapi/routing/join.go

@@ -32,7 +32,7 @@ import (
 func MakeJoin(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
 	roomID, userID string,
 	remoteVersions []gomatrixserverlib.RoomVersion,
@@ -95,7 +95,7 @@ func MakeJoin(
 	queryRes := api.QueryLatestEventsAndStateResponse{
 		RoomVersion: verRes.RoomVersion,
 	}
-	event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes)
+	event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg.Matrix, time.Now(), rsAPI, &queryRes)
 	if err == eventutil.ErrRoomNoExists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
@@ -141,7 +141,7 @@ func MakeJoin(
 func SendJoin(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
 	roomID, eventID string,

federationapi/routing/keys.go

@@ -121,7 +121,7 @@ func ClaimOneTimeKeys(
 
 // LocalKeys returns the local keys for the server.
 // See https://matrix.org/docs/spec/server_server/unstable.html#publishing-keys
-func LocalKeys(cfg *config.Dendrite) util.JSONResponse {
+func LocalKeys(cfg *config.FederationAPI) util.JSONResponse {
 	keys, err := localKeys(cfg, time.Now().Add(cfg.Matrix.KeyValidityPeriod))
 	if err != nil {
 		return util.ErrorResponse(err)
@@ -129,7 +129,7 @@ func LocalKeys(cfg *config.Dendrite) util.JSONResponse {
 	return util.JSONResponse{Code: http.StatusOK, JSON: keys}
 }
 
-func localKeys(cfg *config.Dendrite, validUntil time.Time) (*gomatrixserverlib.ServerKeys, error) {
+func localKeys(cfg *config.FederationAPI, validUntil time.Time) (*gomatrixserverlib.ServerKeys, error) {
 	var keys gomatrixserverlib.ServerKeys
 
 	keys.ServerName = cfg.Matrix.ServerName
@@ -142,7 +142,7 @@ func localKeys(cfg *config.Dendrite, validUntil time.Time) (*gomatrixserverlib.S
 		},
 	}
 
-	keys.TLSFingerprints = cfg.Matrix.TLSFingerPrints
+	keys.TLSFingerprints = cfg.TLSFingerPrints
 	keys.OldVerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.OldVerifyKey{}
 	keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(validUntil)
 

federationapi/routing/leave.go

@@ -28,7 +28,7 @@ import (
 func MakeLeave(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
 	roomID, userID string,
 ) util.JSONResponse {
@@ -60,7 +60,7 @@ func MakeLeave(
 	}
 
 	var queryRes api.QueryLatestEventsAndStateResponse
-	event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes)
+	event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg.Matrix, time.Now(), rsAPI, &queryRes)
 	if err == eventutil.ErrRoomNoExists {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
@@ -102,7 +102,7 @@ func MakeLeave(
 func SendLeave(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
 	keys gomatrixserverlib.JSONVerifier,
 	roomID, eventID string,

federationapi/routing/profile.go

@@ -30,7 +30,7 @@ import (
 func GetProfile(
 	httpReq *http.Request,
 	userAPI userapi.UserInternalAPI,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 ) util.JSONResponse {
 	userID, field := httpReq.FormValue("user_id"), httpReq.FormValue("field")
 

federationapi/routing/query.go

@@ -31,7 +31,7 @@ import (
 func RoomAliasToID(
 	httpReq *http.Request,
 	federation *gomatrixserverlib.FederationClient,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	senderAPI federationSenderAPI.FederationSenderInternalAPI,
 ) util.JSONResponse {

federationapi/routing/routing.go

@@ -47,7 +47,7 @@ const (
 // nolint: gocyclo
 func Setup(
 	publicAPIMux *mux.Router,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI roomserverAPI.RoomserverInternalAPI,
 	eduAPI eduserverAPI.EDUServerInputAPI,
 	fsAPI federationSenderAPI.FederationSenderInternalAPI,

federationapi/routing/send.go

@@ -35,7 +35,7 @@ func Send(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
 	txnID gomatrixserverlib.TransactionID,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	rsAPI api.RoomserverInternalAPI,
 	eduAPI eduserverAPI.EDUServerInputAPI,
 	keyAPI keyapi.KeyInternalAPI,

federationapi/routing/threepid.go

@@ -56,7 +56,7 @@ var (
 // CreateInvitesFrom3PIDInvites implements POST /_matrix/federation/v1/3pid/onbind
 func CreateInvitesFrom3PIDInvites(
 	req *http.Request, rsAPI api.RoomserverInternalAPI,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	federation *gomatrixserverlib.FederationClient,
 	userAPI userapi.UserInternalAPI,
 ) util.JSONResponse {
@@ -106,7 +106,7 @@ func ExchangeThirdPartyInvite(
 	request *gomatrixserverlib.FederationRequest,
 	roomID string,
 	rsAPI api.RoomserverInternalAPI,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
 	var builder gomatrixserverlib.EventBuilder
@@ -196,7 +196,7 @@ func ExchangeThirdPartyInvite(
 // necessary data to do so.
 func createInviteFrom3PIDInvite(
 	ctx context.Context, rsAPI api.RoomserverInternalAPI,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 	inv invite, federation *gomatrixserverlib.FederationClient,
 	userAPI userapi.UserInternalAPI,
 ) (*gomatrixserverlib.Event, error) {
@@ -263,7 +263,7 @@ func createInviteFrom3PIDInvite(
 func buildMembershipEvent(
 	ctx context.Context,
 	builder *gomatrixserverlib.EventBuilder, rsAPI api.RoomserverInternalAPI,
-	cfg *config.Dendrite,
+	cfg *config.FederationAPI,
 ) (*gomatrixserverlib.Event, error) {
 	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
 	if err != nil {
@@ -327,7 +327,7 @@ func buildMembershipEvent(
 // them responded with an error.
 func sendToRemoteServer(
 	ctx context.Context, inv invite,
-	federation *gomatrixserverlib.FederationClient, _ *config.Dendrite,
+	federation *gomatrixserverlib.FederationClient, _ *config.FederationAPI,
 	builder gomatrixserverlib.EventBuilder,
 ) (err error) {
 	remoteServers := make([]gomatrixserverlib.ServerName, 2)

federationsender/consumers/eduserver.go

@@ -43,27 +43,27 @@ type OutputEDUConsumer struct {
 
 // NewOutputEDUConsumer creates a new OutputEDUConsumer. Call Start() to begin consuming from EDU servers.
 func NewOutputEDUConsumer(
-	cfg *config.Dendrite,
+	cfg *config.FederationSender,
 	kafkaConsumer sarama.Consumer,
 	queues *queue.OutgoingQueues,
 	store storage.Database,
 ) *OutputEDUConsumer {
 	c := &OutputEDUConsumer{
 		typingConsumer: &internal.ContinualConsumer{
-			Topic:          string(cfg.Kafka.Topics.OutputTypingEvent),
+			Topic:          string(cfg.Matrix.Kafka.Topics.OutputTypingEvent),
 			Consumer:       kafkaConsumer,
 			PartitionStore: store,
 		},
 		sendToDeviceConsumer: &internal.ContinualConsumer{
-			Topic:          string(cfg.Kafka.Topics.OutputSendToDeviceEvent),
+			Topic:          string(cfg.Matrix.Kafka.Topics.OutputSendToDeviceEvent),
 			Consumer:       kafkaConsumer,
 			PartitionStore: store,
 		},
 		queues:            queues,
 		db:                store,
 		ServerName:        cfg.Matrix.ServerName,
-		TypingTopic:       string(cfg.Kafka.Topics.OutputTypingEvent),
-		SendToDeviceTopic: string(cfg.Kafka.Topics.OutputSendToDeviceEvent),
+		TypingTopic:       string(cfg.Matrix.Kafka.Topics.OutputTypingEvent),
+		SendToDeviceTopic: string(cfg.Matrix.Kafka.Topics.OutputSendToDeviceEvent),
 	}
 	c.typingConsumer.ProcessMessage = c.onTypingEvent
 	c.sendToDeviceConsumer.ProcessMessage = c.onSendToDeviceEvent

federationsender/consumers/keychange.go

@@ -41,7 +41,7 @@ type KeyChangeConsumer struct {
 
 // NewKeyChangeConsumer creates a new KeyChangeConsumer. Call Start() to begin consuming from key servers.
 func NewKeyChangeConsumer(
-	cfg *config.Dendrite,
+	cfg *config.KeyServer,
 	kafkaConsumer sarama.Consumer,
 	queues *queue.OutgoingQueues,
 	store storage.Database,
@@ -49,7 +49,7 @@ func NewKeyChangeConsumer(
 ) *KeyChangeConsumer {
 	c := &KeyChangeConsumer{
 		consumer: &internal.ContinualConsumer{
-			Topic:          string(cfg.Kafka.Topics.OutputKeyChangeEvent),
+			Topic:          string(cfg.Matrix.Kafka.Topics.OutputKeyChangeEvent),
 			Consumer:       kafkaConsumer,
 			PartitionStore: store,
 		},

federationsender/consumers/roomserver.go

@@ -33,7 +33,7 @@ import (
 
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
-	cfg        *config.Dendrite
+	cfg        *config.FederationSender
 	rsAPI      api.RoomserverInternalAPI
 	rsConsumer *internal.ContinualConsumer
 	db         storage.Database
@@ -42,14 +42,14 @@ type OutputRoomEventConsumer struct {
 
 // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
 func NewOutputRoomEventConsumer(
-	cfg *config.Dendrite,
+	cfg *config.FederationSender,
 	kafkaConsumer sarama.Consumer,
 	queues *queue.OutgoingQueues,
 	store storage.Database,
 	rsAPI api.RoomserverInternalAPI,
 ) *OutputRoomEventConsumer {
 	consumer := internal.ContinualConsumer{
-		Topic:          string(cfg.Kafka.Topics.OutputRoomEvent),
+		Topic:          string(cfg.Matrix.Kafka.Topics.OutputRoomEvent),
 		Consumer:       kafkaConsumer,
 		PartitionStore: store,
 	}

federationsender/federationsender.go

@@ -45,27 +45,29 @@ func NewInternalAPI(
 	stateAPI stateapi.CurrentStateInternalAPI,
 	keyRing *gomatrixserverlib.KeyRing,
 ) api.FederationSenderInternalAPI {
-	federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender), base.Cfg.DbProperties())
+	cfg := &base.Cfg.FederationSender
+
+	federationSenderDB, err := storage.NewDatabase(&cfg.Database)
 	if err != nil {
 		logrus.WithError(err).Panic("failed to connect to federation sender db")
 	}
 
 	stats := &statistics.Statistics{
 		DB:                     federationSenderDB,
-		FailuresUntilBlacklist: base.Cfg.Matrix.FederationMaxRetries,
+		FailuresUntilBlacklist: cfg.FederationMaxRetries,
 	}
 
 	queues := queue.NewOutgoingQueues(
-		federationSenderDB, base.Cfg.Matrix.ServerName, federation, rsAPI, stats,
+		federationSenderDB, cfg.Matrix.ServerName, federation, rsAPI, stats,
 		&queue.SigningInfo{
-			KeyID:      base.Cfg.Matrix.KeyID,
-			PrivateKey: base.Cfg.Matrix.PrivateKey,
-			ServerName: base.Cfg.Matrix.ServerName,
+			KeyID:      cfg.Matrix.KeyID,
+			PrivateKey: cfg.Matrix.PrivateKey,
+			ServerName: cfg.Matrix.ServerName,
 		},
 	)
 
 	rsConsumer := consumers.NewOutputRoomEventConsumer(
-		base.Cfg, base.KafkaConsumer, queues,
+		cfg, base.KafkaConsumer, queues,
 		federationSenderDB, rsAPI,
 	)
 	if err = rsConsumer.Start(); err != nil {
@@ -73,17 +75,17 @@ func NewInternalAPI(
 	}
 
 	tsConsumer := consumers.NewOutputEDUConsumer(
-		base.Cfg, base.KafkaConsumer, queues, federationSenderDB,
+		cfg, base.KafkaConsumer, queues, federationSenderDB,
 	)
 	if err := tsConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start typing server consumer")
 	}
 	keyConsumer := consumers.NewKeyChangeConsumer(
-		base.Cfg, base.KafkaConsumer, queues, federationSenderDB, stateAPI,
+		&base.Cfg.KeyServer, base.KafkaConsumer, queues, federationSenderDB, stateAPI,
 	)
 	if err := keyConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start key server consumer")
 	}
 
-	return internal.NewFederationSenderInternalAPI(federationSenderDB, base.Cfg, rsAPI, federation, keyRing, stats, queues)
+	return internal.NewFederationSenderInternalAPI(federationSenderDB, cfg, rsAPI, federation, keyRing, stats, queues)
 }

federationsender/internal/api.go

@@ -12,7 +12,7 @@ import (
 // FederationSenderInternalAPI is an implementation of api.FederationSenderInternalAPI
 type FederationSenderInternalAPI struct {
 	db         storage.Database
-	cfg        *config.Dendrite
+	cfg        *config.FederationSender
 	statistics *statistics.Statistics
 	rsAPI      api.RoomserverInternalAPI
 	federation *gomatrixserverlib.FederationClient
@@ -21,7 +21,7 @@ type FederationSenderInternalAPI struct {
 }
 
 func NewFederationSenderInternalAPI(
-	db storage.Database, cfg *config.Dendrite,
+	db storage.Database, cfg *config.FederationSender,
 	rsAPI api.RoomserverInternalAPI,
 	federation *gomatrixserverlib.FederationClient,
 	keyRing *gomatrixserverlib.KeyRing,

federationsender/storage/postgres/storage.go

@@ -19,6 +19,7 @@ import (
 	"database/sql"
 
 	"github.com/matrix-org/dendrite/federationsender/storage/shared"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
@@ -30,10 +31,10 @@ type Database struct {
 }
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) {
+func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) {
 	var d Database
 	var err error
-	if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
+	if d.db, err = sqlutil.Open(dbProperties); err != nil {
 		return nil, err
 	}
 	joinedHosts, err := NewPostgresJoinedHostsTable(d.db)

federationsender/storage/sqlite3/storage.go

@@ -21,6 +21,7 @@ import (
 	_ "github.com/mattn/go-sqlite3"
 
 	"github.com/matrix-org/dendrite/federationsender/storage/shared"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
@@ -32,14 +33,10 @@ type Database struct {
 }
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string) (*Database, error) {
+func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) {
 	var d Database
 	var err error
-	cs, err := sqlutil.ParseFileURI(dataSourceName)
-	if err != nil {
-		return nil, err
-	}
-	if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil {
+	if d.db, err = sqlutil.Open(dbProperties); err != nil {
 		return nil, err
 	}
 	joinedHosts, err := NewSQLiteJoinedHostsTable(d.db)

federationsender/storage/storage.go

@@ -17,25 +17,21 @@
 package storage
 
 import (
-	"net/url"
+	"fmt"
 
 	"github.com/matrix-org/dendrite/federationsender/storage/postgres"
 	"github.com/matrix-org/dendrite/federationsender/storage/sqlite3"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 )
 
 // NewDatabase opens a new database
-func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return postgres.NewDatabase(dataSourceName, dbProperties)
-	}
-	switch uri.Scheme {
-	case "file":
-		return sqlite3.NewDatabase(dataSourceName)
-	case "postgres":
-		return postgres.NewDatabase(dataSourceName, dbProperties)
+func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.NewDatabase(dbProperties)
+	case dbProperties.ConnectionString.IsPostgres():
+		return postgres.NewDatabase(dbProperties)
 	default:
-		return postgres.NewDatabase(dataSourceName, dbProperties)
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }

federationsender/storage/storage_wasm.go

@@ -16,27 +16,19 @@ package storage
 
 import (
 	"fmt"
-	"net/url"
 
 	"github.com/matrix-org/dendrite/federationsender/storage/sqlite3"
-	"github.com/matrix-org/dendrite/internal/sqlutil"
+	"github.com/matrix-org/dendrite/internal/config"
 )
 
 // NewDatabase opens a new database
-func NewDatabase(
-	dataSourceName string,
-	dbProperties sqlutil.DbProperties, // nolint:unparam
-) (Database, error) {
-	uri, err := url.Parse(dataSourceName)
-	if err != nil {
-		return nil, fmt.Errorf("Cannot use postgres implementation")
-	}
-	switch uri.Scheme {
-	case "file":
-		return sqlite3.NewDatabase(dataSourceName)
-	case "postgres":
-		return nil, fmt.Errorf("Cannot use postgres implementation")
+func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) {
+	switch {
+	case dbProperties.ConnectionString.IsSQLite():
+		return sqlite3.NewDatabase(dbProperties)
+	case dbProperties.ConnectionString.IsPostgres():
+		return nil, fmt.Errorf("can't use Postgres implementation")
 	default:
-		return nil, fmt.Errorf("Cannot use postgres implementation")
+		return nil, fmt.Errorf("unexpected database type")
 	}
 }

go.mod

@@ -2,7 +2,9 @@ module github.com/matrix-org/dendrite
 
 require (
 	github.com/Shopify/sarama v1.26.1
+	github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
+	github.com/ghodss/yaml v1.0.0
 	github.com/gologme/log v1.2.0
 	github.com/gorilla/mux v1.7.3
 	github.com/hashicorp/golang-lru v0.5.4
@@ -40,7 +42,7 @@ require (
 	go.uber.org/atomic v1.4.0
 	golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5
 	gopkg.in/h2non/bimg.v1 v1.0.18
-	gopkg.in/yaml.v2 v2.2.8
+	gopkg.in/yaml.v2 v2.3.0
 )
 
 go 1.13

go.sum

@@ -51,6 +51,9 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/cheggaaa/pb/v3 v3.0.4/go.mod h1:7rgWxLrAUcFMkvJuv09+DYi7mMUYi8nO9iOWcvGJPfw=
+github.com/circonus-labs/circonus-gometrics v1.2.0 h1:Kqa/+nIJhqFJ12B07aeekgC6F95J7yYgEtpD57NQzrE=
+github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible h1:C29Ae4G5GtYyYMm1aztcyj/J5ckgJm2zwdDajFbx1NY=
+github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
@@ -91,6 +94,7 @@ github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3B
 github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
@@ -857,6 +861,8 @@ gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099 h1:XJP7lxbSxWLOMNdBE4B/STaqVy6L73o0knwj2vIlxnw=

internal/config/config.go

@@ -24,7 +24,6 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -38,7 +37,7 @@ import (
 
 // Version is the current version of the config format.
 // This will change whenever we make breaking changes to the config format.
-const Version = 0
+const Version = 1
 
 // Dendrite contains all the config used by a dendrite process.
 // Relative paths are resolved relative to the current working directory
@@ -51,217 +50,19 @@ type Dendrite struct {
 	// been a breaking change to the config file format.
 	Version int `yaml:"version"`
 
-	// The configuration required for a matrix server.
-	Matrix struct {
-		// The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
-		ServerName gomatrixserverlib.ServerName `yaml:"server_name"`
-		// Path to the private key which will be used to sign requests and events.
-		PrivateKeyPath Path `yaml:"private_key"`
-		// The private key which will be used to sign requests and events.
-		PrivateKey ed25519.PrivateKey `yaml:"-"`
-		// An arbitrary string used to uniquely identify the PrivateKey. Must start with the
-		// prefix "ed25519:".
-		KeyID gomatrixserverlib.KeyID `yaml:"-"`
-		// List of paths to X509 certificates used by the external federation listeners.
-		// These are used to calculate the TLS fingerprints to publish for this server.
-		// Other matrix servers talking to this server will expect the x509 certificate
-		// to match one of these certificates.
-		// The certificates should be in PEM format.
-		FederationCertificatePaths []Path `yaml:"federation_certificates"`
-		// A list of SHA256 TLS fingerprints for the X509 certificates used by the
-		// federation listener for this server.
-		TLSFingerPrints []gomatrixserverlib.TLSFingerprint `yaml:"-"`
-		// How long a remote server can cache our server key for before requesting it again.
-		// Increasing this number will reduce the number of requests made by remote servers
-		// for our key, but increases the period a compromised key will be considered valid
-		// by remote servers.
-		// Defaults to 24 hours.
-		KeyValidityPeriod time.Duration `yaml:"key_validity_period"`
-		// List of domains that the server will trust as identity servers to
-		// verify third-party identifiers.
-		// Defaults to an empty array.
-		TrustedIDServers []string `yaml:"trusted_third_party_id_servers"`
-		// If set, allows registration by anyone who also has the shared
-		// secret, even if registration is otherwise disabled.
-		RegistrationSharedSecret string `yaml:"registration_shared_secret"`
-		// This Home Server's ReCAPTCHA public key.
-		RecaptchaPublicKey string `yaml:"recaptcha_public_key"`
-		// This Home Server's ReCAPTCHA private key.
-		RecaptchaPrivateKey string `yaml:"recaptcha_private_key"`
-		// Boolean stating whether catpcha registration is enabled
-		// and required
-		RecaptchaEnabled bool `yaml:"enable_registration_captcha"`
-		// Secret used to bypass the captcha registration entirely
-		RecaptchaBypassSecret string `yaml:"captcha_bypass_secret"`
-		// HTTP API endpoint used to verify whether the captcha response
-		// was successful
-		RecaptchaSiteVerifyAPI string `yaml:"recaptcha_siteverify_api"`
-		// If set disables new users from registering (except via shared
-		// secrets)
-		RegistrationDisabled bool `yaml:"registration_disabled"`
-		// Perspective keyservers, to use as a backup when direct key fetch
-		// requests don't succeed
-		KeyPerspectives KeyPerspectives `yaml:"key_perspectives"`
-		// Federation failure threshold. How many consecutive failures that we should
-		// tolerate when sending federation requests to a specific server. The backoff
-		// is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc.
-		// The default value is 16 if not specified, which is circa 18 hours.
-		FederationMaxRetries uint32 `yaml:"federation_max_retries"`
-		// FederationDisableTLSValidation disables the validation of X.509 TLS certs
-		// on remote federation endpoints. This is not recommended in production!
-		FederationDisableTLSValidation bool `yaml:"federation_disable_tls_validation"`
-	} `yaml:"matrix"`
-
-	// The configuration specific to the media repostitory.
-	Media struct {
-		// The base path to where the media files will be stored. May be relative or absolute.
-		BasePath Path `yaml:"base_path"`
-		// The absolute base path to where media files will be stored.
-		AbsBasePath Path `yaml:"-"`
-		// The maximum file size in bytes that is allowed to be stored on this server.
-		// Note: if max_file_size_bytes is set to 0, the size is unlimited.
-		// Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB)
-		MaxFileSizeBytes *FileSizeBytes `yaml:"max_file_size_bytes,omitempty"`
-		// Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated
-		DynamicThumbnails bool `yaml:"dynamic_thumbnails"`
-		// The maximum number of simultaneous thumbnail generators. default: 10
-		MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"`
-		// A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content
-		ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"`
-	} `yaml:"media"`
-
-	// The configuration to use for Prometheus metrics
-	Metrics struct {
-		// Whether or not the metrics are enabled
-		Enabled bool `yaml:"enabled"`
-		// Use BasicAuth for Authorization
-		BasicAuth struct {
-			// Authorization via Static Username & Password
-			// Hardcoded Username and Password
-			Username string `yaml:"username"`
-			Password string `yaml:"password"`
-		} `yaml:"basic_auth"`
-	} `yaml:"metrics"`
-
-	// The configuration for talking to kafka.
-	Kafka struct {
-		// A list of kafka addresses to connect to.
-		Addresses []string `yaml:"addresses"`
-		// Whether to use naffka instead of kafka.
-		// Naffka can only be used when running dendrite as a single monolithic server.
-		// Kafka can be used both with a monolithic server and when running the
-		// components as separate servers.
-		UseNaffka bool `yaml:"use_naffka,omitempty"`
-		// The names of the topics to use when reading and writing from kafka.
-		Topics struct {
-			// Topic for roomserver/api.OutputRoomEvent events.
-			OutputRoomEvent Topic `yaml:"output_room_event"`
-			// Topic for sending account data from client API to sync API
-			OutputClientData Topic `yaml:"output_client_data"`
-			// Topic for eduserver/api.OutputTypingEvent events.
-			OutputTypingEvent Topic `yaml:"output_typing_event"`
-			// Topic for eduserver/api.OutputSendToDeviceEvent events.
-			OutputSendToDeviceEvent Topic `yaml:"output_send_to_device_event"`
-			// Topic for keyserver when new device keys are added.
-			OutputKeyChangeEvent Topic `yaml:"output_key_change_event"`
-		}
-	} `yaml:"kafka"`
-
-	// Postgres Config
-	Database struct {
-		// The Account database stores the login details and account information
-		// for local users. It is accessed by the UserAPI.
-		Account DataSource `yaml:"account"`
-		// The CurrentState database stores the current state of all rooms.
-		// It is accessed by the CurrentStateServer.
-		CurrentState DataSource `yaml:"current_state"`
-		// The Device database stores session information for the devices of logged
-		// in local users. It is accessed by the UserAPI.
-		Device DataSource `yaml:"device"`
-		// The MediaAPI database stores information about files uploaded and downloaded
-		// by local users. It is only accessed by the MediaAPI.
-		MediaAPI DataSource `yaml:"media_api"`
-		// The ServerKey database caches the public keys of remote servers.
-		// It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI.
-		ServerKey DataSource `yaml:"server_key"`
-		// The E2EKey database stores one-time public keys for devices in addition to
-		// signed device keys. Used for E2E.
-		E2EKey DataSource `yaml:"e2e_key"`
-		// The SyncAPI stores information used by the SyncAPI server.
-		// It is only accessed by the SyncAPI server.
-		SyncAPI DataSource `yaml:"sync_api"`
-		// The RoomServer database stores information about matrix rooms.
-		// It is only accessed by the RoomServer.
-		RoomServer DataSource `yaml:"room_server"`
-		// The FederationSender database stores information used by the FederationSender
-		// It is only accessed by the FederationSender.
-		FederationSender DataSource `yaml:"federation_sender"`
-		// The AppServices database stores information used by the AppService component.
-		// It is only accessed by the AppService component.
-		AppService DataSource `yaml:"appservice"`
-		// The Naffka database is used internally by the naffka library, if used.
-		Naffka DataSource `yaml:"naffka,omitempty"`
-		// Maximum open connections to the DB (0 = use default, negative means unlimited)
-		MaxOpenConns int `yaml:"max_open_conns"`
-		// Maximum idle connections to the DB (0 = use default, negative means unlimited)
-		MaxIdleConns int `yaml:"max_idle_conns"`
-		// maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited)
-		ConnMaxLifetimeSec int `yaml:"conn_max_lifetime"`
-	} `yaml:"database"`
-
-	// TURN Server Config
-	TURN struct {
-		// TODO Guest Support
-		// Whether or not guests can request TURN credentials
-		//AllowGuests bool `yaml:"turn_allow_guests"`
-		// How long the authorization should last
-		UserLifetime string `yaml:"turn_user_lifetime"`
-		// The list of TURN URIs to pass to clients
-		URIs []string `yaml:"turn_uris"`
-
-		// Authorization via Shared Secret
-		// The shared secret from coturn
-		SharedSecret string `yaml:"turn_shared_secret"`
-
-		// Authorization via Static Username & Password
-		// Hardcoded Username and Password
-		Username string `yaml:"turn_username"`
-		Password string `yaml:"turn_password"`
-	} `yaml:"turn"`
-
-	// The internal addresses the components will listen on.
-	// These should not be exposed externally as they expose metrics and debugging APIs.
-	// Falls back to addresses listed in Listen if not specified
-	Bind struct {
-		MediaAPI         Address `yaml:"media_api"`
-		ClientAPI        Address `yaml:"client_api"`
-		CurrentState     Address `yaml:"current_state_server"`
-		FederationAPI    Address `yaml:"federation_api"`
-		ServerKeyAPI     Address `yaml:"server_key_api"`
-		AppServiceAPI    Address `yaml:"appservice_api"`
-		SyncAPI          Address `yaml:"sync_api"`
-		UserAPI          Address `yaml:"user_api"`
-		RoomServer       Address `yaml:"room_server"`
-		FederationSender Address `yaml:"federation_sender"`
-		EDUServer        Address `yaml:"edu_server"`
-		KeyServer        Address `yaml:"key_server"`
-	} `yaml:"bind"`
-
-	// The addresses for talking to other microservices.
-	Listen struct {
-		MediaAPI         Address `yaml:"media_api"`
-		ClientAPI        Address `yaml:"client_api"`
-		CurrentState     Address `yaml:"current_state_server"`
-		FederationAPI    Address `yaml:"federation_api"`
-		ServerKeyAPI     Address `yaml:"server_key_api"`
-		AppServiceAPI    Address `yaml:"appservice_api"`
-		SyncAPI          Address `yaml:"sync_api"`
-		UserAPI          Address `yaml:"user_api"`
-		RoomServer       Address `yaml:"room_server"`
-		FederationSender Address `yaml:"federation_sender"`
-		EDUServer        Address `yaml:"edu_server"`
-		KeyServer        Address `yaml:"key_server"`
-	} `yaml:"listen"`
+	Global             Global             `yaml:"global"`
+	AppServiceAPI      AppServiceAPI      `yaml:"app_service_api"`
+	ClientAPI          ClientAPI          `yaml:"client_api"`
+	CurrentStateServer CurrentStateServer `yaml:"current_state_server"`
+	EDUServer          EDUServer          `yaml:"edu_server"`
+	FederationAPI      FederationAPI      `yaml:"federation_api"`
+	FederationSender   FederationSender   `yaml:"federation_sender"`
+	KeyServer          KeyServer          `yaml:"key_server"`
+	MediaAPI           MediaAPI           `yaml:"media_api"`
+	RoomServer         RoomServer         `yaml:"room_server"`
+	ServerKeyAPI       ServerKeyAPI       `yaml:"server_key_api"`
+	SyncAPI            SyncAPI            `yaml:"sync_api"`
+	UserAPI            UserAPI            `yaml:"user_api"`
 
 	// The config for tracing the dendrite servers.
 	Tracing struct {
@@ -271,28 +72,15 @@ type Dendrite struct {
 		Jaeger jaegerconfig.Configuration `yaml:"jaeger"`
 	} `yaml:"tracing"`
 
-	// Application Services
-	// https://matrix.org/docs/spec/application_service/unstable.html
-	ApplicationServices struct {
-		// Configuration files for various application services
-		ConfigFiles []string `yaml:"config_files"`
-	} `yaml:"application_services"`
-
 	// The config for logging informations. Each hook will be added to logrus.
 	Logging []LogrusHook `yaml:"logging"`
 
-	// The config for setting a proxy to use for server->server requests
-	Proxy *struct {
-		// The protocol for the proxy (http / https / socks5)
-		Protocol string `yaml:"protocol"`
-		// The host where the proxy is listening
-		Host string `yaml:"host"`
-		// The port on which the proxy is listening
-		Port uint16 `yaml:"port"`
-	} `yaml:"proxy"`
-
 	// Any information derived from the configuration options for later use.
-	Derived struct {
+	Derived Derived `yaml:"-"`
+}
+
+// TODO: Kill Derived
+type Derived struct {
 	Registration struct {
 		// Flows is a slice of flows, which represent one possible way that the client can authenticate a request.
 		// http://matrix.org/docs/spec/HEAD/client_server/r0.3.0.html#user-interactive-authentication-api
@@ -320,7 +108,6 @@ type Dendrite struct {
 	ExclusiveApplicationServicesAliasRegexp *regexp.Regexp
 	// Note: An Exclusive Regex for room ID isn't necessary as we aren't blocking
 	// servers from creating RoomIDs in exclusive application service namespaces
-	} `yaml:"-"`
 }
 
 // KeyPerspectives are used to configure perspective key servers for
@@ -344,6 +131,16 @@ type Path string
 // A DataSource for opening a postgresql database using lib/pq.
 type DataSource string
 
+func (d DataSource) IsSQLite() bool {
+	return strings.HasPrefix(string(d), "file:")
+}
+
+func (d DataSource) IsPostgres() bool {
+	// commented line may not always be true?
+	// return strings.HasPrefix(string(d), "postgres:")
+	return !d.IsSQLite()
+}
+
 // A Topic in kafka.
 type Topic string
 
@@ -379,9 +176,9 @@ type LogrusHook struct {
 	Params map[string]interface{} `yaml:"params"`
 }
 
-// configErrors stores problems encountered when parsing a config file.
+// ConfigErrors stores problems encountered when parsing a config file.
 // It implements the error interface.
-type configErrors []string
+type ConfigErrors []string
 
 // Load a yaml config file for a server run as multiple processes or as a monolith.
 // Checks the config to ensure that it is valid.
@@ -405,29 +202,29 @@ func loadConfig(
 	readFile func(string) ([]byte, error),
 	monolithic bool,
 ) (*Dendrite, error) {
-	var config Dendrite
+	var c Dendrite
+	c.Defaults()
+
 	var err error
-	if err = yaml.Unmarshal(configData, &config); err != nil {
+	if err = yaml.Unmarshal(configData, &c); err != nil {
 		return nil, err
 	}
 
-	config.SetDefaults()
-
-	if err = config.check(monolithic); err != nil {
+	if err = c.check(monolithic); err != nil {
 		return nil, err
 	}
 
-	privateKeyPath := absPath(basePath, config.Matrix.PrivateKeyPath)
+	privateKeyPath := absPath(basePath, c.Global.PrivateKeyPath)
 	privateKeyData, err := readFile(privateKeyPath)
 	if err != nil {
 		return nil, err
 	}
 
-	if config.Matrix.KeyID, config.Matrix.PrivateKey, err = readKeyPEM(privateKeyPath, privateKeyData); err != nil {
+	if c.Global.KeyID, c.Global.PrivateKey, err = readKeyPEM(privateKeyPath, privateKeyData); err != nil {
 		return nil, err
 	}
 
-	for _, certPath := range config.Matrix.FederationCertificatePaths {
+	for _, certPath := range c.FederationAPI.FederationCertificatePaths {
 		absCertPath := absPath(basePath, certPath)
 		var pemData []byte
 		pemData, err = readFile(absCertPath)
@@ -438,18 +235,19 @@ func loadConfig(
 		if fingerprint == nil {
 			return nil, fmt.Errorf("no certificate PEM data in %q", absCertPath)
 		}
-		config.Matrix.TLSFingerPrints = append(config.Matrix.TLSFingerPrints, *fingerprint)
+		c.FederationAPI.TLSFingerPrints = append(c.FederationAPI.TLSFingerPrints, *fingerprint)
 	}
 
-	config.Media.AbsBasePath = Path(absPath(basePath, config.Media.BasePath))
+	c.MediaAPI.AbsBasePath = Path(absPath(basePath, c.MediaAPI.BasePath))
 
 	// Generate data from config options
-	err = config.Derive()
+	err = c.Derive()
 	if err != nil {
 		return nil, err
 	}
 
-	return &config, nil
+	c.Wiring()
+	return &c, nil
 }
 
 // Derive generates data that is derived from various values provided in
@@ -462,8 +260,8 @@ func (config *Dendrite) Derive() error {
 	// TODO: Add email auth type
 	// TODO: Add MSISDN auth type
 
-	if config.Matrix.RecaptchaEnabled {
-		config.Derived.Registration.Params[authtypes.LoginTypeRecaptcha] = map[string]string{"public_key": config.Matrix.RecaptchaPublicKey}
+	if config.ClientAPI.RecaptchaEnabled {
+		config.Derived.Registration.Params[authtypes.LoginTypeRecaptcha] = map[string]string{"public_key": config.ClientAPI.RecaptchaPublicKey}
 		config.Derived.Registration.Flows = append(config.Derived.Registration.Flows,
 			authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeRecaptcha}})
 	} else {
@@ -472,7 +270,7 @@ func (config *Dendrite) Derive() error {
 	}
 
 	// Load application service configuration files
-	if err := loadAppServices(config); err != nil {
+	if err := loadAppServices(&config.AppServiceAPI, &config.Derived); err != nil {
 		return err
 	}
 
@@ -480,41 +278,62 @@ func (config *Dendrite) Derive() error {
 }
 
 // SetDefaults sets default config values if they are not explicitly set.
-func (config *Dendrite) SetDefaults() {
-	if config.Matrix.KeyValidityPeriod == 0 {
-		config.Matrix.KeyValidityPeriod = 24 * time.Hour
+func (c *Dendrite) Defaults() {
+	c.Version = 1
+
+	c.Global.Defaults()
+	c.ClientAPI.Defaults()
+	c.CurrentStateServer.Defaults()
+	c.EDUServer.Defaults()
+	c.FederationAPI.Defaults()
+	c.FederationSender.Defaults()
+	c.KeyServer.Defaults()
+	c.MediaAPI.Defaults()
+	c.RoomServer.Defaults()
+	c.ServerKeyAPI.Defaults()
+	c.SyncAPI.Defaults()
+	c.UserAPI.Defaults()
+	c.AppServiceAPI.Defaults()
+
+	c.Wiring()
 }
 
-	if config.Matrix.TrustedIDServers == nil {
-		config.Matrix.TrustedIDServers = []string{}
+func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	type verifiable interface {
+		Verify(configErrs *ConfigErrors, isMonolith bool)
+	}
+	for _, c := range []verifiable{
+		&c.Global, &c.ClientAPI, &c.CurrentStateServer,
+		&c.EDUServer, &c.FederationAPI, &c.FederationSender,
+		&c.KeyServer, &c.MediaAPI, &c.RoomServer,
+		&c.ServerKeyAPI, &c.SyncAPI, &c.UserAPI,
+		&c.AppServiceAPI,
+	} {
+		c.Verify(configErrs, isMonolith)
+	}
 }
 
-	if config.Matrix.FederationMaxRetries == 0 {
-		config.Matrix.FederationMaxRetries = 16
-	}
-
-	if config.Media.MaxThumbnailGenerators == 0 {
-		config.Media.MaxThumbnailGenerators = 10
-	}
-
-	if config.Media.MaxFileSizeBytes == nil {
-		defaultMaxFileSizeBytes := FileSizeBytes(10485760)
-		config.Media.MaxFileSizeBytes = &defaultMaxFileSizeBytes
-	}
-
-	if config.Database.MaxIdleConns == 0 {
-		config.Database.MaxIdleConns = 2
-	}
-
-	if config.Database.MaxOpenConns == 0 {
-		config.Database.MaxOpenConns = 100
-	}
+func (c *Dendrite) Wiring() {
+	c.ClientAPI.Matrix = &c.Global
+	c.CurrentStateServer.Matrix = &c.Global
+	c.EDUServer.Matrix = &c.Global
+	c.FederationAPI.Matrix = &c.Global
+	c.FederationSender.Matrix = &c.Global
+	c.KeyServer.Matrix = &c.Global
+	c.MediaAPI.Matrix = &c.Global
+	c.RoomServer.Matrix = &c.Global
+	c.ServerKeyAPI.Matrix = &c.Global
+	c.SyncAPI.Matrix = &c.Global
+	c.UserAPI.Matrix = &c.Global
+	c.AppServiceAPI.Matrix = &c.Global
 
+	c.ClientAPI.Derived = &c.Derived
+	c.AppServiceAPI.Derived = &c.Derived
 }
 
 // Error returns a string detailing how many errors were contained within a
 // configErrors type.
-func (errs configErrors) Error() string {
+func (errs ConfigErrors) Error() string {
 	if len(errs) == 1 {
 		return errs[0]
 	}
@@ -528,13 +347,13 @@ func (errs configErrors) Error() string {
 // the client code.
 // This method is safe to use with an uninitialized configErrors because
 // if it is nil, it will be properly allocated.
-func (errs *configErrors) Add(str string) {
+func (errs *ConfigErrors) Add(str string) {
 	*errs = append(*errs, str)
 }
 
 // checkNotEmpty verifies the given value is not empty in the configuration.
 // If it is, adds an error to the list.
-func checkNotEmpty(configErrs *configErrors, key, value string) {
+func checkNotEmpty(configErrs *ConfigErrors, key, value string) {
 	if value == "" {
 		configErrs.Add(fmt.Sprintf("missing config key %q", key))
 	}
@@ -542,7 +361,7 @@ func checkNotEmpty(configErrs *configErrors, key, value string) {
 
 // checkNotZero verifies the given value is not zero in the configuration.
 // If it is, adds an error to the list.
-func checkNotZero(configErrs *configErrors, key string, value int64) {
+func checkNotZero(configErrs *ConfigErrors, key string, value int64) {
 	if value == 0 {
 		configErrs.Add(fmt.Sprintf("missing config key %q", key))
 	}
@@ -550,96 +369,14 @@ func checkNotZero(configErrs *configErrors, key string, value int64) {
 
 // checkPositive verifies the given value is positive (zero included)
 // in the configuration. If it is not, adds an error to the list.
-func checkPositive(configErrs *configErrors, key string, value int64) {
+func checkPositive(configErrs *ConfigErrors, key string, value int64) {
 	if value < 0 {
 		configErrs.Add(fmt.Sprintf("invalid value for config key %q: %d", key, value))
 	}
 }
 
-// checkTurn verifies the parameters turn.* are valid.
-func (config *Dendrite) checkTurn(configErrs *configErrors) {
-	value := config.TURN.UserLifetime
-	if value != "" {
-		if _, err := time.ParseDuration(value); err != nil {
-			configErrs.Add(fmt.Sprintf("invalid duration for config key %q: %s", "turn.turn_user_lifetime", value))
-		}
-	}
-}
-
-// checkMatrix verifies the parameters matrix.* are valid.
-func (config *Dendrite) checkMatrix(configErrs *configErrors) {
-	checkNotEmpty(configErrs, "matrix.server_name", string(config.Matrix.ServerName))
-	checkNotEmpty(configErrs, "matrix.private_key", string(config.Matrix.PrivateKeyPath))
-	checkNotZero(configErrs, "matrix.federation_certificates", int64(len(config.Matrix.FederationCertificatePaths)))
-	if config.Matrix.RecaptchaEnabled {
-		checkNotEmpty(configErrs, "matrix.recaptcha_public_key", string(config.Matrix.RecaptchaPublicKey))
-		checkNotEmpty(configErrs, "matrix.recaptcha_private_key", string(config.Matrix.RecaptchaPrivateKey))
-		checkNotEmpty(configErrs, "matrix.recaptcha_siteverify_api", string(config.Matrix.RecaptchaSiteVerifyAPI))
-	}
-}
-
-// checkMedia verifies the parameters media.* are valid.
-func (config *Dendrite) checkMedia(configErrs *configErrors) {
-	checkNotEmpty(configErrs, "media.base_path", string(config.Media.BasePath))
-	checkPositive(configErrs, "media.max_file_size_bytes", int64(*config.Media.MaxFileSizeBytes))
-	checkPositive(configErrs, "media.max_thumbnail_generators", int64(config.Media.MaxThumbnailGenerators))
-
-	for i, size := range config.Media.ThumbnailSizes {
-		checkPositive(configErrs, fmt.Sprintf("media.thumbnail_sizes[%d].width", i), int64(size.Width))
-		checkPositive(configErrs, fmt.Sprintf("media.thumbnail_sizes[%d].height", i), int64(size.Height))
-	}
-}
-
-// checkKafka verifies the parameters kafka.* and the related
-// database.naffka are valid.
-func (config *Dendrite) checkKafka(configErrs *configErrors, monolithic bool) {
-
-	if config.Kafka.UseNaffka {
-		if !monolithic {
-			configErrs.Add(fmt.Sprintf("naffka can only be used in a monolithic server"))
-		}
-
-		checkNotEmpty(configErrs, "database.naffka", string(config.Database.Naffka))
-	} else {
-		// If we aren't using naffka then we need to have at least one kafka
-		// server to talk to.
-		checkNotZero(configErrs, "kafka.addresses", int64(len(config.Kafka.Addresses)))
-	}
-	checkNotEmpty(configErrs, "kafka.topics.output_room_event", string(config.Kafka.Topics.OutputRoomEvent))
-	checkNotEmpty(configErrs, "kafka.topics.output_client_data", string(config.Kafka.Topics.OutputClientData))
-	checkNotEmpty(configErrs, "kafka.topics.output_typing_event", string(config.Kafka.Topics.OutputTypingEvent))
-	checkNotEmpty(configErrs, "kafka.topics.output_send_to_device_event", string(config.Kafka.Topics.OutputSendToDeviceEvent))
-	checkNotEmpty(configErrs, "kafka.topics.output_key_change_event", string(config.Kafka.Topics.OutputKeyChangeEvent))
-}
-
-// checkDatabase verifies the parameters database.* are valid.
-func (config *Dendrite) checkDatabase(configErrs *configErrors) {
-	checkNotEmpty(configErrs, "database.account", string(config.Database.Account))
-	checkNotEmpty(configErrs, "database.device", string(config.Database.Device))
-	checkNotEmpty(configErrs, "database.server_key", string(config.Database.ServerKey))
-	checkNotEmpty(configErrs, "database.media_api", string(config.Database.MediaAPI))
-	checkNotEmpty(configErrs, "database.sync_api", string(config.Database.SyncAPI))
-	checkNotEmpty(configErrs, "database.room_server", string(config.Database.RoomServer))
-	checkNotEmpty(configErrs, "database.current_state", string(config.Database.CurrentState))
-	checkNotEmpty(configErrs, "database.e2e_key", string(config.Database.E2EKey))
-}
-
-// checkListen verifies the parameters listen.* are valid.
-func (config *Dendrite) checkListen(configErrs *configErrors) {
-	checkNotEmpty(configErrs, "listen.media_api", string(config.Listen.MediaAPI))
-	checkNotEmpty(configErrs, "listen.client_api", string(config.Listen.ClientAPI))
-	checkNotEmpty(configErrs, "listen.federation_api", string(config.Listen.FederationAPI))
-	checkNotEmpty(configErrs, "listen.sync_api", string(config.Listen.SyncAPI))
-	checkNotEmpty(configErrs, "listen.room_server", string(config.Listen.RoomServer))
-	checkNotEmpty(configErrs, "listen.edu_server", string(config.Listen.EDUServer))
-	checkNotEmpty(configErrs, "listen.server_key_api", string(config.Listen.EDUServer))
-	checkNotEmpty(configErrs, "listen.user_api", string(config.Listen.UserAPI))
-	checkNotEmpty(configErrs, "listen.current_state_server", string(config.Listen.CurrentState))
-	checkNotEmpty(configErrs, "listen.key_server", string(config.Listen.KeyServer))
-}
-
 // checkLogging verifies the parameters logging.* are valid.
-func (config *Dendrite) checkLogging(configErrs *configErrors) {
+func (config *Dendrite) checkLogging(configErrs *ConfigErrors) {
 	for _, logrusHook := range config.Logging {
 		checkNotEmpty(configErrs, "logging.type", string(logrusHook.Type))
 		checkNotEmpty(configErrs, "logging.level", string(logrusHook.Level))
@@ -648,8 +385,8 @@ func (config *Dendrite) checkLogging(configErrs *configErrors) {
 
 // check returns an error type containing all errors found within the config
 // file.
-func (config *Dendrite) check(monolithic bool) error {
-	var configErrs configErrors
+func (config *Dendrite) check(_ bool) error { // monolithic
+	var configErrs ConfigErrors
 
 	if config.Version != Version {
 		configErrs.Add(fmt.Sprintf(
@@ -658,17 +395,8 @@ func (config *Dendrite) check(monolithic bool) error {
 		return configErrs
 	}
 
-	config.checkMatrix(&configErrs)
-	config.checkMedia(&configErrs)
-	config.checkTurn(&configErrs)
-	config.checkKafka(&configErrs, monolithic)
-	config.checkDatabase(&configErrs)
 	config.checkLogging(&configErrs)
 
-	if !monolithic {
-		config.checkListen(&configErrs)
-	}
-
 	// Due to how Golang manages its interface types, this condition is not redundant.
 	// In order to get the proper behaviour, it is necessary to return an explicit nil
 	// and not a nil configErrors.
@@ -737,7 +465,7 @@ func (config *Dendrite) AppServiceURL() string {
 	// If we support HTTPS we need to think of a practical way to do certificate validation.
 	// People setting up servers shouldn't need to get a certificate valid for the public
 	// internet for an internal API.
-	return "http://" + string(config.Listen.AppServiceAPI)
+	return "http://" + string(config.AppServiceAPI.Listen)
 }
 
 // RoomServerURL returns an HTTP URL for where the roomserver is listening.
@@ -746,7 +474,7 @@ func (config *Dendrite) RoomServerURL() string {
 	// If we support HTTPS we need to think of a practical way to do certificate validation.
 	// People setting up servers shouldn't need to get a certificate valid for the public
 	// internet for an internal API.
-	return "http://" + string(config.Listen.RoomServer)
+	return "http://" + string(config.RoomServer.Listen)
 }
 
 // UserAPIURL returns an HTTP URL for where the userapi is listening.
@@ -755,7 +483,7 @@ func (config *Dendrite) UserAPIURL() string {
 	// If we support HTTPS we need to think of a practical way to do certificate validation.
 	// People setting up servers shouldn't need to get a certificate valid for the public
 	// internet for an internal API.
-	return "http://" + string(config.Listen.UserAPI)
+	return "http://" + string(config.UserAPI.Listen)
 }
 
 // CurrentStateAPIURL returns an HTTP URL for where the currentstateserver is listening.
@@ -764,7 +492,7 @@ func (config *Dendrite) CurrentStateAPIURL() string {
 	// If we support HTTPS we need to think of a practical way to do certificate validation.
 	// People setting up servers shouldn't need to get a certificate valid for the public
 	// internet for an internal API.
-	return "http://" + string(config.Listen.CurrentState)
+	return "http://" + string(config.CurrentStateServer.Listen)
 }
 
 // EDUServerURL returns an HTTP URL for where the EDU server is listening.
@@ -773,7 +501,7 @@ func (config *Dendrite) EDUServerURL() string {
 	// If we support HTTPS we need to think of a practical way to do certificate validation.
 	// People setting up servers shouldn't need to get a certificate valid for the public
 	// internet for an internal API.
-	return "http://" + string(config.Listen.EDUServer)
+	return "http://" + string(config.EDUServer.Listen)
 }
 
 // FederationSenderURL returns an HTTP URL for where the federation sender is listening.
@@ -782,7 +510,7 @@ func (config *Dendrite) FederationSenderURL() string {
 	// If we support HTTPS we need to think of a practical way to do certificate validation.
 	// People setting up servers shouldn't need to get a certificate valid for the public
 	// internet for an internal API.
-	return "http://" + string(config.Listen.FederationSender)
+	return "http://" + string(config.FederationSender.Listen)
 }
 
 // ServerKeyAPIURL returns an HTTP URL for where the server key API is listening.
@@ -791,7 +519,7 @@ func (config *Dendrite) ServerKeyAPIURL() string {
 	// If we support HTTPS we need to think of a practical way to do certificate validation.
 	// People setting up servers shouldn't need to get a certificate valid for the public
 	// internet for an internal API.
-	return "http://" + string(config.Listen.ServerKeyAPI)
+	return "http://" + string(config.ServerKeyAPI.Listen)
 }
 
 // KeyServerURL returns an HTTP URL for where the key server is listening.
@@ -800,7 +528,7 @@ func (config *Dendrite) KeyServerURL() string {
 	// If we support HTTPS we need to think of a practical way to do certificate validation.
 	// People setting up servers shouldn't need to get a certificate valid for the public
 	// internet for an internal API.
-	return "http://" + string(config.Listen.KeyServer)
+	return "http://" + string(config.KeyServer.Listen)
 }
 
 // SetupTracing configures the opentracing using the supplied configuration.
@@ -815,33 +543,6 @@ func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err
 	)
 }
 
-// MaxIdleConns returns maximum idle connections to the DB
-func (config Dendrite) MaxIdleConns() int {
-	return config.Database.MaxIdleConns
-}
-
-// MaxOpenConns returns maximum open connections to the DB
-func (config Dendrite) MaxOpenConns() int {
-	return config.Database.MaxOpenConns
-}
-
-// ConnMaxLifetime returns maximum amount of time a connection may be reused
-func (config Dendrite) ConnMaxLifetime() time.Duration {
-	return time.Duration(config.Database.ConnMaxLifetimeSec) * time.Second
-}
-
-// DbProperties functions return properties used by database/sql/DB
-type DbProperties interface {
-	MaxIdleConns() int
-	MaxOpenConns() int
-	ConnMaxLifetime() time.Duration
-}
-
-// DbProperties returns cfg as a DbProperties interface
-func (config Dendrite) DbProperties() DbProperties {
-	return config
-}
-
 // logrusLogger is a small wrapper that implements jaeger.Logger using logrus.
 type logrusLogger struct {
 	l *logrus.Logger

internal/config/config_appservice.go

@@ -25,6 +25,31 @@ import (
 	yaml "gopkg.in/yaml.v2"
 )
 
+type AppServiceAPI struct {
+	Matrix  *Global  `yaml:"-"`
+	Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit
+
+	Listen Address `yaml:"listen"`
+	Bind   Address `yaml:"bind"`
+
+	Database DatabaseOptions `yaml:"database"`
+
+	ConfigFiles []string `yaml:"config_files"`
+}
+
+func (c *AppServiceAPI) Defaults() {
+	c.Listen = "localhost:7777"
+	c.Bind = "localhost:7777"
+	c.Database.Defaults()
+	c.Database.ConnectionString = "file:appservice.db"
+}
+
+func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkNotEmpty(configErrs, "app_service_api.listen", string(c.Listen))
+	checkNotEmpty(configErrs, "app_service_api.bind", string(c.Bind))
+	checkNotEmpty(configErrs, "app_service_api.database.connection_string", string(c.Database.ConnectionString))
+}
+
 // ApplicationServiceNamespace is the namespace that a specific application
 // service has management over.
 type ApplicationServiceNamespace struct {
@@ -132,8 +157,8 @@ func (a *ApplicationService) IsInterestedInRoomAlias(
 
 // loadAppServices iterates through all application service config files
 // and loads their data into the config object for later access.
-func loadAppServices(config *Dendrite) error {
-	for _, configPath := range config.ApplicationServices.ConfigFiles {
+func loadAppServices(config *AppServiceAPI, derived *Derived) error {
+	for _, configPath := range config.ConfigFiles {
 		// Create a new application service with default options
 		appservice := ApplicationService{
 			RateLimited: true,
@@ -157,26 +182,26 @@ func loadAppServices(config *Dendrite) error {
 		}
 
 		// Append the parsed application service to the global config
-		config.Derived.ApplicationServices = append(
-			config.Derived.ApplicationServices, appservice,
+		derived.ApplicationServices = append(
+			derived.ApplicationServices, appservice,
 		)
 	}
 
 	// Check for any errors in the loaded application services
-	return checkErrors(config)
+	return checkErrors(config, derived)
 }
 
 // setupRegexps will create regex objects for exclusive and non-exclusive
 // usernames, aliases and rooms of all application services, so that other
 // methods can quickly check if a particular string matches any of them.
-func setupRegexps(cfg *Dendrite) (err error) {
+func setupRegexps(_ *AppServiceAPI, derived *Derived) (err error) {
 	// Combine all exclusive namespaces for later string checking
 	var exclusiveUsernameStrings, exclusiveAliasStrings []string
 
 	// If an application service's regex is marked as exclusive, add
 	// its contents to the overall exlusive regex string. Room regex
 	// not necessary as we aren't denying exclusive room ID creation
-	for _, appservice := range cfg.Derived.ApplicationServices {
+	for _, appservice := range derived.ApplicationServices {
 		for key, namespaceSlice := range appservice.NamespaceMap {
 			switch key {
 			case "users":
@@ -204,10 +229,10 @@ func setupRegexps(cfg *Dendrite) (err error) {
 	}
 
 	// Store compiled Regex
-	if cfg.Derived.ExclusiveApplicationServicesUsernameRegexp, err = regexp.Compile(exclusiveUsernames); err != nil {
+	if derived.ExclusiveApplicationServicesUsernameRegexp, err = regexp.Compile(exclusiveUsernames); err != nil {
 		return err
 	}
-	if cfg.Derived.ExclusiveApplicationServicesAliasRegexp, err = regexp.Compile(exclusiveAliases); err != nil {
+	if derived.ExclusiveApplicationServicesAliasRegexp, err = regexp.Compile(exclusiveAliases); err != nil {
 		return err
 	}
 
@@ -234,7 +259,7 @@ func appendExclusiveNamespaceRegexs(
 
 // checkErrors checks for any configuration errors amongst the loaded
 // application services according to the application service spec.
-func checkErrors(config *Dendrite) (err error) {
+func checkErrors(config *AppServiceAPI, derived *Derived) (err error) {
 	var idMap = make(map[string]bool)
 	var tokenMap = make(map[string]bool)
 
@@ -242,7 +267,7 @@ func checkErrors(config *Dendrite) (err error) {
 	groupIDRegexp := regexp.MustCompile(`\+.*:.*`)
 
 	// Check each application service for any config errors
-	for _, appservice := range config.Derived.ApplicationServices {
+	for _, appservice := range derived.ApplicationServices {
 		// Namespace-related checks
 		for key, namespaceSlice := range appservice.NamespaceMap {
 			for _, namespace := range namespaceSlice {
@@ -258,13 +283,13 @@ func checkErrors(config *Dendrite) (err error) {
 		// Check if we've already seen this ID. No two application services
 		// can have the same ID or token.
 		if idMap[appservice.ID] {
-			return configErrors([]string{fmt.Sprintf(
+			return ConfigErrors([]string{fmt.Sprintf(
 				"Application service ID %s must be unique", appservice.ID,
 			)})
 		}
 		// Check if we've already seen this token
 		if tokenMap[appservice.ASToken] {
-			return configErrors([]string{fmt.Sprintf(
+			return ConfigErrors([]string{fmt.Sprintf(
 				"Application service Token %s must be unique", appservice.ASToken,
 			)})
 		}
@@ -284,7 +309,7 @@ func checkErrors(config *Dendrite) (err error) {
 		}
 	}
 
-	return setupRegexps(config)
+	return setupRegexps(config, derived)
 }
 
 // validateNamespace returns nil or an error based on whether a given
@@ -298,7 +323,7 @@ func validateNamespace(
 ) error {
 	// Check that namespace(s) are valid regex
 	if !IsValidRegex(namespace.Regex) {
-		return configErrors([]string{fmt.Sprintf(
+		return ConfigErrors([]string{fmt.Sprintf(
 			"Invalid regex string for Application Service %s", appservice.ID,
 		)})
 	}
@@ -310,7 +335,7 @@ func validateNamespace(
 
 		correctFormat := groupIDRegexp.MatchString(namespace.GroupID)
 		if !correctFormat {
-			return configErrors([]string{fmt.Sprintf(
+			return ConfigErrors([]string{fmt.Sprintf(
 				"Invalid user group_id field for application service %s.",
 				appservice.ID,
 			)})

internal/config/config_clientapi.go

@@ -0,0 +1,87 @@
+package config
+
+import (
+	"fmt"
+	"time"
+)
+
+type ClientAPI struct {
+	Matrix  *Global  `yaml:"-"`
+	Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit
+
+	Listen Address `yaml:"listen"`
+	Bind   Address `yaml:"bind"`
+
+	// If set, allows registration by anyone who also has the shared
+	// secret, even if registration is otherwise disabled.
+	RegistrationSharedSecret string `yaml:"registration_shared_secret"`
+	// This Home Server's ReCAPTCHA public key.
+	RecaptchaPublicKey string `yaml:"recaptcha_public_key"`
+	// This Home Server's ReCAPTCHA private key.
+	RecaptchaPrivateKey string `yaml:"recaptcha_private_key"`
+	// Boolean stating whether catpcha registration is enabled
+	// and required
+	RecaptchaEnabled bool `yaml:"enable_registration_captcha"`
+	// Secret used to bypass the captcha registration entirely
+	RecaptchaBypassSecret string `yaml:"captcha_bypass_secret"`
+	// HTTP API endpoint used to verify whether the captcha response
+	// was successful
+	RecaptchaSiteVerifyAPI string `yaml:"recaptcha_siteverify_api"`
+	// If set disables new users from registering (except via shared
+	// secrets)
+	RegistrationDisabled bool `yaml:"registration_disabled"`
+
+	// TURN options
+	TURN TURN `yaml:"turn"`
+}
+
+func (c *ClientAPI) Defaults() {
+	c.Listen = "localhost:7771"
+	c.Bind = "localhost:7771"
+	c.RegistrationSharedSecret = ""
+	c.RecaptchaPublicKey = ""
+	c.RecaptchaPrivateKey = ""
+	c.RecaptchaEnabled = false
+	c.RecaptchaBypassSecret = ""
+	c.RecaptchaSiteVerifyAPI = ""
+	c.RegistrationDisabled = false
+}
+
+func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkNotEmpty(configErrs, "client_api.listen", string(c.Listen))
+	checkNotEmpty(configErrs, "client_api.bind", string(c.Bind))
+	if c.RecaptchaEnabled {
+		checkNotEmpty(configErrs, "client_api.recaptcha_public_key", string(c.RecaptchaPublicKey))
+		checkNotEmpty(configErrs, "client_api.recaptcha_private_key", string(c.RecaptchaPrivateKey))
+		checkNotEmpty(configErrs, "client_api.recaptcha_siteverify_api", string(c.RecaptchaSiteVerifyAPI))
+	}
+	c.TURN.Verify(configErrs)
+}
+
+type TURN struct {
+	// TODO Guest Support
+	// Whether or not guests can request TURN credentials
+	// AllowGuests bool `yaml:"turn_allow_guests"`
+	// How long the authorization should last
+	UserLifetime string `yaml:"turn_user_lifetime"`
+	// The list of TURN URIs to pass to clients
+	URIs []string `yaml:"turn_uris"`
+
+	// Authorization via Shared Secret
+	// The shared secret from coturn
+	SharedSecret string `yaml:"turn_shared_secret"`
+
+	// Authorization via Static Username & Password
+	// Hardcoded Username and Password
+	Username string `yaml:"turn_username"`
+	Password string `yaml:"turn_password"`
+}
+
+func (c *TURN) Verify(configErrs *ConfigErrors) {
+	value := c.UserLifetime
+	if value != "" {
+		if _, err := time.ParseDuration(value); err != nil {
+			configErrs.Add(fmt.Sprintf("invalid duration for config key %q: %s", "client_api.turn.turn_user_lifetime", value))
+		}
+	}
+}

internal/config/config_currentstate.go

@@ -0,0 +1,25 @@
+package config
+
+type CurrentStateServer struct {
+	Matrix *Global `yaml:"-"`
+
+	Listen Address `yaml:"listen"`
+	Bind   Address `yaml:"bind"`
+
+	// The CurrentState database stores the current state of all rooms.
+	// It is accessed by the CurrentStateServer.
+	Database DatabaseOptions `yaml:"database"`
+}
+
+func (c *CurrentStateServer) Defaults() {
+	c.Listen = "localhost:7782"
+	c.Bind = "localhost:7782"
+	c.Database.Defaults()
+	c.Database.ConnectionString = "file:currentstate.db"
+}
+
+func (c *CurrentStateServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkNotEmpty(configErrs, "current_state_server.listen", string(c.Listen))
+	checkNotEmpty(configErrs, "current_state_server.bind", string(c.Bind))
+	checkNotEmpty(configErrs, "current_state_server.database.connection_string", string(c.Database.ConnectionString))
+}

internal/config/config_eduserver.go

@@ -0,0 +1,18 @@
+package config
+
+type EDUServer struct {
+	Matrix *Global `yaml:"-"`
+
+	Listen Address `yaml:"listen"`
+	Bind   Address `yaml:"bind"`
+}
+
+func (c *EDUServer) Defaults() {
+	c.Listen = "localhost:7778"
+	c.Bind = "localhost:7778"
+}
+
+func (c *EDUServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkNotEmpty(configErrs, "edu_server.listen", string(c.Listen))
+	checkNotEmpty(configErrs, "edu_server.bind", string(c.Bind))
+}

internal/config/config_federationapi.go

@@ -0,0 +1,33 @@
+package config
+
+import "github.com/matrix-org/gomatrixserverlib"
+
+type FederationAPI struct {
+	Matrix *Global `yaml:"-"`
+
+	Listen Address `yaml:"listen"`
+	Bind   Address `yaml:"bind"`
+
+	// List of paths to X509 certificates used by the external federation listeners.
+	// These are used to calculate the TLS fingerprints to publish for this server.
+	// Other matrix servers talking to this server will expect the x509 certificate
+	// to match one of these certificates.
+	// The certificates should be in PEM format.
+	FederationCertificatePaths []Path `yaml:"federation_certificates"`
+
+	// A list of SHA256 TLS fingerprints for the X509 certificates used by the
+	// federation listener for this server.
+	TLSFingerPrints []gomatrixserverlib.TLSFingerprint `yaml:"-"`
+}
+
+func (c *FederationAPI) Defaults() {
+	c.Listen = "localhost:7772"
+	c.Bind = "localhost:7772"
+}
+
+func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkNotEmpty(configErrs, "federation_api.listen", string(c.Listen))
+	checkNotEmpty(configErrs, "federation_api.bind", string(c.Bind))
+	// TODO: not applicable always, e.g. in demos
+	//checkNotZero(configErrs, "federation_api.federation_certificates", int64(len(c.FederationCertificatePaths)))
+}

internal/config/config_federationsender.go

@@ -0,0 +1,64 @@
+package config
+
+type FederationSender struct {
+	Matrix *Global `yaml:"-"`
+
+	Listen Address `yaml:"listen"`
+	Bind   Address `yaml:"bind"`
+
+	// The FederationSender database stores information used by the FederationSender
+	// It is only accessed by the FederationSender.
+	Database DatabaseOptions `yaml:"database"`
+
+	// Federation failure threshold. How many consecutive failures that we should
+	// tolerate when sending federation requests to a specific server. The backoff
+	// is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc.
+	// The default value is 16 if not specified, which is circa 18 hours.
+	FederationMaxRetries uint32 `yaml:"send_max_retries"`
+
+	// FederationDisableTLSValidation disables the validation of X.509 TLS certs
+	// on remote federation endpoints. This is not recommended in production!
+	DisableTLSValidation bool `yaml:"disable_tls_validation"`
+
+	Proxy Proxy `yaml:"proxy_outbound"`
+}
+
+func (c *FederationSender) Defaults() {
+	c.Listen = "localhost:7775"
+	c.Bind = "localhost:7775"
+	c.Database.Defaults()
+	c.Database.ConnectionString = "file:federationsender.db"
+
+	c.FederationMaxRetries = 16
+	c.DisableTLSValidation = false
+
+	c.Proxy.Defaults()
+}
+
+func (c *FederationSender) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkNotEmpty(configErrs, "federation_sender.listen", string(c.Listen))
+	checkNotEmpty(configErrs, "federation_sender.bind", string(c.Bind))
+	checkNotEmpty(configErrs, "federation_sender.database.connection_string", string(c.Database.ConnectionString))
+}
+
+// The config for setting a proxy to use for server->server requests
+type Proxy struct {
+	// Is the proxy enabled?
+	Enabled bool `yaml:"enabled"`
+	// The protocol for the proxy (http / https / socks5)
+	Protocol string `yaml:"protocol"`
+	// The host where the proxy is listening
+	Host string `yaml:"host"`
+	// The port on which the proxy is listening
+	Port uint16 `yaml:"port"`
+}
+
+func (c *Proxy) Defaults() {
+	c.Enabled = false
+	c.Protocol = "http"
+	c.Host = "localhost"
+	c.Port = 8080
+}
+
+func (c *Proxy) Verify(configErrs *ConfigErrors) {
+}

internal/config/config_global.go

@@ -0,0 +1,172 @@
+package config
+
+import (
+	"math/rand"
+	"time"
+
+	"github.com/matrix-org/gomatrixserverlib"
+	"golang.org/x/crypto/ed25519"
+)
+
+type Global struct {
+	// The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
+	ServerName gomatrixserverlib.ServerName `yaml:"server_name"`
+
+	// Path to the private key which will be used to sign requests and events.
+	PrivateKeyPath Path `yaml:"private_key"`
+
+	// The private key which will be used to sign requests and events.
+	PrivateKey ed25519.PrivateKey `yaml:"-"`
+
+	// An arbitrary string used to uniquely identify the PrivateKey. Must start with the
+	// prefix "ed25519:".
+	KeyID gomatrixserverlib.KeyID `yaml:"-"`
+
+	// How long a remote server can cache our server key for before requesting it again.
+	// Increasing this number will reduce the number of requests made by remote servers
+	// for our key, but increases the period a compromised key will be considered valid
+	// by remote servers.
+	// Defaults to 24 hours.
+	KeyValidityPeriod time.Duration `yaml:"key_validity_period"`
+
+	// List of domains that the server will trust as identity servers to
+	// verify third-party identifiers.
+	// Defaults to an empty array.
+	TrustedIDServers []string `yaml:"trusted_third_party_id_servers"`
+
+	// Kafka/Naffka configuration
+	Kafka Kafka `yaml:"kafka"`
+
+	// Metrics configuration
+	Metrics Metrics `yaml:"metrics"`
+}
+
+func (c *Global) Defaults() {
+	c.ServerName = "localhost"
+	c.PrivateKeyPath = "matrix.pem"
+	_, c.PrivateKey, _ = ed25519.GenerateKey(rand.New(rand.NewSource(0)))
+	c.KeyID = "ed25519:auto"
+	c.KeyValidityPeriod = time.Hour * 24 * 7
+
+	c.Kafka.Defaults()
+	c.Metrics.Defaults()
+}
+
+func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkNotEmpty(configErrs, "global.server_name", string(c.ServerName))
+	checkNotEmpty(configErrs, "global.private_key", string(c.PrivateKeyPath))
+
+	c.Kafka.Verify(configErrs, isMonolith)
+	c.Metrics.Verify(configErrs, isMonolith)
+}
+
+type Kafka struct {
+	// A list of kafka addresses to connect to.
+	Addresses []string `yaml:"addresses"`
+	// Whether to use naffka instead of kafka.
+	// Naffka can only be used when running dendrite as a single monolithic server.
+	// Kafka can be used both with a monolithic server and when running the
+	// components as separate servers.
+	UseNaffka bool `yaml:"use_naffka"`
+	// The Naffka database is used internally by the naffka library, if used.
+	Database DatabaseOptions `yaml:"naffka_database"`
+	// The names of the topics to use when reading and writing from kafka.
+	Topics struct {
+		// Topic for roomserver/api.OutputRoomEvent events.
+		OutputRoomEvent Topic `yaml:"output_room_event"`
+		// Topic for sending account data from client API to sync API
+		OutputClientData Topic `yaml:"output_client_data"`
+		// Topic for eduserver/api.OutputTypingEvent events.
+		OutputTypingEvent Topic `yaml:"output_typing_event"`
+		// Topic for eduserver/api.OutputSendToDeviceEvent events.
+		OutputSendToDeviceEvent Topic `yaml:"output_send_to_device_event"`
+		// Topic for keyserver when new device keys are added.
+		OutputKeyChangeEvent Topic `yaml:"output_key_change_event"`
+	}
+}
+
+func (c *Kafka) Defaults() {
+	c.UseNaffka = true
+	c.Database.Defaults()
+	c.Database.ConnectionString = DataSource("file:naffka.db")
+	c.Topics.OutputRoomEvent = "OutputRoomEventTopic"
+	c.Topics.OutputClientData = "OutputClientDataTopic"
+	c.Topics.OutputTypingEvent = "OutputTypingEventTopic"
+	c.Topics.OutputSendToDeviceEvent = "OutputSendToDeviceEventTopic"
+	c.Topics.OutputKeyChangeEvent = "OutputKeyChangeEventTopic"
+}
+
+func (c *Kafka) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	if c.UseNaffka {
+		if !isMonolith {
+			configErrs.Add("naffka can only be used in a monolithic server")
+		}
+		checkNotEmpty(configErrs, "global.kafka.database.connection_string", string(c.Database.ConnectionString))
+	} else {
+		// If we aren't using naffka then we need to have at least one kafka
+		// server to talk to.
+		checkNotZero(configErrs, "global.kafka.addresses", int64(len(c.Addresses)))
+	}
+	checkNotEmpty(configErrs, "global.kafka.topics.output_room_event", string(c.Topics.OutputRoomEvent))
+	checkNotEmpty(configErrs, "global.kafka.topics.output_client_data", string(c.Topics.OutputClientData))
+	checkNotEmpty(configErrs, "global.kafka.topics.output_typing_event", string(c.Topics.OutputTypingEvent))
+	checkNotEmpty(configErrs, "global.kafka.topics.output_send_to_device_event", string(c.Topics.OutputSendToDeviceEvent))
+	checkNotEmpty(configErrs, "global.kafka.topics.output_key_change_event", string(c.Topics.OutputKeyChangeEvent))
+}
+
+// The configuration to use for Prometheus metrics
+type Metrics struct {
+	// Whether or not the metrics are enabled
+	Enabled bool `yaml:"enabled"`
+	// Use BasicAuth for Authorization
+	BasicAuth struct {
+		// Authorization via Static Username & Password
+		// Hardcoded Username and Password
+		Username string `yaml:"username"`
+		Password string `yaml:"password"`
+	} `yaml:"basic_auth"`
+}
+
+func (c *Metrics) Defaults() {
+	c.Enabled = false
+	c.BasicAuth.Username = "metrics"
+	c.BasicAuth.Password = "metrics"
+}
+
+func (c *Metrics) Verify(configErrs *ConfigErrors, isMonolith bool) {
+}
+
+type DatabaseOptions struct {
+	// The connection string, file:filename.db or postgres://server....
+	ConnectionString DataSource `yaml:"connection_string"`
+	// Maximum open connections to the DB (0 = use default, negative means unlimited)
+	MaxOpenConnections int `yaml:"max_open_conns"`
+	// Maximum idle connections to the DB (0 = use default, negative means unlimited)
+	MaxIdleConnections int `yaml:"max_idle_conns"`
+	// maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited)
+	ConnMaxLifetimeSeconds int `yaml:"conn_max_lifetime"`
+}
+
+func (c *DatabaseOptions) Defaults() {
+	c.MaxOpenConnections = 100
+	c.MaxIdleConnections = 2
+	c.ConnMaxLifetimeSeconds = -1
+}
+
+func (c *DatabaseOptions) Verify(configErrs *ConfigErrors, isMonolith bool) {
+}
+
+// MaxIdleConns returns maximum idle connections to the DB
+func (c DatabaseOptions) MaxIdleConns() int {
+	return c.MaxIdleConnections
+}
+
+// MaxOpenConns returns maximum open connections to the DB
+func (c DatabaseOptions) MaxOpenConns() int {
+	return c.MaxOpenConnections
+}
+
+// ConnMaxLifetime returns maximum amount of time a connection may be reused
+func (c DatabaseOptions) ConnMaxLifetime() time.Duration {
+	return time.Duration(c.ConnMaxLifetimeSeconds) * time.Second
+}

internal/config/config_keyserver.go

@@ -0,0 +1,23 @@
+package config
+
+type KeyServer struct {
+	Matrix *Global `yaml:"-"`
+
+	Listen Address `yaml:"listen"`
+	Bind   Address `yaml:"bind"`
+
+	Database DatabaseOptions `yaml:"database"`
+}
+
+func (c *KeyServer) Defaults() {
+	c.Listen = "localhost:7779"
+	c.Bind = "localhost:7779"
+	c.Database.Defaults()
+	c.Database.ConnectionString = "file:keyserver.db"
+}
+
+func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkNotEmpty(configErrs, "key_server.listen", string(c.Listen))
+	checkNotEmpty(configErrs, "key_server.bind", string(c.Bind))
+	checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString))
+}

internal/config/config_mediaapi.go

@@ -0,0 +1,63 @@
+package config
+
+import (
+	"fmt"
+)
+
+type MediaAPI struct {
+	Matrix *Global `yaml:"-"`
+
+	Listen Address `yaml:"listen"`
+	Bind   Address `yaml:"bind"`
+
+	// The MediaAPI database stores information about files uploaded and downloaded
+	// by local users. It is only accessed by the MediaAPI.
+	Database DatabaseOptions `yaml:"database"`
+
+	// The base path to where the media files will be stored. May be relative or absolute.
+	BasePath Path `yaml:"base_path"`
+
+	// The absolute base path to where media files will be stored.
+	AbsBasePath Path `yaml:"-"`
+
+	// The maximum file size in bytes that is allowed to be stored on this server.
+	// Note: if max_file_size_bytes is set to 0, the size is unlimited.
+	// Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB)
+	MaxFileSizeBytes *FileSizeBytes `yaml:"max_file_size_bytes,omitempty"`
+
+	// Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated
+	DynamicThumbnails bool `yaml:"dynamic_thumbnails"`
+
+	// The maximum number of simultaneous thumbnail generators. default: 10
+	MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"`
+
+	// A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content
+	ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"`
+}
+
+func (c *MediaAPI) Defaults() {
+	c.Listen = "localhost:7774"
+	c.Bind = "localhost:7774"
+	c.Database.Defaults()
+	c.Database.ConnectionString = "file:mediaapi.db"
+
+	defaultMaxFileSizeBytes := FileSizeBytes(10485760)
+	c.MaxFileSizeBytes = &defaultMaxFileSizeBytes
+	c.MaxThumbnailGenerators = 10
+	c.BasePath = "./media_store"
+}
+
+func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkNotEmpty(configErrs, "media_api.listen", string(c.Listen))
+	checkNotEmpty(configErrs, "media_api.bind", string(c.Bind))
+	checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString))
+
+	checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath))
+	checkPositive(configErrs, "media_api.max_file_size_bytes", int64(*c.MaxFileSizeBytes))
+	checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators))
+
+	for i, size := range c.ThumbnailSizes {
+		checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].width", i), int64(size.Width))
+		checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].height", i), int64(size.Height))
+	}
+}

internal/config/config_roomserver.go

@@ -0,0 +1,23 @@
+package config
+
+type RoomServer struct {
+	Matrix *Global `yaml:"-"`
+
+	Listen Address `yaml:"listen"`
+	Bind   Address `yaml:"bind"`
+
+	Database DatabaseOptions `yaml:"database"`
+}
+
+func (c *RoomServer) Defaults() {
+	c.Listen = "localhost:7770"
+	c.Bind = "localhost:7770"
+	c.Database.Defaults()
+	c.Database.ConnectionString = "file:roomserver.db"
+}
+
+func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkNotEmpty(configErrs, "room_server.listen", string(c.Listen))
+	checkNotEmpty(configErrs, "room_server.bind", string(c.Bind))
+	checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString))
+}

internal/config/config_serverkey.go

@@ -0,0 +1,29 @@
+package config
+
+type ServerKeyAPI struct {
+	Matrix *Global `yaml:"-"`
+
+	Listen Address `yaml:"listen"`
+	Bind   Address `yaml:"bind"`
+
+	// The ServerKey database caches the public keys of remote servers.
+	// It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI.
+	Database DatabaseOptions `yaml:"database"`
+
+	// Perspective keyservers, to use as a backup when direct key fetch
+	// requests don't succeed
+	KeyPerspectives KeyPerspectives `yaml:"key_perspectives"`
+}
+
+func (c *ServerKeyAPI) Defaults() {
+	c.Listen = "localhost:7780"
+	c.Bind = "localhost:7780"
+	c.Database.Defaults()
+	c.Database.ConnectionString = "file:serverkeyapi.db"
+}
+
+func (c *ServerKeyAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkNotEmpty(configErrs, "server_key_api.listen", string(c.Listen))
+	checkNotEmpty(configErrs, "server_key_api.bind", string(c.Bind))
+	checkNotEmpty(configErrs, "server_key_api.database.connection_string", string(c.Database.ConnectionString))
+}

internal/config/config_syncapi.go

@@ -0,0 +1,23 @@
+package config
+
+type SyncAPI struct {
+	Matrix *Global `yaml:"-"`
+
+	Listen Address `yaml:"listen"`
+	Bind   Address `yaml:"bind"`
+
+	Database DatabaseOptions `yaml:"database"`
+}
+
+func (c *SyncAPI) Defaults() {
+	c.Listen = "localhost:7773"
+	c.Bind = "localhost:7773"
+	c.Database.Defaults()
+	c.Database.ConnectionString = "file:syncapi.db"
+}
+
+func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkNotEmpty(configErrs, "sync_api.listen", string(c.Listen))
+	checkNotEmpty(configErrs, "sync_api.bind", string(c.Bind))
+	checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString))
+}

internal/config/config_test.go

@@ -33,48 +33,157 @@ func TestLoadConfigRelative(t *testing.T) {
 }
 
 const testConfig = `
-version: 0
-matrix:
+version: 1
+global:
   server_name: localhost
   private_key: matrix_key.pem
-  federation_certificates: [tls_cert.pem]
-media:
-  base_path: media_store
+  key_validity_period: 168h0m0s
+  trusted_third_party_id_servers: []
   kafka:
-  addresses: ["localhost:9092"]
+    addresses: []
+    use_naffka: true
+    naffka_database:
+      connection_string: file:naffka.db
+      max_open_conns: 0
+      max_idle_conns: 0
+      conn_max_lifetime: -1
     topics:
-    output_room_event: output.room
-    output_client_data: output.client
-    output_typing_event: output.typing
-    output_send_to_device_event: output.std
-    output_key_change_event: output.key_change
-    user_updates: output.user
+      output_room_event: OutputRoomEventTopic
+      output_client_data: OutputClientDataTopic
+      output_typing_event: OutputTypingEventTopic
+      output_send_to_device_event: OutputSendToDeviceEventTopic
+      output_key_change_event: OutputKeyChangeEventTopic
+  metrics:
+    enabled: false
+    basic_auth:
+      username: metrics
+      password: metrics
+app_service_api:
+  listen: localhost:7777
+  bind: localhost:7777
   database:
-  media_api: "postgresql:///media_api"
-  account: "postgresql:///account"
-  device: "postgresql:///device"
-  server_key: "postgresql:///server_keys"
-  sync_api: "postgresql:///syn_api"
-  room_server: "postgresql:///room_server"
-  appservice: "postgresql:///appservice"
-  current_state: "postgresql:///current_state"
-  e2e_key: "postgresql:///e2e_key"
-listen:
-  room_server: "localhost:7770"
-  client_api: "localhost:7771"
-  federation_api: "localhost:7772"
-  sync_api: "localhost:7773"
-  media_api: "localhost:7774"
-  appservice_api: "localhost:7777"
-  edu_server: "localhost:7778"
-  user_api: "localhost:7779"
-  current_state_server: "localhost:7775"
-  key_server: "localhost:7776"
-logging:
-  - type: "file"
-    level: "info"
-    params:
-      path: "/my/log/dir"
+    connection_string: file:appservice.db
+    max_open_conns: 0
+    max_idle_conns: 0
+    conn_max_lifetime: -1
+  config_files: []
+client_api:
+  listen: localhost:7771
+  bind: localhost:7771
+  registration_shared_secret: ""
+  recaptcha_public_key: ""
+  recaptcha_private_key: ""
+  enable_registration_captcha: false
+  captcha_bypass_secret: ""
+  recaptcha_siteverify_api: ""
+  registration_disabled: false
+  turn:
+    turn_user_lifetime: ""
+    turn_uris: []
+    turn_shared_secret: ""
+    turn_username: ""
+    turn_password: ""
+current_state_server:
+  listen: localhost:7782
+  bind: localhost:7782
+  database:
+    connection_string: file:currentstate.db
+    max_open_conns: 0
+    max_idle_conns: 0
+    conn_max_lifetime: -1
+edu_server:
+  listen: localhost:7778
+  bind: localhost:7778
+federation_api:
+  listen: localhost:7772
+  bind: localhost:7772
+  federation_certificates: []
+federation_sender:
+  listen: localhost:7775
+  bind: localhost:7775
+  database:
+    connection_string: file:federationsender.db
+    max_open_conns: 0
+    max_idle_conns: 0
+    conn_max_lifetime: -1
+  federation_max_retries: 16
+  proxy_outbound:
+    enabled: false
+    protocol: http
+    host: localhost
+    port: 8080
+key_server:
+  listen: localhost:7779
+  bind: localhost:7779
+  database:
+    connection_string: file:keyserver.db
+    max_open_conns: 0
+    max_idle_conns: 0
+    conn_max_lifetime: -1
+media_api:
+  listen: localhost:7774
+  bind: localhost:7774
+  database:
+    connection_string: file:mediaapi.db
+    max_open_conns: 0
+    max_idle_conns: 0
+    conn_max_lifetime: -1
+  base_path: ""
+  max_file_size_bytes: 10485760
+  dynamic_thumbnails: false
+  max_thumbnail_generators: 10
+  thumbnail_sizes: []
+room_server:
+  listen: localhost:7770
+  bind: localhost:7770
+  database:
+    connection_string: file:roomserver.db
+    max_open_conns: 0
+    max_idle_conns: 0
+    conn_max_lifetime: -1
+server_key_api:
+  listen: localhost:7780
+  bind: localhost:7780
+  database:
+    connection_string: file:serverkeyapi.db
+    max_open_conns: 0
+    max_idle_conns: 0
+    conn_max_lifetime: -1
+  key_perspectives: []
+sync_api:
+  listen: localhost:7773
+  bind: localhost:7773
+  database:
+    connection_string: file:syncapi.db
+    max_open_conns: 0
+    max_idle_conns: 0
+    conn_max_lifetime: -1
+user_api:
+  listen: localhost:7781
+  bind: localhost:7781
+  account_database:
+    connection_string: file:userapi_accounts.db
+    max_open_conns: 0
+    max_idle_conns: 0
+    conn_max_lifetime: -1
+  device_database:
+    connection_string: file:userapi_devices.db
+    max_open_conns: 0
+    max_idle_conns: 0
+    conn_max_lifetime: -1
+tracing:
+  enabled: false
+  jaeger:
+    serviceName: ""
+    disabled: false
+    rpc_metrics: false
+    tags: []
+    sampler: null
+    reporter: null
+    headers: null
+    baggage_restrictions: null
+    throttler: null
+logging: []
 `
 
 type mockReadFile map[string]string

internal/config/config_userapi.go

@@ -0,0 +1,31 @@
+package config
+
+type UserAPI struct {
+	Matrix *Global `yaml:"-"`
+
+	Listen Address `yaml:"listen"`
+	Bind   Address `yaml:"bind"`
+
+	// The Account database stores the login details and account information
+	// for local users. It is accessed by the UserAPI.
+	AccountDatabase DatabaseOptions `yaml:"account_database"`
+	// The Device database stores session information for the devices of logged
+	// in local users. It is accessed by the UserAPI.
+	DeviceDatabase DatabaseOptions `yaml:"device_database"`
+}
+
+func (c *UserAPI) Defaults() {
+	c.Listen = "localhost:7781"
+	c.Bind = "localhost:7781"
+	c.AccountDatabase.Defaults()
+	c.DeviceDatabase.Defaults()
+	c.AccountDatabase.ConnectionString = "file:userapi_accounts.db"
+	c.DeviceDatabase.ConnectionString = "file:userapi_devices.db"
+}
+
+func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
+	checkNotEmpty(configErrs, "user_api.listen", string(c.Listen))
+	checkNotEmpty(configErrs, "user_api.bind", string(c.Bind))
+	checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString))
+	checkNotEmpty(configErrs, "user_api.device_database.connection_string", string(c.DeviceDatabase.ConnectionString))
+}

internal/eventutil/events.go

@@ -38,7 +38,7 @@ var ErrRoomNoExists = errors.New("Room does not exist")
 // Returns an error if something else went wrong
 func BuildEvent(
 	ctx context.Context,
-	builder *gomatrixserverlib.EventBuilder, cfg *config.Dendrite, evTime time.Time,
+	builder *gomatrixserverlib.EventBuilder, cfg *config.Global, evTime time.Time,
 	rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse,
 ) (*gomatrixserverlib.HeaderedEvent, error) {
 	if queryRes == nil {
@@ -52,8 +52,8 @@ func BuildEvent(
 	}
 
 	event, err := builder.Build(
-		evTime, cfg.Matrix.ServerName, cfg.Matrix.KeyID,
-		cfg.Matrix.PrivateKey, queryRes.RoomVersion,
+		evTime, cfg.ServerName, cfg.KeyID,
+		cfg.PrivateKey, queryRes.RoomVersion,
 	)
 	if err != nil {
 		return nil, err

internal/httputil/httpapi.go

@@ -234,7 +234,7 @@ func (f *FederationWakeups) Wakeup(ctx context.Context, origin gomatrixserverlib
 }
 
 // SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics listener
-func SetupHTTPAPI(servMux, publicApiMux, internalApiMux *mux.Router, cfg *config.Dendrite, enableHTTPAPIs bool) {
+func SetupHTTPAPI(servMux, publicApiMux, internalApiMux *mux.Router, cfg *config.Global, enableHTTPAPIs bool) {
 	if cfg.Metrics.Enabled {
 		servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth))
 	}

internal/setup/base.go

@@ -15,7 +15,6 @@
 package setup
 
 import (
-	"database/sql"
 	"fmt"
 	"io"
 	"net/http"
@@ -85,6 +84,15 @@ const HTTPClientTimeout = time.Second * 30
 // The componentName is used for logging purposes, and should be a friendly name
 // of the compontent running, e.g. "SyncAPI"
 func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs bool) *BaseDendrite {
+	configErrors := &config.ConfigErrors{}
+	cfg.Verify(configErrors, componentName == "Monolith") // TODO: better way?
+	if len(*configErrors) > 0 {
+		for _, err := range *configErrors {
+			logrus.Errorf("Configuration error: %s", err)
+		}
+		logrus.Fatalf("Failed to start due to configuration errors")
+	}
+
 	internal.SetupStdLogging()
 	internal.SetupHookLogging(cfg.Logging, componentName)
 	internal.SetupPprof()
@@ -96,7 +104,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo
 
 	var kafkaConsumer sarama.Consumer
 	var kafkaProducer sarama.SyncProducer
-	if cfg.Kafka.UseNaffka {
+	if cfg.Global.Kafka.UseNaffka {
 		kafkaConsumer, kafkaProducer = setupNaffka(cfg)
 	} else {
 		kafkaConsumer, kafkaProducer = setupKafka(cfg)
@@ -108,10 +116,10 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo
 	}
 
 	client := http.Client{Timeout: HTTPClientTimeout}
-	if cfg.Proxy != nil {
+	if cfg.FederationSender.Proxy.Enabled {
 		client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{
-			Scheme: cfg.Proxy.Protocol,
-			Host:   fmt.Sprintf("%s:%d", cfg.Proxy.Host, cfg.Proxy.Port),
+			Scheme: cfg.FederationSender.Proxy.Protocol,
+			Host:   fmt.Sprintf("%s:%d", cfg.FederationSender.Proxy.Host, cfg.FederationSender.Proxy.Port),
 		})}
 	}
 
@@ -228,7 +236,7 @@ func (b *BaseDendrite) KeyServerHTTPClient() keyserverAPI.KeyInternalAPI {
 // CreateDeviceDB creates a new instance of the device database. Should only be
 // called once per component.
 func (b *BaseDendrite) CreateDeviceDB() devices.Database {
-	db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName)
+	db, err := devices.NewDatabase(&b.Cfg.UserAPI.DeviceDatabase, b.Cfg.Global.ServerName)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to devices db")
 	}
@@ -239,7 +247,7 @@ func (b *BaseDendrite) CreateDeviceDB() devices.Database {
 // CreateAccountsDB creates a new instance of the accounts database. Should only
 // be called once per component.
 func (b *BaseDendrite) CreateAccountsDB() accounts.Database {
-	db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName)
+	db, err := accounts.NewDatabase(&b.Cfg.UserAPI.AccountDatabase, b.Cfg.Global.ServerName)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to accounts db")
 	}
@@ -251,8 +259,8 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database {
 // once per component.
 func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationClient {
 	return gomatrixserverlib.NewFederationClient(
-		b.Cfg.Matrix.ServerName, b.Cfg.Matrix.KeyID, b.Cfg.Matrix.PrivateKey,
-		b.Cfg.Matrix.FederationDisableTLSValidation,
+		b.Cfg.Global.ServerName, b.Cfg.Global.KeyID, b.Cfg.Global.PrivateKey,
+		b.Cfg.FederationSender.DisableTLSValidation,
 	)
 }
 
@@ -277,7 +285,7 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) {
 		b.BaseMux,
 		b.PublicAPIMux,
 		b.InternalAPIMux,
-		b.Cfg,
+		&b.Cfg.Global,
 		b.UseHTTPAPIs,
 	)
 	serv.Handler = b.BaseMux
@@ -293,12 +301,12 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) {
 
 // setupKafka creates kafka consumer/producer pair from the config.
 func setupKafka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) {
-	consumer, err := sarama.NewConsumer(cfg.Kafka.Addresses, nil)
+	consumer, err := sarama.NewConsumer(cfg.Global.Kafka.Addresses, nil)
 	if err != nil {
 		logrus.WithError(err).Panic("failed to start kafka consumer")
 	}
 
-	producer, err := sarama.NewSyncProducer(cfg.Kafka.Addresses, nil)
+	producer, err := sarama.NewSyncProducer(cfg.Global.Kafka.Addresses, nil)
 	if err != nil {
 		logrus.WithError(err).Panic("failed to setup kafka producers")
 	}
@@ -308,36 +316,26 @@ func setupKafka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) {
 
 // setupNaffka creates kafka consumer/producer pair from the config.
 func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) {
-	var err error
-	var db *sql.DB
 	var naffkaDB *naffka.DatabaseImpl
 
-	uri, err := url.Parse(string(cfg.Database.Naffka))
-	if err != nil || uri.Scheme == "file" {
-		var cs string
-		cs, err = sqlutil.ParseFileURI(string(cfg.Database.Naffka))
-		if err != nil {
-			logrus.WithError(err).Panic("Failed to parse naffka database file URI")
-		}
-		db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil)
+	db, err := sqlutil.Open(&cfg.Global.Kafka.Database)
 	if err != nil {
 		logrus.WithError(err).Panic("Failed to open naffka database")
 	}
 
+	switch {
+	case cfg.Global.Kafka.Database.ConnectionString.IsSQLite():
 		naffkaDB, err = naffka.NewSqliteDatabase(db)
 		if err != nil {
 			logrus.WithError(err).Panic("Failed to setup naffka database")
 		}
-	} else {
-		db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka), nil)
-		if err != nil {
-			logrus.WithError(err).Panic("Failed to open naffka database")
-		}
-
+	case cfg.Global.Kafka.Database.ConnectionString.IsPostgres():
 		naffkaDB, err = naffka.NewPostgresqlDatabase(db)
 		if err != nil {
 			logrus.WithError(err).Panic("Failed to setup naffka database")
 		}
+	default:
+		panic("unknown naffka database type")
 	}
 
 	if naffkaDB == nil {