Validate config, reinstate appservice derived in directory, tweaks

clientapi/routing/directory.go

@@ -140,25 +140,22 @@ func SetLocalAlias(
 	// 1. The new method for checking for things matching an AS's namespace
 	// 2. Using an overall Regex object for all AS's just like we did for usernames
 
-	// TODO: What to do with derived?
+	for _, appservice := range cfg.Derived.ApplicationServices {
-	/*
+		// Don't prevent AS from creating aliases in its own namespace
-		for _, appservice := range cfg.Derived.ApplicationServices {
+		// Note that Dendrite uses SenderLocalpart as UserID for AS users
-			// Don't prevent AS from creating aliases in its own namespace
+		if device.UserID != appservice.SenderLocalpart {
-			// Note that Dendrite uses SenderLocalpart as UserID for AS users
+			if aliasNamespaces, ok := appservice.NamespaceMap["aliases"]; ok {
-			if device.UserID != appservice.SenderLocalpart {
+				for _, namespace := range aliasNamespaces {
-				if aliasNamespaces, ok := appservice.NamespaceMap["aliases"]; ok {
+					if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) {
-					for _, namespace := range aliasNamespaces {
+						return util.JSONResponse{
-						if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) {
+							Code: http.StatusBadRequest,
-							return util.JSONResponse{
+							JSON: jsonerror.ASExclusive("Alias is reserved by an application service"),
-								Code: http.StatusBadRequest,
-								JSON: jsonerror.ASExclusive("Alias is reserved by an application service"),
-							}
 						}
 					}
 				}
 			}
 		}
-	*/
+	}
 
 	var r struct {
 		RoomID string `json:"room_id"`

internal/config/config.go

@@ -176,9 +176,9 @@ type LogrusHook struct {
 	Params map[string]interface{} `yaml:"params"`
 }
 
-// configErrors stores problems encountered when parsing a config file.
+// ConfigErrors stores problems encountered when parsing a config file.
 // It implements the error interface.
-type configErrors []string
+type ConfigErrors []string
 
 // Load a yaml config file for a server run as multiple processes or as a monolith.
 // Checks the config to ensure that it is valid.
@@ -298,9 +298,9 @@ func (c *Dendrite) Defaults() {
 	c.Wiring()
 }
 
-func (c *Dendrite) Verify(configErrs *configErrors) {
+func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	type verifiable interface {
-		Verify(configErrs *configErrors)
+		Verify(configErrs *ConfigErrors, isMonolith bool)
 	}
 	for _, c := range []verifiable{
 		&c.Global, &c.ClientAPI, &c.CurrentStateServer,
@@ -309,7 +309,7 @@ func (c *Dendrite) Verify(configErrs *configErrors) {
 		&c.ServerKeyAPI, &c.SyncAPI, &c.UserAPI,
 		&c.AppServiceAPI,
 	} {
-		c.Verify(configErrs)
+		c.Verify(configErrs, isMonolith)
 	}
 }
 
@@ -333,7 +333,7 @@ func (c *Dendrite) Wiring() {
 
 // Error returns a string detailing how many errors were contained within a
 // configErrors type.
-func (errs configErrors) Error() string {
+func (errs ConfigErrors) Error() string {
 	if len(errs) == 1 {
 		return errs[0]
 	}
@@ -347,13 +347,13 @@ func (errs configErrors) Error() string {
 // the client code.
 // This method is safe to use with an uninitialized configErrors because
 // if it is nil, it will be properly allocated.
-func (errs *configErrors) Add(str string) {
+func (errs *ConfigErrors) Add(str string) {
 	*errs = append(*errs, str)
 }
 
 // checkNotEmpty verifies the given value is not empty in the configuration.
 // If it is, adds an error to the list.
-func checkNotEmpty(configErrs *configErrors, key, value string) {
+func checkNotEmpty(configErrs *ConfigErrors, key, value string) {
 	if value == "" {
 		configErrs.Add(fmt.Sprintf("missing config key %q", key))
 	}
@@ -361,7 +361,7 @@ func checkNotEmpty(configErrs *configErrors, key, value string) {
 
 // checkNotZero verifies the given value is not zero in the configuration.
 // If it is, adds an error to the list.
-func checkNotZero(configErrs *configErrors, key string, value int64) {
+func checkNotZero(configErrs *ConfigErrors, key string, value int64) {
 	if value == 0 {
 		configErrs.Add(fmt.Sprintf("missing config key %q", key))
 	}
@@ -369,14 +369,14 @@ func checkNotZero(configErrs *configErrors, key string, value int64) {
 
 // checkPositive verifies the given value is positive (zero included)
 // in the configuration. If it is not, adds an error to the list.
-func checkPositive(configErrs *configErrors, key string, value int64) {
+func checkPositive(configErrs *ConfigErrors, key string, value int64) {
 	if value < 0 {
 		configErrs.Add(fmt.Sprintf("invalid value for config key %q: %d", key, value))
 	}
 }
 
 // checkLogging verifies the parameters logging.* are valid.
-func (config *Dendrite) checkLogging(configErrs *configErrors) {
+func (config *Dendrite) checkLogging(configErrs *ConfigErrors) {
 	for _, logrusHook := range config.Logging {
 		checkNotEmpty(configErrs, "logging.type", string(logrusHook.Type))
 		checkNotEmpty(configErrs, "logging.level", string(logrusHook.Level))
@@ -386,7 +386,7 @@ func (config *Dendrite) checkLogging(configErrs *configErrors) {
 // check returns an error type containing all errors found within the config
 // file.
 func (config *Dendrite) check(_ bool) error { // monolithic
-	var configErrs configErrors
+	var configErrs ConfigErrors
 
 	if config.Version != Version {
 		configErrs.Add(fmt.Sprintf(

internal/config/config_appservice.go

@@ -44,7 +44,7 @@ func (c *AppServiceAPI) Defaults() {
 	c.Database.ConnectionString = "file:appservice.db"
 }
 
-func (c *AppServiceAPI) Verify(configErrs *configErrors) {
+func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkNotEmpty(configErrs, "app_service_api.listen", string(c.Listen))
 	checkNotEmpty(configErrs, "app_service_api.bind", string(c.Bind))
 	checkNotEmpty(configErrs, "app_service_api.database.connection_string", string(c.Database.ConnectionString))
@@ -283,13 +283,13 @@ func checkErrors(config *AppServiceAPI, derived *Derived) (err error) {
 		// Check if we've already seen this ID. No two application services
 		// can have the same ID or token.
 		if idMap[appservice.ID] {
-			return configErrors([]string{fmt.Sprintf(
+			return ConfigErrors([]string{fmt.Sprintf(
 				"Application service ID %s must be unique", appservice.ID,
 			)})
 		}
 		// Check if we've already seen this token
 		if tokenMap[appservice.ASToken] {
-			return configErrors([]string{fmt.Sprintf(
+			return ConfigErrors([]string{fmt.Sprintf(
 				"Application service Token %s must be unique", appservice.ASToken,
 			)})
 		}
@@ -323,7 +323,7 @@ func validateNamespace(
 ) error {
 	// Check that namespace(s) are valid regex
 	if !IsValidRegex(namespace.Regex) {
-		return configErrors([]string{fmt.Sprintf(
+		return ConfigErrors([]string{fmt.Sprintf(
 			"Invalid regex string for Application Service %s", appservice.ID,
 		)})
 	}
@@ -335,7 +335,7 @@ func validateNamespace(
 
 		correctFormat := groupIDRegexp.MatchString(namespace.GroupID)
 		if !correctFormat {
-			return configErrors([]string{fmt.Sprintf(
+			return ConfigErrors([]string{fmt.Sprintf(
 				"Invalid user group_id field for application service %s.",
 				appservice.ID,
 			)})

internal/config/config_clientapi.go

@@ -47,7 +47,7 @@ func (c *ClientAPI) Defaults() {
 	c.RegistrationDisabled = false
 }
 
-func (c *ClientAPI) Verify(configErrs *configErrors) {
+func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkNotEmpty(configErrs, "client_api.listen", string(c.Listen))
 	checkNotEmpty(configErrs, "client_api.bind", string(c.Bind))
 	if c.RecaptchaEnabled {
@@ -77,7 +77,7 @@ type TURN struct {
 	Password string `yaml:"turn_password"`
 }
 
-func (c *TURN) Verify(configErrs *configErrors) {
+func (c *TURN) Verify(configErrs *ConfigErrors) {
 	value := c.UserLifetime
 	if value != "" {
 		if _, err := time.ParseDuration(value); err != nil {

internal/config/config_currentstate.go

@@ -18,7 +18,7 @@ func (c *CurrentStateServer) Defaults() {
 	c.Database.ConnectionString = "file:currentstate.db"
 }
 
-func (c *CurrentStateServer) Verify(configErrs *configErrors) {
+func (c *CurrentStateServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkNotEmpty(configErrs, "current_state_server.listen", string(c.Listen))
 	checkNotEmpty(configErrs, "current_state_server.bind", string(c.Bind))
 	checkNotEmpty(configErrs, "current_state_server.database.connection_string", string(c.Database.ConnectionString))

internal/config/config_eduserver.go

@@ -12,7 +12,7 @@ func (c *EDUServer) Defaults() {
 	c.Bind = "localhost:7778"
 }
 
-func (c *EDUServer) Verify(configErrs *configErrors) {
+func (c *EDUServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkNotEmpty(configErrs, "edu_server.listen", string(c.Listen))
 	checkNotEmpty(configErrs, "edu_server.bind", string(c.Bind))
 }

internal/config/config_federationapi.go

@@ -25,8 +25,9 @@ func (c *FederationAPI) Defaults() {
 	c.Bind = "localhost:7772"
 }
 
-func (c *FederationAPI) Verify(configErrs *configErrors) {
+func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkNotEmpty(configErrs, "federation_api.listen", string(c.Listen))
 	checkNotEmpty(configErrs, "federation_api.bind", string(c.Bind))
-	checkNotZero(configErrs, "federation_api.federation_certificates", int64(len(c.FederationCertificatePaths)))
+	// TODO: not applicable always, e.g. in demos
+	//checkNotZero(configErrs, "federation_api.federation_certificates", int64(len(c.FederationCertificatePaths)))
 }

internal/config/config_federationsender.go

@@ -35,7 +35,7 @@ func (c *FederationSender) Defaults() {
 	c.Proxy.Defaults()
 }
 
-func (c *FederationSender) Verify(configErrs *configErrors) {
+func (c *FederationSender) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkNotEmpty(configErrs, "federation_sender.listen", string(c.Listen))
 	checkNotEmpty(configErrs, "federation_sender.bind", string(c.Bind))
 	checkNotEmpty(configErrs, "federation_sender.database.connection_string", string(c.Database.ConnectionString))
@@ -60,5 +60,5 @@ func (c *Proxy) Defaults() {
 	c.Port = 8080
 }
 
-func (c *Proxy) Verify(configErrs *configErrors) {
+func (c *Proxy) Verify(configErrs *ConfigErrors) {
 }

internal/config/config_global.go

@@ -52,12 +52,12 @@ func (c *Global) Defaults() {
 	c.Metrics.Defaults()
 }
 
-func (c *Global) Verify(configErrs *configErrors) {
+func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkNotEmpty(configErrs, "global.server_name", string(c.ServerName))
 	checkNotEmpty(configErrs, "global.private_key", string(c.PrivateKeyPath))
 
-	c.Kafka.Verify(configErrs)
+	c.Kafka.Verify(configErrs, isMonolith)
-	c.Metrics.Verify(configErrs)
+	c.Metrics.Verify(configErrs, isMonolith)
 }
 
 type Kafka struct {
@@ -96,14 +96,11 @@ func (c *Kafka) Defaults() {
 	c.Topics.OutputKeyChangeEvent = "OutputKeyChangeEventTopic"
 }
 
-func (c *Kafka) Verify(configErrs *configErrors) {
+func (c *Kafka) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	if c.UseNaffka {
-		// TODO: monolithic check
+		if !isMonolith {
-		/*
+			configErrs.Add("naffka can only be used in a monolithic server")
-			if !monolithic {
+		}
-				configErrs.Add(fmt.Sprintf("naffka can only be used in a monolithic server"))
-			}
-		*/
 		checkNotEmpty(configErrs, "global.kafka.database.connection_string", string(c.Database.ConnectionString))
 	} else {
 		// If we aren't using naffka then we need to have at least one kafka
@@ -136,7 +133,7 @@ func (c *Metrics) Defaults() {
 	c.BasicAuth.Password = "metrics"
 }
 
-func (c *Metrics) Verify(configErrs *configErrors) {
+func (c *Metrics) Verify(configErrs *ConfigErrors, isMonolith bool) {
 }
 
 type DatabaseOptions struct {
@@ -156,7 +153,7 @@ func (c *DatabaseOptions) Defaults() {
 	c.ConnMaxLifetimeSeconds = -1
 }
 
-func (c *DatabaseOptions) Verify(configErrs *configErrors) {
+func (c *DatabaseOptions) Verify(configErrs *ConfigErrors, isMonolith bool) {
 }
 
 // MaxIdleConns returns maximum idle connections to the DB

internal/config/config_keyserver.go

@@ -16,7 +16,7 @@ func (c *KeyServer) Defaults() {
 	c.Database.ConnectionString = "file:keyserver.db"
 }
 
-func (c *KeyServer) Verify(configErrs *configErrors) {
+func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkNotEmpty(configErrs, "key_server.listen", string(c.Listen))
 	checkNotEmpty(configErrs, "key_server.bind", string(c.Bind))
 	checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString))

internal/config/config_mediaapi.go

@@ -44,9 +44,10 @@ func (c *MediaAPI) Defaults() {
 	defaultMaxFileSizeBytes := FileSizeBytes(10485760)
 	c.MaxFileSizeBytes = &defaultMaxFileSizeBytes
 	c.MaxThumbnailGenerators = 10
+	c.BasePath = "./media_store"
 }
 
-func (c *MediaAPI) Verify(configErrs *configErrors) {
+func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkNotEmpty(configErrs, "media_api.listen", string(c.Listen))
 	checkNotEmpty(configErrs, "media_api.bind", string(c.Bind))
 	checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString))

internal/config/config_roomserver.go

@@ -16,7 +16,7 @@ func (c *RoomServer) Defaults() {
 	c.Database.ConnectionString = "file:roomserver.db"
 }
 
-func (c *RoomServer) Verify(configErrs *configErrors) {
+func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkNotEmpty(configErrs, "room_server.listen", string(c.Listen))
 	checkNotEmpty(configErrs, "room_server.bind", string(c.Bind))
 	checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString))

internal/config/config_serverkey.go

@@ -22,7 +22,7 @@ func (c *ServerKeyAPI) Defaults() {
 	c.Database.ConnectionString = "file:serverkeyapi.db"
 }
 
-func (c *ServerKeyAPI) Verify(configErrs *configErrors) {
+func (c *ServerKeyAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkNotEmpty(configErrs, "server_key_api.listen", string(c.Listen))
 	checkNotEmpty(configErrs, "server_key_api.bind", string(c.Bind))
 	checkNotEmpty(configErrs, "server_key_api.database.connection_string", string(c.Database.ConnectionString))

internal/config/config_syncapi.go

@@ -16,7 +16,7 @@ func (c *SyncAPI) Defaults() {
 	c.Database.ConnectionString = "file:syncapi.db"
 }
 
-func (c *SyncAPI) Verify(configErrs *configErrors) {
+func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkNotEmpty(configErrs, "sync_api.listen", string(c.Listen))
 	checkNotEmpty(configErrs, "sync_api.bind", string(c.Bind))
 	checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString))

internal/config/config_userapi.go

@@ -23,7 +23,7 @@ func (c *UserAPI) Defaults() {
 	c.DeviceDatabase.ConnectionString = "file:userapi_devices.db"
 }
 
-func (c *UserAPI) Verify(configErrs *configErrors) {
+func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) {
 	checkNotEmpty(configErrs, "user_api.listen", string(c.Listen))
 	checkNotEmpty(configErrs, "user_api.bind", string(c.Bind))
 	checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString))

internal/setup/base.go

@@ -84,6 +84,15 @@ const HTTPClientTimeout = time.Second * 30
 // The componentName is used for logging purposes, and should be a friendly name
 // of the compontent running, e.g. "SyncAPI"
 func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs bool) *BaseDendrite {
+	configErrors := &config.ConfigErrors{}
+	cfg.Verify(configErrors, componentName == "Monolith") // TODO: better way?
+	if len(*configErrors) > 0 {
+		for _, err := range *configErrors {
+			logrus.Errorf("Configuration error: %s", err)
+		}
+		logrus.Fatalf("Failed to start due to configuration errors")
+	}
+
 	internal.SetupStdLogging()
 	internal.SetupHookLogging(cfg.Logging, componentName)
 	internal.SetupPprof()