mediaapi: Add YAML config file support

media-api-server-config.yaml

@@ -0,0 +1,38 @@
+# The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
+server_name: "example.com"
+
+# The base path to where the media files will be stored. May be relative or absolute.
+base_path: /var/dendrite/media
+
+# The maximum file size in bytes that is allowed to be stored on this server.
+# Note: if max_file_size_bytes is set to 0, the size is unlimited.
+# Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB)
+max_file_size_bytes: 10485760
+
+# The postgres connection config for connecting to the database e.g a postgres:// URI
+database: "postgres://dendrite:itsasecret@localhost/mediaapi?sslmode=disable"
+
+# Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated
+# NOTE: This is a possible denial-of-service attack vector - use at your own risk
+dynamic_thumbnails: false
+
+# A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content
+# method is one of crop or scale. If omitted, it will default to scale.
+# crop scales to fill the requested dimensions and crops the excess.
+# scale scales to fit the requested dimensions and one dimension may be smaller than requested.
+thumbnail_sizes:
+- width: 32
+  height: 32
+  method: crop
+- width: 96
+  height: 96
+  method: crop
+- width: 320
+  height: 240
+  method: scale
+- width: 640
+  height: 480
+  method: scale
+- width: 800
+  height: 600
+  method: scale

src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go

@@ -15,10 +15,14 @@
 package main
 
 import (
+	"fmt"
+	"io/ioutil"
 	"net/http"
 	"os"
+	"os/user"
 	"path/filepath"
 	"strconv"
+	"strings"
 
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/mediaapi/config"
@@ -28,6 +32,7 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 
 	log "github.com/Sirupsen/logrus"
+	yaml "gopkg.in/yaml.v2"
 )
 
 var (
@@ -38,36 +43,25 @@ var (
 	basePath   = os.Getenv("BASE_PATH")
 	// Note: if the MAX_FILE_SIZE_BYTES is set to 0, it will be unlimited
 	maxFileSizeBytesString = os.Getenv("MAX_FILE_SIZE_BYTES")
+	configPath             = os.Getenv("CONFIG_PATH")
 )
 
 func main() {
 	common.SetupLogging(logDir)
 
-	if bindAddr == "" {
+	log.WithFields(log.Fields{
-		log.Panic("No BIND_ADDRESS environment variable found.")
+		"BIND_ADDRESS":        bindAddr,
-	}
+		"DATABASE":            dataSource,
-	if basePath == "" {
+		"LOG_DIR":             logDir,
-		log.Panic("No BASE_PATH environment variable found.")
+		"SERVER_NAME":         serverName,
-	}
+		"BASE_PATH":           basePath,
-	absBasePath, err := filepath.Abs(basePath)
+		"MAX_FILE_SIZE_BYTES": maxFileSizeBytesString,
-	if err != nil {
+		"CONFIG_PATH":         configPath,
-		log.WithError(err).WithField("BASE_PATH", basePath).Panic("BASE_PATH is invalid (must be able to make absolute)")
+	}).Info("Loading configuration based on config file and environment variables")
-	}
 
-	if serverName == "" {
+	cfg, err := configureServer()
-		serverName = "localhost"
-	}
-	maxFileSizeBytes, err := strconv.ParseInt(maxFileSizeBytesString, 10, 64)
 	if err != nil {
-		maxFileSizeBytes = 10 * 1024 * 1024
+		log.WithError(err).Fatal("Invalid configuration")
-		log.WithError(err).WithField("MAX_FILE_SIZE_BYTES", maxFileSizeBytesString).Warnf("Failed to parse MAX_FILE_SIZE_BYTES. Defaulting to %v bytes.", maxFileSizeBytes)
-	}
-
-	cfg := &config.MediaAPI{
-		ServerName:       gomatrixserverlib.ServerName(serverName),
-		AbsBasePath:      types.Path(absBasePath),
-		MaxFileSizeBytes: types.FileSizeBytes(maxFileSizeBytes),
-		DataSource:       dataSource,
 	}
 
 	db, err := storage.Open(cfg.DataSource)
@@ -76,14 +70,174 @@ func main() {
 	}
 
 	log.WithFields(log.Fields{
-		"BASE_PATH":           absBasePath,
+		"BIND_ADDRESS":      bindAddr,
-		"BIND_ADDRESS":        bindAddr,
+		"LOG_DIR":           logDir,
-		"DATABASE":            dataSource,
+		"CONFIG_PATH":       configPath,
-		"LOG_DIR":             logDir,
+		"ServerName":        cfg.ServerName,
-		"MAX_FILE_SIZE_BYTES": maxFileSizeBytes,
+		"AbsBasePath":       cfg.AbsBasePath,
-		"SERVER_NAME":         serverName,
+		"MaxFileSizeBytes":  *cfg.MaxFileSizeBytes,
-	}).Info("Starting mediaapi")
+		"DataSource":        cfg.DataSource,
+		"DynamicThumbnails": cfg.DynamicThumbnails,
+		"ThumbnailSizes":    cfg.ThumbnailSizes,
+	}).Info("Starting mediaapi server with configuration")
 
 	routing.Setup(http.DefaultServeMux, http.DefaultClient, cfg, db)
 	log.Fatal(http.ListenAndServe(bindAddr, nil))
 }
+
+// configureServer loads configuration from a yaml file and overrides with environment variables
+func configureServer() (*config.MediaAPI, error) {
+	cfg, err := loadConfig(configPath)
+	if err != nil {
+		log.WithError(err).Fatal("Invalid config file")
+	}
+
+	// override values from environment variables
+	applyOverrides(cfg)
+
+	if err := validateConfig(cfg); err != nil {
+		return nil, err
+	}
+
+	return cfg, nil
+}
+
+// FIXME: make common somehow? copied from sync api
+func loadConfig(configPath string) (*config.MediaAPI, error) {
+	contents, err := ioutil.ReadFile(configPath)
+	if err != nil {
+		return nil, err
+	}
+	var cfg config.MediaAPI
+	if err = yaml.Unmarshal(contents, &cfg); err != nil {
+		return nil, err
+	}
+	return &cfg, nil
+}
+
+func applyOverrides(cfg *config.MediaAPI) {
+	if serverName != "" {
+		if cfg.ServerName != "" {
+			log.WithFields(log.Fields{
+				"server_name": cfg.ServerName,
+				"SERVER_NAME": serverName,
+			}).Info("Overriding server_name from config file with environment variable")
+		}
+		cfg.ServerName = gomatrixserverlib.ServerName(serverName)
+	}
+	if cfg.ServerName == "" {
+		log.Info("ServerName not set. Defaulting to 'localhost'.")
+		cfg.ServerName = "localhost"
+	}
+
+	if basePath != "" {
+		if cfg.BasePath != "" {
+			log.WithFields(log.Fields{
+				"base_path": cfg.BasePath,
+				"BASE_PATH": basePath,
+			}).Info("Overriding base_path from config file with environment variable")
+		}
+		cfg.BasePath = types.Path(basePath)
+	}
+
+	if maxFileSizeBytesString != "" {
+		if cfg.MaxFileSizeBytes != nil {
+			log.WithFields(log.Fields{
+				"max_file_size_bytes": *cfg.MaxFileSizeBytes,
+				"MAX_FILE_SIZE_BYTES": maxFileSizeBytesString,
+			}).Info("Overriding max_file_size_bytes from config file with environment variable")
+		}
+		maxFileSizeBytesInt, err := strconv.ParseInt(maxFileSizeBytesString, 10, 64)
+		if err != nil {
+			maxFileSizeBytesInt = 10 * 1024 * 1024
+			log.WithError(err).WithField(
+				"MAX_FILE_SIZE_BYTES", maxFileSizeBytesString,
+			).Infof("MAX_FILE_SIZE_BYTES not set? Defaulting to %v bytes.", maxFileSizeBytesInt)
+		}
+		maxFileSizeBytes := types.FileSizeBytes(maxFileSizeBytesInt)
+		cfg.MaxFileSizeBytes = &maxFileSizeBytes
+	}
+
+	if dataSource != "" {
+		if cfg.DataSource != "" {
+			log.WithFields(log.Fields{
+				"database": cfg.DataSource,
+				"DATABASE": dataSource,
+			}).Info("Overriding database from config file with environment variable")
+		}
+		cfg.DataSource = dataSource
+	}
+}
+
+func validateConfig(cfg *config.MediaAPI) error {
+	if bindAddr == "" {
+		return fmt.Errorf("no BIND_ADDRESS environment variable found")
+	}
+
+	absBasePath, err := getAbsolutePath(cfg.BasePath)
+	if err != nil {
+		return fmt.Errorf("invalid base path (%v): %q", cfg.BasePath, err)
+	}
+	cfg.AbsBasePath = types.Path(absBasePath)
+
+	if *cfg.MaxFileSizeBytes < 0 {
+		return fmt.Errorf("invalid max file size bytes (%v)", *cfg.MaxFileSizeBytes)
+	}
+
+	if cfg.DataSource == "" {
+		return fmt.Errorf("invalid database (%v)", cfg.DataSource)
+	}
+
+	for _, config := range cfg.ThumbnailSizes {
+		if config.Width <= 0 || config.Height <= 0 {
+			return fmt.Errorf("invalid thumbnail size %vx%v", config.Width, config.Height)
+		}
+	}
+
+	return nil
+}
+
+func getAbsolutePath(basePath types.Path) (types.Path, error) {
+	var err error
+	if basePath == "" {
+		var wd string
+		wd, err = os.Getwd()
+		return types.Path(wd), err
+	}
+	// Note: If we got here len(basePath) >= 1
+	if basePath[0] == '~' {
+		basePath, err = expandHomeDir(basePath)
+		if err != nil {
+			return "", err
+		}
+	}
+	absBasePath, err := filepath.Abs(string(basePath))
+	return types.Path(absBasePath), err
+}
+
+// expandHomeDir parses paths beginning with ~/path or ~user/path and replaces the home directory part
+func expandHomeDir(basePath types.Path) (types.Path, error) {
+	slash := strings.Index(string(basePath), "/")
+	if slash == -1 {
+		// pretend the slash is after the path as none was found within the string
+		// simplifies code using slash below
+		slash = len(basePath)
+	}
+	var usr *user.User
+	var err error
+	if slash == 1 {
+		// basePath is ~ or ~/path
+		usr, err = user.Current()
+		if err != nil {
+			return "", fmt.Errorf("failed to get user's home directory: %q", err)
+		}
+	} else {
+		// slash > 1
+		// basePath is ~user or ~user/path
+		usr, err = user.Lookup(string(basePath[1:slash]))
+		if err != nil {
+			return "", fmt.Errorf("failed to get user's home directory: %q", err)
+		}
+	}
+	return types.Path(filepath.Join(usr.HomeDir, string(basePath[slash:]))), nil
+}

src/github.com/matrix-org/dendrite/mediaapi/config/config.go

@@ -23,12 +23,18 @@ import (
 type MediaAPI struct {
 	// The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
 	ServerName gomatrixserverlib.ServerName `yaml:"server_name"`
+	// The base path to where the media files will be stored. May be relative or absolute.
+	BasePath types.Path `yaml:"base_path"`
 	// The absolute base path to where media files will be stored.
-	AbsBasePath types.Path `yaml:"abs_base_path"`
+	AbsBasePath types.Path `yaml:"-"`
 	// The maximum file size in bytes that is allowed to be stored on this server.
 	// Note: if MaxFileSizeBytes is set to 0, the size is unlimited.
 	// Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB)
-	MaxFileSizeBytes types.FileSizeBytes `yaml:"max_file_size_bytes"`
+	MaxFileSizeBytes *types.FileSizeBytes `yaml:"max_file_size_bytes,omitempty"`
 	// The postgres connection config for connecting to the database e.g a postgres:// URI
 	DataSource string `yaml:"database"`
+	// Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated
+	DynamicThumbnails bool `yaml:"dynamic_thumbnails"`
+	// A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content
+	ThumbnailSizes []types.ThumbnailSize `yaml:"thumbnail_sizes"`
 }

src/github.com/matrix-org/dendrite/mediaapi/types/types.go

@@ -77,3 +77,15 @@ type ActiveRemoteRequests struct {
 	// The string key is an mxc:// URL
 	MXCToResult map[string]*RemoteRequestResult
 }
+
+// ThumbnailSize contains a single thumbnail size configuration
+type ThumbnailSize struct {
+	// Maximum width of the thumbnail image
+	Width int `yaml:"width"`
+	// Maximum height of the thumbnail image
+	Height int `yaml:"height"`
+	// ResizeMethod is one of crop or scale.
+	// crop scales to fill the requested dimensions and crops the excess.
+	// scale scales to fit the requested dimensions and one dimension may be smaller than requested.
+	ResizeMethod string `yaml:"method,omitempty"`
+}

src/github.com/matrix-org/dendrite/mediaapi/writers/download.go

@@ -245,7 +245,7 @@ func (r *downloadRequest) getRemoteFile(cfg *config.MediaAPI, db *storage.Databa
 
 		if mediaMetadata == nil {
 			// If we do not have a record, we need to fetch the remote file first and then respond from the local file
-			resErr := r.fetchRemoteFileAndStoreMetadata(cfg.AbsBasePath, cfg.MaxFileSizeBytes, db)
+			resErr := r.fetchRemoteFileAndStoreMetadata(cfg.AbsBasePath, *cfg.MaxFileSizeBytes, db)
 			if resErr != nil {
 				return resErr
 			}

src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go

@@ -89,7 +89,7 @@ func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRe
 		Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.ServerName),
 	}
 
-	if resErr := r.Validate(cfg.MaxFileSizeBytes); resErr != nil {
+	if resErr := r.Validate(*cfg.MaxFileSizeBytes); resErr != nil {
 		return nil, resErr
 	}
 
@@ -107,10 +107,10 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db *
 	// method of deduplicating files to save storage, as well as a way to conduct
 	// integrity checks on the file data in the repository.
 	// Data is truncated to maxFileSizeBytes. Content-Length was reported as 0 < Content-Length <= maxFileSizeBytes so this is OK.
-	hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, cfg.MaxFileSizeBytes, cfg.AbsBasePath)
+	hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.MaxFileSizeBytes, cfg.AbsBasePath)
 	if err != nil {
 		r.Logger.WithError(err).WithFields(log.Fields{
-			"MaxFileSizeBytes": cfg.MaxFileSizeBytes,
+			"MaxFileSizeBytes": *cfg.MaxFileSizeBytes,
 		}).Warn("Error while transferring file")
 		fileutils.RemoveDir(tmpDir, r.Logger)
 		return &util.JSONResponse{