From e05f1af49aabac7b28b9abc9fc0b55dcc91fa45a Mon Sep 17 00:00:00 2001 From: Robert Swain Date: Sat, 3 Jun 2017 10:42:49 +0200 Subject: [PATCH] mediaapi: Add YAML config file support --- media-api-server-config.yaml | 38 ++++ .../cmd/dendrite-media-api-server/main.go | 214 +++++++++++++++--- .../dendrite/mediaapi/config/config.go | 10 +- .../dendrite/mediaapi/types/types.go | 12 + .../dendrite/mediaapi/writers/download.go | 2 +- .../dendrite/mediaapi/writers/upload.go | 6 +- 6 files changed, 246 insertions(+), 36 deletions(-) create mode 100644 media-api-server-config.yaml diff --git a/media-api-server-config.yaml b/media-api-server-config.yaml new file mode 100644 index 000000000..c222fe8fb --- /dev/null +++ b/media-api-server-config.yaml @@ -0,0 +1,38 @@ +# The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. +server_name: "example.com" + +# The base path to where the media files will be stored. May be relative or absolute. +base_path: /var/dendrite/media + +# The maximum file size in bytes that is allowed to be stored on this server. +# Note: if max_file_size_bytes is set to 0, the size is unlimited. +# Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) +max_file_size_bytes: 10485760 + +# The postgres connection config for connecting to the database e.g a postgres:// URI +database: "postgres://dendrite:itsasecret@localhost/mediaapi?sslmode=disable" + +# Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated +# NOTE: This is a possible denial-of-service attack vector - use at your own risk +dynamic_thumbnails: false + +# A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content +# method is one of crop or scale. If omitted, it will default to scale. +# crop scales to fill the requested dimensions and crops the excess. +# scale scales to fit the requested dimensions and one dimension may be smaller than requested. +thumbnail_sizes: +- width: 32 + height: 32 + method: crop +- width: 96 + height: 96 + method: crop +- width: 320 + height: 240 + method: scale +- width: 640 + height: 480 + method: scale +- width: 800 + height: 600 + method: scale diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go index 298762482..2ce9226da 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go @@ -15,10 +15,14 @@ package main import ( + "fmt" + "io/ioutil" "net/http" "os" + "os/user" "path/filepath" "strconv" + "strings" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/mediaapi/config" @@ -28,6 +32,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" log "github.com/Sirupsen/logrus" + yaml "gopkg.in/yaml.v2" ) var ( @@ -38,36 +43,25 @@ var ( basePath = os.Getenv("BASE_PATH") // Note: if the MAX_FILE_SIZE_BYTES is set to 0, it will be unlimited maxFileSizeBytesString = os.Getenv("MAX_FILE_SIZE_BYTES") + configPath = os.Getenv("CONFIG_PATH") ) func main() { common.SetupLogging(logDir) - if bindAddr == "" { - log.Panic("No BIND_ADDRESS environment variable found.") - } - if basePath == "" { - log.Panic("No BASE_PATH environment variable found.") - } - absBasePath, err := filepath.Abs(basePath) - if err != nil { - log.WithError(err).WithField("BASE_PATH", basePath).Panic("BASE_PATH is invalid (must be able to make absolute)") - } + log.WithFields(log.Fields{ + "BIND_ADDRESS": bindAddr, + "DATABASE": dataSource, + "LOG_DIR": logDir, + "SERVER_NAME": serverName, + "BASE_PATH": basePath, + "MAX_FILE_SIZE_BYTES": maxFileSizeBytesString, + "CONFIG_PATH": configPath, + }).Info("Loading configuration based on config file and environment variables") - if serverName == "" { - serverName = "localhost" - } - maxFileSizeBytes, err := strconv.ParseInt(maxFileSizeBytesString, 10, 64) + cfg, err := configureServer() if err != nil { - maxFileSizeBytes = 10 * 1024 * 1024 - log.WithError(err).WithField("MAX_FILE_SIZE_BYTES", maxFileSizeBytesString).Warnf("Failed to parse MAX_FILE_SIZE_BYTES. Defaulting to %v bytes.", maxFileSizeBytes) - } - - cfg := &config.MediaAPI{ - ServerName: gomatrixserverlib.ServerName(serverName), - AbsBasePath: types.Path(absBasePath), - MaxFileSizeBytes: types.FileSizeBytes(maxFileSizeBytes), - DataSource: dataSource, + log.WithError(err).Fatal("Invalid configuration") } db, err := storage.Open(cfg.DataSource) @@ -76,14 +70,174 @@ func main() { } log.WithFields(log.Fields{ - "BASE_PATH": absBasePath, - "BIND_ADDRESS": bindAddr, - "DATABASE": dataSource, - "LOG_DIR": logDir, - "MAX_FILE_SIZE_BYTES": maxFileSizeBytes, - "SERVER_NAME": serverName, - }).Info("Starting mediaapi") + "BIND_ADDRESS": bindAddr, + "LOG_DIR": logDir, + "CONFIG_PATH": configPath, + "ServerName": cfg.ServerName, + "AbsBasePath": cfg.AbsBasePath, + "MaxFileSizeBytes": *cfg.MaxFileSizeBytes, + "DataSource": cfg.DataSource, + "DynamicThumbnails": cfg.DynamicThumbnails, + "ThumbnailSizes": cfg.ThumbnailSizes, + }).Info("Starting mediaapi server with configuration") routing.Setup(http.DefaultServeMux, http.DefaultClient, cfg, db) log.Fatal(http.ListenAndServe(bindAddr, nil)) } + +// configureServer loads configuration from a yaml file and overrides with environment variables +func configureServer() (*config.MediaAPI, error) { + cfg, err := loadConfig(configPath) + if err != nil { + log.WithError(err).Fatal("Invalid config file") + } + + // override values from environment variables + applyOverrides(cfg) + + if err := validateConfig(cfg); err != nil { + return nil, err + } + + return cfg, nil +} + +// FIXME: make common somehow? copied from sync api +func loadConfig(configPath string) (*config.MediaAPI, error) { + contents, err := ioutil.ReadFile(configPath) + if err != nil { + return nil, err + } + var cfg config.MediaAPI + if err = yaml.Unmarshal(contents, &cfg); err != nil { + return nil, err + } + return &cfg, nil +} + +func applyOverrides(cfg *config.MediaAPI) { + if serverName != "" { + if cfg.ServerName != "" { + log.WithFields(log.Fields{ + "server_name": cfg.ServerName, + "SERVER_NAME": serverName, + }).Info("Overriding server_name from config file with environment variable") + } + cfg.ServerName = gomatrixserverlib.ServerName(serverName) + } + if cfg.ServerName == "" { + log.Info("ServerName not set. Defaulting to 'localhost'.") + cfg.ServerName = "localhost" + } + + if basePath != "" { + if cfg.BasePath != "" { + log.WithFields(log.Fields{ + "base_path": cfg.BasePath, + "BASE_PATH": basePath, + }).Info("Overriding base_path from config file with environment variable") + } + cfg.BasePath = types.Path(basePath) + } + + if maxFileSizeBytesString != "" { + if cfg.MaxFileSizeBytes != nil { + log.WithFields(log.Fields{ + "max_file_size_bytes": *cfg.MaxFileSizeBytes, + "MAX_FILE_SIZE_BYTES": maxFileSizeBytesString, + }).Info("Overriding max_file_size_bytes from config file with environment variable") + } + maxFileSizeBytesInt, err := strconv.ParseInt(maxFileSizeBytesString, 10, 64) + if err != nil { + maxFileSizeBytesInt = 10 * 1024 * 1024 + log.WithError(err).WithField( + "MAX_FILE_SIZE_BYTES", maxFileSizeBytesString, + ).Infof("MAX_FILE_SIZE_BYTES not set? Defaulting to %v bytes.", maxFileSizeBytesInt) + } + maxFileSizeBytes := types.FileSizeBytes(maxFileSizeBytesInt) + cfg.MaxFileSizeBytes = &maxFileSizeBytes + } + + if dataSource != "" { + if cfg.DataSource != "" { + log.WithFields(log.Fields{ + "database": cfg.DataSource, + "DATABASE": dataSource, + }).Info("Overriding database from config file with environment variable") + } + cfg.DataSource = dataSource + } +} + +func validateConfig(cfg *config.MediaAPI) error { + if bindAddr == "" { + return fmt.Errorf("no BIND_ADDRESS environment variable found") + } + + absBasePath, err := getAbsolutePath(cfg.BasePath) + if err != nil { + return fmt.Errorf("invalid base path (%v): %q", cfg.BasePath, err) + } + cfg.AbsBasePath = types.Path(absBasePath) + + if *cfg.MaxFileSizeBytes < 0 { + return fmt.Errorf("invalid max file size bytes (%v)", *cfg.MaxFileSizeBytes) + } + + if cfg.DataSource == "" { + return fmt.Errorf("invalid database (%v)", cfg.DataSource) + } + + for _, config := range cfg.ThumbnailSizes { + if config.Width <= 0 || config.Height <= 0 { + return fmt.Errorf("invalid thumbnail size %vx%v", config.Width, config.Height) + } + } + + return nil +} + +func getAbsolutePath(basePath types.Path) (types.Path, error) { + var err error + if basePath == "" { + var wd string + wd, err = os.Getwd() + return types.Path(wd), err + } + // Note: If we got here len(basePath) >= 1 + if basePath[0] == '~' { + basePath, err = expandHomeDir(basePath) + if err != nil { + return "", err + } + } + absBasePath, err := filepath.Abs(string(basePath)) + return types.Path(absBasePath), err +} + +// expandHomeDir parses paths beginning with ~/path or ~user/path and replaces the home directory part +func expandHomeDir(basePath types.Path) (types.Path, error) { + slash := strings.Index(string(basePath), "/") + if slash == -1 { + // pretend the slash is after the path as none was found within the string + // simplifies code using slash below + slash = len(basePath) + } + var usr *user.User + var err error + if slash == 1 { + // basePath is ~ or ~/path + usr, err = user.Current() + if err != nil { + return "", fmt.Errorf("failed to get user's home directory: %q", err) + } + } else { + // slash > 1 + // basePath is ~user or ~user/path + usr, err = user.Lookup(string(basePath[1:slash])) + if err != nil { + return "", fmt.Errorf("failed to get user's home directory: %q", err) + } + } + return types.Path(filepath.Join(usr.HomeDir, string(basePath[slash:]))), nil +} diff --git a/src/github.com/matrix-org/dendrite/mediaapi/config/config.go b/src/github.com/matrix-org/dendrite/mediaapi/config/config.go index 5c514194a..069befb00 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/config/config.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/config/config.go @@ -23,12 +23,18 @@ import ( type MediaAPI struct { // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. ServerName gomatrixserverlib.ServerName `yaml:"server_name"` + // The base path to where the media files will be stored. May be relative or absolute. + BasePath types.Path `yaml:"base_path"` // The absolute base path to where media files will be stored. - AbsBasePath types.Path `yaml:"abs_base_path"` + AbsBasePath types.Path `yaml:"-"` // The maximum file size in bytes that is allowed to be stored on this server. // Note: if MaxFileSizeBytes is set to 0, the size is unlimited. // Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) - MaxFileSizeBytes types.FileSizeBytes `yaml:"max_file_size_bytes"` + MaxFileSizeBytes *types.FileSizeBytes `yaml:"max_file_size_bytes,omitempty"` // The postgres connection config for connecting to the database e.g a postgres:// URI DataSource string `yaml:"database"` + // Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated + DynamicThumbnails bool `yaml:"dynamic_thumbnails"` + // A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content + ThumbnailSizes []types.ThumbnailSize `yaml:"thumbnail_sizes"` } diff --git a/src/github.com/matrix-org/dendrite/mediaapi/types/types.go b/src/github.com/matrix-org/dendrite/mediaapi/types/types.go index d54bcdf67..0729c25dc 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/types/types.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/types/types.go @@ -77,3 +77,15 @@ type ActiveRemoteRequests struct { // The string key is an mxc:// URL MXCToResult map[string]*RemoteRequestResult } + +// ThumbnailSize contains a single thumbnail size configuration +type ThumbnailSize struct { + // Maximum width of the thumbnail image + Width int `yaml:"width"` + // Maximum height of the thumbnail image + Height int `yaml:"height"` + // ResizeMethod is one of crop or scale. + // crop scales to fill the requested dimensions and crops the excess. + // scale scales to fit the requested dimensions and one dimension may be smaller than requested. + ResizeMethod string `yaml:"method,omitempty"` +} diff --git a/src/github.com/matrix-org/dendrite/mediaapi/writers/download.go b/src/github.com/matrix-org/dendrite/mediaapi/writers/download.go index af31535b4..32f4c9c4e 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/writers/download.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/writers/download.go @@ -245,7 +245,7 @@ func (r *downloadRequest) getRemoteFile(cfg *config.MediaAPI, db *storage.Databa if mediaMetadata == nil { // If we do not have a record, we need to fetch the remote file first and then respond from the local file - resErr := r.fetchRemoteFileAndStoreMetadata(cfg.AbsBasePath, cfg.MaxFileSizeBytes, db) + resErr := r.fetchRemoteFileAndStoreMetadata(cfg.AbsBasePath, *cfg.MaxFileSizeBytes, db) if resErr != nil { return resErr } diff --git a/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go b/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go index f1838a559..aeda72fb2 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go @@ -89,7 +89,7 @@ func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRe Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.ServerName), } - if resErr := r.Validate(cfg.MaxFileSizeBytes); resErr != nil { + if resErr := r.Validate(*cfg.MaxFileSizeBytes); resErr != nil { return nil, resErr } @@ -107,10 +107,10 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db * // method of deduplicating files to save storage, as well as a way to conduct // integrity checks on the file data in the repository. // Data is truncated to maxFileSizeBytes. Content-Length was reported as 0 < Content-Length <= maxFileSizeBytes so this is OK. - hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, cfg.MaxFileSizeBytes, cfg.AbsBasePath) + hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.MaxFileSizeBytes, cfg.AbsBasePath) if err != nil { r.Logger.WithError(err).WithFields(log.Fields{ - "MaxFileSizeBytes": cfg.MaxFileSizeBytes, + "MaxFileSizeBytes": *cfg.MaxFileSizeBytes, }).Warn("Error while transferring file") fileutils.RemoveDir(tmpDir, r.Logger) return &util.JSONResponse{