diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go index fb39a03a6..df3af4797 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go @@ -16,35 +16,20 @@ package main import ( "flag" - "fmt" - "io/ioutil" "net/http" "os" - "os/user" - "path/filepath" - "strconv" - "strings" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/mediaapi/config" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/routing" "github.com/matrix-org/dendrite/mediaapi/storage" - "github.com/matrix-org/dendrite/mediaapi/types" - "github.com/matrix-org/gomatrixserverlib" log "github.com/Sirupsen/logrus" - yaml "gopkg.in/yaml.v2" ) var ( - bindAddr = flag.String("listen", "", "The port to listen on.") - dataSource = os.Getenv("DATABASE") logDir = os.Getenv("LOG_DIR") - serverName = os.Getenv("SERVER_NAME") - basePath = os.Getenv("BASE_PATH") - // Note: if the MAX_FILE_SIZE_BYTES is set to 0, it will be unlimited - maxFileSizeBytesString = os.Getenv("MAX_FILE_SIZE_BYTES") - configPath = flag.String("config", "", "The path to the config file. For more information, see the config file in this repository.") + configPath = flag.String("config", "", "The path to the config file. For more information, see the config file in this repository.") ) func main() { @@ -52,206 +37,21 @@ func main() { flag.Parse() - log.WithFields(log.Fields{ - "listen": *bindAddr, - "DATABASE": dataSource, - "LOG_DIR": logDir, - "SERVER_NAME": serverName, - "BASE_PATH": basePath, - "MAX_FILE_SIZE_BYTES": maxFileSizeBytesString, - "config": *configPath, - }).Info("Loading configuration based on config file and environment variables") - - cfg, err := configureServer() + if *configPath == "" { + log.Fatal("--config must be supplied") + } + cfg, err := config.Load(*configPath) if err != nil { - log.WithError(err).Fatal("Invalid configuration") + log.Fatalf("Invalid config file: %s", err) } - log.WithFields(log.Fields{ - "listen": *bindAddr, - "LOG_DIR": logDir, - "CONFIG_PATH": *configPath, - "ServerName": cfg.ServerName, - "AbsBasePath": cfg.AbsBasePath, - "MaxFileSizeBytes": *cfg.MaxFileSizeBytes, - "DataSource": cfg.DataSource, - "DynamicThumbnails": cfg.DynamicThumbnails, - "MaxThumbnailGenerators": cfg.MaxThumbnailGenerators, - "ThumbnailSizes": cfg.ThumbnailSizes, - }).Info("Starting mediaapi server with configuration") - - db, err := storage.Open(cfg.DataSource) + db, err := storage.Open(string(cfg.Database.MediaServer)) if err != nil { log.WithError(err).Panic("Failed to open database") } + log.Info("Starting media API server on ", cfg.Listen.MediaAPI) + routing.Setup(http.DefaultServeMux, http.DefaultClient, cfg, db) - log.Fatal(http.ListenAndServe(*bindAddr, nil)) -} - -// configureServer loads configuration from a yaml file and overrides with environment variables -func configureServer() (*config.MediaAPI, error) { - if *configPath == "" { - log.Fatal("--config must be supplied") - } - cfg, err := loadConfig(*configPath) - if err != nil { - log.WithError(err).Fatal("Invalid config file") - } - - // override values from environment variables - applyOverrides(cfg) - - if err := validateConfig(cfg); err != nil { - return nil, err - } - - return cfg, nil -} - -// FIXME: make common somehow? copied from sync api -func loadConfig(configPath string) (*config.MediaAPI, error) { - contents, err := ioutil.ReadFile(configPath) - if err != nil { - return nil, err - } - var cfg config.MediaAPI - if err = yaml.Unmarshal(contents, &cfg); err != nil { - return nil, err - } - return &cfg, nil -} - -func applyOverrides(cfg *config.MediaAPI) { - if serverName != "" { - if cfg.ServerName != "" { - log.WithFields(log.Fields{ - "server_name": cfg.ServerName, - "SERVER_NAME": serverName, - }).Info("Overriding server_name from config file with environment variable") - } - cfg.ServerName = gomatrixserverlib.ServerName(serverName) - } - if cfg.ServerName == "" { - log.Info("ServerName not set. Defaulting to 'localhost'.") - cfg.ServerName = "localhost" - } - - if basePath != "" { - if cfg.BasePath != "" { - log.WithFields(log.Fields{ - "base_path": cfg.BasePath, - "BASE_PATH": basePath, - }).Info("Overriding base_path from config file with environment variable") - } - cfg.BasePath = types.Path(basePath) - } - - if maxFileSizeBytesString != "" { - if cfg.MaxFileSizeBytes != nil { - log.WithFields(log.Fields{ - "max_file_size_bytes": *cfg.MaxFileSizeBytes, - "MAX_FILE_SIZE_BYTES": maxFileSizeBytesString, - }).Info("Overriding max_file_size_bytes from config file with environment variable") - } - maxFileSizeBytesInt, err := strconv.ParseInt(maxFileSizeBytesString, 10, 64) - if err != nil { - maxFileSizeBytesInt = 10 * 1024 * 1024 - log.WithError(err).WithField( - "MAX_FILE_SIZE_BYTES", maxFileSizeBytesString, - ).Infof("MAX_FILE_SIZE_BYTES not set? Defaulting to %v bytes.", maxFileSizeBytesInt) - } - maxFileSizeBytes := types.FileSizeBytes(maxFileSizeBytesInt) - cfg.MaxFileSizeBytes = &maxFileSizeBytes - } - - if dataSource != "" { - if cfg.DataSource != "" { - log.WithFields(log.Fields{ - "database": cfg.DataSource, - "DATABASE": dataSource, - }).Info("Overriding database from config file with environment variable") - } - cfg.DataSource = dataSource - } - - if cfg.MaxThumbnailGenerators == 0 { - log.WithField( - "max_thumbnail_generators", cfg.MaxThumbnailGenerators, - ).Info("Using default max_thumbnail_generators value of 10") - cfg.MaxThumbnailGenerators = 10 - } -} - -func validateConfig(cfg *config.MediaAPI) error { - if *bindAddr == "" { - log.Fatal("--listen must be supplied") - } - - absBasePath, err := getAbsolutePath(cfg.BasePath) - if err != nil { - return fmt.Errorf("invalid base path (%v): %q", cfg.BasePath, err) - } - cfg.AbsBasePath = types.Path(absBasePath) - - if *cfg.MaxFileSizeBytes < 0 { - return fmt.Errorf("invalid max file size bytes (%v)", *cfg.MaxFileSizeBytes) - } - - if cfg.DataSource == "" { - return fmt.Errorf("invalid database (%v)", cfg.DataSource) - } - - for _, config := range cfg.ThumbnailSizes { - if config.Width <= 0 || config.Height <= 0 { - return fmt.Errorf("invalid thumbnail size %vx%v", config.Width, config.Height) - } - } - - return nil -} - -func getAbsolutePath(basePath types.Path) (types.Path, error) { - var err error - if basePath == "" { - var wd string - wd, err = os.Getwd() - return types.Path(wd), err - } - // Note: If we got here len(basePath) >= 1 - if basePath[0] == '~' { - basePath, err = expandHomeDir(basePath) - if err != nil { - return "", err - } - } - absBasePath, err := filepath.Abs(string(basePath)) - return types.Path(absBasePath), err -} - -// expandHomeDir parses paths beginning with ~/path or ~user/path and replaces the home directory part -func expandHomeDir(basePath types.Path) (types.Path, error) { - slash := strings.Index(string(basePath), "/") - if slash == -1 { - // pretend the slash is after the path as none was found within the string - // simplifies code using slash below - slash = len(basePath) - } - var usr *user.User - var err error - if slash == 1 { - // basePath is ~ or ~/path - usr, err = user.Current() - if err != nil { - return "", fmt.Errorf("failed to get user's home directory: %q", err) - } - } else { - // slash > 1 - // basePath is ~user or ~user/path - usr, err = user.Lookup(string(basePath[1:slash])) - if err != nil { - return "", fmt.Errorf("failed to get user's home directory: %q", err) - } - } - return types.Path(filepath.Join(usr.HomeDir, string(basePath[slash:]))), nil + log.Fatal(http.ListenAndServe(string(cfg.Listen.MediaAPI), nil)) } diff --git a/src/github.com/matrix-org/dendrite/common/config/config.go b/src/github.com/matrix-org/dendrite/common/config/config.go index 29d5c539c..6d8fc1b16 100644 --- a/src/github.com/matrix-org/dendrite/common/config/config.go +++ b/src/github.com/matrix-org/dendrite/common/config/config.go @@ -178,6 +178,8 @@ func loadConfig( return nil, err } + config.setDefaults() + if err = config.check(); err != nil { return nil, err } @@ -249,17 +251,31 @@ func (config *Dendrite) check() error { } } - checkNotZero := func(key string, value int) { + checkNotZero := func(key string, value int64) { if value == 0 { problems = append(problems, fmt.Sprintf("missing config key %q", key)) } } + checkPositive := func(key string, value int64) { + if value < 0 { + problems = append(problems, fmt.Sprintf("invalid value for config key %q: %d", key, value)) + } + } + checkNotEmpty("matrix.server_name", string(config.Matrix.ServerName)) checkNotEmpty("matrix.private_key", string(config.Matrix.PrivateKeyPath)) - checkNotZero("matrix.federation_certificates", len(config.Matrix.FederationCertificatePaths)) + checkNotZero("matrix.federation_certificates", int64(len(config.Matrix.FederationCertificatePaths))) + checkNotEmpty("media.base_path", string(config.Media.BasePath)) - checkNotZero("kafka.addresses", len(config.Kafka.Addresses)) + checkPositive("media.max_file_size_bytes", int64(*config.Media.MaxFileSizeBytes)) + checkPositive("media.max_thumbnail_generators", int64(config.Media.MaxThumbnailGenerators)) + for i, size := range config.Media.ThumbnailSizes { + checkPositive(fmt.Sprintf("media.thumbnail_sizes[%d].width", i), int64(size.Width)) + checkPositive(fmt.Sprintf("media.thumbnail_sizes[%d].height", i), int64(size.Height)) + } + + checkNotZero("kafka.addresses", int64(len(config.Kafka.Addresses))) checkNotEmpty("kafka.topics.input_room_event", string(config.Kafka.Topics.InputRoomEvent)) checkNotEmpty("kafka.topics.output_room_event", string(config.Kafka.Topics.OutputRoomEvent)) checkNotEmpty("database.media_server", string(config.Database.MediaServer)) diff --git a/src/github.com/matrix-org/dendrite/mediaapi/config/config.go b/src/github.com/matrix-org/dendrite/mediaapi/config/config.go deleted file mode 100644 index 14c12e861..000000000 --- a/src/github.com/matrix-org/dendrite/mediaapi/config/config.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "github.com/matrix-org/dendrite/mediaapi/types" - "github.com/matrix-org/gomatrixserverlib" -) - -// MediaAPI contains the config information necessary to spin up a mediaapi process. -type MediaAPI struct { - // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. - ServerName gomatrixserverlib.ServerName `yaml:"server_name"` - // The base path to where the media files will be stored. May be relative or absolute. - BasePath types.Path `yaml:"base_path"` - // The absolute base path to where media files will be stored. - AbsBasePath types.Path `yaml:"-"` - // The maximum file size in bytes that is allowed to be stored on this server. - // Note: if MaxFileSizeBytes is set to 0, the size is unlimited. - // Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) - MaxFileSizeBytes *types.FileSizeBytes `yaml:"max_file_size_bytes,omitempty"` - // The postgres connection config for connecting to the database e.g a postgres:// URI - DataSource string `yaml:"database"` - // Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated - DynamicThumbnails bool `yaml:"dynamic_thumbnails"` - // The maximum number of simultaneous thumbnail generators. default: 10 - MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"` - // A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content - ThumbnailSizes []types.ThumbnailSize `yaml:"thumbnail_sizes"` -} diff --git a/src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go b/src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go index a166f4765..706afe0d5 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go @@ -26,13 +26,14 @@ import ( "strings" log "github.com/Sirupsen/logrus" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/types" ) // GetPathFromBase64Hash evaluates the path to a media file from its Base64Hash // 3 subdirectories are created for more manageable browsing and use the remainder as the file name. // For example, if Base64Hash is 'qwerty', the path will be 'q/w/erty/file'. -func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath types.Path) (string, error) { +func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath config.Path) (string, error) { if len(base64Hash) < 3 { return "", fmt.Errorf("Invalid filePath (Base64Hash too short - min 3 characters): %q", base64Hash) } @@ -66,7 +67,7 @@ func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath types.Path) // If the final path exists and the file size matches, the file does not need to be moved. // In error cases where the file is not a duplicate, the caller may decide to remove the final path. // Returns the final path of the file, whether it is a duplicate and an error. -func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata, absBasePath types.Path, logger *log.Entry) (types.Path, bool, error) { +func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata, absBasePath config.Path, logger *log.Entry) (types.Path, bool, error) { // Note: in all error and success cases, we need to remove the temporary directory defer RemoveDir(tmpDir, logger) duplicate := false @@ -104,7 +105,7 @@ func RemoveDir(dir types.Path, logger *log.Entry) { } // WriteTempFile writes to a new temporary file -func WriteTempFile(reqReader io.Reader, maxFileSizeBytes types.FileSizeBytes, absBasePath types.Path) (types.Base64Hash, types.FileSizeBytes, types.Path, error) { +func WriteTempFile(reqReader io.Reader, maxFileSizeBytes config.FileSizeBytes, absBasePath config.Path) (types.Base64Hash, types.FileSizeBytes, types.Path, error) { tmpFileWriter, tmpFile, tmpDir, err := createTempFileWriter(absBasePath) if err != nil { return "", -1, "", err @@ -144,7 +145,7 @@ func moveFile(src types.Path, dst types.Path) error { return nil } -func createTempFileWriter(absBasePath types.Path) (*bufio.Writer, *os.File, types.Path, error) { +func createTempFileWriter(absBasePath config.Path) (*bufio.Writer, *os.File, types.Path, error) { tmpDir, err := createTempDir(absBasePath) if err != nil { return nil, nil, "", fmt.Errorf("Failed to create temp dir: %q", err) @@ -157,7 +158,7 @@ func createTempFileWriter(absBasePath types.Path) (*bufio.Writer, *os.File, type } // createTempDir creates a tmp/ directory within baseDirectory and returns its path -func createTempDir(baseDirectory types.Path) (types.Path, error) { +func createTempDir(baseDirectory config.Path) (types.Path, error) { baseTmpDir := filepath.Join(string(baseDirectory), "tmp") if err := os.MkdirAll(baseTmpDir, 0770); err != nil { return "", fmt.Errorf("Failed to create base temp dir: %v", err) diff --git a/src/github.com/matrix-org/dendrite/mediaapi/routing/routing.go b/src/github.com/matrix-org/dendrite/mediaapi/routing/routing.go index 47985fc16..1241bd0ee 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/routing/routing.go @@ -19,7 +19,7 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/mediaapi/config" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/dendrite/mediaapi/writers" @@ -32,7 +32,7 @@ const pathPrefixR0 = "/_matrix/media/v1" // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. -func Setup(servMux *http.ServeMux, httpClient *http.Client, cfg *config.MediaAPI, db *storage.Database) { +func Setup(servMux *http.ServeMux, httpClient *http.Client, cfg *config.Dendrite, db *storage.Database) { apiMux := mux.NewRouter() r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() @@ -59,7 +59,7 @@ func Setup(servMux *http.ServeMux, httpClient *http.Client, cfg *config.MediaAPI servMux.Handle("/api/", http.StripPrefix("/api", apiMux)) } -func makeDownloadAPI(name string, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) http.HandlerFunc { +func makeDownloadAPI(name string, cfg *config.Dendrite, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) http.HandlerFunc { return prometheus.InstrumentHandler(name, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { req = util.RequestWithLogging(req) diff --git a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go index 16268d13b..ded71b651 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go @@ -22,6 +22,7 @@ import ( "sync" log "github.com/Sirupsen/logrus" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" ) @@ -56,7 +57,7 @@ func GetThumbnailPath(src types.Path, config types.ThumbnailSize) types.Path { // * has a small file size // If a pre-generated thumbnail size is the best match, but it has not been generated yet, the caller can use the returned size to generate it. // Returns nil if no thumbnail matches the criteria -func SelectThumbnail(desired types.ThumbnailSize, thumbnails []*types.ThumbnailMetadata, thumbnailSizes []types.ThumbnailSize) (*types.ThumbnailMetadata, *types.ThumbnailSize) { +func SelectThumbnail(desired types.ThumbnailSize, thumbnails []*types.ThumbnailMetadata, thumbnailSizes []config.ThumbnailSize) (*types.ThumbnailMetadata, *types.ThumbnailSize) { var chosenThumbnail *types.ThumbnailMetadata var chosenThumbnailSize *types.ThumbnailSize bestFit := newThumbnailFitness() @@ -76,7 +77,7 @@ func SelectThumbnail(desired types.ThumbnailSize, thumbnails []*types.ThumbnailM if desired.ResizeMethod == "scale" && thumbnailSize.ResizeMethod != "scale" { continue } - fitness := calcThumbnailFitness(thumbnailSize, nil, desired) + fitness := calcThumbnailFitness(types.ThumbnailSize(thumbnailSize), nil, desired) if isBetter := fitness.betterThan(bestFit, desired.ResizeMethod == "crop"); isBetter { bestFit = fitness chosenThumbnailSize = &types.ThumbnailSize{ diff --git a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go index 41edce7cb..ad3b8b00d 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go @@ -21,13 +21,14 @@ import ( "time" log "github.com/Sirupsen/logrus" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" "gopkg.in/h2non/bimg.v1" ) // GenerateThumbnails generates the configured thumbnail sizes for the source file -func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) { +func GenerateThumbnails(src types.Path, configs []config.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) { buffer, err := bimg.Read(string(src)) if err != nil { logger.WithError(err).WithField("src", src).Error("Failed to read src file") diff --git a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go index 0b75ea52a..3e822dd02 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go @@ -28,13 +28,14 @@ import ( "time" log "github.com/Sirupsen/logrus" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/nfnt/resize" ) // GenerateThumbnails generates the configured thumbnail sizes for the source file -func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) { +func GenerateThumbnails(src types.Path, configs []config.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) { img, err := readFile(string(src)) if err != nil { logger.WithError(err).WithField("src", src).Error("Failed to read src file") @@ -42,7 +43,7 @@ func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMeta } for _, config := range configs { // Note: createThumbnail does locking based on activeThumbnailGeneration - busy, err = createThumbnail(src, img, config, mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger) + busy, err = createThumbnail(src, img, types.ThumbnailSize(config), mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger) if err != nil { logger.WithError(err).WithField("src", src).Error("Failed to generate thumbnails") return false, err diff --git a/src/github.com/matrix-org/dendrite/mediaapi/types/types.go b/src/github.com/matrix-org/dendrite/mediaapi/types/types.go index 3a0888f88..26b09e4f5 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/types/types.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/types/types.go @@ -17,6 +17,7 @@ package types import ( "sync" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -79,16 +80,7 @@ type ActiveRemoteRequests struct { } // ThumbnailSize contains a single thumbnail size configuration -type ThumbnailSize struct { - // Maximum width of the thumbnail image - Width int `yaml:"width"` - // Maximum height of the thumbnail image - Height int `yaml:"height"` - // ResizeMethod is one of crop or scale. - // crop scales to fill the requested dimensions and crops the excess. - // scale scales to fit the requested dimensions and one dimension may be smaller than requested. - ResizeMethod string `yaml:"method,omitempty"` -} +type ThumbnailSize config.ThumbnailSize // ThumbnailMetadata contains the metadata about an individual thumbnail type ThumbnailMetadata struct { diff --git a/src/github.com/matrix-org/dendrite/mediaapi/writers/download.go b/src/github.com/matrix-org/dendrite/mediaapi/writers/download.go index 90f6ca35b..8066327ca 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/writers/download.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/writers/download.go @@ -29,7 +29,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/mediaapi/config" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/fileutils" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/thumbnailer" @@ -59,7 +59,7 @@ type downloadRequest struct { // If they are present in the cache, they are served directly. // If they are not present in the cache, they are obtained from the remote server and // simultaneously served back to the client and written into the cache. -func Download(w http.ResponseWriter, req *http.Request, origin gomatrixserverlib.ServerName, mediaID types.MediaID, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, isThumbnailRequest bool) { +func Download(w http.ResponseWriter, req *http.Request, origin gomatrixserverlib.ServerName, mediaID types.MediaID, cfg *config.Dendrite, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, isThumbnailRequest bool) { r := &downloadRequest{ MediaMetadata: &types.MediaMetadata{ MediaID: mediaID, @@ -167,7 +167,7 @@ func (r *downloadRequest) Validate() *util.JSONResponse { return nil } -func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse { +func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.Dendrite, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse { // check if we have a record of the media in our database mediaMetadata, err := db.GetMediaMetadata(r.MediaMetadata.MediaID, r.MediaMetadata.Origin) if err != nil { @@ -176,7 +176,7 @@ func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI return &resErr } if mediaMetadata == nil { - if r.MediaMetadata.Origin == cfg.ServerName { + if r.MediaMetadata.Origin == cfg.Matrix.ServerName { // If we do not have a record and the origin is local, the file is not found return &util.JSONResponse{ Code: 404, @@ -192,12 +192,12 @@ func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI // If we have a record, we can respond from the local file r.MediaMetadata = mediaMetadata } - return r.respondFromLocalFile(w, cfg.AbsBasePath, activeThumbnailGeneration, cfg.MaxThumbnailGenerators, db, cfg.DynamicThumbnails, cfg.ThumbnailSizes) + return r.respondFromLocalFile(w, cfg.Media.AbsBasePath, activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators, db, cfg.Media.DynamicThumbnails, cfg.Media.ThumbnailSizes) } // respondFromLocalFile reads a file from local storage and writes it to the http.ResponseWriter // Returns a util.JSONResponse error in case of error -func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []types.ThumbnailSize) *util.JSONResponse { +func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePath config.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []config.ThumbnailSize) *util.JSONResponse { filePath, err := fileutils.GetPathFromBase64Hash(r.MediaMetadata.Base64Hash, absBasePath) if err != nil { r.Logger.WithError(err).Error("Failed to get file path from metadata") @@ -284,7 +284,7 @@ func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePat } // Note: Thumbnail generation may be ongoing asynchronously. -func (r *downloadRequest) getThumbnailFile(filePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []types.ThumbnailSize) (*os.File, *types.ThumbnailMetadata, *util.JSONResponse) { +func (r *downloadRequest) getThumbnailFile(filePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []config.ThumbnailSize) (*os.File, *types.ThumbnailMetadata, *util.JSONResponse) { var thumbnail *types.ThumbnailMetadata var resErr *util.JSONResponse @@ -383,7 +383,7 @@ func (r *downloadRequest) generateThumbnail(filePath types.Path, thumbnailSize t // regardless of how many download requests are received. // Note: The named errorResponse return variable is used in a deferred broadcast of the metadata and error response to waiting goroutines. // Returns a util.JSONResponse error in case of error -func (r *downloadRequest) getRemoteFile(cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) (errorResponse *util.JSONResponse) { +func (r *downloadRequest) getRemoteFile(cfg *config.Dendrite, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) (errorResponse *util.JSONResponse) { // Note: getMediaMetadataFromActiveRequest uses mutexes and conditions from activeRemoteRequests mediaMetadata, resErr := r.getMediaMetadataFromActiveRequest(activeRemoteRequests) if resErr != nil { @@ -414,7 +414,7 @@ func (r *downloadRequest) getRemoteFile(cfg *config.MediaAPI, db *storage.Databa if mediaMetadata == nil { // If we do not have a record, we need to fetch the remote file first and then respond from the local file - resErr := r.fetchRemoteFileAndStoreMetadata(cfg.AbsBasePath, *cfg.MaxFileSizeBytes, db, cfg.ThumbnailSizes, activeThumbnailGeneration, cfg.MaxThumbnailGenerators) + resErr := r.fetchRemoteFileAndStoreMetadata(cfg.Media.AbsBasePath, *cfg.Media.MaxFileSizeBytes, db, cfg.Media.ThumbnailSizes, activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators) if resErr != nil { return resErr } @@ -476,7 +476,7 @@ func (r *downloadRequest) broadcastMediaMetadata(activeRemoteRequests *types.Act } // fetchRemoteFileAndStoreMetadata fetches the file from the remote server and stores its metadata in the database -func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath types.Path, maxFileSizeBytes types.FileSizeBytes, db *storage.Database, thumbnailSizes []types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse { +func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath config.Path, maxFileSizeBytes config.FileSizeBytes, db *storage.Database, thumbnailSizes []config.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse { finalPath, duplicate, resErr := r.fetchRemoteFile(absBasePath, maxFileSizeBytes) if resErr != nil { return resErr @@ -524,7 +524,7 @@ func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath types.Path return nil } -func (r *downloadRequest) fetchRemoteFile(absBasePath types.Path, maxFileSizeBytes types.FileSizeBytes) (types.Path, bool, *util.JSONResponse) { +func (r *downloadRequest) fetchRemoteFile(absBasePath config.Path, maxFileSizeBytes config.FileSizeBytes) (types.Path, bool, *util.JSONResponse) { r.Logger.Info("Fetching remote file") // create request for remote file diff --git a/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go b/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go index 7071b4179..163ea7b68 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go @@ -24,7 +24,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/mediaapi/config" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/fileutils" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/thumbnailer" @@ -51,7 +51,7 @@ type uploadResponse struct { // This implementation supports a configurable maximum file size limit in bytes. If a user tries to upload more than this, they will receive an error that their upload is too large. // Uploaded files are processed piece-wise to avoid DoS attacks which would starve the server of memory. // TODO: We should time out requests if they have not received any data within a configured timeout period. -func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse { +func Upload(req *http.Request, cfg *config.Dendrite, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse { r, resErr := parseAndValidateRequest(req, cfg) if resErr != nil { return *resErr @@ -64,7 +64,7 @@ func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database, activ return util.JSONResponse{ Code: 200, JSON: uploadResponse{ - ContentURI: fmt.Sprintf("mxc://%s/%s", cfg.ServerName, r.MediaMetadata.MediaID), + ContentURI: fmt.Sprintf("mxc://%s/%s", cfg.Matrix.ServerName, r.MediaMetadata.MediaID), }, } } @@ -72,7 +72,7 @@ func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database, activ // parseAndValidateRequest parses the incoming upload request to validate and extract // all the metadata about the media being uploaded. // Returns either an uploadRequest or an error formatted as a util.JSONResponse -func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRequest, *util.JSONResponse) { +func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRequest, *util.JSONResponse) { if req.Method != "POST" { return nil, &util.JSONResponse{ Code: 405, @@ -82,22 +82,22 @@ func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRe r := &uploadRequest{ MediaMetadata: &types.MediaMetadata{ - Origin: cfg.ServerName, + Origin: cfg.Matrix.ServerName, FileSizeBytes: types.FileSizeBytes(req.ContentLength), ContentType: types.ContentType(req.Header.Get("Content-Type")), UploadName: types.Filename(url.PathEscape(req.FormValue("filename"))), }, - Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.ServerName), + Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.Matrix.ServerName), } - if resErr := r.Validate(*cfg.MaxFileSizeBytes); resErr != nil { + if resErr := r.Validate(*cfg.Media.MaxFileSizeBytes); resErr != nil { return nil, resErr } return r, nil } -func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse { +func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.Dendrite, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse { r.Logger.WithFields(log.Fields{ "UploadName": r.MediaMetadata.UploadName, "FileSizeBytes": r.MediaMetadata.FileSizeBytes, @@ -108,10 +108,10 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db * // method of deduplicating files to save storage, as well as a way to conduct // integrity checks on the file data in the repository. // Data is truncated to maxFileSizeBytes. Content-Length was reported as 0 < Content-Length <= maxFileSizeBytes so this is OK. - hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.MaxFileSizeBytes, cfg.AbsBasePath) + hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.Media.MaxFileSizeBytes, cfg.Media.AbsBasePath) if err != nil { r.Logger.WithError(err).WithFields(log.Fields{ - "MaxFileSizeBytes": *cfg.MaxFileSizeBytes, + "MaxFileSizeBytes": *cfg.Media.MaxFileSizeBytes, }).Warn("Error while transferring file") fileutils.RemoveDir(tmpDir, r.Logger) return &util.JSONResponse{ @@ -147,12 +147,12 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db * return &util.JSONResponse{ Code: 200, JSON: uploadResponse{ - ContentURI: fmt.Sprintf("mxc://%s/%s", cfg.ServerName, r.MediaMetadata.MediaID), + ContentURI: fmt.Sprintf("mxc://%s/%s", cfg.Matrix.ServerName, r.MediaMetadata.MediaID), }, } } - if resErr := r.storeFileAndMetadata(tmpDir, cfg.AbsBasePath, db, cfg.ThumbnailSizes, activeThumbnailGeneration, cfg.MaxThumbnailGenerators); resErr != nil { + if resErr := r.storeFileAndMetadata(tmpDir, cfg.Media.AbsBasePath, db, cfg.Media.ThumbnailSizes, activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators); resErr != nil { return resErr } @@ -160,14 +160,14 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db * } // Validate validates the uploadRequest fields -func (r *uploadRequest) Validate(maxFileSizeBytes types.FileSizeBytes) *util.JSONResponse { +func (r *uploadRequest) Validate(maxFileSizeBytes config.FileSizeBytes) *util.JSONResponse { if r.MediaMetadata.FileSizeBytes < 1 { return &util.JSONResponse{ Code: 411, JSON: jsonerror.Unknown("HTTP Content-Length request header must be greater than zero."), } } - if maxFileSizeBytes > 0 && r.MediaMetadata.FileSizeBytes > maxFileSizeBytes { + if maxFileSizeBytes > 0 && r.MediaMetadata.FileSizeBytes > types.FileSizeBytes(maxFileSizeBytes) { return &util.JSONResponse{ Code: 413, JSON: jsonerror.Unknown(fmt.Sprintf("HTTP Content-Length is greater than the maximum allowed upload size (%v).", maxFileSizeBytes)), @@ -215,7 +215,7 @@ func (r *uploadRequest) Validate(maxFileSizeBytes types.FileSizeBytes) *util.JSO // The order of operations is important as it avoids metadata entering the database before the file // is ready, and if we fail to move the file, it never gets added to the database. // Returns a util.JSONResponse error and cleans up directories in case of error. -func (r *uploadRequest) storeFileAndMetadata(tmpDir types.Path, absBasePath types.Path, db *storage.Database, thumbnailSizes []types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse { +func (r *uploadRequest) storeFileAndMetadata(tmpDir types.Path, absBasePath config.Path, db *storage.Database, thumbnailSizes []config.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse { finalPath, duplicate, err := fileutils.MoveFileWithHashCheck(tmpDir, r.MediaMetadata, absBasePath, r.Logger) if err != nil { r.Logger.WithError(err).Error("Failed to move file.")