mirror of
https://github.com/matrix-org/dendrite.git
synced 2025-12-10 16:33:11 -06:00
Replace media api config with common config
This commit is contained in:
Mark Haines
parent
ec66788d04
commit
928cde17e0
|
|
@ -16,34 +16,19 @@ package main
|
|||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/matrix-org/dendrite/common"
|
||||
"github.com/matrix-org/dendrite/mediaapi/config"
|
||||
"github.com/matrix-org/dendrite/common/config"
|
||||
"github.com/matrix-org/dendrite/mediaapi/routing"
|
||||
"github.com/matrix-org/dendrite/mediaapi/storage"
|
||||
"github.com/matrix-org/dendrite/mediaapi/types"
|
||||
"github.com/matrix-org/gomatrixserverlib"
|
||||
|
||||
log "github.com/Sirupsen/logrus"
|
||||
yaml "gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
var (
|
||||
bindAddr = flag.String("listen", "", "The port to listen on.")
|
||||
dataSource = os.Getenv("DATABASE")
|
||||
logDir = os.Getenv("LOG_DIR")
|
||||
serverName = os.Getenv("SERVER_NAME")
|
||||
basePath = os.Getenv("BASE_PATH")
|
||||
// Note: if the MAX_FILE_SIZE_BYTES is set to 0, it will be unlimited
|
||||
maxFileSizeBytesString = os.Getenv("MAX_FILE_SIZE_BYTES")
|
||||
configPath = flag.String("config", "", "The path to the config file. For more information, see the config file in this repository.")
|
||||
)
|
||||
|
||||
|
|
@ -52,206 +37,21 @@ func main() {
|
|||
|
||||
flag.Parse()
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"listen": *bindAddr,
|
||||
"DATABASE": dataSource,
|
||||
"LOG_DIR": logDir,
|
||||
"SERVER_NAME": serverName,
|
||||
"BASE_PATH": basePath,
|
||||
"MAX_FILE_SIZE_BYTES": maxFileSizeBytesString,
|
||||
"config": *configPath,
|
||||
}).Info("Loading configuration based on config file and environment variables")
|
||||
|
||||
cfg, err := configureServer()
|
||||
if *configPath == "" {
|
||||
log.Fatal("--config must be supplied")
|
||||
}
|
||||
cfg, err := config.Load(*configPath)
|
||||
if err != nil {
|
||||
log.WithError(err).Fatal("Invalid configuration")
|
||||
log.Fatalf("Invalid config file: %s", err)
|
||||
}
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"listen": *bindAddr,
|
||||
"LOG_DIR": logDir,
|
||||
"CONFIG_PATH": *configPath,
|
||||
"ServerName": cfg.ServerName,
|
||||
"AbsBasePath": cfg.AbsBasePath,
|
||||
"MaxFileSizeBytes": *cfg.MaxFileSizeBytes,
|
||||
"DataSource": cfg.DataSource,
|
||||
"DynamicThumbnails": cfg.DynamicThumbnails,
|
||||
"MaxThumbnailGenerators": cfg.MaxThumbnailGenerators,
|
||||
"ThumbnailSizes": cfg.ThumbnailSizes,
|
||||
}).Info("Starting mediaapi server with configuration")
|
||||
|
||||
db, err := storage.Open(cfg.DataSource)
|
||||
db, err := storage.Open(string(cfg.Database.MediaServer))
|
||||
if err != nil {
|
||||
log.WithError(err).Panic("Failed to open database")
|
||||
}
|
||||
|
||||
log.Info("Starting media API server on ", cfg.Listen.MediaAPI)
|
||||
|
||||
routing.Setup(http.DefaultServeMux, http.DefaultClient, cfg, db)
|
||||
log.Fatal(http.ListenAndServe(*bindAddr, nil))
|
||||
}
|
||||
|
||||
// configureServer loads configuration from a yaml file and overrides with environment variables
|
||||
func configureServer() (*config.MediaAPI, error) {
|
||||
if *configPath == "" {
|
||||
log.Fatal("--config must be supplied")
|
||||
}
|
||||
cfg, err := loadConfig(*configPath)
|
||||
if err != nil {
|
||||
log.WithError(err).Fatal("Invalid config file")
|
||||
}
|
||||
|
||||
// override values from environment variables
|
||||
applyOverrides(cfg)
|
||||
|
||||
if err := validateConfig(cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// FIXME: make common somehow? copied from sync api
|
||||
func loadConfig(configPath string) (*config.MediaAPI, error) {
|
||||
contents, err := ioutil.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var cfg config.MediaAPI
|
||||
if err = yaml.Unmarshal(contents, &cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
func applyOverrides(cfg *config.MediaAPI) {
|
||||
if serverName != "" {
|
||||
if cfg.ServerName != "" {
|
||||
log.WithFields(log.Fields{
|
||||
"server_name": cfg.ServerName,
|
||||
"SERVER_NAME": serverName,
|
||||
}).Info("Overriding server_name from config file with environment variable")
|
||||
}
|
||||
cfg.ServerName = gomatrixserverlib.ServerName(serverName)
|
||||
}
|
||||
if cfg.ServerName == "" {
|
||||
log.Info("ServerName not set. Defaulting to 'localhost'.")
|
||||
cfg.ServerName = "localhost"
|
||||
}
|
||||
|
||||
if basePath != "" {
|
||||
if cfg.BasePath != "" {
|
||||
log.WithFields(log.Fields{
|
||||
"base_path": cfg.BasePath,
|
||||
"BASE_PATH": basePath,
|
||||
}).Info("Overriding base_path from config file with environment variable")
|
||||
}
|
||||
cfg.BasePath = types.Path(basePath)
|
||||
}
|
||||
|
||||
if maxFileSizeBytesString != "" {
|
||||
if cfg.MaxFileSizeBytes != nil {
|
||||
log.WithFields(log.Fields{
|
||||
"max_file_size_bytes": *cfg.MaxFileSizeBytes,
|
||||
"MAX_FILE_SIZE_BYTES": maxFileSizeBytesString,
|
||||
}).Info("Overriding max_file_size_bytes from config file with environment variable")
|
||||
}
|
||||
maxFileSizeBytesInt, err := strconv.ParseInt(maxFileSizeBytesString, 10, 64)
|
||||
if err != nil {
|
||||
maxFileSizeBytesInt = 10 * 1024 * 1024
|
||||
log.WithError(err).WithField(
|
||||
"MAX_FILE_SIZE_BYTES", maxFileSizeBytesString,
|
||||
).Infof("MAX_FILE_SIZE_BYTES not set? Defaulting to %v bytes.", maxFileSizeBytesInt)
|
||||
}
|
||||
maxFileSizeBytes := types.FileSizeBytes(maxFileSizeBytesInt)
|
||||
cfg.MaxFileSizeBytes = &maxFileSizeBytes
|
||||
}
|
||||
|
||||
if dataSource != "" {
|
||||
if cfg.DataSource != "" {
|
||||
log.WithFields(log.Fields{
|
||||
"database": cfg.DataSource,
|
||||
"DATABASE": dataSource,
|
||||
}).Info("Overriding database from config file with environment variable")
|
||||
}
|
||||
cfg.DataSource = dataSource
|
||||
}
|
||||
|
||||
if cfg.MaxThumbnailGenerators == 0 {
|
||||
log.WithField(
|
||||
"max_thumbnail_generators", cfg.MaxThumbnailGenerators,
|
||||
).Info("Using default max_thumbnail_generators value of 10")
|
||||
cfg.MaxThumbnailGenerators = 10
|
||||
}
|
||||
}
|
||||
|
||||
func validateConfig(cfg *config.MediaAPI) error {
|
||||
if *bindAddr == "" {
|
||||
log.Fatal("--listen must be supplied")
|
||||
}
|
||||
|
||||
absBasePath, err := getAbsolutePath(cfg.BasePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid base path (%v): %q", cfg.BasePath, err)
|
||||
}
|
||||
cfg.AbsBasePath = types.Path(absBasePath)
|
||||
|
||||
if *cfg.MaxFileSizeBytes < 0 {
|
||||
return fmt.Errorf("invalid max file size bytes (%v)", *cfg.MaxFileSizeBytes)
|
||||
}
|
||||
|
||||
if cfg.DataSource == "" {
|
||||
return fmt.Errorf("invalid database (%v)", cfg.DataSource)
|
||||
}
|
||||
|
||||
for _, config := range cfg.ThumbnailSizes {
|
||||
if config.Width <= 0 || config.Height <= 0 {
|
||||
return fmt.Errorf("invalid thumbnail size %vx%v", config.Width, config.Height)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getAbsolutePath(basePath types.Path) (types.Path, error) {
|
||||
var err error
|
||||
if basePath == "" {
|
||||
var wd string
|
||||
wd, err = os.Getwd()
|
||||
return types.Path(wd), err
|
||||
}
|
||||
// Note: If we got here len(basePath) >= 1
|
||||
if basePath[0] == '~' {
|
||||
basePath, err = expandHomeDir(basePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
absBasePath, err := filepath.Abs(string(basePath))
|
||||
return types.Path(absBasePath), err
|
||||
}
|
||||
|
||||
// expandHomeDir parses paths beginning with ~/path or ~user/path and replaces the home directory part
|
||||
func expandHomeDir(basePath types.Path) (types.Path, error) {
|
||||
slash := strings.Index(string(basePath), "/")
|
||||
if slash == -1 {
|
||||
// pretend the slash is after the path as none was found within the string
|
||||
// simplifies code using slash below
|
||||
slash = len(basePath)
|
||||
}
|
||||
var usr *user.User
|
||||
var err error
|
||||
if slash == 1 {
|
||||
// basePath is ~ or ~/path
|
||||
usr, err = user.Current()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get user's home directory: %q", err)
|
||||
}
|
||||
} else {
|
||||
// slash > 1
|
||||
// basePath is ~user or ~user/path
|
||||
usr, err = user.Lookup(string(basePath[1:slash]))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get user's home directory: %q", err)
|
||||
}
|
||||
}
|
||||
return types.Path(filepath.Join(usr.HomeDir, string(basePath[slash:]))), nil
|
||||
log.Fatal(http.ListenAndServe(string(cfg.Listen.MediaAPI), nil))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -178,6 +178,8 @@ func loadConfig(
|
|||
return nil, err
|
||||
}
|
||||
|
||||
config.setDefaults()
|
||||
|
||||
if err = config.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -249,17 +251,31 @@ func (config *Dendrite) check() error {
|
|||
}
|
||||
}
|
||||
|
||||
checkNotZero := func(key string, value int) {
|
||||
checkNotZero := func(key string, value int64) {
|
||||
if value == 0 {
|
||||
problems = append(problems, fmt.Sprintf("missing config key %q", key))
|
||||
}
|
||||
}
|
||||
|
||||
checkPositive := func(key string, value int64) {
|
||||
if value < 0 {
|
||||
problems = append(problems, fmt.Sprintf("invalid value for config key %q: %d", key, value))
|
||||
}
|
||||
}
|
||||
|
||||
checkNotEmpty("matrix.server_name", string(config.Matrix.ServerName))
|
||||
checkNotEmpty("matrix.private_key", string(config.Matrix.PrivateKeyPath))
|
||||
checkNotZero("matrix.federation_certificates", len(config.Matrix.FederationCertificatePaths))
|
||||
checkNotZero("matrix.federation_certificates", int64(len(config.Matrix.FederationCertificatePaths)))
|
||||
|
||||
checkNotEmpty("media.base_path", string(config.Media.BasePath))
|
||||
checkNotZero("kafka.addresses", len(config.Kafka.Addresses))
|
||||
checkPositive("media.max_file_size_bytes", int64(*config.Media.MaxFileSizeBytes))
|
||||
checkPositive("media.max_thumbnail_generators", int64(config.Media.MaxThumbnailGenerators))
|
||||
for i, size := range config.Media.ThumbnailSizes {
|
||||
checkPositive(fmt.Sprintf("media.thumbnail_sizes[%d].width", i), int64(size.Width))
|
||||
checkPositive(fmt.Sprintf("media.thumbnail_sizes[%d].height", i), int64(size.Height))
|
||||
}
|
||||
|
||||
checkNotZero("kafka.addresses", int64(len(config.Kafka.Addresses)))
|
||||
checkNotEmpty("kafka.topics.input_room_event", string(config.Kafka.Topics.InputRoomEvent))
|
||||
checkNotEmpty("kafka.topics.output_room_event", string(config.Kafka.Topics.OutputRoomEvent))
|
||||
checkNotEmpty("database.media_server", string(config.Database.MediaServer))
|
||||
|
|
|
|||
|
|
@ -1,42 +0,0 @@
|
|||
// Copyright 2017 Vector Creations Ltd
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"github.com/matrix-org/dendrite/mediaapi/types"
|
||||
"github.com/matrix-org/gomatrixserverlib"
|
||||
)
|
||||
|
||||
// MediaAPI contains the config information necessary to spin up a mediaapi process.
|
||||
type MediaAPI struct {
|
||||
// The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
|
||||
ServerName gomatrixserverlib.ServerName `yaml:"server_name"`
|
||||
// The base path to where the media files will be stored. May be relative or absolute.
|
||||
BasePath types.Path `yaml:"base_path"`
|
||||
// The absolute base path to where media files will be stored.
|
||||
AbsBasePath types.Path `yaml:"-"`
|
||||
// The maximum file size in bytes that is allowed to be stored on this server.
|
||||
// Note: if MaxFileSizeBytes is set to 0, the size is unlimited.
|
||||
// Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB)
|
||||
MaxFileSizeBytes *types.FileSizeBytes `yaml:"max_file_size_bytes,omitempty"`
|
||||
// The postgres connection config for connecting to the database e.g a postgres:// URI
|
||||
DataSource string `yaml:"database"`
|
||||
// Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated
|
||||
DynamicThumbnails bool `yaml:"dynamic_thumbnails"`
|
||||
// The maximum number of simultaneous thumbnail generators. default: 10
|
||||
MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"`
|
||||
// A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content
|
||||
ThumbnailSizes []types.ThumbnailSize `yaml:"thumbnail_sizes"`
|
||||
}
|
||||
|
|
@ -26,13 +26,14 @@ import (
|
|||
"strings"
|
||||
|
||||
log "github.com/Sirupsen/logrus"
|
||||
"github.com/matrix-org/dendrite/common/config"
|
||||
"github.com/matrix-org/dendrite/mediaapi/types"
|
||||
)
|
||||
|
||||
// GetPathFromBase64Hash evaluates the path to a media file from its Base64Hash
|
||||
// 3 subdirectories are created for more manageable browsing and use the remainder as the file name.
|
||||
// For example, if Base64Hash is 'qwerty', the path will be 'q/w/erty/file'.
|
||||
func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath types.Path) (string, error) {
|
||||
func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath config.Path) (string, error) {
|
||||
if len(base64Hash) < 3 {
|
||||
return "", fmt.Errorf("Invalid filePath (Base64Hash too short - min 3 characters): %q", base64Hash)
|
||||
}
|
||||
|
|
@ -66,7 +67,7 @@ func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath types.Path)
|
|||
// If the final path exists and the file size matches, the file does not need to be moved.
|
||||
// In error cases where the file is not a duplicate, the caller may decide to remove the final path.
|
||||
// Returns the final path of the file, whether it is a duplicate and an error.
|
||||
func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata, absBasePath types.Path, logger *log.Entry) (types.Path, bool, error) {
|
||||
func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata, absBasePath config.Path, logger *log.Entry) (types.Path, bool, error) {
|
||||
// Note: in all error and success cases, we need to remove the temporary directory
|
||||
defer RemoveDir(tmpDir, logger)
|
||||
duplicate := false
|
||||
|
|
@ -104,7 +105,7 @@ func RemoveDir(dir types.Path, logger *log.Entry) {
|
|||
}
|
||||
|
||||
// WriteTempFile writes to a new temporary file
|
||||
func WriteTempFile(reqReader io.Reader, maxFileSizeBytes types.FileSizeBytes, absBasePath types.Path) (types.Base64Hash, types.FileSizeBytes, types.Path, error) {
|
||||
func WriteTempFile(reqReader io.Reader, maxFileSizeBytes config.FileSizeBytes, absBasePath config.Path) (types.Base64Hash, types.FileSizeBytes, types.Path, error) {
|
||||
tmpFileWriter, tmpFile, tmpDir, err := createTempFileWriter(absBasePath)
|
||||
if err != nil {
|
||||
return "", -1, "", err
|
||||
|
|
@ -144,7 +145,7 @@ func moveFile(src types.Path, dst types.Path) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func createTempFileWriter(absBasePath types.Path) (*bufio.Writer, *os.File, types.Path, error) {
|
||||
func createTempFileWriter(absBasePath config.Path) (*bufio.Writer, *os.File, types.Path, error) {
|
||||
tmpDir, err := createTempDir(absBasePath)
|
||||
if err != nil {
|
||||
return nil, nil, "", fmt.Errorf("Failed to create temp dir: %q", err)
|
||||
|
|
@ -157,7 +158,7 @@ func createTempFileWriter(absBasePath types.Path) (*bufio.Writer, *os.File, type
|
|||
}
|
||||
|
||||
// createTempDir creates a tmp/<random string> directory within baseDirectory and returns its path
|
||||
func createTempDir(baseDirectory types.Path) (types.Path, error) {
|
||||
func createTempDir(baseDirectory config.Path) (types.Path, error) {
|
||||
baseTmpDir := filepath.Join(string(baseDirectory), "tmp")
|
||||
if err := os.MkdirAll(baseTmpDir, 0770); err != nil {
|
||||
return "", fmt.Errorf("Failed to create base temp dir: %v", err)
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import (
|
|||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/matrix-org/dendrite/common"
|
||||
"github.com/matrix-org/dendrite/mediaapi/config"
|
||||
"github.com/matrix-org/dendrite/common/config"
|
||||
"github.com/matrix-org/dendrite/mediaapi/storage"
|
||||
"github.com/matrix-org/dendrite/mediaapi/types"
|
||||
"github.com/matrix-org/dendrite/mediaapi/writers"
|
||||
|
|
@ -32,7 +32,7 @@ const pathPrefixR0 = "/_matrix/media/v1"
|
|||
|
||||
// Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
|
||||
// to clients which need to make outbound HTTP requests.
|
||||
func Setup(servMux *http.ServeMux, httpClient *http.Client, cfg *config.MediaAPI, db *storage.Database) {
|
||||
func Setup(servMux *http.ServeMux, httpClient *http.Client, cfg *config.Dendrite, db *storage.Database) {
|
||||
apiMux := mux.NewRouter()
|
||||
r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
|
||||
|
||||
|
|
@ -59,7 +59,7 @@ func Setup(servMux *http.ServeMux, httpClient *http.Client, cfg *config.MediaAPI
|
|||
servMux.Handle("/api/", http.StripPrefix("/api", apiMux))
|
||||
}
|
||||
|
||||
func makeDownloadAPI(name string, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) http.HandlerFunc {
|
||||
func makeDownloadAPI(name string, cfg *config.Dendrite, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) http.HandlerFunc {
|
||||
return prometheus.InstrumentHandler(name, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
req = util.RequestWithLogging(req)
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import (
|
|||
"sync"
|
||||
|
||||
log "github.com/Sirupsen/logrus"
|
||||
"github.com/matrix-org/dendrite/common/config"
|
||||
"github.com/matrix-org/dendrite/mediaapi/storage"
|
||||
"github.com/matrix-org/dendrite/mediaapi/types"
|
||||
)
|
||||
|
|
@ -56,7 +57,7 @@ func GetThumbnailPath(src types.Path, config types.ThumbnailSize) types.Path {
|
|||
// * has a small file size
|
||||
// If a pre-generated thumbnail size is the best match, but it has not been generated yet, the caller can use the returned size to generate it.
|
||||
// Returns nil if no thumbnail matches the criteria
|
||||
func SelectThumbnail(desired types.ThumbnailSize, thumbnails []*types.ThumbnailMetadata, thumbnailSizes []types.ThumbnailSize) (*types.ThumbnailMetadata, *types.ThumbnailSize) {
|
||||
func SelectThumbnail(desired types.ThumbnailSize, thumbnails []*types.ThumbnailMetadata, thumbnailSizes []config.ThumbnailSize) (*types.ThumbnailMetadata, *types.ThumbnailSize) {
|
||||
var chosenThumbnail *types.ThumbnailMetadata
|
||||
var chosenThumbnailSize *types.ThumbnailSize
|
||||
bestFit := newThumbnailFitness()
|
||||
|
|
@ -76,7 +77,7 @@ func SelectThumbnail(desired types.ThumbnailSize, thumbnails []*types.ThumbnailM
|
|||
if desired.ResizeMethod == "scale" && thumbnailSize.ResizeMethod != "scale" {
|
||||
continue
|
||||
}
|
||||
fitness := calcThumbnailFitness(thumbnailSize, nil, desired)
|
||||
fitness := calcThumbnailFitness(types.ThumbnailSize(thumbnailSize), nil, desired)
|
||||
if isBetter := fitness.betterThan(bestFit, desired.ResizeMethod == "crop"); isBetter {
|
||||
bestFit = fitness
|
||||
chosenThumbnailSize = &types.ThumbnailSize{
|
||||
|
|
|
|||
|
|
@ -21,13 +21,14 @@ import (
|
|||
"time"
|
||||
|
||||
log "github.com/Sirupsen/logrus"
|
||||
"github.com/matrix-org/dendrite/common/config"
|
||||
"github.com/matrix-org/dendrite/mediaapi/storage"
|
||||
"github.com/matrix-org/dendrite/mediaapi/types"
|
||||
"gopkg.in/h2non/bimg.v1"
|
||||
)
|
||||
|
||||
// GenerateThumbnails generates the configured thumbnail sizes for the source file
|
||||
func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
|
||||
func GenerateThumbnails(src types.Path, configs []config.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
|
||||
buffer, err := bimg.Read(string(src))
|
||||
if err != nil {
|
||||
logger.WithError(err).WithField("src", src).Error("Failed to read src file")
|
||||
|
|
|
|||
|
|
@ -28,13 +28,14 @@ import (
|
|||
"time"
|
||||
|
||||
log "github.com/Sirupsen/logrus"
|
||||
"github.com/matrix-org/dendrite/common/config"
|
||||
"github.com/matrix-org/dendrite/mediaapi/storage"
|
||||
"github.com/matrix-org/dendrite/mediaapi/types"
|
||||
"github.com/nfnt/resize"
|
||||
)
|
||||
|
||||
// GenerateThumbnails generates the configured thumbnail sizes for the source file
|
||||
func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
|
||||
func GenerateThumbnails(src types.Path, configs []config.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
|
||||
img, err := readFile(string(src))
|
||||
if err != nil {
|
||||
logger.WithError(err).WithField("src", src).Error("Failed to read src file")
|
||||
|
|
@ -42,7 +43,7 @@ func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMeta
|
|||
}
|
||||
for _, config := range configs {
|
||||
// Note: createThumbnail does locking based on activeThumbnailGeneration
|
||||
busy, err = createThumbnail(src, img, config, mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger)
|
||||
busy, err = createThumbnail(src, img, types.ThumbnailSize(config), mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger)
|
||||
if err != nil {
|
||||
logger.WithError(err).WithField("src", src).Error("Failed to generate thumbnails")
|
||||
return false, err
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ package types
|
|||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/matrix-org/dendrite/common/config"
|
||||
"github.com/matrix-org/gomatrixserverlib"
|
||||
"github.com/matrix-org/util"
|
||||
)
|
||||
|
|
@ -79,16 +80,7 @@ type ActiveRemoteRequests struct {
|
|||
}
|
||||
|
||||
// ThumbnailSize contains a single thumbnail size configuration
|
||||
type ThumbnailSize struct {
|
||||
// Maximum width of the thumbnail image
|
||||
Width int `yaml:"width"`
|
||||
// Maximum height of the thumbnail image
|
||||
Height int `yaml:"height"`
|
||||
// ResizeMethod is one of crop or scale.
|
||||
// crop scales to fill the requested dimensions and crops the excess.
|
||||
// scale scales to fit the requested dimensions and one dimension may be smaller than requested.
|
||||
ResizeMethod string `yaml:"method,omitempty"`
|
||||
}
|
||||
type ThumbnailSize config.ThumbnailSize
|
||||
|
||||
// ThumbnailMetadata contains the metadata about an individual thumbnail
|
||||
type ThumbnailMetadata struct {
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ import (
|
|||
|
||||
log "github.com/Sirupsen/logrus"
|
||||
"github.com/matrix-org/dendrite/clientapi/jsonerror"
|
||||
"github.com/matrix-org/dendrite/mediaapi/config"
|
||||
"github.com/matrix-org/dendrite/common/config"
|
||||
"github.com/matrix-org/dendrite/mediaapi/fileutils"
|
||||
"github.com/matrix-org/dendrite/mediaapi/storage"
|
||||
"github.com/matrix-org/dendrite/mediaapi/thumbnailer"
|
||||
|
|
@ -59,7 +59,7 @@ type downloadRequest struct {
|
|||
// If they are present in the cache, they are served directly.
|
||||
// If they are not present in the cache, they are obtained from the remote server and
|
||||
// simultaneously served back to the client and written into the cache.
|
||||
func Download(w http.ResponseWriter, req *http.Request, origin gomatrixserverlib.ServerName, mediaID types.MediaID, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, isThumbnailRequest bool) {
|
||||
func Download(w http.ResponseWriter, req *http.Request, origin gomatrixserverlib.ServerName, mediaID types.MediaID, cfg *config.Dendrite, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, isThumbnailRequest bool) {
|
||||
r := &downloadRequest{
|
||||
MediaMetadata: &types.MediaMetadata{
|
||||
MediaID: mediaID,
|
||||
|
|
@ -167,7 +167,7 @@ func (r *downloadRequest) Validate() *util.JSONResponse {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse {
|
||||
func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.Dendrite, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse {
|
||||
// check if we have a record of the media in our database
|
||||
mediaMetadata, err := db.GetMediaMetadata(r.MediaMetadata.MediaID, r.MediaMetadata.Origin)
|
||||
if err != nil {
|
||||
|
|
@ -176,7 +176,7 @@ func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI
|
|||
return &resErr
|
||||
}
|
||||
if mediaMetadata == nil {
|
||||
if r.MediaMetadata.Origin == cfg.ServerName {
|
||||
if r.MediaMetadata.Origin == cfg.Matrix.ServerName {
|
||||
// If we do not have a record and the origin is local, the file is not found
|
||||
return &util.JSONResponse{
|
||||
Code: 404,
|
||||
|
|
@ -192,12 +192,12 @@ func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI
|
|||
// If we have a record, we can respond from the local file
|
||||
r.MediaMetadata = mediaMetadata
|
||||
}
|
||||
return r.respondFromLocalFile(w, cfg.AbsBasePath, activeThumbnailGeneration, cfg.MaxThumbnailGenerators, db, cfg.DynamicThumbnails, cfg.ThumbnailSizes)
|
||||
return r.respondFromLocalFile(w, cfg.Media.AbsBasePath, activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators, db, cfg.Media.DynamicThumbnails, cfg.Media.ThumbnailSizes)
|
||||
}
|
||||
|
||||
// respondFromLocalFile reads a file from local storage and writes it to the http.ResponseWriter
|
||||
// Returns a util.JSONResponse error in case of error
|
||||
func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []types.ThumbnailSize) *util.JSONResponse {
|
||||
func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePath config.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []config.ThumbnailSize) *util.JSONResponse {
|
||||
filePath, err := fileutils.GetPathFromBase64Hash(r.MediaMetadata.Base64Hash, absBasePath)
|
||||
if err != nil {
|
||||
r.Logger.WithError(err).Error("Failed to get file path from metadata")
|
||||
|
|
@ -284,7 +284,7 @@ func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePat
|
|||
}
|
||||
|
||||
// Note: Thumbnail generation may be ongoing asynchronously.
|
||||
func (r *downloadRequest) getThumbnailFile(filePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []types.ThumbnailSize) (*os.File, *types.ThumbnailMetadata, *util.JSONResponse) {
|
||||
func (r *downloadRequest) getThumbnailFile(filePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []config.ThumbnailSize) (*os.File, *types.ThumbnailMetadata, *util.JSONResponse) {
|
||||
var thumbnail *types.ThumbnailMetadata
|
||||
var resErr *util.JSONResponse
|
||||
|
||||
|
|
@ -383,7 +383,7 @@ func (r *downloadRequest) generateThumbnail(filePath types.Path, thumbnailSize t
|
|||
// regardless of how many download requests are received.
|
||||
// Note: The named errorResponse return variable is used in a deferred broadcast of the metadata and error response to waiting goroutines.
|
||||
// Returns a util.JSONResponse error in case of error
|
||||
func (r *downloadRequest) getRemoteFile(cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) (errorResponse *util.JSONResponse) {
|
||||
func (r *downloadRequest) getRemoteFile(cfg *config.Dendrite, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) (errorResponse *util.JSONResponse) {
|
||||
// Note: getMediaMetadataFromActiveRequest uses mutexes and conditions from activeRemoteRequests
|
||||
mediaMetadata, resErr := r.getMediaMetadataFromActiveRequest(activeRemoteRequests)
|
||||
if resErr != nil {
|
||||
|
|
@ -414,7 +414,7 @@ func (r *downloadRequest) getRemoteFile(cfg *config.MediaAPI, db *storage.Databa
|
|||
|
||||
if mediaMetadata == nil {
|
||||
// If we do not have a record, we need to fetch the remote file first and then respond from the local file
|
||||
resErr := r.fetchRemoteFileAndStoreMetadata(cfg.AbsBasePath, *cfg.MaxFileSizeBytes, db, cfg.ThumbnailSizes, activeThumbnailGeneration, cfg.MaxThumbnailGenerators)
|
||||
resErr := r.fetchRemoteFileAndStoreMetadata(cfg.Media.AbsBasePath, *cfg.Media.MaxFileSizeBytes, db, cfg.Media.ThumbnailSizes, activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators)
|
||||
if resErr != nil {
|
||||
return resErr
|
||||
}
|
||||
|
|
@ -476,7 +476,7 @@ func (r *downloadRequest) broadcastMediaMetadata(activeRemoteRequests *types.Act
|
|||
}
|
||||
|
||||
// fetchRemoteFileAndStoreMetadata fetches the file from the remote server and stores its metadata in the database
|
||||
func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath types.Path, maxFileSizeBytes types.FileSizeBytes, db *storage.Database, thumbnailSizes []types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse {
|
||||
func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath config.Path, maxFileSizeBytes config.FileSizeBytes, db *storage.Database, thumbnailSizes []config.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse {
|
||||
finalPath, duplicate, resErr := r.fetchRemoteFile(absBasePath, maxFileSizeBytes)
|
||||
if resErr != nil {
|
||||
return resErr
|
||||
|
|
@ -524,7 +524,7 @@ func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath types.Path
|
|||
return nil
|
||||
}
|
||||
|
||||
func (r *downloadRequest) fetchRemoteFile(absBasePath types.Path, maxFileSizeBytes types.FileSizeBytes) (types.Path, bool, *util.JSONResponse) {
|
||||
func (r *downloadRequest) fetchRemoteFile(absBasePath config.Path, maxFileSizeBytes config.FileSizeBytes) (types.Path, bool, *util.JSONResponse) {
|
||||
r.Logger.Info("Fetching remote file")
|
||||
|
||||
// create request for remote file
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ import (
|
|||
|
||||
log "github.com/Sirupsen/logrus"
|
||||
"github.com/matrix-org/dendrite/clientapi/jsonerror"
|
||||
"github.com/matrix-org/dendrite/mediaapi/config"
|
||||
"github.com/matrix-org/dendrite/common/config"
|
||||
"github.com/matrix-org/dendrite/mediaapi/fileutils"
|
||||
"github.com/matrix-org/dendrite/mediaapi/storage"
|
||||
"github.com/matrix-org/dendrite/mediaapi/thumbnailer"
|
||||
|
|
@ -51,7 +51,7 @@ type uploadResponse struct {
|
|||
// This implementation supports a configurable maximum file size limit in bytes. If a user tries to upload more than this, they will receive an error that their upload is too large.
|
||||
// Uploaded files are processed piece-wise to avoid DoS attacks which would starve the server of memory.
|
||||
// TODO: We should time out requests if they have not received any data within a configured timeout period.
|
||||
func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse {
|
||||
func Upload(req *http.Request, cfg *config.Dendrite, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse {
|
||||
r, resErr := parseAndValidateRequest(req, cfg)
|
||||
if resErr != nil {
|
||||
return *resErr
|
||||
|
|
@ -64,7 +64,7 @@ func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database, activ
|
|||
return util.JSONResponse{
|
||||
Code: 200,
|
||||
JSON: uploadResponse{
|
||||
ContentURI: fmt.Sprintf("mxc://%s/%s", cfg.ServerName, r.MediaMetadata.MediaID),
|
||||
ContentURI: fmt.Sprintf("mxc://%s/%s", cfg.Matrix.ServerName, r.MediaMetadata.MediaID),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
@ -72,7 +72,7 @@ func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database, activ
|
|||
// parseAndValidateRequest parses the incoming upload request to validate and extract
|
||||
// all the metadata about the media being uploaded.
|
||||
// Returns either an uploadRequest or an error formatted as a util.JSONResponse
|
||||
func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRequest, *util.JSONResponse) {
|
||||
func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRequest, *util.JSONResponse) {
|
||||
if req.Method != "POST" {
|
||||
return nil, &util.JSONResponse{
|
||||
Code: 405,
|
||||
|
|
@ -82,22 +82,22 @@ func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRe
|
|||
|
||||
r := &uploadRequest{
|
||||
MediaMetadata: &types.MediaMetadata{
|
||||
Origin: cfg.ServerName,
|
||||
Origin: cfg.Matrix.ServerName,
|
||||
FileSizeBytes: types.FileSizeBytes(req.ContentLength),
|
||||
ContentType: types.ContentType(req.Header.Get("Content-Type")),
|
||||
UploadName: types.Filename(url.PathEscape(req.FormValue("filename"))),
|
||||
},
|
||||
Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.ServerName),
|
||||
Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.Matrix.ServerName),
|
||||
}
|
||||
|
||||
if resErr := r.Validate(*cfg.MaxFileSizeBytes); resErr != nil {
|
||||
if resErr := r.Validate(*cfg.Media.MaxFileSizeBytes); resErr != nil {
|
||||
return nil, resErr
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse {
|
||||
func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.Dendrite, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse {
|
||||
r.Logger.WithFields(log.Fields{
|
||||
"UploadName": r.MediaMetadata.UploadName,
|
||||
"FileSizeBytes": r.MediaMetadata.FileSizeBytes,
|
||||
|
|
@ -108,10 +108,10 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db *
|
|||
// method of deduplicating files to save storage, as well as a way to conduct
|
||||
// integrity checks on the file data in the repository.
|
||||
// Data is truncated to maxFileSizeBytes. Content-Length was reported as 0 < Content-Length <= maxFileSizeBytes so this is OK.
|
||||
hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.MaxFileSizeBytes, cfg.AbsBasePath)
|
||||
hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.Media.MaxFileSizeBytes, cfg.Media.AbsBasePath)
|
||||
if err != nil {
|
||||
r.Logger.WithError(err).WithFields(log.Fields{
|
||||
"MaxFileSizeBytes": *cfg.MaxFileSizeBytes,
|
||||
"MaxFileSizeBytes": *cfg.Media.MaxFileSizeBytes,
|
||||
}).Warn("Error while transferring file")
|
||||
fileutils.RemoveDir(tmpDir, r.Logger)
|
||||
return &util.JSONResponse{
|
||||
|
|
@ -147,12 +147,12 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db *
|
|||
return &util.JSONResponse{
|
||||
Code: 200,
|
||||
JSON: uploadResponse{
|
||||
ContentURI: fmt.Sprintf("mxc://%s/%s", cfg.ServerName, r.MediaMetadata.MediaID),
|
||||
ContentURI: fmt.Sprintf("mxc://%s/%s", cfg.Matrix.ServerName, r.MediaMetadata.MediaID),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if resErr := r.storeFileAndMetadata(tmpDir, cfg.AbsBasePath, db, cfg.ThumbnailSizes, activeThumbnailGeneration, cfg.MaxThumbnailGenerators); resErr != nil {
|
||||
if resErr := r.storeFileAndMetadata(tmpDir, cfg.Media.AbsBasePath, db, cfg.Media.ThumbnailSizes, activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators); resErr != nil {
|
||||
return resErr
|
||||
}
|
||||
|
||||
|
|
@ -160,14 +160,14 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db *
|
|||
}
|
||||
|
||||
// Validate validates the uploadRequest fields
|
||||
func (r *uploadRequest) Validate(maxFileSizeBytes types.FileSizeBytes) *util.JSONResponse {
|
||||
func (r *uploadRequest) Validate(maxFileSizeBytes config.FileSizeBytes) *util.JSONResponse {
|
||||
if r.MediaMetadata.FileSizeBytes < 1 {
|
||||
return &util.JSONResponse{
|
||||
Code: 411,
|
||||
JSON: jsonerror.Unknown("HTTP Content-Length request header must be greater than zero."),
|
||||
}
|
||||
}
|
||||
if maxFileSizeBytes > 0 && r.MediaMetadata.FileSizeBytes > maxFileSizeBytes {
|
||||
if maxFileSizeBytes > 0 && r.MediaMetadata.FileSizeBytes > types.FileSizeBytes(maxFileSizeBytes) {
|
||||
return &util.JSONResponse{
|
||||
Code: 413,
|
||||
JSON: jsonerror.Unknown(fmt.Sprintf("HTTP Content-Length is greater than the maximum allowed upload size (%v).", maxFileSizeBytes)),
|
||||
|
|
@ -215,7 +215,7 @@ func (r *uploadRequest) Validate(maxFileSizeBytes types.FileSizeBytes) *util.JSO
|
|||
// The order of operations is important as it avoids metadata entering the database before the file
|
||||
// is ready, and if we fail to move the file, it never gets added to the database.
|
||||
// Returns a util.JSONResponse error and cleans up directories in case of error.
|
||||
func (r *uploadRequest) storeFileAndMetadata(tmpDir types.Path, absBasePath types.Path, db *storage.Database, thumbnailSizes []types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse {
|
||||
func (r *uploadRequest) storeFileAndMetadata(tmpDir types.Path, absBasePath config.Path, db *storage.Database, thumbnailSizes []config.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse {
|
||||
finalPath, duplicate, err := fileutils.MoveFileWithHashCheck(tmpDir, r.MediaMetadata, absBasePath, r.Logger)
|
||||
if err != nil {
|
||||
r.Logger.WithError(err).Error("Failed to move file.")
|
||||
|
|
|
|||
Loading…
Reference in a new issue