mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2025-12-08 15:33:09 -06:00
Code Issues Packages Projects Releases Wiki Activity

mediaapi/thumbnailer: Limit number of parallel generators

Browse source 
Fall back to selecting from already-/pre-generated thumbnails or serving
the original.
This commit is contained in:
Robert Swain 2017-06-06 00:49:15 +02:00
parent 4afc4bd3eb
commit d40f54dc9d
5 changed files with 93 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go
View file

@ -70,15 +70,16 @@ func main() {
} }
log.WithFields(log.Fields{ log.WithFields(log.Fields{
"BIND_ADDRESS": bindAddr, "BIND_ADDRESS": bindAddr,
"LOG_DIR": logDir, "LOG_DIR": logDir,
"CONFIG_PATH": configPath, "CONFIG_PATH": configPath,
"ServerName": cfg.ServerName, "ServerName": cfg.ServerName,
"AbsBasePath": cfg.AbsBasePath, "AbsBasePath": cfg.AbsBasePath,
"MaxFileSizeBytes": *cfg.MaxFileSizeBytes, "MaxFileSizeBytes": *cfg.MaxFileSizeBytes,
"DataSource": cfg.DataSource, "DataSource": cfg.DataSource,
"DynamicThumbnails": cfg.DynamicThumbnails, "DynamicThumbnails": cfg.DynamicThumbnails,
"ThumbnailSizes": cfg.ThumbnailSizes, "MaxThumbnailGenerators": cfg.MaxThumbnailGenerators,
"ThumbnailSizes": cfg.ThumbnailSizes,
}).Info("Starting mediaapi server with configuration") }).Info("Starting mediaapi server with configuration")
routing.Setup(http.DefaultServeMux, http.DefaultClient, cfg, db) routing.Setup(http.DefaultServeMux, http.DefaultClient, cfg, db)
@ -167,6 +168,13 @@ func applyOverrides(cfg *config.MediaAPI) {
} }
cfg.DataSource = dataSource cfg.DataSource = dataSource
} }
if cfg.MaxThumbnailGenerators == 0 {
log.WithField(
"max_thumbnail_generators", cfg.MaxThumbnailGenerators,
).Info("Using default max_thumbnail_generators")
cfg.MaxThumbnailGenerators = 10
}
} }
func validateConfig(cfg *config.MediaAPI) error { func validateConfig(cfg *config.MediaAPI) error {

2
src/github.com/matrix-org/dendrite/mediaapi/config/config.go
View file

@ -35,6 +35,8 @@ type MediaAPI struct {
DataSource string `yaml:"database"` DataSource string `yaml:"database"`
// Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated // Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated
DynamicThumbnails bool `yaml:"dynamic_thumbnails"` DynamicThumbnails bool `yaml:"dynamic_thumbnails"`
// The maximum number of simultaneous thumbnail generators. default: 10
MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"`
// A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content // A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content
ThumbnailSizes []types.ThumbnailSize `yaml:"thumbnail_sizes"` ThumbnailSizes []types.ThumbnailSize `yaml:"thumbnail_sizes"`
} }

68
src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go
View file

@ -40,39 +40,47 @@ type thumbnailFitness struct {
const thumbnailTemplate = "thumbnail-%vx%v-%v" const thumbnailTemplate = "thumbnail-%vx%v-%v"
// GenerateThumbnails generates the configured thumbnail sizes for the source file // GenerateThumbnails generates the configured thumbnail sizes for the source file
func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, db *storage.Database, logger *log.Entry) error { func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
buffer, err := bimg.Read(string(src)) buffer, err := bimg.Read(string(src))
if err != nil { if err != nil {
logger.WithError(err).WithField("src", src).Error("Failed to read src file") logger.WithError(err).WithField("src", src).Error("Failed to read src file")
return err return false, err
} }
for _, config := range configs { for _, config := range configs {
// Note: createThumbnail does locking based on activeThumbnailGeneration // Note: createThumbnail does locking based on activeThumbnailGeneration
if err = createThumbnail(src, buffer, config, mediaMetadata, activeThumbnailGeneration, db, logger); err != nil { busy, err = createThumbnail(src, buffer, config, mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger)
if err != nil {
logger.WithError(err).WithField("src", src).Error("Failed to generate thumbnails") logger.WithError(err).WithField("src", src).Error("Failed to generate thumbnails")
return err return false, err
}
if busy {
return true, nil
} }
} }
return nil return false, nil
} }
// GenerateThumbnail generates the configured thumbnail size for the source file // GenerateThumbnail generates the configured thumbnail size for the source file
func GenerateThumbnail(src types.Path, config types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, db *storage.Database, logger *log.Entry) error { func GenerateThumbnail(src types.Path, config types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
buffer, err := bimg.Read(string(src)) buffer, err := bimg.Read(string(src))
if err != nil { if err != nil {
logger.WithError(err).WithFields(log.Fields{ logger.WithError(err).WithFields(log.Fields{
"src": src, "src": src,
}).Error("Failed to read src file") }).Error("Failed to read src file")
return err return false, err
} }
// Note: createThumbnail does locking based on activeThumbnailGeneration // Note: createThumbnail does locking based on activeThumbnailGeneration
if err = createThumbnail(src, buffer, config, mediaMetadata, activeThumbnailGeneration, db, logger); err != nil { busy, err = createThumbnail(src, buffer, config, mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger)
if err != nil {
logger.WithError(err).WithFields(log.Fields{ logger.WithError(err).WithFields(log.Fields{
"src": src, "src": src,
}).Error("Failed to generate thumbnails") }).Error("Failed to generate thumbnails")
return err return false, err
} }
return nil if busy {
return true, nil
}
return false, nil
} }
// GetThumbnailPath returns the path to a thumbnail given the absolute src path and thumbnail size configuration // GetThumbnailPath returns the path to a thumbnail given the absolute src path and thumbnail size configuration
@ -130,7 +138,7 @@ func SelectThumbnail(desired types.ThumbnailSize, thumbnails []*types.ThumbnailM
// createThumbnail checks if the thumbnail exists, and if not, generates it // createThumbnail checks if the thumbnail exists, and if not, generates it
// Thumbnail generation is only done once for each non-existing thumbnail. // Thumbnail generation is only done once for each non-existing thumbnail.
func createThumbnail(src types.Path, buffer []byte, config types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, db *storage.Database, logger *log.Entry) (errorReturn error) { func createThumbnail(src types.Path, buffer []byte, config types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
logger = logger.WithFields(log.Fields{ logger = logger.WithFields(log.Fields{
"Width": config.Width, "Width": config.Width,
"Height": config.Height, "Height": config.Height,
@ -140,9 +148,12 @@ func createThumbnail(src types.Path, buffer []byte, config types.ThumbnailSize,
dst := GetThumbnailPath(src, config) dst := GetThumbnailPath(src, config)
// Note: getActiveThumbnailGeneration uses mutexes and conditions from activeThumbnailGeneration // Note: getActiveThumbnailGeneration uses mutexes and conditions from activeThumbnailGeneration
isActive, err := getActiveThumbnailGeneration(dst, config, activeThumbnailGeneration, logger) isActive, busy, err := getActiveThumbnailGeneration(dst, config, activeThumbnailGeneration, maxThumbnailGenerators, logger)
if err != nil { if err != nil {
return err return false, err
}
if busy {
return true, nil
} }
if isActive { if isActive {
@ -162,28 +173,28 @@ func createThumbnail(src types.Path, buffer []byte, config types.ThumbnailSize,
thumbnailMetadata, err := db.GetThumbnail(mediaMetadata.MediaID, mediaMetadata.Origin, config.Width, config.Height, config.ResizeMethod) thumbnailMetadata, err := db.GetThumbnail(mediaMetadata.MediaID, mediaMetadata.Origin, config.Width, config.Height, config.ResizeMethod)
if err != nil { if err != nil {
logger.Error("Failed to query database for thumbnail.") logger.Error("Failed to query database for thumbnail.")
return err return false, err
} }
if thumbnailMetadata != nil { if thumbnailMetadata != nil {
return nil return false, nil
} }
// Note: The double-negative is intentional as os.IsExist(err) != !os.IsNotExist(err). // Note: The double-negative is intentional as os.IsExist(err) != !os.IsNotExist(err).
// The functions are error checkers to be used in different cases. // The functions are error checkers to be used in different cases.
if _, err = os.Stat(string(dst)); !os.IsNotExist(err) { if _, err = os.Stat(string(dst)); !os.IsNotExist(err) {
// Thumbnail exists // Thumbnail exists
return nil return false, nil
} }
if isActive == false { if isActive == false {
// Note: This should not happen, but we check just in case. // Note: This should not happen, but we check just in case.
logger.Error("Failed to stat file but this is not the active thumbnail generator. This should not happen.") logger.Error("Failed to stat file but this is not the active thumbnail generator. This should not happen.")
return fmt.Errorf("Not active thumbnail generator. Stat error: %q", err) return false, fmt.Errorf("Not active thumbnail generator. Stat error: %q", err)
} }
start := time.Now() start := time.Now()
width, height, err := resize(dst, buffer, config.Width, config.Height, config.ResizeMethod == "crop", logger) width, height, err := resize(dst, buffer, config.Width, config.Height, config.ResizeMethod == "crop", logger)
if err != nil { if err != nil {
return err return false, err
} }
logger.WithFields(log.Fields{ logger.WithFields(log.Fields{
"ActualWidth": width, "ActualWidth": width,
@ -193,7 +204,7 @@ func createThumbnail(src types.Path, buffer []byte, config types.ThumbnailSize,
stat, err := os.Stat(string(dst)) stat, err := os.Stat(string(dst))
if err != nil { if err != nil {
return err return false, err
} }
thumbnailMetadata = &types.ThumbnailMetadata{ thumbnailMetadata = &types.ThumbnailMetadata{
@ -217,14 +228,14 @@ func createThumbnail(src types.Path, buffer []byte, config types.ThumbnailSize,
"ActualWidth": width, "ActualWidth": width,
"ActualHeight": height, "ActualHeight": height,
}).Error("Failed to store thumbnail metadata in database.") }).Error("Failed to store thumbnail metadata in database.")
return err return false, err
} }
return nil return false, nil
} }
// getActiveThumbnailGeneration checks for active thumbnail generation // getActiveThumbnailGeneration checks for active thumbnail generation
func getActiveThumbnailGeneration(dst types.Path, config types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, logger *log.Entry) (bool, error) { func getActiveThumbnailGeneration(dst types.Path, config types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, logger *log.Entry) (isActive bool, busy bool, errorReturn error) {
// Check if there is active thumbnail generation. // Check if there is active thumbnail generation.
activeThumbnailGeneration.Lock() activeThumbnailGeneration.Lock()
defer activeThumbnailGeneration.Unlock() defer activeThumbnailGeneration.Unlock()
@ -234,7 +245,14 @@ func getActiveThumbnailGeneration(dst types.Path, config types.ThumbnailSize, ac
// NOTE: Wait unlocks and locks again internally. There is still a deferred Unlock() that will unlock this. // NOTE: Wait unlocks and locks again internally. There is still a deferred Unlock() that will unlock this.
activeThumbnailGenerationResult.Cond.Wait() activeThumbnailGenerationResult.Cond.Wait()
// Note: either there is an error or it is nil, either way returning it is correct // Note: either there is an error or it is nil, either way returning it is correct
return false, activeThumbnailGenerationResult.Err return false, false, activeThumbnailGenerationResult.Err
}
// Only allow thumbnail generation up to a maximum configured number. Above this we fall back to serving the
// original. Or in the case of pre-generation, they maybe get generated on the first request for a thumbnail if
// load has subsided.
if len(activeThumbnailGeneration.PathToResult) >= maxThumbnailGenerators {
return false, true, nil
} }
// No active thumbnail generation so create one // No active thumbnail generation so create one
@ -242,7 +260,7 @@ func getActiveThumbnailGeneration(dst types.Path, config types.ThumbnailSize, ac
Cond: &sync.Cond{L: activeThumbnailGeneration}, Cond: &sync.Cond{L: activeThumbnailGeneration},
} }
return true, nil return true, false, nil
} }
// broadcastGeneration broadcasts that thumbnail generation completed and the error to all waiting goroutines // broadcastGeneration broadcasts that thumbnail generation completed and the error to all waiting goroutines
@ -252,7 +270,7 @@ func broadcastGeneration(dst types.Path, activeThumbnailGeneration *types.Active
defer activeThumbnailGeneration.Unlock() defer activeThumbnailGeneration.Unlock()
if activeThumbnailGenerationResult, ok := activeThumbnailGeneration.PathToResult[string(dst)]; ok { if activeThumbnailGenerationResult, ok := activeThumbnailGeneration.PathToResult[string(dst)]; ok {
logger.Info("Signalling other goroutines waiting for this goroutine to generate the thumbnail.") logger.Info("Signalling other goroutines waiting for this goroutine to generate the thumbnail.")
// Note: retErr is a named return value error that is signalled from here to waiting goroutines // Note: errorReturn is a named return value error that is signalled from here to waiting goroutines
activeThumbnailGenerationResult.Err = errorReturn activeThumbnailGenerationResult.Err = errorReturn
activeThumbnailGenerationResult.Cond.Broadcast() activeThumbnailGenerationResult.Cond.Broadcast()
} }

39
src/github.com/matrix-org/dendrite/mediaapi/writers/download.go
View file

@ -192,12 +192,12 @@ func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI
// If we have a record, we can respond from the local file // If we have a record, we can respond from the local file
r.MediaMetadata = mediaMetadata r.MediaMetadata = mediaMetadata
} }
return r.respondFromLocalFile(w, cfg.AbsBasePath, activeThumbnailGeneration, db, cfg.DynamicThumbnails, cfg.ThumbnailSizes) return r.respondFromLocalFile(w, cfg.AbsBasePath, activeThumbnailGeneration, cfg.MaxThumbnailGenerators, db, cfg.DynamicThumbnails, cfg.ThumbnailSizes)
} }
// respondFromLocalFile reads a file from local storage and writes it to the http.ResponseWriter // respondFromLocalFile reads a file from local storage and writes it to the http.ResponseWriter
// Returns a util.JSONResponse error in case of error // Returns a util.JSONResponse error in case of error
func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []types.ThumbnailSize) *util.JSONResponse { func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []types.ThumbnailSize) *util.JSONResponse {
filePath, err := fileutils.GetPathFromBase64Hash(r.MediaMetadata.Base64Hash, absBasePath) filePath, err := fileutils.GetPathFromBase64Hash(r.MediaMetadata.Base64Hash, absBasePath)
if err != nil { if err != nil {
r.Logger.WithError(err).Error("Failed to get file path from metadata") r.Logger.WithError(err).Error("Failed to get file path from metadata")
@ -230,7 +230,7 @@ func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePat
var responseFile *os.File var responseFile *os.File
var responseMetadata *types.MediaMetadata var responseMetadata *types.MediaMetadata
if r.IsThumbnailRequest { if r.IsThumbnailRequest {
thumbFile, thumbMetadata, resErr := r.getThumbnailFile(types.Path(filePath), activeThumbnailGeneration, db, dynamicThumbnails, thumbnailSizes) thumbFile, thumbMetadata, resErr := r.getThumbnailFile(types.Path(filePath), activeThumbnailGeneration, maxThumbnailGenerators, db, dynamicThumbnails, thumbnailSizes)
if thumbFile != nil { if thumbFile != nil {
defer thumbFile.Close() defer thumbFile.Close()
} }
@ -284,16 +284,19 @@ func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePat
} }
// Note: Thumbnail generation may be ongoing asynchronously. // Note: Thumbnail generation may be ongoing asynchronously.
func (r *downloadRequest) getThumbnailFile(filePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []types.ThumbnailSize) (*os.File, *types.ThumbnailMetadata, *util.JSONResponse) { func (r *downloadRequest) getThumbnailFile(filePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []types.ThumbnailSize) (*os.File, *types.ThumbnailMetadata, *util.JSONResponse) {
var thumbnail *types.ThumbnailMetadata var thumbnail *types.ThumbnailMetadata
var resErr *util.JSONResponse var resErr *util.JSONResponse
if dynamicThumbnails { if dynamicThumbnails {
thumbnail, resErr = r.generateThumbnail(filePath, r.ThumbnailSize, activeThumbnailGeneration, db) thumbnail, resErr = r.generateThumbnail(filePath, r.ThumbnailSize, activeThumbnailGeneration, maxThumbnailGenerators, db)
if resErr != nil { if resErr != nil {
return nil, nil, resErr return nil, nil, resErr
} }
} else { }
// If dynamicThumbnails is true but there are too many thumbnails being actively generated, we can fall back
// to trying to use a pre-generated thumbnail
if thumbnail == nil {
thumbnails, err := db.GetThumbnails(r.MediaMetadata.MediaID, r.MediaMetadata.Origin) thumbnails, err := db.GetThumbnails(r.MediaMetadata.MediaID, r.MediaMetadata.Origin)
if err != nil { if err != nil {
r.Logger.WithError(err).Error("Error looking up thumbnails") r.Logger.WithError(err).Error("Error looking up thumbnails")
@ -305,13 +308,15 @@ func (r *downloadRequest) getThumbnailFile(filePath types.Path, activeThumbnailG
// If we get a thumbnail, we're done. // If we get a thumbnail, we're done.
var thumbnailSize *types.ThumbnailSize var thumbnailSize *types.ThumbnailSize
thumbnail, thumbnailSize = thumbnailer.SelectThumbnail(r.ThumbnailSize, thumbnails, thumbnailSizes) thumbnail, thumbnailSize = thumbnailer.SelectThumbnail(r.ThumbnailSize, thumbnails, thumbnailSizes)
if thumbnailSize != nil { // If dynamicThumbnails is true and we are not over-loaded then we would have generated what was requested above.
// So we don't try to generate a pre-generated thumbnail here.
if thumbnailSize != nil && dynamicThumbnails == false {
r.Logger.WithFields(log.Fields{ r.Logger.WithFields(log.Fields{
"Width": thumbnailSize.Width, "Width": thumbnailSize.Width,
"Height": thumbnailSize.Height, "Height": thumbnailSize.Height,
"ResizeMethod": thumbnailSize.ResizeMethod, "ResizeMethod": thumbnailSize.ResizeMethod,
}).Info("Pre-generating thumbnail for immediate response.") }).Info("Pre-generating thumbnail for immediate response.")
thumbnail, resErr = r.generateThumbnail(filePath, *thumbnailSize, activeThumbnailGeneration, db) thumbnail, resErr = r.generateThumbnail(filePath, *thumbnailSize, activeThumbnailGeneration, maxThumbnailGenerators, db)
if resErr != nil { if resErr != nil {
return nil, nil, resErr return nil, nil, resErr
} }
@ -348,18 +353,21 @@ func (r *downloadRequest) getThumbnailFile(filePath types.Path, activeThumbnailG
return thumbFile, thumbnail, nil return thumbFile, thumbnail, nil
} }
func (r *downloadRequest) generateThumbnail(filePath types.Path, thumbnailSize types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, db *storage.Database) (*types.ThumbnailMetadata, *util.JSONResponse) { func (r *downloadRequest) generateThumbnail(filePath types.Path, thumbnailSize types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database) (*types.ThumbnailMetadata, *util.JSONResponse) {
logger := r.Logger.WithFields(log.Fields{ logger := r.Logger.WithFields(log.Fields{
"Width": thumbnailSize.Width, "Width": thumbnailSize.Width,
"Height": thumbnailSize.Height, "Height": thumbnailSize.Height,
"ResizeMethod": thumbnailSize.ResizeMethod, "ResizeMethod": thumbnailSize.ResizeMethod,
}) })
var err error busy, err := thumbnailer.GenerateThumbnail(filePath, thumbnailSize, r.MediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, r.Logger)
if err = thumbnailer.GenerateThumbnail(filePath, thumbnailSize, r.MediaMetadata, activeThumbnailGeneration, db, logger); err != nil { if err != nil {
logger.WithError(err).Error("Error creating thumbnail") logger.WithError(err).Error("Error creating thumbnail")
resErr := jsonerror.InternalServerError() resErr := jsonerror.InternalServerError()
return nil, &resErr return nil, &resErr
} }
if busy {
return nil, nil
}
var thumbnail *types.ThumbnailMetadata var thumbnail *types.ThumbnailMetadata
thumbnail, err = db.GetThumbnail(r.MediaMetadata.MediaID, r.MediaMetadata.Origin, thumbnailSize.Width, thumbnailSize.Height, thumbnailSize.ResizeMethod) thumbnail, err = db.GetThumbnail(r.MediaMetadata.MediaID, r.MediaMetadata.Origin, thumbnailSize.Width, thumbnailSize.Height, thumbnailSize.ResizeMethod)
if err != nil { if err != nil {
@ -406,7 +414,7 @@ func (r *downloadRequest) getRemoteFile(cfg *config.MediaAPI, db *storage.Databa
if mediaMetadata == nil { if mediaMetadata == nil {
// If we do not have a record, we need to fetch the remote file first and then respond from the local file // If we do not have a record, we need to fetch the remote file first and then respond from the local file
resErr := r.fetchRemoteFileAndStoreMetadata(cfg.AbsBasePath, *cfg.MaxFileSizeBytes, db, cfg.ThumbnailSizes, activeThumbnailGeneration) resErr := r.fetchRemoteFileAndStoreMetadata(cfg.AbsBasePath, *cfg.MaxFileSizeBytes, db, cfg.ThumbnailSizes, activeThumbnailGeneration, cfg.MaxThumbnailGenerators)
if resErr != nil { if resErr != nil {
return resErr return resErr
} }
@ -468,7 +476,7 @@ func (r *downloadRequest) broadcastMediaMetadata(activeRemoteRequests *types.Act
} }
// fetchRemoteFileAndStoreMetadata fetches the file from the remote server and stores its metadata in the database // fetchRemoteFileAndStoreMetadata fetches the file from the remote server and stores its metadata in the database
func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath types.Path, maxFileSizeBytes types.FileSizeBytes, db *storage.Database, thumbnailSizes []types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse { func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath types.Path, maxFileSizeBytes types.FileSizeBytes, db *storage.Database, thumbnailSizes []types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse {
finalPath, duplicate, resErr := r.fetchRemoteFile(absBasePath, maxFileSizeBytes) finalPath, duplicate, resErr := r.fetchRemoteFile(absBasePath, maxFileSizeBytes)
if resErr != nil { if resErr != nil {
return resErr return resErr
@ -497,10 +505,13 @@ func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath types.Path
} }
go func() { go func() {
err := thumbnailer.GenerateThumbnails(finalPath, thumbnailSizes, r.MediaMetadata, activeThumbnailGeneration, db, r.Logger) busy, err := thumbnailer.GenerateThumbnails(finalPath, thumbnailSizes, r.MediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, r.Logger)
if err != nil { if err != nil {
r.Logger.WithError(err).Warn("Error generating thumbnails") r.Logger.WithError(err).Warn("Error generating thumbnails")
} }
if busy {
r.Logger.Warn("Maximum number of active thumbnail generators reached. Skipping pre-generation.")
}
}() }()
r.Logger.WithFields(log.Fields{ r.Logger.WithFields(log.Fields{

9
src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go
View file

@ -152,7 +152,7 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db *
} }
} }
if resErr := r.storeFileAndMetadata(tmpDir, cfg.AbsBasePath, db, cfg.ThumbnailSizes, activeThumbnailGeneration); resErr != nil { if resErr := r.storeFileAndMetadata(tmpDir, cfg.AbsBasePath, db, cfg.ThumbnailSizes, activeThumbnailGeneration, cfg.MaxThumbnailGenerators); resErr != nil {
return resErr return resErr
} }
@ -215,7 +215,7 @@ func (r *uploadRequest) Validate(maxFileSizeBytes types.FileSizeBytes) *util.JSO
// The order of operations is important as it avoids metadata entering the database before the file // The order of operations is important as it avoids metadata entering the database before the file
// is ready, and if we fail to move the file, it never gets added to the database. // is ready, and if we fail to move the file, it never gets added to the database.
// Returns a util.JSONResponse error and cleans up directories in case of error. // Returns a util.JSONResponse error and cleans up directories in case of error.
func (r *uploadRequest) storeFileAndMetadata(tmpDir types.Path, absBasePath types.Path, db *storage.Database, thumbnailSizes []types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse { func (r *uploadRequest) storeFileAndMetadata(tmpDir types.Path, absBasePath types.Path, db *storage.Database, thumbnailSizes []types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse {
finalPath, duplicate, err := fileutils.MoveFileWithHashCheck(tmpDir, r.MediaMetadata, absBasePath, r.Logger) finalPath, duplicate, err := fileutils.MoveFileWithHashCheck(tmpDir, r.MediaMetadata, absBasePath, r.Logger)
if err != nil { if err != nil {
r.Logger.WithError(err).Error("Failed to move file.") r.Logger.WithError(err).Error("Failed to move file.")
@ -243,10 +243,13 @@ func (r *uploadRequest) storeFileAndMetadata(tmpDir types.Path, absBasePath type
} }
go func() { go func() {
err := thumbnailer.GenerateThumbnails(finalPath, thumbnailSizes, r.MediaMetadata, activeThumbnailGeneration, db, r.Logger) busy, err := thumbnailer.GenerateThumbnails(finalPath, thumbnailSizes, r.MediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, r.Logger)
if err != nil { if err != nil {
r.Logger.WithError(err).Warn("Error generating thumbnails") r.Logger.WithError(err).Warn("Error generating thumbnails")
} }
if busy {
r.Logger.Warn("Maximum number of active thumbnail generators reached. Skipping pre-generation.")
}
}() }()
return nil return nil