diff --git a/src/github.com/matrix-org/dendrite/mediaapi/README.md b/src/github.com/matrix-org/dendrite/mediaapi/README.md new file mode 100644 index 000000000..ab62fae23 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/mediaapi/README.md @@ -0,0 +1,7 @@ +# Media API + +This server is responsible for serving `/media` requests as per: + +http://matrix.org/docs/spec/client_server/r0.2.0.html#id43 + +Thumbnailing uses bimg from https://github.com/h2non/bimg (MIT-licensed) which uses libvips from https://github.com/jcupitt/libvips (LGPL v2.1+ -licensed). libvips is a C library and must be installed/built separately. See the github page for details. diff --git a/src/github.com/matrix-org/dendrite/mediaapi/storage/thumbnail_table.go b/src/github.com/matrix-org/dendrite/mediaapi/storage/thumbnail_table.go new file mode 100644 index 000000000..1284dbd48 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/mediaapi/storage/thumbnail_table.go @@ -0,0 +1,157 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "database/sql" + "time" + + "github.com/matrix-org/dendrite/mediaapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +const thumbnailSchema = ` +-- The thumbnail table holds metadata for each thumbnail file stored and accessible to the local server, +-- the actual file is stored separately. +CREATE TABLE IF NOT EXISTS thumbnail ( + -- The id used to refer to the media. + -- For uploads to this server this is a base64-encoded sha256 hash of the file data + -- For media from remote servers, this can be any unique identifier string + media_id TEXT NOT NULL, + -- The origin of the media as requested by the client. Should be a homeserver domain. + media_origin TEXT NOT NULL, + -- The MIME-type of the thumbnail file. + content_type TEXT NOT NULL, + -- Size of the thumbnail file in bytes. + file_size_bytes BIGINT NOT NULL, + -- When the thumbnail was generated in UNIX epoch ms. + creation_ts BIGINT NOT NULL, + -- The width of the thumbnail + width INTEGER NOT NULL, + -- The height of the thumbnail + height INTEGER NOT NULL, + -- The resize method used to generate the thumbnail. Can be crop or scale. + resize_method TEXT NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS thumbnail_index ON thumbnail (media_id, media_origin, width, height, resize_method); +` + +const insertThumbnailSQL = ` +INSERT INTO thumbnail (media_id, media_origin, content_type, file_size_bytes, creation_ts, width, height, resize_method) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) +` + +// Note: this selects one specific thumbnail +const selectThumbnailSQL = ` +SELECT content_type, file_size_bytes, creation_ts FROM thumbnail WHERE media_id = $1 AND media_origin = $2 AND width = $3 AND height = $4 AND resize_method = $5 +` + +// Note: this selects all thumbnails for a media_origin and media_id +const selectThumbnailsSQL = ` +SELECT content_type, file_size_bytes, creation_ts, width, height, resize_method FROM thumbnail WHERE media_id = $1 AND media_origin = $2 +` + +type thumbnailStatements struct { + insertThumbnailStmt *sql.Stmt + selectThumbnailStmt *sql.Stmt + selectThumbnailsStmt *sql.Stmt +} + +func (s *thumbnailStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(thumbnailSchema) + if err != nil { + return + } + + return statementList{ + {&s.insertThumbnailStmt, insertThumbnailSQL}, + {&s.selectThumbnailStmt, selectThumbnailSQL}, + {&s.selectThumbnailsStmt, selectThumbnailsSQL}, + }.prepare(db) +} + +func (s *thumbnailStatements) insertThumbnail(thumbnailMetadata *types.ThumbnailMetadata) error { + thumbnailMetadata.MediaMetadata.CreationTimestamp = types.UnixMs(time.Now().UnixNano() / 1000000) + _, err := s.insertThumbnailStmt.Exec( + thumbnailMetadata.MediaMetadata.MediaID, + thumbnailMetadata.MediaMetadata.Origin, + thumbnailMetadata.MediaMetadata.ContentType, + thumbnailMetadata.MediaMetadata.FileSizeBytes, + thumbnailMetadata.MediaMetadata.CreationTimestamp, + thumbnailMetadata.ThumbnailSize.Width, + thumbnailMetadata.ThumbnailSize.Height, + thumbnailMetadata.ThumbnailSize.ResizeMethod, + ) + return err +} + +func (s *thumbnailStatements) selectThumbnail(mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error) { + thumbnailMetadata := types.ThumbnailMetadata{ + MediaMetadata: &types.MediaMetadata{ + MediaID: mediaID, + Origin: mediaOrigin, + }, + ThumbnailSize: types.ThumbnailSize{ + Width: width, + Height: height, + ResizeMethod: resizeMethod, + }, + } + err := s.selectThumbnailStmt.QueryRow( + thumbnailMetadata.MediaMetadata.MediaID, + thumbnailMetadata.MediaMetadata.Origin, + thumbnailMetadata.ThumbnailSize.Width, + thumbnailMetadata.ThumbnailSize.Height, + thumbnailMetadata.ThumbnailSize.ResizeMethod, + ).Scan( + &thumbnailMetadata.MediaMetadata.ContentType, + &thumbnailMetadata.MediaMetadata.FileSizeBytes, + &thumbnailMetadata.MediaMetadata.CreationTimestamp, + ) + return &thumbnailMetadata, err +} + +func (s *thumbnailStatements) selectThumbnails(mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) ([]*types.ThumbnailMetadata, error) { + rows, err := s.selectThumbnailsStmt.Query( + mediaID, mediaOrigin, + ) + if err != nil { + return nil, err + } + + var thumbnails []*types.ThumbnailMetadata + for rows.Next() { + thumbnailMetadata := types.ThumbnailMetadata{ + MediaMetadata: &types.MediaMetadata{ + MediaID: mediaID, + Origin: mediaOrigin, + }, + } + err = rows.Scan( + &thumbnailMetadata.MediaMetadata.ContentType, + &thumbnailMetadata.MediaMetadata.FileSizeBytes, + &thumbnailMetadata.MediaMetadata.CreationTimestamp, + &thumbnailMetadata.ThumbnailSize.Width, + &thumbnailMetadata.ThumbnailSize.Height, + &thumbnailMetadata.ThumbnailSize.ResizeMethod, + ) + if err != nil { + return nil, err + } + thumbnails = append(thumbnails, &thumbnailMetadata) + } + + return thumbnails, err +} diff --git a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go new file mode 100644 index 000000000..f2716b6fb --- /dev/null +++ b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go @@ -0,0 +1,416 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package thumbnailer + +import ( + "fmt" + "math" + "os" + "path/filepath" + "sync" + "time" + + log "github.com/Sirupsen/logrus" + "github.com/matrix-org/dendrite/mediaapi/storage" + "github.com/matrix-org/dendrite/mediaapi/types" + "gopkg.in/h2non/bimg.v1" +) + +type thumbnailMetrics struct { + isSmaller int + aspect float64 + size float64 + methodMismatch int + fileSize types.FileSizeBytes +} + +// thumbnailTemplate is the filename template for thumbnails +const thumbnailTemplate = "thumbnail-%vx%v-%v" + +// GenerateThumbnails generates the configured thumbnail sizes for the source file +func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, db *storage.Database, logger *log.Entry) error { + buffer, err := bimg.Read(string(src)) + if err != nil { + logger.WithError(err).WithField("src", src).Error("Failed to read src file") + return err + } + for _, config := range configs { + // Note: createThumbnail does locking based on activeThumbnailGeneration + if err = createThumbnail(src, buffer, config, mediaMetadata, activeThumbnailGeneration, db, logger); err != nil { + logger.WithError(err).WithField("src", src).Error("Failed to generate thumbnails") + return err + } + } + return nil +} + +// GenerateThumbnail generates the configured thumbnail size for the source file +func GenerateThumbnail(src types.Path, config types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, db *storage.Database, logger *log.Entry) error { + buffer, err := bimg.Read(string(src)) + if err != nil { + logger.WithError(err).WithFields(log.Fields{ + "src": src, + }).Error("Failed to read src file") + return err + } + // Note: createThumbnail does locking based on activeThumbnailGeneration + if err = createThumbnail(src, buffer, config, mediaMetadata, activeThumbnailGeneration, db, logger); err != nil { + logger.WithError(err).WithFields(log.Fields{ + "src": src, + }).Error("Failed to generate thumbnails") + return err + } + return nil +} + +// GetThumbnailPath returns the path to a thumbnail given the absolute src path and thumbnail size configuration +func GetThumbnailPath(src types.Path, config types.ThumbnailSize) types.Path { + srcDir := filepath.Dir(string(src)) + return types.Path(filepath.Join( + srcDir, + fmt.Sprintf(thumbnailTemplate, config.Width, config.Height, config.ResizeMethod), + )) +} + +// SelectThumbnail compares the (potentially) available thumbnails with the desired thumbnail and returns the best match +// The algorithm is very similar to what was implemented in Synapse +// In order of priority unless absolute, the following metrics are compared; the image is: +// * the same size or larger than requested +// * if a cropped image is desired, has an aspect ratio close to requested +// * has a size close to requested +// * if a cropped image is desired, prefer the same method, if scaled is desired, absolutely require scaled +// * has a small file size +// If a pre-generated thumbnail size is the best match, but it has not been generated yet, the caller can use the returned size to generate it. +// Returns nil if no thumbnail matches the criteria +func SelectThumbnail(desired types.ThumbnailSize, thumbnails []*types.ThumbnailMetadata, thumbnailSizes []types.ThumbnailSize) (*types.ThumbnailMetadata, *types.ThumbnailSize) { + var chosenThumbnail *types.ThumbnailMetadata + var chosenThumbnailSize *types.ThumbnailSize + chosenMetrics := newThumbnailMetrics() + + for _, thumbnail := range thumbnails { + if desired.ResizeMethod == "scale" && thumbnail.ThumbnailSize.ResizeMethod != "scale" { + continue + } + metrics := calcThumbnailMetrics(thumbnail.ThumbnailSize, thumbnail.MediaMetadata, desired) + // Note: the result will be -1 for better than, 0 for the same as and 1 for worse than. Take better results only. + result := compareThumbnailMetrics(metrics, chosenMetrics, desired.ResizeMethod == "crop") + if result == -1 { + chosenMetrics = metrics + chosenThumbnail = thumbnail + } + } + + for _, thumbnailSize := range thumbnailSizes { + if desired.ResizeMethod == "scale" && thumbnailSize.ResizeMethod != "scale" { + continue + } + metrics := calcThumbnailMetrics(thumbnailSize, nil, desired) + // Note: the result will be -1 for better than, 0 for the same as and 1 for worse than. Take better results only. + result := compareThumbnailMetrics(metrics, chosenMetrics, desired.ResizeMethod == "crop") + if result == -1 { + chosenMetrics = metrics + chosenThumbnailSize = &types.ThumbnailSize{ + Width: thumbnailSize.Width, + Height: thumbnailSize.Height, + ResizeMethod: thumbnailSize.ResizeMethod, + } + } + } + + return chosenThumbnail, chosenThumbnailSize +} + +// createThumbnail checks if the thumbnail exists, and if not, generates it +// Thumbnail generation is only done once for each non-existing thumbnail. +func createThumbnail(src types.Path, buffer []byte, config types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, db *storage.Database, logger *log.Entry) (errorReturn error) { + dst := GetThumbnailPath(src, config) + + // Note: getActiveThumbnailGeneration uses mutexes and conditions from activeThumbnailGeneration + isActive, err := getActiveThumbnailGeneration(dst, config, activeThumbnailGeneration, logger) + if err != nil { + return err + } + + if isActive { + // Note: This is an active request that MUST broadcastGeneration to wake up waiting goroutines! + // Note: broadcastGeneration uses mutexes and conditions from activeThumbnailGeneration + defer func() { + // Note: errorReturn is the named return variable so we wrap this in a closure to re-evaluate the arguments at defer-time + if err := recover(); err != nil { + broadcastGeneration(dst, activeThumbnailGeneration, config, err.(error), logger) + panic(err) + } + broadcastGeneration(dst, activeThumbnailGeneration, config, errorReturn, logger) + }() + } + + // Check if the thumbnail exists. + thumbnailMetadata, err := db.GetThumbnail(mediaMetadata.MediaID, mediaMetadata.Origin, config.Width, config.Height, config.ResizeMethod) + if err != nil { + logger.WithFields(log.Fields{ + "Width": config.Width, + "Height": config.Height, + "ResizeMethod": config.ResizeMethod, + }).Error("Failed to query database for thumbnail.") + return err + } + if thumbnailMetadata != nil { + return nil + } + // Note: The double-negative is intentional as os.IsExist(err) != !os.IsNotExist(err). + // The functions are error checkers to be used in different cases. + if _, err = os.Stat(string(dst)); !os.IsNotExist(err) { + // Thumbnail exists + return nil + } + + if isActive == false { + // Note: This should not happen, but we check just in case. + logger.WithFields(log.Fields{ + "Width": config.Width, + "Height": config.Height, + "ResizeMethod": config.ResizeMethod, + }).Error("Failed to stat file but this is not the active thumbnail generator. This should not happen.") + return fmt.Errorf("Not active thumbnail generator. Stat error: %q", err) + } + + start := time.Now() + width, height, err := resize(dst, buffer, config.Width, config.Height, config.ResizeMethod == "crop", logger) + if err != nil { + return err + } + logger.WithFields(log.Fields{ + "Width": config.Width, + "Height": config.Height, + "ActualWidth": width, + "ActualHeight": height, + "ResizeMethod": config.ResizeMethod, + "processTime": time.Now().Sub(start), + }).Info("Generated thumbnail") + + stat, err := os.Stat(string(dst)) + if err != nil { + return err + } + + thumbnailMetadata = &types.ThumbnailMetadata{ + MediaMetadata: &types.MediaMetadata{ + MediaID: mediaMetadata.MediaID, + Origin: mediaMetadata.Origin, + // Note: the code currently always creates a JPEG thumbnail + ContentType: types.ContentType("image/jpeg"), + FileSizeBytes: types.FileSizeBytes(stat.Size()), + }, + ThumbnailSize: types.ThumbnailSize{ + Width: width, + Height: height, + ResizeMethod: config.ResizeMethod, + }, + } + + err = db.StoreThumbnail(thumbnailMetadata) + if err != nil { + logger.WithError(err).WithFields(log.Fields{ + "Width": config.Width, + "Height": config.Height, + "ActualWidth": width, + "ActualHeight": height, + "ResizeMethod": config.ResizeMethod, + }).Error("Failed to store thumbnail metadata in database.") + return err + } + + return nil +} + +// getActiveThumbnailGeneration checks for active thumbnail generation +func getActiveThumbnailGeneration(dst types.Path, config types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, logger *log.Entry) (bool, error) { + // Check if there is active thumbnail generation. + activeThumbnailGeneration.Lock() + defer activeThumbnailGeneration.Unlock() + if activeThumbnailGenerationResult, ok := activeThumbnailGeneration.PathToResult[string(dst)]; ok { + logger.WithFields(log.Fields{ + "Width": config.Width, + "Height": config.Height, + "ResizeMethod": config.ResizeMethod, + }).Info("Waiting for another goroutine to generate the thumbnail.") + + // NOTE: Wait unlocks and locks again internally. There is still a deferred Unlock() that will unlock this. + activeThumbnailGenerationResult.Cond.Wait() + // Note: either there is an error or it is nil, either way returning it is correct + return false, activeThumbnailGenerationResult.Err + } + + // No active thumbnail generation so create one + activeThumbnailGeneration.PathToResult[string(dst)] = &types.ThumbnailGenerationResult{ + Cond: &sync.Cond{L: activeThumbnailGeneration}, + } + + return true, nil +} + +// broadcastGeneration broadcasts that thumbnail generation completed and the error to all waiting goroutines +// Note: This should only be called by the owner of the activeThumbnailGenerationResult +func broadcastGeneration(dst types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, config types.ThumbnailSize, errorReturn error, logger *log.Entry) { + activeThumbnailGeneration.Lock() + defer activeThumbnailGeneration.Unlock() + if activeThumbnailGenerationResult, ok := activeThumbnailGeneration.PathToResult[string(dst)]; ok { + logger.WithFields(log.Fields{ + "Width": config.Width, + "Height": config.Height, + "ResizeMethod": config.ResizeMethod, + }).Info("Signalling other goroutines waiting for this goroutine to generate the thumbnail.") + // Note: retErr is a named return value error that is signalled from here to waiting goroutines + activeThumbnailGenerationResult.Err = errorReturn + activeThumbnailGenerationResult.Cond.Broadcast() + } + delete(activeThumbnailGeneration.PathToResult, string(dst)) +} + +// resize scales an image to fit within the provided width and height +// If the source aspect ratio is different to the target dimensions, one edge will be smaller than requested +// If crop is set to true, the image will be scaled to fill the width and height with any excess being cropped off +func resize(dst types.Path, buffer []byte, w, h int, crop bool, logger *log.Entry) (int, int, error) { + inImage := bimg.NewImage(buffer) + + inSize, err := inImage.Size() + if err != nil { + return -1, -1, err + } + + options := bimg.Options{ + Type: bimg.JPEG, + Quality: 85, + } + if crop { + options.Width = w + options.Height = h + options.Crop = true + } else { + inAR := float64(inSize.Width) / float64(inSize.Height) + outAR := float64(w) / float64(h) + + if inAR > outAR { + // input has wider AR than requested output so use requested width and calculate height to match input AR + options.Width = w + options.Height = int(float64(w) / inAR) + } else { + // input has narrower AR than requested output so use requested height and calculate width to match input AR + options.Width = int(float64(h) * inAR) + options.Height = h + } + } + + newImage, err := inImage.Process(options) + if err != nil { + return -1, -1, err + } + + if err = bimg.Write(string(dst), newImage); err != nil { + logger.WithError(err).Error("Failed to resize image") + return -1, -1, err + } + + return options.Width, options.Height, nil +} + +// init with worst values +func newThumbnailMetrics() thumbnailMetrics { + return thumbnailMetrics{ + isSmaller: 1, + aspect: float64(16384 * 16384), + size: float64(16384 * 16384), + methodMismatch: 0, + fileSize: types.FileSizeBytes(math.MaxInt64), + } +} + +func calcThumbnailMetrics(size types.ThumbnailSize, metadata *types.MediaMetadata, desired types.ThumbnailSize) thumbnailMetrics { + dW := desired.Width + dH := desired.Height + tW := size.Width + tH := size.Height + + metrics := newThumbnailMetrics() + // In all cases, a larger metric value is a worse fit. + // compare size: thumbnail smaller is true and gives 1, larger is false and gives 0 + metrics.isSmaller = boolToInt(tW < dW || tH < dH) + // comparison of aspect ratios only makes sense for a request for desired cropped + metrics.aspect = math.Abs(float64(dW*tH - dH*tW)) + // compare sizes + metrics.size = math.Abs(float64((dW - tW) * (dH - tH))) + // compare resize method + metrics.methodMismatch = boolToInt(size.ResizeMethod != desired.ResizeMethod) + if metadata != nil { + // file size + metrics.fileSize = metadata.FileSizeBytes + } + + return metrics +} + +func boolToInt(b bool) int { + if b { + return 1 + } + return 0 +} + +func compareThumbnailMetrics(a thumbnailMetrics, b thumbnailMetrics, desiredCrop bool) int { + // preference means returning -1 + + // prefer images that are not smaller + // e.g. isSmallerDiff > 0 means b is smaller than desired and a is not smaller + if a.isSmaller > b.isSmaller { + return 1 + } else if a.isSmaller < b.isSmaller { + return -1 + } + + // prefer aspect ratios closer to desired only if desired cropped + // only cropped images have differing aspect ratios + // desired scaled only accepts scaled images + if desiredCrop { + if a.aspect > b.aspect { + return 1 + } else if a.aspect < b.aspect { + return -1 + } + } + + // prefer closer in size + if a.size > b.size { + return 1 + } else if a.size < b.size { + return -1 + } + + // prefer images using the same method + // e.g. methodMismatchDiff > 0 means b's method is different from desired and a's matches the desired method + if a.methodMismatch > b.methodMismatch { + return 1 + } else if a.methodMismatch < b.methodMismatch { + return -1 + } + + // prefer smaller files + if a.fileSize > b.fileSize { + return 1 + } else if a.fileSize < b.fileSize { + return -1 + } + + return 0 +}