diff --git a/federationsender/storage/storage.go b/federationsender/storage/storage.go index bafb5226a..85c03c8b0 100644 --- a/federationsender/storage/storage.go +++ b/federationsender/storage/storage.go @@ -1,3 +1,17 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package storage import ( diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 80ad8418d..8544bd64f 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -67,7 +67,7 @@ func Download( origin gomatrixserverlib.ServerName, mediaID types.MediaID, cfg *config.Dendrite, - db *storage.Database, + db storage.Database, client *gomatrixserverlib.Client, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, @@ -192,7 +192,7 @@ func (r *downloadRequest) doDownload( ctx context.Context, w http.ResponseWriter, cfg *config.Dendrite, - db *storage.Database, + db storage.Database, client *gomatrixserverlib.Client, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, @@ -235,7 +235,7 @@ func (r *downloadRequest) respondFromLocalFile( absBasePath config.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, - db *storage.Database, + db storage.Database, dynamicThumbnails bool, thumbnailSizes []config.ThumbnailSize, ) (*types.MediaMetadata, error) { @@ -325,7 +325,7 @@ func (r *downloadRequest) getThumbnailFile( filePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, - db *storage.Database, + db storage.Database, dynamicThumbnails bool, thumbnailSizes []config.ThumbnailSize, ) (*os.File, *types.ThumbnailMetadata, error) { @@ -407,7 +407,7 @@ func (r *downloadRequest) generateThumbnail( thumbnailSize types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, - db *storage.Database, + db storage.Database, ) (*types.ThumbnailMetadata, error) { r.Logger.WithFields(log.Fields{ "Width": thumbnailSize.Width, @@ -443,7 +443,7 @@ func (r *downloadRequest) getRemoteFile( ctx context.Context, client *gomatrixserverlib.Client, cfg *config.Dendrite, - db *storage.Database, + db storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, ) (errorResponse error) { @@ -545,7 +545,7 @@ func (r *downloadRequest) fetchRemoteFileAndStoreMetadata( client *gomatrixserverlib.Client, absBasePath config.Path, maxFileSizeBytes config.FileSizeBytes, - db *storage.Database, + db storage.Database, thumbnailSizes []config.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 0f2266640..dcc6ac06c 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -43,7 +43,7 @@ const pathPrefixR0 = "/_matrix/media/r0" func Setup( apiMux *mux.Router, cfg *config.Dendrite, - db *storage.Database, + db storage.Database, deviceDB *devices.Database, client *gomatrixserverlib.Client, ) { @@ -80,7 +80,7 @@ func Setup( func makeDownloadAPI( name string, cfg *config.Dendrite, - db *storage.Database, + db storage.Database, client *gomatrixserverlib.Client, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, diff --git a/mediaapi/routing/upload.go b/mediaapi/routing/upload.go index 2cb0d8757..91a453190 100644 --- a/mediaapi/routing/upload.go +++ b/mediaapi/routing/upload.go @@ -53,7 +53,7 @@ type uploadResponse struct { // This implementation supports a configurable maximum file size limit in bytes. If a user tries to upload more than this, they will receive an error that their upload is too large. // Uploaded files are processed piece-wise to avoid DoS attacks which would starve the server of memory. // TODO: We should time out requests if they have not received any data within a configured timeout period. -func Upload(req *http.Request, cfg *config.Dendrite, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse { +func Upload(req *http.Request, cfg *config.Dendrite, db storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse { r, resErr := parseAndValidateRequest(req, cfg) if resErr != nil { return *resErr @@ -96,7 +96,7 @@ func (r *uploadRequest) doUpload( ctx context.Context, reqReader io.Reader, cfg *config.Dendrite, - db *storage.Database, + db storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration, ) *util.JSONResponse { r.Logger.WithFields(log.Fields{ @@ -214,7 +214,7 @@ func (r *uploadRequest) storeFileAndMetadata( ctx context.Context, tmpDir types.Path, absBasePath config.Path, - db *storage.Database, + db storage.Database, thumbnailSizes []config.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, diff --git a/mediaapi/storage/media_repository_table.go b/mediaapi/storage/postgres/media_repository_table.go similarity index 99% rename from mediaapi/storage/media_repository_table.go rename to mediaapi/storage/postgres/media_repository_table.go index addd47b49..259901695 100644 --- a/mediaapi/storage/media_repository_table.go +++ b/mediaapi/storage/postgres/media_repository_table.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package storage +package postgres import ( "context" diff --git a/mediaapi/storage/prepare.go b/mediaapi/storage/postgres/prepare.go similarity index 98% rename from mediaapi/storage/prepare.go rename to mediaapi/storage/postgres/prepare.go index a30586de4..cfc320d29 100644 --- a/mediaapi/storage/prepare.go +++ b/mediaapi/storage/postgres/prepare.go @@ -14,7 +14,7 @@ // FIXME: This should be made common! -package storage +package postgres import ( "database/sql" diff --git a/mediaapi/storage/sql.go b/mediaapi/storage/postgres/sql.go similarity index 98% rename from mediaapi/storage/sql.go rename to mediaapi/storage/postgres/sql.go index 1f8c7be30..7f59d22f1 100644 --- a/mediaapi/storage/sql.go +++ b/mediaapi/storage/postgres/sql.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package storage +package postgres import ( "database/sql" diff --git a/mediaapi/storage/postgres/storage.go b/mediaapi/storage/postgres/storage.go new file mode 100644 index 000000000..f21d4d945 --- /dev/null +++ b/mediaapi/storage/postgres/storage.go @@ -0,0 +1,105 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + + // Import the postgres database driver. + _ "github.com/lib/pq" + "github.com/matrix-org/dendrite/mediaapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +// Database is used to store metadata about a repository of media files. +type Database struct { + statements statements + db *sql.DB +} + +// Open opens a postgres database. +func Open(dataSourceName string) (*Database, error) { + var d Database + var err error + if d.db, err = sql.Open("postgres", dataSourceName); err != nil { + return nil, err + } + if err = d.statements.prepare(d.db); err != nil { + return nil, err + } + return &d, nil +} + +// StoreMediaMetadata inserts the metadata about the uploaded media into the database. +// Returns an error if the combination of MediaID and Origin are not unique in the table. +func (d *Database) StoreMediaMetadata( + ctx context.Context, mediaMetadata *types.MediaMetadata, +) error { + return d.statements.media.insertMedia(ctx, mediaMetadata) +} + +// GetMediaMetadata returns metadata about media stored on this server. +// The media could have been uploaded to this server or fetched from another server and cached here. +// Returns nil metadata if there is no metadata associated with this media. +func (d *Database) GetMediaMetadata( + ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, +) (*types.MediaMetadata, error) { + mediaMetadata, err := d.statements.media.selectMedia(ctx, mediaID, mediaOrigin) + if err != nil && err == sql.ErrNoRows { + return nil, nil + } + return mediaMetadata, err +} + +// StoreThumbnail inserts the metadata about the thumbnail into the database. +// Returns an error if the combination of MediaID and Origin are not unique in the table. +func (d *Database) StoreThumbnail( + ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata, +) error { + return d.statements.thumbnail.insertThumbnail(ctx, thumbnailMetadata) +} + +// GetThumbnail returns metadata about a specific thumbnail. +// The media could have been uploaded to this server or fetched from another server and cached here. +// Returns nil metadata if there is no metadata associated with this thumbnail. +func (d *Database) GetThumbnail( + ctx context.Context, + mediaID types.MediaID, + mediaOrigin gomatrixserverlib.ServerName, + width, height int, + resizeMethod string, +) (*types.ThumbnailMetadata, error) { + thumbnailMetadata, err := d.statements.thumbnail.selectThumbnail( + ctx, mediaID, mediaOrigin, width, height, resizeMethod, + ) + if err != nil && err == sql.ErrNoRows { + return nil, nil + } + return thumbnailMetadata, err +} + +// GetThumbnails returns metadata about all thumbnails for a specific media stored on this server. +// The media could have been uploaded to this server or fetched from another server and cached here. +// Returns nil metadata if there are no thumbnails associated with this media. +func (d *Database) GetThumbnails( + ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, +) ([]*types.ThumbnailMetadata, error) { + thumbnails, err := d.statements.thumbnail.selectThumbnails(ctx, mediaID, mediaOrigin) + if err != nil && err == sql.ErrNoRows { + return nil, nil + } + return thumbnails, err +} diff --git a/mediaapi/storage/thumbnail_table.go b/mediaapi/storage/postgres/thumbnail_table.go similarity index 99% rename from mediaapi/storage/thumbnail_table.go rename to mediaapi/storage/postgres/thumbnail_table.go index f100485f8..53a82c54e 100644 --- a/mediaapi/storage/thumbnail_table.go +++ b/mediaapi/storage/postgres/thumbnail_table.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package storage +package postgres import ( "context" diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index bef134a95..874ec4cd4 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -16,90 +16,32 @@ package storage import ( "context" - "database/sql" + "errors" + "net/url" - // Import the postgres database driver. - _ "github.com/lib/pq" + "github.com/matrix-org/dendrite/mediaapi/storage/postgres" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" ) -// Database is used to store metadata about a repository of media files. -type Database struct { - statements statements - db *sql.DB +type Database interface { + StoreMediaMetadata(ctx context.Context, mediaMetadata *types.MediaMetadata) error + GetMediaMetadata(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error) + StoreThumbnail(ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata) error + GetThumbnail(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error) + GetThumbnails(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) ([]*types.ThumbnailMetadata, error) } // Open opens a postgres database. -func Open(dataSourceName string) (*Database, error) { - var d Database - var err error - if d.db, err = sql.Open("postgres", dataSourceName); err != nil { +func Open(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { return nil, err } - if err = d.statements.prepare(d.db); err != nil { - return nil, err + switch uri.Scheme { + case "postgres": + return postgres.Open(dataSourceName) + default: + return nil, errors.New("unknown schema") } - return &d, nil -} - -// StoreMediaMetadata inserts the metadata about the uploaded media into the database. -// Returns an error if the combination of MediaID and Origin are not unique in the table. -func (d *Database) StoreMediaMetadata( - ctx context.Context, mediaMetadata *types.MediaMetadata, -) error { - return d.statements.media.insertMedia(ctx, mediaMetadata) -} - -// GetMediaMetadata returns metadata about media stored on this server. -// The media could have been uploaded to this server or fetched from another server and cached here. -// Returns nil metadata if there is no metadata associated with this media. -func (d *Database) GetMediaMetadata( - ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, -) (*types.MediaMetadata, error) { - mediaMetadata, err := d.statements.media.selectMedia(ctx, mediaID, mediaOrigin) - if err != nil && err == sql.ErrNoRows { - return nil, nil - } - return mediaMetadata, err -} - -// StoreThumbnail inserts the metadata about the thumbnail into the database. -// Returns an error if the combination of MediaID and Origin are not unique in the table. -func (d *Database) StoreThumbnail( - ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata, -) error { - return d.statements.thumbnail.insertThumbnail(ctx, thumbnailMetadata) -} - -// GetThumbnail returns metadata about a specific thumbnail. -// The media could have been uploaded to this server or fetched from another server and cached here. -// Returns nil metadata if there is no metadata associated with this thumbnail. -func (d *Database) GetThumbnail( - ctx context.Context, - mediaID types.MediaID, - mediaOrigin gomatrixserverlib.ServerName, - width, height int, - resizeMethod string, -) (*types.ThumbnailMetadata, error) { - thumbnailMetadata, err := d.statements.thumbnail.selectThumbnail( - ctx, mediaID, mediaOrigin, width, height, resizeMethod, - ) - if err != nil && err == sql.ErrNoRows { - return nil, nil - } - return thumbnailMetadata, err -} - -// GetThumbnails returns metadata about all thumbnails for a specific media stored on this server. -// The media could have been uploaded to this server or fetched from another server and cached here. -// Returns nil metadata if there are no thumbnails associated with this media. -func (d *Database) GetThumbnails( - ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, -) ([]*types.ThumbnailMetadata, error) { - thumbnails, err := d.statements.thumbnail.selectThumbnails(ctx, mediaID, mediaOrigin) - if err != nil && err == sql.ErrNoRows { - return nil, nil - } - return thumbnails, err } diff --git a/mediaapi/thumbnailer/thumbnailer.go b/mediaapi/thumbnailer/thumbnailer.go index 61b66ebc6..ebf5138c5 100644 --- a/mediaapi/thumbnailer/thumbnailer.go +++ b/mediaapi/thumbnailer/thumbnailer.go @@ -136,7 +136,7 @@ func isThumbnailExists( dst types.Path, config types.ThumbnailSize, mediaMetadata *types.MediaMetadata, - db *storage.Database, + db storage.Database, logger *log.Entry, ) (bool, error) { thumbnailMetadata, err := db.GetThumbnail( diff --git a/mediaapi/thumbnailer/thumbnailer_nfnt.go b/mediaapi/thumbnailer/thumbnailer_nfnt.go index 5df6ce4be..4f1e98aa0 100644 --- a/mediaapi/thumbnailer/thumbnailer_nfnt.go +++ b/mediaapi/thumbnailer/thumbnailer_nfnt.go @@ -45,7 +45,7 @@ func GenerateThumbnails( mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, - db *storage.Database, + db storage.Database, logger *log.Entry, ) (busy bool, errorReturn error) { img, err := readFile(string(src)) @@ -78,7 +78,7 @@ func GenerateThumbnail( mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, - db *storage.Database, + db storage.Database, logger *log.Entry, ) (busy bool, errorReturn error) { img, err := readFile(string(src)) @@ -142,7 +142,7 @@ func createThumbnail( mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, - db *storage.Database, + db storage.Database, logger *log.Entry, ) (busy bool, errorReturn error) { logger = logger.WithFields(log.Fields{