From 2004dfcfcf4389a9ad3d454c2d7f58d5be39af83 Mon Sep 17 00:00:00 2001 From: Robert Swain Date: Mon, 22 May 2017 15:07:45 +0200 Subject: [PATCH] mediaapi: ContentLength -> FileSizeBytes --- .../cmd/dendrite-media-api-server/main.go | 2 +- .../dendrite/mediaapi/config/config.go | 2 +- .../dendrite/mediaapi/fileutils/fileutils.go | 12 +++++----- .../storage/media_repository_table.go | 10 ++++----- .../dendrite/mediaapi/types/types.go | 6 ++--- .../dendrite/mediaapi/writers/download.go | 22 +++++++++---------- .../dendrite/mediaapi/writers/upload.go | 14 ++++++------ 7 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go index b053076a2..298762482 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go @@ -66,7 +66,7 @@ func main() { cfg := &config.MediaAPI{ ServerName: gomatrixserverlib.ServerName(serverName), AbsBasePath: types.Path(absBasePath), - MaxFileSizeBytes: types.ContentLength(maxFileSizeBytes), + MaxFileSizeBytes: types.FileSizeBytes(maxFileSizeBytes), DataSource: dataSource, } diff --git a/src/github.com/matrix-org/dendrite/mediaapi/config/config.go b/src/github.com/matrix-org/dendrite/mediaapi/config/config.go index 6163f2315..a2d8f43c6 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/config/config.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/config/config.go @@ -28,7 +28,7 @@ type MediaAPI struct { // The maximum file size in bytes that is allowed to be stored on this server. // Note that remote files larger than this can still be proxied to a client, they will just not be cached. // Note: if MaxFileSizeBytes is set to 0, the size is unlimited. - MaxFileSizeBytes types.ContentLength `yaml:"max_file_size_bytes"` + MaxFileSizeBytes types.FileSizeBytes `yaml:"max_file_size_bytes"` // The postgres connection config for connecting to the database e.g a postgres:// URI DataSource string `yaml:"database"` } diff --git a/src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go b/src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go index 99d6dbd2c..5db185e8d 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go @@ -103,7 +103,7 @@ func writeToResponse(respWriter http.ResponseWriter, buffer []byte, bytesToWrite // writeToDiskAndHasher takes bytesToWrite bytes from buffer and writes them to tmpFileWriter and hasher. // Returns bytes written and an error. In case of error, including if writing would exceed maxFileSizeBytes, // the number of bytes written will be 0. -func writeToDiskAndHasher(tmpFileWriter *bufio.Writer, hasher hash.Hash, bytesWritten int64, maxFileSizeBytes types.ContentLength, buffer []byte, bytesToWrite int) (int64, error) { +func writeToDiskAndHasher(tmpFileWriter *bufio.Writer, hasher hash.Hash, bytesWritten int64, maxFileSizeBytes types.FileSizeBytes, buffer []byte, bytesToWrite int) (int64, error) { // if larger than maxFileSizeBytes then stop writing to disk and discard cached file if bytesWritten+int64(bytesToWrite) > int64(maxFileSizeBytes) { return 0, ErrFileIsTooLarge @@ -129,7 +129,7 @@ func writeToDiskAndHasher(tmpFileWriter *bufio.Writer, hasher hash.Hash, bytesWr // If a respWriter is provided, all the data will be proxied from the reqReader to // the respWriter, regardless of errors or limits on writing to disk. // Returns all of the hash sum, bytes written to disk, and temporary directory path, or an error. -func ReadAndHashAndWriteWithLimit(reqReader io.Reader, maxFileSizeBytes types.ContentLength, absBasePath types.Path, respWriter http.ResponseWriter) (types.Base64Hash, types.ContentLength, types.ContentLength, types.Path, error) { +func ReadAndHashAndWriteWithLimit(reqReader io.Reader, maxFileSizeBytes types.FileSizeBytes, absBasePath types.Path, respWriter http.ResponseWriter) (types.Base64Hash, types.FileSizeBytes, types.FileSizeBytes, types.Path, error) { // create the temporary file writer tmpFileWriter, tmpFile, tmpDir, err := createTempFileWriter(absBasePath) if err != nil { @@ -185,7 +185,7 @@ func ReadAndHashAndWriteWithLimit(reqReader io.Reader, maxFileSizeBytes types.Co tmpFileWriter.Flush() hash := hasher.Sum(nil) - return types.Base64Hash(base64.URLEncoding.EncodeToString(hash[:])), types.ContentLength(bytesResponded), types.ContentLength(bytesWritten), tmpDir, nil + return types.Base64Hash(base64.URLEncoding.EncodeToString(hash[:])), types.FileSizeBytes(bytesResponded), types.FileSizeBytes(bytesWritten), tmpDir, nil } // GetPathFromMediaMetadata validates and constructs the on-disk path to the media @@ -257,8 +257,8 @@ func moveFile(src types.Path, dst types.Path) error { // MoveFileWithHashCheck attempts to move the file src to dst and checks for hash collisions based on metadata // Check if destination file exists. As the destination is based on a hash of the file data, -// if it exists and the content length does not match then there is a hash collision for two different files. If -// it exists and the content length matches, it is believable that it is the same file and we can just +// if it exists and the file size does not match then there is a hash collision for two different files. If +// it exists and the file size matches, it is believable that it is the same file and we can just // discard the temporary file. func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata, absBasePath types.Path, logger *log.Entry) (string, bool, error) { duplicate := false @@ -271,7 +271,7 @@ func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata var stat os.FileInfo if stat, err = os.Stat(finalPath); os.IsExist(err) { duplicate = true - if stat.Size() == int64(mediaMetadata.ContentLength) { + if stat.Size() == int64(mediaMetadata.FileSizeBytes) { RemoveDir(tmpDir, logger) return finalPath, duplicate, nil } diff --git a/src/github.com/matrix-org/dendrite/mediaapi/storage/media_repository_table.go b/src/github.com/matrix-org/dendrite/mediaapi/storage/media_repository_table.go index c19a9d9b3..a3b1c7594 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/storage/media_repository_table.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/storage/media_repository_table.go @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS media_repository ( -- The HTTP Content-Disposition header for the media file as specified when uploading. content_disposition TEXT NOT NULL, -- Size of the media file in bytes. - content_length BIGINT NOT NULL, + file_size_bytes BIGINT NOT NULL, -- When the content was uploaded in UNIX epoch ms. creation_ts BIGINT NOT NULL, -- The file name with which the media was uploaded. @@ -51,12 +51,12 @@ CREATE UNIQUE INDEX IF NOT EXISTS media_repository_index ON media_repository (me ` const insertMediaSQL = ` -INSERT INTO media_repository (media_id, media_origin, content_type, content_disposition, content_length, creation_ts, upload_name, base64hash, user_id) +INSERT INTO media_repository (media_id, media_origin, content_type, content_disposition, file_size_bytes, creation_ts, upload_name, base64hash, user_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ` const selectMediaSQL = ` -SELECT content_type, content_disposition, content_length, creation_ts, upload_name, base64hash, user_id FROM media_repository WHERE media_id = $1 AND media_origin = $2 +SELECT content_type, content_disposition, file_size_bytes, creation_ts, upload_name, base64hash, user_id FROM media_repository WHERE media_id = $1 AND media_origin = $2 ` type mediaStatements struct { @@ -83,7 +83,7 @@ func (s *mediaStatements) insertMedia(mediaMetadata *types.MediaMetadata) error mediaMetadata.Origin, mediaMetadata.ContentType, mediaMetadata.ContentDisposition, - mediaMetadata.ContentLength, + mediaMetadata.FileSizeBytes, mediaMetadata.CreationTimestamp, mediaMetadata.UploadName, mediaMetadata.Base64Hash, @@ -102,7 +102,7 @@ func (s *mediaStatements) selectMedia(mediaID types.MediaID, mediaOrigin gomatri ).Scan( &mediaMetadata.ContentType, &mediaMetadata.ContentDisposition, - &mediaMetadata.ContentLength, + &mediaMetadata.FileSizeBytes, &mediaMetadata.CreationTimestamp, &mediaMetadata.UploadName, &mediaMetadata.Base64Hash, diff --git a/src/github.com/matrix-org/dendrite/mediaapi/types/types.go b/src/github.com/matrix-org/dendrite/mediaapi/types/types.go index bef86adcc..cef390cf0 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/types/types.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/types/types.go @@ -23,8 +23,8 @@ import ( // ContentDisposition is an HTTP Content-Disposition header string type ContentDisposition string -// ContentLength is an HTTP Content-Length header which is a number of bytes to be expected in a request body -type ContentLength int64 +// FileSizeBytes is a file size in bytes +type FileSizeBytes int64 // ContentType is an HTTP Content-Type header string representing the MIME type of a request body type ContentType string @@ -56,7 +56,7 @@ type MediaMetadata struct { Origin gomatrixserverlib.ServerName ContentType ContentType ContentDisposition ContentDisposition - ContentLength ContentLength + FileSizeBytes FileSizeBytes CreationTimestamp UnixMs UploadName Filename Base64Hash Base64Hash diff --git a/src/github.com/matrix-org/dendrite/mediaapi/writers/download.go b/src/github.com/matrix-org/dendrite/mediaapi/writers/download.go index 4c682ce99..98083c756 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/writers/download.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/writers/download.go @@ -179,7 +179,7 @@ func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePat "Origin": r.MediaMetadata.Origin, "UploadName": r.MediaMetadata.UploadName, "Base64Hash": r.MediaMetadata.Base64Hash, - "Content-Length": r.MediaMetadata.ContentLength, + "FileSizeBytes": r.MediaMetadata.FileSizeBytes, "Content-Type": r.MediaMetadata.ContentType, "Content-Disposition": r.MediaMetadata.ContentDisposition, }).Infof("Downloading file") @@ -216,11 +216,11 @@ func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePat return } - if r.MediaMetadata.ContentLength > 0 && int64(r.MediaMetadata.ContentLength) != stat.Size() { + if r.MediaMetadata.FileSizeBytes > 0 && int64(r.MediaMetadata.FileSizeBytes) != stat.Size() { r.Logger.WithFields(log.Fields{ - "contentLength": r.MediaMetadata.ContentLength, - "fileSize": stat.Size(), - }).Warn("Content-Length in database and on-disk file size differ.") + "fileSizeDatabase": r.MediaMetadata.FileSizeBytes, + "fileSizeDisk": stat.Size(), + }).Warn("File size in database and on-disk differ.") // FIXME: Remove erroneous file from database? } @@ -336,7 +336,7 @@ func (r *downloadRequest) commitFileAndMetadata(tmpDir types.Path, absBasePath t "Origin": r.MediaMetadata.Origin, "Base64Hash": r.MediaMetadata.Base64Hash, "UploadName": r.MediaMetadata.UploadName, - "Content-Length": r.MediaMetadata.ContentLength, + "FileSizeBytes": r.MediaMetadata.FileSizeBytes, "Content-Type": r.MediaMetadata.ContentType, "Content-Disposition": r.MediaMetadata.ContentDisposition, }).Info("Storing file metadata to media repository database") @@ -381,7 +381,7 @@ func (r *downloadRequest) commitFileAndMetadata(tmpDir types.Path, absBasePath t return updateActiveRemoteRequests } -func (r *downloadRequest) respondFromRemoteFile(w http.ResponseWriter, absBasePath types.Path, maxFileSizeBytes types.ContentLength, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests) { +func (r *downloadRequest) respondFromRemoteFile(w http.ResponseWriter, absBasePath types.Path, maxFileSizeBytes types.FileSizeBytes, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests) { r.Logger.WithFields(log.Fields{ "Origin": r.MediaMetadata.Origin, "MediaID": r.MediaMetadata.MediaID, @@ -415,7 +415,7 @@ func (r *downloadRequest) respondFromRemoteFile(w http.ResponseWriter, absBasePa if err != nil { r.Logger.WithError(err).Warn("Failed to parse content length") } - r.MediaMetadata.ContentLength = types.ContentLength(contentLength) + r.MediaMetadata.FileSizeBytes = types.FileSizeBytes(contentLength) r.MediaMetadata.ContentType = types.ContentType(resp.Header.Get("Content-Type")) r.MediaMetadata.ContentDisposition = types.ContentDisposition(resp.Header.Get("Content-Disposition")) @@ -428,7 +428,7 @@ func (r *downloadRequest) respondFromRemoteFile(w http.ResponseWriter, absBasePa }).Infof("Connected to remote") w.Header().Set("Content-Type", string(r.MediaMetadata.ContentType)) - w.Header().Set("Content-Length", strconv.FormatInt(int64(r.MediaMetadata.ContentLength), 10)) + w.Header().Set("Content-Length", strconv.FormatInt(int64(r.MediaMetadata.FileSizeBytes), 10)) contentSecurityPolicy := "default-src 'none';" + " script-src 'none';" + " plugin-types application/pdf;" + @@ -483,7 +483,7 @@ func (r *downloadRequest) respondFromRemoteFile(w http.ResponseWriter, absBasePa // It's possible the bytesWritten to the temporary file is different to the reported Content-Length from the remote // request's response. bytesWritten is therefore used as it is what would be sent to clients when reading from the local // file. - r.MediaMetadata.ContentLength = types.ContentLength(bytesWritten) + r.MediaMetadata.FileSizeBytes = types.FileSizeBytes(bytesWritten) r.MediaMetadata.Base64Hash = hash r.MediaMetadata.UserID = types.MatrixUserID("@:" + string(r.MediaMetadata.Origin)) @@ -496,7 +496,7 @@ func (r *downloadRequest) respondFromRemoteFile(w http.ResponseWriter, absBasePa "Origin": r.MediaMetadata.Origin, "UploadName": r.MediaMetadata.UploadName, "Base64Hash": r.MediaMetadata.Base64Hash, - "Content-Length": r.MediaMetadata.ContentLength, + "FileSizeBytes": r.MediaMetadata.FileSizeBytes, "Content-Type": r.MediaMetadata.ContentType, "Content-Disposition": r.MediaMetadata.ContentDisposition, }).Infof("Remote file cached") diff --git a/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go b/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go index f8e635ece..a515e09bc 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go @@ -40,16 +40,16 @@ type uploadRequest struct { } // Validate validates the uploadRequest fields -func (r uploadRequest) Validate(maxFileSizeBytes types.ContentLength) *util.JSONResponse { +func (r uploadRequest) Validate(maxFileSizeBytes types.FileSizeBytes) *util.JSONResponse { // TODO: Any validation to be done on ContentDisposition? - if r.MediaMetadata.ContentLength < 1 { + if r.MediaMetadata.FileSizeBytes < 1 { return &util.JSONResponse{ Code: 400, JSON: jsonerror.Unknown("HTTP Content-Length request header must be greater than zero."), } } - if maxFileSizeBytes > 0 && r.MediaMetadata.ContentLength > maxFileSizeBytes { + if maxFileSizeBytes > 0 && r.MediaMetadata.FileSizeBytes > maxFileSizeBytes { return &util.JSONResponse{ Code: 400, JSON: jsonerror.Unknown(fmt.Sprintf("HTTP Content-Length is greater than the maximum allowed upload size (%v).", maxFileSizeBytes)), @@ -113,7 +113,7 @@ func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRe MediaMetadata: &types.MediaMetadata{ Origin: cfg.ServerName, ContentDisposition: types.ContentDisposition(req.Header.Get("Content-Disposition")), - ContentLength: types.ContentLength(req.ContentLength), + FileSizeBytes: types.FileSizeBytes(req.ContentLength), ContentType: types.ContentType(req.Header.Get("Content-Type")), UploadName: types.Filename(req.FormValue("filename")), UserID: types.MatrixUserID(userID), @@ -189,7 +189,7 @@ func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database) util. logger.WithFields(log.Fields{ "Origin": r.MediaMetadata.Origin, "UploadName": r.MediaMetadata.UploadName, - "Content-Length": r.MediaMetadata.ContentLength, + "FileSizeBytes": r.MediaMetadata.FileSizeBytes, "Content-Type": r.MediaMetadata.ContentType, "Content-Disposition": r.MediaMetadata.ContentDisposition, }).Info("Uploading file") @@ -216,7 +216,7 @@ func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database) util. } } - r.MediaMetadata.ContentLength = bytesWritten + r.MediaMetadata.FileSizeBytes = bytesWritten r.MediaMetadata.Base64Hash = hash r.MediaMetadata.MediaID = types.MediaID(hash) @@ -225,7 +225,7 @@ func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database) util. "Origin": r.MediaMetadata.Origin, "Base64Hash": r.MediaMetadata.Base64Hash, "UploadName": r.MediaMetadata.UploadName, - "Content-Length": r.MediaMetadata.ContentLength, + "FileSizeBytes": r.MediaMetadata.FileSizeBytes, "Content-Type": r.MediaMetadata.ContentType, "Content-Disposition": r.MediaMetadata.ContentDisposition, }).Info("File uploaded")