mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2025-12-10 08:23:11 -06:00
Code Issues Packages Projects Releases Wiki Activity

mediaapi: ContentLength -> FileSizeBytes

Browse source
This commit is contained in:
Robert Swain 2017-05-22 15:07:45 +02:00
parent 1e639b0868
commit 2004dfcfcf
7 changed files with 34 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go
View file

@ -66,7 +66,7 @@ func main() {
cfg := &config.MediaAPI{ cfg := &config.MediaAPI{
ServerName: gomatrixserverlib.ServerName(serverName), ServerName: gomatrixserverlib.ServerName(serverName),
AbsBasePath: types.Path(absBasePath), AbsBasePath: types.Path(absBasePath),
MaxFileSizeBytes: types.ContentLength(maxFileSizeBytes), MaxFileSizeBytes: types.FileSizeBytes(maxFileSizeBytes),
DataSource: dataSource, DataSource: dataSource,
} }

2
src/github.com/matrix-org/dendrite/mediaapi/config/config.go
View file

@ -28,7 +28,7 @@ type MediaAPI struct {
// The maximum file size in bytes that is allowed to be stored on this server. // The maximum file size in bytes that is allowed to be stored on this server.
// Note that remote files larger than this can still be proxied to a client, they will just not be cached. // Note that remote files larger than this can still be proxied to a client, they will just not be cached.
// Note: if MaxFileSizeBytes is set to 0, the size is unlimited. // Note: if MaxFileSizeBytes is set to 0, the size is unlimited.
MaxFileSizeBytes types.ContentLength `yaml:"max_file_size_bytes"` MaxFileSizeBytes types.FileSizeBytes `yaml:"max_file_size_bytes"`
// The postgres connection config for connecting to the database e.g a postgres:// URI // The postgres connection config for connecting to the database e.g a postgres:// URI
DataSource string `yaml:"database"` DataSource string `yaml:"database"`
} }

12
src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go
View file

@ -103,7 +103,7 @@ func writeToResponse(respWriter http.ResponseWriter, buffer []byte, bytesToWrite
// writeToDiskAndHasher takes bytesToWrite bytes from buffer and writes them to tmpFileWriter and hasher. // writeToDiskAndHasher takes bytesToWrite bytes from buffer and writes them to tmpFileWriter and hasher.
// Returns bytes written and an error. In case of error, including if writing would exceed maxFileSizeBytes, // Returns bytes written and an error. In case of error, including if writing would exceed maxFileSizeBytes,
// the number of bytes written will be 0. // the number of bytes written will be 0.
func writeToDiskAndHasher(tmpFileWriter *bufio.Writer, hasher hash.Hash, bytesWritten int64, maxFileSizeBytes types.ContentLength, buffer []byte, bytesToWrite int) (int64, error) { func writeToDiskAndHasher(tmpFileWriter *bufio.Writer, hasher hash.Hash, bytesWritten int64, maxFileSizeBytes types.FileSizeBytes, buffer []byte, bytesToWrite int) (int64, error) {
// if larger than maxFileSizeBytes then stop writing to disk and discard cached file // if larger than maxFileSizeBytes then stop writing to disk and discard cached file
if bytesWritten+int64(bytesToWrite) > int64(maxFileSizeBytes) { if bytesWritten+int64(bytesToWrite) > int64(maxFileSizeBytes) {
return 0, ErrFileIsTooLarge return 0, ErrFileIsTooLarge
@ -129,7 +129,7 @@ func writeToDiskAndHasher(tmpFileWriter *bufio.Writer, hasher hash.Hash, bytesWr
// If a respWriter is provided, all the data will be proxied from the reqReader to // If a respWriter is provided, all the data will be proxied from the reqReader to
// the respWriter, regardless of errors or limits on writing to disk. // the respWriter, regardless of errors or limits on writing to disk.
// Returns all of the hash sum, bytes written to disk, and temporary directory path, or an error. // Returns all of the hash sum, bytes written to disk, and temporary directory path, or an error.
func ReadAndHashAndWriteWithLimit(reqReader io.Reader, maxFileSizeBytes types.ContentLength, absBasePath types.Path, respWriter http.ResponseWriter) (types.Base64Hash, types.ContentLength, types.ContentLength, types.Path, error) { func ReadAndHashAndWriteWithLimit(reqReader io.Reader, maxFileSizeBytes types.FileSizeBytes, absBasePath types.Path, respWriter http.ResponseWriter) (types.Base64Hash, types.FileSizeBytes, types.FileSizeBytes, types.Path, error) {
// create the temporary file writer // create the temporary file writer
tmpFileWriter, tmpFile, tmpDir, err := createTempFileWriter(absBasePath) tmpFileWriter, tmpFile, tmpDir, err := createTempFileWriter(absBasePath)
if err != nil { if err != nil {
@ -185,7 +185,7 @@ func ReadAndHashAndWriteWithLimit(reqReader io.Reader, maxFileSizeBytes types.Co
tmpFileWriter.Flush() tmpFileWriter.Flush()
hash := hasher.Sum(nil) hash := hasher.Sum(nil)
return types.Base64Hash(base64.URLEncoding.EncodeToString(hash[:])), types.ContentLength(bytesResponded), types.ContentLength(bytesWritten), tmpDir, nil return types.Base64Hash(base64.URLEncoding.EncodeToString(hash[:])), types.FileSizeBytes(bytesResponded), types.FileSizeBytes(bytesWritten), tmpDir, nil
} }
// GetPathFromMediaMetadata validates and constructs the on-disk path to the media // GetPathFromMediaMetadata validates and constructs the on-disk path to the media
@ -257,8 +257,8 @@ func moveFile(src types.Path, dst types.Path) error {
// MoveFileWithHashCheck attempts to move the file src to dst and checks for hash collisions based on metadata // MoveFileWithHashCheck attempts to move the file src to dst and checks for hash collisions based on metadata
// Check if destination file exists. As the destination is based on a hash of the file data, // Check if destination file exists. As the destination is based on a hash of the file data,
// if it exists and the content length does not match then there is a hash collision for two different files. If // if it exists and the file size does not match then there is a hash collision for two different files. If
// it exists and the content length matches, it is believable that it is the same file and we can just // it exists and the file size matches, it is believable that it is the same file and we can just
// discard the temporary file. // discard the temporary file.
func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata, absBasePath types.Path, logger *log.Entry) (string, bool, error) { func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata, absBasePath types.Path, logger *log.Entry) (string, bool, error) {
duplicate := false duplicate := false
@ -271,7 +271,7 @@ func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata
var stat os.FileInfo var stat os.FileInfo
if stat, err = os.Stat(finalPath); os.IsExist(err) { if stat, err = os.Stat(finalPath); os.IsExist(err) {
duplicate = true duplicate = true
if stat.Size() == int64(mediaMetadata.ContentLength) { if stat.Size() == int64(mediaMetadata.FileSizeBytes) {
RemoveDir(tmpDir, logger) RemoveDir(tmpDir, logger)
return finalPath, duplicate, nil return finalPath, duplicate, nil
} }

10
src/github.com/matrix-org/dendrite/mediaapi/storage/media_repository_table.go
View file

@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS media_repository (
-- The HTTP Content-Disposition header for the media file as specified when uploading. -- The HTTP Content-Disposition header for the media file as specified when uploading.
content_disposition TEXT NOT NULL, content_disposition TEXT NOT NULL,
-- Size of the media file in bytes. -- Size of the media file in bytes.
content_length BIGINT NOT NULL, file_size_bytes BIGINT NOT NULL,
-- When the content was uploaded in UNIX epoch ms. -- When the content was uploaded in UNIX epoch ms.
creation_ts BIGINT NOT NULL, creation_ts BIGINT NOT NULL,
-- The file name with which the media was uploaded. -- The file name with which the media was uploaded.
@ -51,12 +51,12 @@ CREATE UNIQUE INDEX IF NOT EXISTS media_repository_index ON media_repository (me
` `
const insertMediaSQL = ` const insertMediaSQL = `
INSERT INTO media_repository (media_id, media_origin, content_type, content_disposition, content_length, creation_ts, upload_name, base64hash, user_id) INSERT INTO media_repository (media_id, media_origin, content_type, content_disposition, file_size_bytes, creation_ts, upload_name, base64hash, user_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
` `
const selectMediaSQL = ` const selectMediaSQL = `
SELECT content_type, content_disposition, content_length, creation_ts, upload_name, base64hash, user_id FROM media_repository WHERE media_id = $1 AND media_origin = $2 SELECT content_type, content_disposition, file_size_bytes, creation_ts, upload_name, base64hash, user_id FROM media_repository WHERE media_id = $1 AND media_origin = $2
` `
type mediaStatements struct { type mediaStatements struct {
@ -83,7 +83,7 @@ func (s *mediaStatements) insertMedia(mediaMetadata *types.MediaMetadata) error
mediaMetadata.Origin, mediaMetadata.Origin,
mediaMetadata.ContentType, mediaMetadata.ContentType,
mediaMetadata.ContentDisposition, mediaMetadata.ContentDisposition,
mediaMetadata.ContentLength, mediaMetadata.FileSizeBytes,
mediaMetadata.CreationTimestamp, mediaMetadata.CreationTimestamp,
mediaMetadata.UploadName, mediaMetadata.UploadName,
mediaMetadata.Base64Hash, mediaMetadata.Base64Hash,
@ -102,7 +102,7 @@ func (s *mediaStatements) selectMedia(mediaID types.MediaID, mediaOrigin gomatri
).Scan( ).Scan(
&mediaMetadata.ContentType, &mediaMetadata.ContentType,
&mediaMetadata.ContentDisposition, &mediaMetadata.ContentDisposition,
&mediaMetadata.ContentLength, &mediaMetadata.FileSizeBytes,
&mediaMetadata.CreationTimestamp, &mediaMetadata.CreationTimestamp,
&mediaMetadata.UploadName, &mediaMetadata.UploadName,
&mediaMetadata.Base64Hash, &mediaMetadata.Base64Hash,

6
src/github.com/matrix-org/dendrite/mediaapi/types/types.go
View file

@ -23,8 +23,8 @@ import (
// ContentDisposition is an HTTP Content-Disposition header string // ContentDisposition is an HTTP Content-Disposition header string
type ContentDisposition string type ContentDisposition string
// ContentLength is an HTTP Content-Length header which is a number of bytes to be expected in a request body // FileSizeBytes is a file size in bytes
type ContentLength int64 type FileSizeBytes int64
// ContentType is an HTTP Content-Type header string representing the MIME type of a request body // ContentType is an HTTP Content-Type header string representing the MIME type of a request body
type ContentType string type ContentType string
@ -56,7 +56,7 @@ type MediaMetadata struct {
Origin gomatrixserverlib.ServerName Origin gomatrixserverlib.ServerName
ContentType ContentType ContentType ContentType
ContentDisposition ContentDisposition ContentDisposition ContentDisposition
ContentLength ContentLength FileSizeBytes FileSizeBytes
CreationTimestamp UnixMs CreationTimestamp UnixMs
UploadName Filename UploadName Filename
Base64Hash Base64Hash Base64Hash Base64Hash

22
src/github.com/matrix-org/dendrite/mediaapi/writers/download.go
View file

@ -179,7 +179,7 @@ func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePat
"Origin": r.MediaMetadata.Origin, "Origin": r.MediaMetadata.Origin,
"UploadName": r.MediaMetadata.UploadName, "UploadName": r.MediaMetadata.UploadName,
"Base64Hash": r.MediaMetadata.Base64Hash, "Base64Hash": r.MediaMetadata.Base64Hash,
"Content-Length": r.MediaMetadata.ContentLength, "FileSizeBytes": r.MediaMetadata.FileSizeBytes,
"Content-Type": r.MediaMetadata.ContentType, "Content-Type": r.MediaMetadata.ContentType,
"Content-Disposition": r.MediaMetadata.ContentDisposition, "Content-Disposition": r.MediaMetadata.ContentDisposition,
}).Infof("Downloading file") }).Infof("Downloading file")
@ -216,11 +216,11 @@ func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePat
return return
} }
if r.MediaMetadata.ContentLength > 0 && int64(r.MediaMetadata.ContentLength) != stat.Size() { if r.MediaMetadata.FileSizeBytes > 0 && int64(r.MediaMetadata.FileSizeBytes) != stat.Size() {
r.Logger.WithFields(log.Fields{ r.Logger.WithFields(log.Fields{
"contentLength": r.MediaMetadata.ContentLength, "fileSizeDatabase": r.MediaMetadata.FileSizeBytes,
"fileSize": stat.Size(), "fileSizeDisk": stat.Size(),
}).Warn("Content-Length in database and on-disk file size differ.") }).Warn("File size in database and on-disk differ.")
// FIXME: Remove erroneous file from database? // FIXME: Remove erroneous file from database?
} }
@ -336,7 +336,7 @@ func (r *downloadRequest) commitFileAndMetadata(tmpDir types.Path, absBasePath t
"Origin": r.MediaMetadata.Origin, "Origin": r.MediaMetadata.Origin,
"Base64Hash": r.MediaMetadata.Base64Hash, "Base64Hash": r.MediaMetadata.Base64Hash,
"UploadName": r.MediaMetadata.UploadName, "UploadName": r.MediaMetadata.UploadName,
"Content-Length": r.MediaMetadata.ContentLength, "FileSizeBytes": r.MediaMetadata.FileSizeBytes,
"Content-Type": r.MediaMetadata.ContentType, "Content-Type": r.MediaMetadata.ContentType,
"Content-Disposition": r.MediaMetadata.ContentDisposition, "Content-Disposition": r.MediaMetadata.ContentDisposition,
}).Info("Storing file metadata to media repository database") }).Info("Storing file metadata to media repository database")
@ -381,7 +381,7 @@ func (r *downloadRequest) commitFileAndMetadata(tmpDir types.Path, absBasePath t
return updateActiveRemoteRequests return updateActiveRemoteRequests
} }
func (r *downloadRequest) respondFromRemoteFile(w http.ResponseWriter, absBasePath types.Path, maxFileSizeBytes types.ContentLength, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests) { func (r *downloadRequest) respondFromRemoteFile(w http.ResponseWriter, absBasePath types.Path, maxFileSizeBytes types.FileSizeBytes, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests) {
r.Logger.WithFields(log.Fields{ r.Logger.WithFields(log.Fields{
"Origin": r.MediaMetadata.Origin, "Origin": r.MediaMetadata.Origin,
"MediaID": r.MediaMetadata.MediaID, "MediaID": r.MediaMetadata.MediaID,
@ -415,7 +415,7 @@ func (r *downloadRequest) respondFromRemoteFile(w http.ResponseWriter, absBasePa
if err != nil { if err != nil {
r.Logger.WithError(err).Warn("Failed to parse content length") r.Logger.WithError(err).Warn("Failed to parse content length")
} }
r.MediaMetadata.ContentLength = types.ContentLength(contentLength) r.MediaMetadata.FileSizeBytes = types.FileSizeBytes(contentLength)
r.MediaMetadata.ContentType = types.ContentType(resp.Header.Get("Content-Type")) r.MediaMetadata.ContentType = types.ContentType(resp.Header.Get("Content-Type"))
r.MediaMetadata.ContentDisposition = types.ContentDisposition(resp.Header.Get("Content-Disposition")) r.MediaMetadata.ContentDisposition = types.ContentDisposition(resp.Header.Get("Content-Disposition"))
@ -428,7 +428,7 @@ func (r *downloadRequest) respondFromRemoteFile(w http.ResponseWriter, absBasePa
}).Infof("Connected to remote") }).Infof("Connected to remote")
w.Header().Set("Content-Type", string(r.MediaMetadata.ContentType)) w.Header().Set("Content-Type", string(r.MediaMetadata.ContentType))
w.Header().Set("Content-Length", strconv.FormatInt(int64(r.MediaMetadata.ContentLength), 10)) w.Header().Set("Content-Length", strconv.FormatInt(int64(r.MediaMetadata.FileSizeBytes), 10))
contentSecurityPolicy := "default-src 'none';" + contentSecurityPolicy := "default-src 'none';" +
" script-src 'none';" + " script-src 'none';" +
" plugin-types application/pdf;" + " plugin-types application/pdf;" +
@ -483,7 +483,7 @@ func (r *downloadRequest) respondFromRemoteFile(w http.ResponseWriter, absBasePa
// It's possible the bytesWritten to the temporary file is different to the reported Content-Length from the remote // It's possible the bytesWritten to the temporary file is different to the reported Content-Length from the remote
// request's response. bytesWritten is therefore used as it is what would be sent to clients when reading from the local // request's response. bytesWritten is therefore used as it is what would be sent to clients when reading from the local
// file. // file.
r.MediaMetadata.ContentLength = types.ContentLength(bytesWritten) r.MediaMetadata.FileSizeBytes = types.FileSizeBytes(bytesWritten)
r.MediaMetadata.Base64Hash = hash r.MediaMetadata.Base64Hash = hash
r.MediaMetadata.UserID = types.MatrixUserID("@:" + string(r.MediaMetadata.Origin)) r.MediaMetadata.UserID = types.MatrixUserID("@:" + string(r.MediaMetadata.Origin))
@ -496,7 +496,7 @@ func (r *downloadRequest) respondFromRemoteFile(w http.ResponseWriter, absBasePa
"Origin": r.MediaMetadata.Origin, "Origin": r.MediaMetadata.Origin,
"UploadName": r.MediaMetadata.UploadName, "UploadName": r.MediaMetadata.UploadName,
"Base64Hash": r.MediaMetadata.Base64Hash, "Base64Hash": r.MediaMetadata.Base64Hash,
"Content-Length": r.MediaMetadata.ContentLength, "FileSizeBytes": r.MediaMetadata.FileSizeBytes,
"Content-Type": r.MediaMetadata.ContentType, "Content-Type": r.MediaMetadata.ContentType,
"Content-Disposition": r.MediaMetadata.ContentDisposition, "Content-Disposition": r.MediaMetadata.ContentDisposition,
}).Infof("Remote file cached") }).Infof("Remote file cached")

14
src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go
View file

@ -40,16 +40,16 @@ type uploadRequest struct {
} }
// Validate validates the uploadRequest fields // Validate validates the uploadRequest fields
func (r uploadRequest) Validate(maxFileSizeBytes types.ContentLength) *util.JSONResponse { func (r uploadRequest) Validate(maxFileSizeBytes types.FileSizeBytes) *util.JSONResponse {
// TODO: Any validation to be done on ContentDisposition? // TODO: Any validation to be done on ContentDisposition?
if r.MediaMetadata.ContentLength < 1 { if r.MediaMetadata.FileSizeBytes < 1 {
return &util.JSONResponse{ return &util.JSONResponse{
Code: 400, Code: 400,
JSON: jsonerror.Unknown("HTTP Content-Length request header must be greater than zero."), JSON: jsonerror.Unknown("HTTP Content-Length request header must be greater than zero."),
} }
} }
if maxFileSizeBytes > 0 && r.MediaMetadata.ContentLength > maxFileSizeBytes { if maxFileSizeBytes > 0 && r.MediaMetadata.FileSizeBytes > maxFileSizeBytes {
return &util.JSONResponse{ return &util.JSONResponse{
Code: 400, Code: 400,
JSON: jsonerror.Unknown(fmt.Sprintf("HTTP Content-Length is greater than the maximum allowed upload size (%v).", maxFileSizeBytes)), JSON: jsonerror.Unknown(fmt.Sprintf("HTTP Content-Length is greater than the maximum allowed upload size (%v).", maxFileSizeBytes)),
@ -113,7 +113,7 @@ func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRe
MediaMetadata: &types.MediaMetadata{ MediaMetadata: &types.MediaMetadata{
Origin: cfg.ServerName, Origin: cfg.ServerName,
ContentDisposition: types.ContentDisposition(req.Header.Get("Content-Disposition")), ContentDisposition: types.ContentDisposition(req.Header.Get("Content-Disposition")),
ContentLength: types.ContentLength(req.ContentLength), FileSizeBytes: types.FileSizeBytes(req.ContentLength),
ContentType: types.ContentType(req.Header.Get("Content-Type")), ContentType: types.ContentType(req.Header.Get("Content-Type")),
UploadName: types.Filename(req.FormValue("filename")), UploadName: types.Filename(req.FormValue("filename")),
UserID: types.MatrixUserID(userID), UserID: types.MatrixUserID(userID),
@ -189,7 +189,7 @@ func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database) util.
logger.WithFields(log.Fields{ logger.WithFields(log.Fields{
"Origin": r.MediaMetadata.Origin, "Origin": r.MediaMetadata.Origin,
"UploadName": r.MediaMetadata.UploadName, "UploadName": r.MediaMetadata.UploadName,
"Content-Length": r.MediaMetadata.ContentLength, "FileSizeBytes": r.MediaMetadata.FileSizeBytes,
"Content-Type": r.MediaMetadata.ContentType, "Content-Type": r.MediaMetadata.ContentType,
"Content-Disposition": r.MediaMetadata.ContentDisposition, "Content-Disposition": r.MediaMetadata.ContentDisposition,
}).Info("Uploading file") }).Info("Uploading file")
@ -216,7 +216,7 @@ func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database) util.
} }
} }
r.MediaMetadata.ContentLength = bytesWritten r.MediaMetadata.FileSizeBytes = bytesWritten
r.MediaMetadata.Base64Hash = hash r.MediaMetadata.Base64Hash = hash
r.MediaMetadata.MediaID = types.MediaID(hash) r.MediaMetadata.MediaID = types.MediaID(hash)
@ -225,7 +225,7 @@ func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database) util.
"Origin": r.MediaMetadata.Origin, "Origin": r.MediaMetadata.Origin,
"Base64Hash": r.MediaMetadata.Base64Hash, "Base64Hash": r.MediaMetadata.Base64Hash,
"UploadName": r.MediaMetadata.UploadName, "UploadName": r.MediaMetadata.UploadName,
"Content-Length": r.MediaMetadata.ContentLength, "FileSizeBytes": r.MediaMetadata.FileSizeBytes,
"Content-Type": r.MediaMetadata.ContentType, "Content-Type": r.MediaMetadata.ContentType,
"Content-Disposition": r.MediaMetadata.ContentDisposition, "Content-Disposition": r.MediaMetadata.ContentDisposition,
}).Info("File uploaded") }).Info("File uploaded")