From 4d1bff2f61b48b1209e1d2aadc9b1d0fb15b2bbf Mon Sep 17 00:00:00 2001 From: Robert Swain Date: Mon, 24 Apr 2017 18:31:44 +0200 Subject: [PATCH] mediaapi: Initial commit for /upload HTTP infra --- .../cmd/dendrite-media-api-server/main.go | 54 +++++++ .../matrix-org/dendrite/mediaapi/README.md | 3 + .../dendrite/mediaapi/config/config.go | 25 ++++ .../dendrite/mediaapi/routing/routing.go | 45 ++++++ .../dendrite/mediaapi/storage/media.go | 47 ++++++ .../dendrite/mediaapi/storage/prepare.go | 37 +++++ .../dendrite/mediaapi/storage/sql.go | 33 +++++ .../dendrite/mediaapi/storage/storage.go | 38 +++++ .../dendrite/mediaapi/writers/upload.go | 139 ++++++++++++++++++ 9 files changed, 421 insertions(+) create mode 100644 src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go create mode 100644 src/github.com/matrix-org/dendrite/mediaapi/README.md create mode 100644 src/github.com/matrix-org/dendrite/mediaapi/config/config.go create mode 100644 src/github.com/matrix-org/dendrite/mediaapi/routing/routing.go create mode 100644 src/github.com/matrix-org/dendrite/mediaapi/storage/media.go create mode 100644 src/github.com/matrix-org/dendrite/mediaapi/storage/prepare.go create mode 100644 src/github.com/matrix-org/dendrite/mediaapi/storage/sql.go create mode 100644 src/github.com/matrix-org/dendrite/mediaapi/storage/storage.go create mode 100644 src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go new file mode 100644 index 000000000..884eac80f --- /dev/null +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go @@ -0,0 +1,54 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "net/http" + "os" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/mediaapi/config" + "github.com/matrix-org/dendrite/mediaapi/routing" + + log "github.com/Sirupsen/logrus" +) + +var ( + bindAddr = os.Getenv("BIND_ADDRESS") + database = os.Getenv("DATABASE") + logDir = os.Getenv("LOG_DIR") +) + +func main() { + common.SetupLogging(logDir) + if bindAddr == "" { + log.Panic("No BIND_ADDRESS environment variable found.") + } + // db, err := storage.Open(database) + // if err != nil { + // panic(err) + // } + + cfg := config.MediaAPI{ + ServerName: "localhost", + BasePath: "/Users/robertsw/dendrite", + DataSource: database, + } + + log.Info("Starting mediaapi") + + routing.Setup(http.DefaultServeMux, http.DefaultClient, cfg) + log.Fatal(http.ListenAndServe(bindAddr, nil)) +} diff --git a/src/github.com/matrix-org/dendrite/mediaapi/README.md b/src/github.com/matrix-org/dendrite/mediaapi/README.md new file mode 100644 index 000000000..2f51e8fe6 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/mediaapi/README.md @@ -0,0 +1,3 @@ +# Media API + +This server is responsible for serving `/media` requests diff --git a/src/github.com/matrix-org/dendrite/mediaapi/config/config.go b/src/github.com/matrix-org/dendrite/mediaapi/config/config.go new file mode 100644 index 000000000..5900d9d56 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/mediaapi/config/config.go @@ -0,0 +1,25 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +// MediaAPI contains the config information necessary to spin up a mediaapi process. +type MediaAPI struct { + // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. + ServerName string `yaml:"server_name"` + // The base path to where media files will be stored. + BasePath string `yaml:"base_path"` + // The postgres connection config for connecting to the database e.g a postgres:// URI + DataSource string `yaml:"database"` +} diff --git a/src/github.com/matrix-org/dendrite/mediaapi/routing/routing.go b/src/github.com/matrix-org/dendrite/mediaapi/routing/routing.go new file mode 100644 index 000000000..fd5ff7384 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/mediaapi/routing/routing.go @@ -0,0 +1,45 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/mediaapi/config" + "github.com/matrix-org/dendrite/mediaapi/writers" + "github.com/matrix-org/util" + "github.com/prometheus/client_golang/prometheus" +) + +const pathPrefixR0 = "/_matrix/media/v1" + +// Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client +// to clients which need to make outbound HTTP requests. +func Setup(servMux *http.ServeMux, httpClient *http.Client, cfg config.MediaAPI) { + apiMux := mux.NewRouter() + r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() + r0mux.Handle("/upload", make("upload", util.NewJSONRequestHandler(func(req *http.Request) util.JSONResponse { + return writers.Upload(req, cfg) + }))) + + servMux.Handle("/metrics", prometheus.Handler()) + servMux.Handle("/api/", http.StripPrefix("/api", apiMux)) +} + +// make a util.JSONRequestHandler into an http.Handler +func make(metricsName string, h util.JSONRequestHandler) http.Handler { + return prometheus.InstrumentHandler(metricsName, util.MakeJSONAPI(h)) +} diff --git a/src/github.com/matrix-org/dendrite/mediaapi/storage/media.go b/src/github.com/matrix-org/dendrite/mediaapi/storage/media.go new file mode 100644 index 000000000..f87401d73 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/mediaapi/storage/media.go @@ -0,0 +1,47 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "database/sql" +) + +const mediaSchema = ` +` + +const insertMediaSQL = "" + + "INSERT INTO events (room_nid, event_type_nid, event_state_key_nid, event_id, reference_sha256, auth_event_nids)" + + " VALUES ($1, $2, $3, $4, $5, $6)" + + " ON CONFLICT ON CONSTRAINT event_id_unique" + + " DO NOTHING" + + " RETURNING event_nid, state_snapshot_nid" + +type mediaStatements struct { + insertMediaStmt *sql.Stmt +} + +func (s *mediaStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(mediaSchema) + if err != nil { + return + } + + return statementList{ + {&s.insertMediaStmt, insertMediaSQL}, + }.prepare(db) +} + +func (s *mediaStatements) insertMedia() { +} diff --git a/src/github.com/matrix-org/dendrite/mediaapi/storage/prepare.go b/src/github.com/matrix-org/dendrite/mediaapi/storage/prepare.go new file mode 100644 index 000000000..a30586de4 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/mediaapi/storage/prepare.go @@ -0,0 +1,37 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// FIXME: This should be made common! + +package storage + +import ( + "database/sql" +) + +// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement. +type statementList []struct { + statement **sql.Stmt + sql string +} + +// prepare the SQL for each statement in the list and assign the result to the prepared statement. +func (s statementList) prepare(db *sql.DB) (err error) { + for _, statement := range s { + if *statement.statement, err = db.Prepare(statement.sql); err != nil { + return + } + } + return +} diff --git a/src/github.com/matrix-org/dendrite/mediaapi/storage/sql.go b/src/github.com/matrix-org/dendrite/mediaapi/storage/sql.go new file mode 100644 index 000000000..e992e073e --- /dev/null +++ b/src/github.com/matrix-org/dendrite/mediaapi/storage/sql.go @@ -0,0 +1,33 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "database/sql" +) + +type statements struct { + mediaStatements +} + +func (s *statements) prepare(db *sql.DB) error { + var err error + + if err = s.mediaStatements.prepare(db); err != nil { + return err + } + + return nil +} diff --git a/src/github.com/matrix-org/dendrite/mediaapi/storage/storage.go b/src/github.com/matrix-org/dendrite/mediaapi/storage/storage.go new file mode 100644 index 000000000..0c881a480 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/mediaapi/storage/storage.go @@ -0,0 +1,38 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "database/sql" +) + +// A Database is used to store room events and stream offsets. +type Database struct { + statements statements + db *sql.DB +} + +// Open a postgres database. +func Open(dataSourceName string) (*Database, error) { + var d Database + var err error + if d.db, err = sql.Open("postgres", dataSourceName); err != nil { + return nil, err + } + if err = d.statements.prepare(d.db); err != nil { + return nil, err + } + return &d, nil +} diff --git a/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go b/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go new file mode 100644 index 000000000..0e0d16c8b --- /dev/null +++ b/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go @@ -0,0 +1,139 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package writers + +import ( + "net/http" + "strconv" + "strings" + + log "github.com/Sirupsen/logrus" + "github.com/matrix-org/dendrite/clientapi/auth" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/mediaapi/config" + "github.com/matrix-org/util" +) + +// https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-media-r0-upload +// NOTE: ContentType is an HTTP request header and Filename is passed as a query parameter +type uploadRequest struct { + ContentDisposition string + ContentLength int + ContentType string + Filename string + Method string + UserID string +} + +func (r uploadRequest) Validate() *util.JSONResponse { + // TODO: Any validation to be done on ContentDisposition? + if r.ContentLength < 1 { + return &util.JSONResponse{ + Code: 400, + JSON: jsonerror.BadJSON("HTTP Content-Length request header must be greater than zero."), + } + } + // TODO: Check if the Content-Type is a valid type? + if r.ContentType == "" { + return &util.JSONResponse{ + Code: 400, + JSON: jsonerror.BadJSON("HTTP Content-Type request header must be set."), + } + } + // TODO: Validate filename - what are the valid characters? + if r.Method != "POST" { + return &util.JSONResponse{ + Code: 400, + JSON: jsonerror.BadJSON("HTTP request method must be POST."), + } + } + if r.UserID != "" { + // TODO: We should put user ID parsing code into gomatrixserverlib and use that instead + // (see https://github.com/matrix-org/gomatrixserverlib/blob/3394e7c7003312043208aa73727d2256eea3d1f6/eventcontent.go#L347 ) + // It should be a struct (with pointers into a single string to avoid copying) and + // we should update all refs to use UserID types rather than strings. + // https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/types.py#L92 + if len(r.UserID) == 0 || r.UserID[0] != '@' { + return &util.JSONResponse{ + Code: 400, + JSON: jsonerror.BadJSON("user id must start with '@'"), + } + } + parts := strings.SplitN(r.UserID[1:], ":", 2) + if len(parts) != 2 { + return &util.JSONResponse{ + Code: 400, + JSON: jsonerror.BadJSON("user id must be in the form @localpart:domain"), + } + } + } + return nil +} + +// https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-media-r0-upload +type uploadResponse struct { + ContentURI string `json:"content_uri"` +} + +// Upload implements /upload +func Upload(req *http.Request, cfg config.MediaAPI) util.JSONResponse { + logger := util.GetLogger(req.Context()) + + // FIXME: This will require querying some other component/db but currently + // just accepts a user id for auth + userID, resErr := auth.VerifyAccessToken(req) + if resErr != nil { + return *resErr + } + + // req.Header.Get() returns "" if no header + // strconv.Atoi() returns 0 when parsing "" + contentLength, _ := strconv.Atoi(req.Header.Get("Content-Length")) + + r := uploadRequest{ + ContentDisposition: req.Header.Get("Content-Disposition"), + ContentLength: contentLength, + ContentType: req.Header.Get("Content-Type"), + Filename: req.FormValue("filename"), + Method: req.Method, + UserID: userID, + } + + if resErr = r.Validate(); resErr != nil { + return *resErr + } + + logger.WithFields(log.Fields{ + "ContentType": r.ContentType, + "Filename": r.Filename, + "UserID": r.UserID, + }).Info("Uploading file") + + // TODO: Store file to disk + // - make path to file + // - progressive writing (could support Content-Length 0 and cut off + // after some max upload size is exceeded) + // - generate id (ideally a hash but a random string to start with) + // - generate thumbnails + // TODO: Write metadata to database + // TODO: Respond to request + + return util.JSONResponse{ + Code: 200, + JSON: uploadResponse{ + ContentURI: "mxc://example.com/AQwafuaFswefuhsfAFAgsw", + }, + } +}