mediaapi: Initial commit for /upload HTTP infra

src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go

@@ -0,0 +1,54 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"net/http"
+	"os"
+
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/mediaapi/config"
+	"github.com/matrix-org/dendrite/mediaapi/routing"
+
+	log "github.com/Sirupsen/logrus"
+)
+
+var (
+	bindAddr = os.Getenv("BIND_ADDRESS")
+	database = os.Getenv("DATABASE")
+	logDir   = os.Getenv("LOG_DIR")
+)
+
+func main() {
+	common.SetupLogging(logDir)
+	if bindAddr == "" {
+		log.Panic("No BIND_ADDRESS environment variable found.")
+	}
+	// db, err := storage.Open(database)
+	// if err != nil {
+	// 	panic(err)
+	// }
+
+	cfg := config.MediaAPI{
+		ServerName: "localhost",
+		BasePath:   "/Users/robertsw/dendrite",
+		DataSource: database,
+	}
+
+	log.Info("Starting mediaapi")
+
+	routing.Setup(http.DefaultServeMux, http.DefaultClient, cfg)
+	log.Fatal(http.ListenAndServe(bindAddr, nil))
+}

src/github.com/matrix-org/dendrite/mediaapi/README.md

@@ -0,0 +1,3 @@
+# Media API
+
+This server is responsible for serving `/media` requests

src/github.com/matrix-org/dendrite/mediaapi/config/config.go

@@ -0,0 +1,25 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+// MediaAPI contains the config information necessary to spin up a mediaapi process.
+type MediaAPI struct {
+	// The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
+	ServerName string `yaml:"server_name"`
+	// The base path to where media files will be stored.
+	BasePath string `yaml:"base_path"`
+	// The postgres connection config for connecting to the database e.g a postgres:// URI
+	DataSource string `yaml:"database"`
+}

src/github.com/matrix-org/dendrite/mediaapi/routing/routing.go

@@ -0,0 +1,45 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/matrix-org/dendrite/mediaapi/config"
+	"github.com/matrix-org/dendrite/mediaapi/writers"
+	"github.com/matrix-org/util"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const pathPrefixR0 = "/_matrix/media/v1"
+
+// Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
+// to clients which need to make outbound HTTP requests.
+func Setup(servMux *http.ServeMux, httpClient *http.Client, cfg config.MediaAPI) {
+	apiMux := mux.NewRouter()
+	r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
+	r0mux.Handle("/upload", make("upload", util.NewJSONRequestHandler(func(req *http.Request) util.JSONResponse {
+		return writers.Upload(req, cfg)
+	})))
+
+	servMux.Handle("/metrics", prometheus.Handler())
+	servMux.Handle("/api/", http.StripPrefix("/api", apiMux))
+}
+
+// make a util.JSONRequestHandler into an http.Handler
+func make(metricsName string, h util.JSONRequestHandler) http.Handler {
+	return prometheus.InstrumentHandler(metricsName, util.MakeJSONAPI(h))
+}

src/github.com/matrix-org/dendrite/mediaapi/storage/media.go

@@ -0,0 +1,47 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"database/sql"
+)
+
+const mediaSchema = `
+`
+
+const insertMediaSQL = "" +
+	"INSERT INTO events (room_nid, event_type_nid, event_state_key_nid, event_id, reference_sha256, auth_event_nids)" +
+	" VALUES ($1, $2, $3, $4, $5, $6)" +
+	" ON CONFLICT ON CONSTRAINT event_id_unique" +
+	" DO NOTHING" +
+	" RETURNING event_nid, state_snapshot_nid"
+
+type mediaStatements struct {
+	insertMediaStmt *sql.Stmt
+}
+
+func (s *mediaStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(mediaSchema)
+	if err != nil {
+		return
+	}
+
+	return statementList{
+		{&s.insertMediaStmt, insertMediaSQL},
+	}.prepare(db)
+}
+
+func (s *mediaStatements) insertMedia() {
+}

src/github.com/matrix-org/dendrite/mediaapi/storage/prepare.go

@@ -0,0 +1,37 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// FIXME: This should be made common!
+
+package storage
+
+import (
+	"database/sql"
+)
+
+// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement.
+type statementList []struct {
+	statement **sql.Stmt
+	sql       string
+}
+
+// prepare the SQL for each statement in the list and assign the result to the prepared statement.
+func (s statementList) prepare(db *sql.DB) (err error) {
+	for _, statement := range s {
+		if *statement.statement, err = db.Prepare(statement.sql); err != nil {
+			return
+		}
+	}
+	return
+}

src/github.com/matrix-org/dendrite/mediaapi/storage/sql.go

@@ -0,0 +1,33 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"database/sql"
+)
+
+type statements struct {
+	mediaStatements
+}
+
+func (s *statements) prepare(db *sql.DB) error {
+	var err error
+
+	if err = s.mediaStatements.prepare(db); err != nil {
+		return err
+	}
+
+	return nil
+}

src/github.com/matrix-org/dendrite/mediaapi/storage/storage.go

@@ -0,0 +1,38 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"database/sql"
+)
+
+// A Database is used to store room events and stream offsets.
+type Database struct {
+	statements statements
+	db         *sql.DB
+}
+
+// Open a postgres database.
+func Open(dataSourceName string) (*Database, error) {
+	var d Database
+	var err error
+	if d.db, err = sql.Open("postgres", dataSourceName); err != nil {
+		return nil, err
+	}
+	if err = d.statements.prepare(d.db); err != nil {
+		return nil, err
+	}
+	return &d, nil
+}

src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go

@@ -0,0 +1,139 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package writers
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/matrix-org/dendrite/clientapi/auth"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/mediaapi/config"
+	"github.com/matrix-org/util"
+)
+
+// https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-media-r0-upload
+// NOTE: ContentType is an HTTP request header and Filename is passed as a query parameter
+type uploadRequest struct {
+	ContentDisposition string
+	ContentLength      int
+	ContentType        string
+	Filename           string
+	Method             string
+	UserID             string
+}
+
+func (r uploadRequest) Validate() *util.JSONResponse {
+	// TODO: Any validation to be done on ContentDisposition?
+	if r.ContentLength < 1 {
+		return &util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.BadJSON("HTTP Content-Length request header must be greater than zero."),
+		}
+	}
+	// TODO: Check if the Content-Type is a valid type?
+	if r.ContentType == "" {
+		return &util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.BadJSON("HTTP Content-Type request header must be set."),
+		}
+	}
+	// TODO: Validate filename - what are the valid characters?
+	if r.Method != "POST" {
+		return &util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.BadJSON("HTTP request method must be POST."),
+		}
+	}
+	if r.UserID != "" {
+		// TODO: We should put user ID parsing code into gomatrixserverlib and use that instead
+		//       (see https://github.com/matrix-org/gomatrixserverlib/blob/3394e7c7003312043208aa73727d2256eea3d1f6/eventcontent.go#L347 )
+		//       It should be a struct (with pointers into a single string to avoid copying) and
+		//       we should update all refs to use UserID types rather than strings.
+		// https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/types.py#L92
+		if len(r.UserID) == 0 || r.UserID[0] != '@' {
+			return &util.JSONResponse{
+				Code: 400,
+				JSON: jsonerror.BadJSON("user id must start with '@'"),
+			}
+		}
+		parts := strings.SplitN(r.UserID[1:], ":", 2)
+		if len(parts) != 2 {
+			return &util.JSONResponse{
+				Code: 400,
+				JSON: jsonerror.BadJSON("user id must be in the form @localpart:domain"),
+			}
+		}
+	}
+	return nil
+}
+
+// https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-media-r0-upload
+type uploadResponse struct {
+	ContentURI string `json:"content_uri"`
+}
+
+// Upload implements /upload
+func Upload(req *http.Request, cfg config.MediaAPI) util.JSONResponse {
+	logger := util.GetLogger(req.Context())
+
+	// FIXME: This will require querying some other component/db but currently
+	// just accepts a user id for auth
+	userID, resErr := auth.VerifyAccessToken(req)
+	if resErr != nil {
+		return *resErr
+	}
+
+	// req.Header.Get() returns "" if no header
+	// strconv.Atoi() returns 0 when parsing ""
+	contentLength, _ := strconv.Atoi(req.Header.Get("Content-Length"))
+
+	r := uploadRequest{
+		ContentDisposition: req.Header.Get("Content-Disposition"),
+		ContentLength:      contentLength,
+		ContentType:        req.Header.Get("Content-Type"),
+		Filename:           req.FormValue("filename"),
+		Method:             req.Method,
+		UserID:             userID,
+	}
+
+	if resErr = r.Validate(); resErr != nil {
+		return *resErr
+	}
+
+	logger.WithFields(log.Fields{
+		"ContentType": r.ContentType,
+		"Filename":    r.Filename,
+		"UserID":      r.UserID,
+	}).Info("Uploading file")
+
+	// TODO: Store file to disk
+	//       - make path to file
+	//       - progressive writing (could support Content-Length 0 and cut off
+	//         after some max upload size is exceeded)
+	//       - generate id (ideally a hash but a random string to start with)
+	//       - generate thumbnails
+	// TODO: Write metadata to database
+	// TODO: Respond to request
+
+	return util.JSONResponse{
+		Code: 200,
+		JSON: uploadResponse{
+			ContentURI: "mxc://example.com/AQwafuaFswefuhsfAFAgsw",
+		},
+	}
+}