mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2025-12-10 16:33:11 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into rob/docker-compose

Browse source
This commit is contained in:
Robert Swain 2017-06-26 07:41:44 +02:00
parent 54043b9dcc eb029116b0
commit ab16fe54fc
36 changed files with 971 additions and 758 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
src/github.com/matrix-org/dendrite/clientapi/config/config.go
View file

@ -1,37 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"github.com/matrix-org/gomatrixserverlib"
"golang.org/x/crypto/ed25519"
)
// ClientAPI contains the config information necessary to spin up a clientapi process.
type ClientAPI struct {
// The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
ServerName gomatrixserverlib.ServerName
// The private key which will be used to sign events.
PrivateKey ed25519.PrivateKey
// An arbitrary string used to uniquely identify the PrivateKey. Must start with the
// prefix "ed25519:".
KeyID gomatrixserverlib.KeyID
// A list of URIs to send events to. These kafka logs should be consumed by a Room Server.
KafkaProducerURIs []string
// The topic for events which are written to the logs.
ClientAPIOutputTopic string
// The URL of the roomserver which can service Query API requests
RoomserverURL string
}

9
src/github.com/matrix-org/dendrite/clientapi/readers/login.go
View file

@ -21,9 +21,9 @@ import (
"github.com/matrix-org/dendrite/clientapi/auth"
"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
"github.com/matrix-org/dendrite/clientapi/config"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
@ -56,7 +56,10 @@ func passwordLogin() loginFlows {
}
// Login implements GET and POST /login
func Login(req *http.Request, accountDB *accounts.Database, deviceDB *devices.Database, cfg config.ClientAPI) util.JSONResponse {
func Login(
req *http.Request, accountDB *accounts.Database, deviceDB *devices.Database,
cfg config.Dendrite,
) util.JSONResponse {
if req.Method == "GET" { // TODO: support other forms of login other than password, depending on config options
return util.JSONResponse{
Code: 200,
@ -109,7 +112,7 @@ func Login(req *http.Request, accountDB *accounts.Database, deviceDB *devices.Da
JSON: loginResponse{
UserID: dev.UserID,
AccessToken: dev.AccessToken,
HomeServer: cfg.ServerName,
HomeServer: cfg.Matrix.ServerName,
},
}
}

4
src/github.com/matrix-org/dendrite/clientapi/routing/routing.go
View file

@ -22,11 +22,11 @@ import (
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
"github.com/matrix-org/dendrite/clientapi/config"
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/dendrite/clientapi/readers"
"github.com/matrix-org/dendrite/clientapi/writers"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
@ -38,7 +38,7 @@ const pathPrefixR0 = "/_matrix/client/r0"
// Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
// to clients which need to make outbound HTTP requests.
func Setup(
servMux *http.ServeMux, httpClient *http.Client, cfg config.ClientAPI,
servMux *http.ServeMux, httpClient *http.Client, cfg config.Dendrite,
producer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI,
accountDB *accounts.Database,
deviceDB *devices.Database,

14
src/github.com/matrix-org/dendrite/clientapi/writers/createroom.go
View file

@ -23,11 +23,11 @@ import (
log "github.com/Sirupsen/logrus"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/config"
"github.com/matrix-org/dendrite/clientapi/events"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
@ -91,15 +91,15 @@ type fledglingEvent struct {
}
// CreateRoom implements /createRoom
func CreateRoom(req *http.Request, device *authtypes.Device, cfg config.ClientAPI, producer *producers.RoomserverProducer) util.JSONResponse {
func CreateRoom(req *http.Request, device *authtypes.Device, cfg config.Dendrite, producer *producers.RoomserverProducer) util.JSONResponse {
// TODO: Check room ID doesn't clash with an existing one, and we
// probably shouldn't be using pseudo-random strings, maybe GUIDs?
roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.ServerName)
roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.Matrix.ServerName)
return createRoom(req, device, cfg, roomID, producer)
}
// createRoom implements /createRoom
func createRoom(req *http.Request, device *authtypes.Device, cfg config.ClientAPI, roomID string, producer *producers.RoomserverProducer) util.JSONResponse {
func createRoom(req *http.Request, device *authtypes.Device, cfg config.Dendrite, roomID string, producer *producers.RoomserverProducer) util.JSONResponse {
logger := util.GetLogger(req.Context())
userID := device.UserID
var r createRoomRequest
@ -201,7 +201,7 @@ func createRoom(req *http.Request, device *authtypes.Device, cfg config.ClientAP
// buildEvent fills out auth_events for the builder then builds the event
func buildEvent(builder *gomatrixserverlib.EventBuilder,
provider gomatrixserverlib.AuthEventProvider,
cfg config.ClientAPI) (*gomatrixserverlib.Event, error) {
cfg config.Dendrite) (*gomatrixserverlib.Event, error) {
eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
if err != nil {
@ -212,9 +212,9 @@ func buildEvent(builder *gomatrixserverlib.EventBuilder,
return nil, err
}
builder.AuthEvents = refs
eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), cfg.ServerName)
eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), cfg.Matrix.ServerName)
now := time.Now()
event, err := builder.Build(eventID, now, cfg.ServerName, cfg.KeyID, cfg.PrivateKey)
event, err := builder.Build(eventID, now, cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey)
if err != nil {
return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %s", builder.Type, err)
}

12
src/github.com/matrix-org/dendrite/clientapi/writers/joinroom.go
View file

@ -17,10 +17,10 @@ package writers
import (
"fmt"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/config"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/gomatrixserverlib"
@ -36,7 +36,7 @@ func JoinRoomByIDOrAlias(
req *http.Request,
device *authtypes.Device,
roomIDOrAlias string,
cfg config.ClientAPI,
cfg config.Dendrite,
federation *gomatrixserverlib.FederationClient,
producer *producers.RoomserverProducer,
queryAPI api.RoomserverQueryAPI,
@ -67,7 +67,7 @@ type joinRoomReq struct {
req *http.Request
content map[string]interface{}
userID string
cfg config.ClientAPI
cfg config.Dendrite
federation *gomatrixserverlib.FederationClient
producer *producers.RoomserverProducer
queryAPI api.RoomserverQueryAPI
@ -95,7 +95,7 @@ func (r joinRoomReq) joinRoomByAlias(roomAlias string) util.JSONResponse {
JSON: jsonerror.BadJSON("Room alias must be in the form '#localpart:domain'"),
}
}
if domain == r.cfg.ServerName {
if domain == r.cfg.Matrix.ServerName {
// TODO: Implement joining local room aliases.
panic(fmt.Errorf("Joining local room aliases is not implemented"))
} else {
@ -212,9 +212,9 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib
r.writeToBuilder(&respMakeJoin.JoinEvent, roomID)
now := time.Now()
eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), r.cfg.ServerName)
eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), r.cfg.Matrix.ServerName)
event, err := respMakeJoin.JoinEvent.Build(
eventID, now, r.cfg.ServerName, r.cfg.KeyID, r.cfg.PrivateKey,
eventID, now, r.cfg.Matrix.ServerName, r.cfg.Matrix.KeyID, r.cfg.Matrix.PrivateKey,
)
if err != nil {
res := httputil.LogThenError(r.req, err)

17
src/github.com/matrix-org/dendrite/clientapi/writers/sendevent.go
View file

@ -21,10 +21,10 @@ import (
"time"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/config"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
@ -39,7 +39,14 @@ type sendEventResponse struct {
// SendEvent implements:
// /rooms/{roomID}/send/{eventType}/{txnID}
// /rooms/{roomID}/state/{eventType}/{stateKey}
func SendEvent(req *http.Request, device *authtypes.Device, roomID, eventType, txnID string, stateKey *string, cfg config.ClientAPI, queryAPI api.RoomserverQueryAPI, producer *producers.RoomserverProducer) util.JSONResponse {
func SendEvent(
req *http.Request,
device *authtypes.Device,
roomID, eventType, txnID string, stateKey *string,
cfg config.Dendrite,
queryAPI api.RoomserverQueryAPI,
producer *producers.RoomserverProducer,
) util.JSONResponse {
// parse the incoming http request
userID := device.UserID
var r map[string]interface{} // must be a JSON object
@ -86,8 +93,10 @@ func SendEvent(req *http.Request, device *authtypes.Device, roomID, eventType, t
refs = append(refs, e.EventReference())
}
builder.AuthEvents = refs
eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), cfg.ServerName)
e, err := builder.Build(eventID, time.Now(), cfg.ServerName, cfg.KeyID, cfg.PrivateKey)
eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), cfg.Matrix.ServerName)
e, err := builder.Build(
eventID, time.Now(), cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey,
)
if err != nil {
return httputil.LogThenError(req, err)
}

2
src/github.com/matrix-org/dendrite/cmd/client-api-proxy/main.go
View file

@ -106,7 +106,7 @@ func main() {
if *syncServerURL == "" {
flag.Usage()
fmt.Fprintln(os.Stderr, "no --sync-server-url specified.")
fmt.Fprintln(os.Stderr, "no --sync-api-server-url specified.")
os.Exit(1)
}

74
src/github.com/matrix-org/dendrite/cmd/dendrite-client-api-server/main.go
View file

@ -15,16 +15,16 @@
package main
import (
"flag"
"net/http"
"os"
"strings"
"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
"github.com/matrix-org/dendrite/clientapi/config"
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/dendrite/clientapi/routing"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/common/keydb"
"github.com/matrix-org/dendrite/roomserver/api"
@ -34,70 +34,45 @@ import (
)
var (
kafkaURIs = strings.Split(os.Getenv("KAFKA_URIS"), ",")
bindAddr = os.Getenv("BIND_ADDRESS")
logDir = os.Getenv("LOG_DIR")
roomserverURL = os.Getenv("ROOMSERVER_URL")
clientAPIOutputTopic = os.Getenv("CLIENTAPI_OUTPUT_TOPIC")
serverName = gomatrixserverlib.ServerName(os.Getenv("SERVER_NAME"))
serverKey = os.Getenv("SERVER_KEY")
accountDataSource = os.Getenv("ACCOUNT_DATABASE")
keyDataSource = os.Getenv("KEY_DATABASE")
logDir = os.Getenv("LOG_DIR")
configPath = flag.String("config", "dendrite.yaml", "The path to the config file, For more information see the config file in this repository")
)
func main() {
common.SetupLogging(logDir)
if bindAddr == "" {
log.Panic("No BIND_ADDRESS environment variable found.")
}
if len(kafkaURIs) == 0 {
// the kafka default is :9092
kafkaURIs = []string{"localhost:9092"}
}
if roomserverURL == "" {
log.Panic("No ROOMSERVER_URL environment variable found.")
}
if clientAPIOutputTopic == "" {
log.Panic("No CLIENTAPI_OUTPUT_TOPIC environment variable found. This should match the roomserver input topic.")
}
if serverName == "" {
serverName = "localhost"
}
cfg := config.ClientAPI{
ServerName: serverName,
KafkaProducerURIs: kafkaURIs,
ClientAPIOutputTopic: clientAPIOutputTopic,
RoomserverURL: roomserverURL,
}
flag.Parse()
var err error
cfg.KeyID, cfg.PrivateKey, err = common.ReadKey(serverKey)
cfg, err := config.Load(*configPath)
if err != nil {
log.Panicf("Failed to load private key: %s", err)
log.Fatalf("Invalid config file: %s", err)
}
log.Info("Starting clientapi")
log.Info("config: ", cfg)
roomserverProducer, err := producers.NewRoomserverProducer(cfg.KafkaProducerURIs, cfg.ClientAPIOutputTopic)
roomserverProducer, err := producers.NewRoomserverProducer(
cfg.Kafka.Addresses, string(cfg.Kafka.Topics.InputRoomEvent),
)
if err != nil {
log.Panicf("Failed to setup kafka producers(%q): %s", cfg.KafkaProducerURIs, err)
log.Panicf("Failed to setup kafka producers(%q): %s", cfg.Kafka.Addresses, err)
}
federation := gomatrixserverlib.NewFederationClient(cfg.ServerName, cfg.KeyID, cfg.PrivateKey)
federation := gomatrixserverlib.NewFederationClient(
cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey,
)
queryAPI := api.NewRoomserverQueryAPIHTTP(cfg.RoomserverURL, nil)
accountDB, err := accounts.NewDatabase(accountDataSource, serverName)
queryAPI := api.NewRoomserverQueryAPIHTTP(cfg.RoomServerURL(), nil)
accountDB, err := accounts.NewDatabase(string(cfg.Database.Account), cfg.Matrix.ServerName)
if err != nil {
log.Panicf("Failed to setup account database(%q): %s", accountDataSource, err.Error())
log.Panicf("Failed to setup account database(%q): %s", cfg.Database.Account, err.Error())
}
deviceDB, err := devices.NewDatabase(accountDataSource, serverName)
deviceDB, err := devices.NewDatabase(string(cfg.Database.Device), cfg.Matrix.ServerName)
if err != nil {
log.Panicf("Failed to setup device database(%q): %s", accountDataSource, err.Error())
log.Panicf("Failed to setup device database(%q): %s", cfg.Database.Device, err.Error())
}
keyDB, err := keydb.NewDatabase(keyDataSource)
keyDB, err := keydb.NewDatabase(string(cfg.Database.ServerKey))
if err != nil {
log.Panicf("Failed to setup key database(%q): %s", keyDataSource, err.Error())
log.Panicf("Failed to setup key database(%q): %s", cfg.Database.ServerKey, err.Error())
}
keyRing := gomatrixserverlib.KeyRing{
@ -108,9 +83,10 @@ func main() {
KeyDatabase: keyDB,
}
log.Info("Starting client API server on ", cfg.Listen.ClientAPI)
routing.Setup(
http.DefaultServeMux, http.DefaultClient, cfg, roomserverProducer,
http.DefaultServeMux, http.DefaultClient, *cfg, roomserverProducer,
queryAPI, accountDB, deviceDB, federation, keyRing,
)
log.Fatal(http.ListenAndServe(bindAddr, nil))
log.Fatal(http.ListenAndServe(string(cfg.Listen.ClientAPI), nil))
}

87
src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go
View file

@ -15,16 +15,14 @@
package main
import (
"encoding/base64"
"flag"
"net/http"
"os"
"strings"
"time"
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/common/keydb"
"github.com/matrix-org/dendrite/federationapi/config"
"github.com/matrix-org/dendrite/federationapi/routing"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib"
@ -33,73 +31,30 @@ import (
)
var (
bindAddr = os.Getenv("BIND_ADDRESS")
logDir = os.Getenv("LOG_DIR")
serverName = gomatrixserverlib.ServerName(os.Getenv("SERVER_NAME"))
serverKey = os.Getenv("SERVER_KEY")
// Base64 encoded SHA256 TLS fingerprint of the X509 certificate used by
// the public federation listener for this server.
// Can be generated from a PEM certificate called "server.crt" using:
//
// openssl x509 -noout -fingerprint -sha256 -inform pem -in server.crt |\
// python -c 'print raw_input()[19:].replace(":","").decode("hex").encode("base64").rstrip("=\n")'
//
tlsFingerprint = os.Getenv("TLS_FINGERPRINT")
kafkaURIs = strings.Split(os.Getenv("KAFKA_URIS"), ",")
roomserverURL = os.Getenv("ROOMSERVER_URL")
roomserverInputTopic = os.Getenv("TOPIC_INPUT_ROOM_EVENT")
keyDataSource = os.Getenv("KEY_DATABASE")
configPath = flag.String("config", "dendrite.yaml", "The path to the config file. For more information, see the config file in this repository.")
)
func main() {
common.SetupLogging(logDir)
if bindAddr == "" {
log.Panic("No BIND_ADDRESS environment variable found.")
}
if serverName == "" {
serverName = "localhost"
}
flag.Parse()
if tlsFingerprint == "" {
log.Panic("No TLS_FINGERPRINT environment variable found.")
if *configPath == "" {
log.Fatal("--config must be supplied")
}
if len(kafkaURIs) == 0 {
// the kafka default is :9092
kafkaURIs = []string{"localhost:9092"}
}
if roomserverURL == "" {
log.Panic("No ROOMSERVER_URL environment variable found.")
}
if roomserverInputTopic == "" {
log.Panic("No TOPIC_INPUT_ROOM_EVENT environment variable found. This should match the roomserver input topic.")
}
cfg := config.FederationAPI{
ServerName: serverName,
// TODO: make the validity period configurable.
ValidityPeriod: 24 * time.Hour,
}
var err error
cfg.KeyID, cfg.PrivateKey, err = common.ReadKey(serverKey)
cfg, err := config.Load(*configPath)
if err != nil {
log.Panicf("Failed to load private key: %s", err)
log.Fatalf("Invalid config file: %s", err)
}
var fingerprintSHA256 []byte
if fingerprintSHA256, err = base64.RawStdEncoding.DecodeString(tlsFingerprint); err != nil {
log.Panicf("Failed to load TLS fingerprint: %s", err)
}
cfg.TLSFingerPrints = []gomatrixserverlib.TLSFingerprint{{fingerprintSHA256}}
federation := gomatrixserverlib.NewFederationClient(
cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey,
)
federation := gomatrixserverlib.NewFederationClient(cfg.ServerName, cfg.KeyID, cfg.PrivateKey)
keyDB, err := keydb.NewDatabase(keyDataSource)
keyDB, err := keydb.NewDatabase(string(cfg.Database.ServerKey))
if err != nil {
log.Panicf("Failed to setup key database(%q): %s", keyDataSource, err.Error())
log.Panicf("Failed to setup key database(%q): %s", cfg.Database.ServerKey, err.Error())
}
keyRing := gomatrixserverlib.KeyRing{
@ -109,13 +64,19 @@ func main() {
},
KeyDatabase: keyDB,
}
queryAPI := api.NewRoomserverQueryAPIHTTP(roomserverURL, nil)
roomserverProducer, err := producers.NewRoomserverProducer(kafkaURIs, roomserverInputTopic)
queryAPI := api.NewRoomserverQueryAPIHTTP(cfg.RoomServerURL(), nil)
roomserverProducer, err := producers.NewRoomserverProducer(
cfg.Kafka.Addresses, string(cfg.Kafka.Topics.InputRoomEvent),
)
if err != nil {
log.Panicf("Failed to setup kafka producers(%s): %s", kafkaURIs, err)
log.Panicf("Failed to setup kafka producers(%s): %s", cfg.Kafka.Addresses, err)
}
routing.Setup(http.DefaultServeMux, cfg, queryAPI, roomserverProducer, keyRing, federation)
log.Fatal(http.ListenAndServe(bindAddr, nil))
log.Info("Starting federation API server on ", cfg.Listen.FederationAPI)
routing.Setup(http.DefaultServeMux, *cfg, queryAPI, roomserverProducer, keyRing, federation)
log.Fatal(http.ListenAndServe(string(cfg.Listen.FederationAPI), nil))
}

222
src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go
View file

@ -16,35 +16,20 @@ package main
import (
"flag"
"fmt"
"io/ioutil"
"net/http"
"os"
"os/user"
"path/filepath"
"strconv"
"strings"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/mediaapi/config"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/mediaapi/routing"
"github.com/matrix-org/dendrite/mediaapi/storage"
"github.com/matrix-org/dendrite/mediaapi/types"
"github.com/matrix-org/gomatrixserverlib"
log "github.com/Sirupsen/logrus"
yaml "gopkg.in/yaml.v2"
)
var (
bindAddr = flag.String("listen", "", "The port to listen on.")
dataSource = os.Getenv("DATABASE")
logDir = os.Getenv("LOG_DIR")
serverName = os.Getenv("SERVER_NAME")
basePath = os.Getenv("BASE_PATH")
// Note: if the MAX_FILE_SIZE_BYTES is set to 0, it will be unlimited
maxFileSizeBytesString = os.Getenv("MAX_FILE_SIZE_BYTES")
configPath = flag.String("config", "", "The path to the config file. For more information, see the config file in this repository.")
configPath = flag.String("config", "", "The path to the config file. For more information, see the config file in this repository.")
)
func main() {
@ -52,206 +37,21 @@ func main() {
flag.Parse()
log.WithFields(log.Fields{
"listen": *bindAddr,
"DATABASE": dataSource,
"LOG_DIR": logDir,
"SERVER_NAME": serverName,
"BASE_PATH": basePath,
"MAX_FILE_SIZE_BYTES": maxFileSizeBytesString,
"config": *configPath,
}).Info("Loading configuration based on config file and environment variables")
cfg, err := configureServer()
if *configPath == "" {
log.Fatal("--config must be supplied")
}
cfg, err := config.Load(*configPath)
if err != nil {
log.WithError(err).Fatal("Invalid configuration")
log.Fatalf("Invalid config file: %s", err)
}
log.WithFields(log.Fields{
"listen": *bindAddr,
"LOG_DIR": logDir,
"CONFIG_PATH": *configPath,
"ServerName": cfg.ServerName,
"AbsBasePath": cfg.AbsBasePath,
"MaxFileSizeBytes": *cfg.MaxFileSizeBytes,
"DataSource": cfg.DataSource,
"DynamicThumbnails": cfg.DynamicThumbnails,
"MaxThumbnailGenerators": cfg.MaxThumbnailGenerators,
"ThumbnailSizes": cfg.ThumbnailSizes,
}).Info("Starting mediaapi server with configuration")
db, err := storage.Open(cfg.DataSource)
db, err := storage.Open(string(cfg.Database.MediaAPI))
if err != nil {
log.WithError(err).Panic("Failed to open database")
}
log.Info("Starting media API server on ", cfg.Listen.MediaAPI)
routing.Setup(http.DefaultServeMux, http.DefaultClient, cfg, db)
log.Fatal(http.ListenAndServe(*bindAddr, nil))
}
// configureServer loads configuration from a yaml file and overrides with environment variables
func configureServer() (*config.MediaAPI, error) {
if *configPath == "" {
log.Fatal("--config must be supplied")
}
cfg, err := loadConfig(*configPath)
if err != nil {
log.WithError(err).Fatal("Invalid config file")
}
// override values from environment variables
applyOverrides(cfg)
if err := validateConfig(cfg); err != nil {
return nil, err
}
return cfg, nil
}
// FIXME: make common somehow? copied from sync api
func loadConfig(configPath string) (*config.MediaAPI, error) {
contents, err := ioutil.ReadFile(configPath)
if err != nil {
return nil, err
}
var cfg config.MediaAPI
if err = yaml.Unmarshal(contents, &cfg); err != nil {
return nil, err
}
return &cfg, nil
}
func applyOverrides(cfg *config.MediaAPI) {
if serverName != "" {
if cfg.ServerName != "" {
log.WithFields(log.Fields{
"server_name": cfg.ServerName,
"SERVER_NAME": serverName,
}).Info("Overriding server_name from config file with environment variable")
}
cfg.ServerName = gomatrixserverlib.ServerName(serverName)
}
if cfg.ServerName == "" {
log.Info("ServerName not set. Defaulting to 'localhost'.")
cfg.ServerName = "localhost"
}
if basePath != "" {
if cfg.BasePath != "" {
log.WithFields(log.Fields{
"base_path": cfg.BasePath,
"BASE_PATH": basePath,
}).Info("Overriding base_path from config file with environment variable")
}
cfg.BasePath = types.Path(basePath)
}
if maxFileSizeBytesString != "" {
if cfg.MaxFileSizeBytes != nil {
log.WithFields(log.Fields{
"max_file_size_bytes": *cfg.MaxFileSizeBytes,
"MAX_FILE_SIZE_BYTES": maxFileSizeBytesString,
}).Info("Overriding max_file_size_bytes from config file with environment variable")
}
maxFileSizeBytesInt, err := strconv.ParseInt(maxFileSizeBytesString, 10, 64)
if err != nil {
maxFileSizeBytesInt = 10 * 1024 * 1024
log.WithError(err).WithField(
"MAX_FILE_SIZE_BYTES", maxFileSizeBytesString,
).Infof("MAX_FILE_SIZE_BYTES not set? Defaulting to %v bytes.", maxFileSizeBytesInt)
}
maxFileSizeBytes := types.FileSizeBytes(maxFileSizeBytesInt)
cfg.MaxFileSizeBytes = &maxFileSizeBytes
}
if dataSource != "" {
if cfg.DataSource != "" {
log.WithFields(log.Fields{
"database": cfg.DataSource,
"DATABASE": dataSource,
}).Info("Overriding database from config file with environment variable")
}
cfg.DataSource = dataSource
}
if cfg.MaxThumbnailGenerators == 0 {
log.WithField(
"max_thumbnail_generators", cfg.MaxThumbnailGenerators,
).Info("Using default max_thumbnail_generators value of 10")
cfg.MaxThumbnailGenerators = 10
}
}
func validateConfig(cfg *config.MediaAPI) error {
if *bindAddr == "" {
log.Fatal("--listen must be supplied")
}
absBasePath, err := getAbsolutePath(cfg.BasePath)
if err != nil {
return fmt.Errorf("invalid base path (%v): %q", cfg.BasePath, err)
}
cfg.AbsBasePath = types.Path(absBasePath)
if *cfg.MaxFileSizeBytes < 0 {
return fmt.Errorf("invalid max file size bytes (%v)", *cfg.MaxFileSizeBytes)
}
if cfg.DataSource == "" {
return fmt.Errorf("invalid database (%v)", cfg.DataSource)
}
for _, config := range cfg.ThumbnailSizes {
if config.Width <= 0 || config.Height <= 0 {
return fmt.Errorf("invalid thumbnail size %vx%v", config.Width, config.Height)
}
}
return nil
}
func getAbsolutePath(basePath types.Path) (types.Path, error) {
var err error
if basePath == "" {
var wd string
wd, err = os.Getwd()
return types.Path(wd), err
}
// Note: If we got here len(basePath) >= 1
if basePath[0] == '~' {
basePath, err = expandHomeDir(basePath)
if err != nil {
return "", err
}
}
absBasePath, err := filepath.Abs(string(basePath))
return types.Path(absBasePath), err
}
// expandHomeDir parses paths beginning with ~/path or ~user/path and replaces the home directory part
func expandHomeDir(basePath types.Path) (types.Path, error) {
slash := strings.Index(string(basePath), "/")
if slash == -1 {
// pretend the slash is after the path as none was found within the string
// simplifies code using slash below
slash = len(basePath)
}
var usr *user.User
var err error
if slash == 1 {
// basePath is ~ or ~/path
usr, err = user.Current()
if err != nil {
return "", fmt.Errorf("failed to get user's home directory: %q", err)
}
} else {
// slash > 1
// basePath is ~user or ~user/path
usr, err = user.Lookup(string(basePath[1:slash]))
if err != nil {
return "", fmt.Errorf("failed to get user's home directory: %q", err)
}
}
return types.Path(filepath.Join(usr.HomeDir, string(basePath[slash:]))), nil
log.Fatal(http.ListenAndServe(string(cfg.Listen.MediaAPI), nil))
}

40
src/github.com/matrix-org/dendrite/cmd/dendrite-room-server/main.go
View file

@ -15,14 +15,16 @@
package main
import (
"flag"
"fmt"
"net/http"
_ "net/http/pprof"
"os"
"strconv"
"strings"
log "github.com/Sirupsen/logrus"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/roomserver/input"
"github.com/matrix-org/dendrite/roomserver/query"
"github.com/matrix-org/dendrite/roomserver/storage"
@ -31,42 +33,48 @@ import (
)
var (
database = os.Getenv("DATABASE")
kafkaURIs = strings.Split(os.Getenv("KAFKA_URIS"), ",")
inputRoomEventTopic = os.Getenv("TOPIC_INPUT_ROOM_EVENT")
outputRoomEventTopic = os.Getenv("TOPIC_OUTPUT_ROOM_EVENT")
bindAddr = os.Getenv("BIND_ADDRESS")
// Shuts the roomserver down after processing a given number of messages.
// This is useful for running benchmarks for seeing how quickly the server
// can process a given number of messages.
logDir = os.Getenv("LOG_DIR")
configPath = flag.String("config", "", "The path to the config file. For more information, see the config file in this repository.")
stopProcessingAfter = os.Getenv("STOP_AFTER")
)
func main() {
db, err := storage.Open(database)
common.SetupLogging(logDir)
flag.Parse()
if *configPath == "" {
log.Fatal("--config must be supplied")
}
cfg, err := config.Load(*configPath)
if err != nil {
log.Fatalf("Invalid config file: %s", err)
}
db, err := storage.Open(string(cfg.Database.RoomServer))
if err != nil {
panic(err)
}
kafkaConsumer, err := sarama.NewConsumer(kafkaURIs, nil)
kafkaConsumer, err := sarama.NewConsumer(cfg.Kafka.Addresses, nil)
if err != nil {
panic(err)
}
kafkaProducer, err := sarama.NewSyncProducer(kafkaURIs, nil)
kafkaProducer, err := sarama.NewSyncProducer(cfg.Kafka.Addresses, nil)
if err != nil {
panic(err)
}
consumer := input.Consumer{
ContinualConsumer: common.ContinualConsumer{
Topic: inputRoomEventTopic,
Topic: string(cfg.Kafka.Topics.InputRoomEvent),
Consumer: kafkaConsumer,
PartitionStore: db,
},
DB: db,
Producer: kafkaProducer,
OutputRoomEventTopic: outputRoomEventTopic,
OutputRoomEventTopic: string(cfg.Kafka.Topics.OutputRoomEvent),
}
if stopProcessingAfter != "" {
@ -93,10 +101,10 @@ func main() {
http.DefaultServeMux.Handle("/metrics", prometheus.Handler())
fmt.Println("Started roomserver")
log.Info("Started room server on ", cfg.Listen.RoomServer)
// TODO: Implement clean shutdown.
if err := http.ListenAndServe(bindAddr, nil); err != nil {
if err := http.ListenAndServe(string(cfg.Listen.RoomServer), nil); err != nil {
panic(err)
}
}

52
src/github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server/main.go
View file

@ -16,13 +16,12 @@ package main
import (
"flag"
"io/ioutil"
"net/http"
"os"
"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/syncapi/config"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/syncapi/consumers"
"github.com/matrix-org/dendrite/syncapi/routing"
"github.com/matrix-org/dendrite/syncapi/storage"
@ -30,27 +29,9 @@ import (
"github.com/matrix-org/dendrite/syncapi/types"
log "github.com/Sirupsen/logrus"
yaml "gopkg.in/yaml.v2"
)
var configPath = flag.String("config", "sync-server-config.yaml", "The path to the config file. For more information, see the config file in this repository.")
var bindAddr = flag.String("listen", ":4200", "The port to listen on.")
func loadConfig(configPath string) (*config.Sync, error) {
contents, err := ioutil.ReadFile(configPath)
if err != nil {
return nil, err
}
var cfg config.Sync
if err = yaml.Unmarshal(contents, &cfg); err != nil {
return nil, err
}
// check required fields
if cfg.ServerName == "" {
log.Fatalf("'server_name' must be supplied in %s", configPath)
}
return &cfg, nil
}
var configPath = flag.String("config", "dendrite.yaml", "The path to the config file. For more information, see the config file in this repository.")
func main() {
common.SetupLogging(os.Getenv("LOG_DIR"))
@ -60,26 +41,21 @@ func main() {
if *configPath == "" {
log.Fatal("--config must be supplied")
}
cfg, err := loadConfig(*configPath)
cfg, err := config.Load(*configPath)
if err != nil {
log.Fatalf("Invalid config file: %s", err)
}
if *bindAddr == "" {
log.Fatal("--listen must be supplied")
log.Info("config: ", cfg)
db, err := storage.NewSyncServerDatabase(string(cfg.Database.SyncAPI))
if err != nil {
log.Panicf("startup: failed to create sync server database with data source %s : %s", cfg.Database.SyncAPI, err)
}
log.Info("sync server config: ", cfg)
db, err := storage.NewSyncServerDatabase(cfg.DataSource)
deviceDB, err := devices.NewDatabase(string(cfg.Database.Device), cfg.Matrix.ServerName)
if err != nil {
log.Panicf("startup: failed to create sync server database with data source %s : %s", cfg.DataSource, err)
}
// TODO: DO NOT USE THIS DATA SOURCE (it's the sync one, not devices!)
deviceDB, err := devices.NewDatabase(cfg.DataSource, cfg.ServerName)
if err != nil {
log.Panicf("startup: failed to create device database with data source %s : %s", cfg.DataSource, err)
log.Panicf("startup: failed to create device database with data source %s : %s", cfg.Database.Device, err)
}
pos, err := db.SyncStreamPosition()
@ -88,7 +64,7 @@ func main() {
}
n := sync.NewNotifier(types.StreamPosition(pos))
if err := n.Load(db); err != nil {
if err = n.Load(db); err != nil {
log.Panicf("startup: failed to set up notifier: %s", err)
}
consumer, err := consumers.NewOutputRoomEvent(cfg, n, db)
@ -99,7 +75,7 @@ func main() {
log.Panicf("startup: failed to start room server consumer")
}
log.Info("Starting sync server on ", *bindAddr)
routing.SetupSyncServerListeners(http.DefaultServeMux, http.DefaultClient, *cfg, sync.NewRequestPool(db, n), deviceDB)
log.Fatal(http.ListenAndServe(*bindAddr, nil))
log.Info("Starting sync server on ", cfg.Listen.SyncAPI)
routing.SetupSyncServerListeners(http.DefaultServeMux, http.DefaultClient, sync.NewRequestPool(db, n), deviceDB)
log.Fatal(http.ListenAndServe(string(cfg.Listen.SyncAPI), nil))
}

61
src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/main.go
View file

@ -22,10 +22,11 @@ import (
"os/exec"
"path"
"path/filepath"
"strconv"
"time"
"github.com/matrix-org/dendrite/common/test"
"github.com/matrix-org/gomatrixserverlib"
"gopkg.in/yaml.v2"
)
var (
@ -41,10 +42,10 @@ var (
postgresContainerName = os.Getenv("POSTGRES_CONTAINER")
// Test image to be uploaded/downloaded
testJPEG = test.Defaulting(os.Getenv("TEST_JPEG_PATH"), "src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/totem.jpg")
kafkaURI = test.Defaulting(os.Getenv("KAFKA_URIS"), "localhost:9092")
)
var thumbnailPregenerationConfig = (`
thumbnail_sizes:
var thumbnailSizes = (`
- width: 32
height: 32
method: crop
@ -68,65 +69,51 @@ var testDatabaseTemplate = "dbname=%s sslmode=disable binary_parameters=yes"
var timeout time.Duration
var port = 10000
func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error, string, *exec.Cmd, chan error, string, string) {
dir, err := ioutil.TempDir("", serverType+"-server-test"+suffix)
if err != nil {
panic(err)
}
serverAddr := "localhost:177" + suffix + "9"
proxyAddr := "localhost:1800" + suffix
configFilename := serverType + "-server-test-config" + suffix + ".yaml"
configFileContents := makeConfig(proxyAddr, suffix, dir, dynamicThumbnails)
database := fmt.Sprintf(testDatabaseTemplate, testDatabaseName+suffix)
cfg, nextPort, err := test.MakeConfig(dir, kafkaURI, database, "localhost", port)
cfg.Matrix.ServerName = gomatrixserverlib.ServerName(proxyAddr)
if err != nil {
panic(err)
}
if err = yaml.Unmarshal([]byte(thumbnailSizes), &cfg.Media.ThumbnailSizes); err != nil {
panic(err)
}
port = nextPort
if err = test.WriteConfig(cfg, dir); err != nil {
panic(err)
}
serverArgs := []string{
"--config", configFilename,
"--listen", serverAddr,
"--config", filepath.Join(dir, test.ConfigFile),
}
databases := []string{
testDatabaseName + suffix,
}
proxyCmd, proxyCmdChan := test.StartProxy(
proxyAddr,
"http://localhost:177"+suffix+"6",
"http://localhost:177"+suffix+"8",
"http://"+serverAddr,
)
proxyCmd, proxyCmdChan := test.StartProxy(proxyAddr, cfg)
cmd, cmdChan := test.StartServer(
serverType,
serverArgs,
suffix,
configFilename,
configFileContents,
postgresDatabase,
postgresContainerName,
databases,
)
fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, serverAddr, dir)
return cmd, cmdChan, serverAddr, proxyCmd, proxyCmdChan, proxyAddr, dir
}
func makeConfig(serverAddr, suffix, basePath string, dynamicThumbnails bool) string {
return fmt.Sprintf(
`
server_name: "%s"
base_path: %s
max_file_size_bytes: %s
database: "%s"
dynamic_thumbnails: %s
%s`,
serverAddr,
basePath,
"10485760",
fmt.Sprintf(testDatabaseTemplate, testDatabaseName+suffix),
strconv.FormatBool(dynamicThumbnails),
thumbnailPregenerationConfig,
)
fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, cfg.Listen.MediaAPI, dir)
return cmd, cmdChan, string(cfg.Listen.MediaAPI), proxyCmd, proxyCmdChan, proxyAddr, dir
}
func cleanUpServer(cmd *exec.Cmd, dir string) {

36
src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go
View file

@ -16,6 +16,7 @@ package main
import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
@ -34,8 +35,6 @@ var (
zookeeperURI = defaulting(os.Getenv("ZOOKEEPER_URI"), "localhost:2181")
// The URI the kafka server is listening on.
kafkaURI = defaulting(os.Getenv("KAFKA_URIS"), "localhost:9092")
// The address the roomserver should listen on.
roomserverAddr = defaulting(os.Getenv("ROOMSERVER_URI"), "localhost:9876")
// How long to wait for the roomserver to write the expected output messages.
// This needs to be high enough to account for the time it takes to create
// the postgres database tables which can take a while on travis.
@ -164,10 +163,22 @@ func runAndReadFromTopic(runCmd *exec.Cmd, topic string, count int, checkQueryAP
// a api.RoomserverQueryAPI client. The caller can use this function to check the
// behaviour of the query API.
func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverQueryAPI)) {
const (
inputTopic = "roomserverInput"
outputTopic = "roomserverOutput"
)
dir, err := ioutil.TempDir("", "room-server-test")
if err != nil {
panic(err)
}
cfg, _, err := test.MakeConfig(dir, kafkaURI, testDatabase, "localhost", 10000)
if err != nil {
panic(err)
}
if err := test.WriteConfig(cfg, dir); err != nil {
panic(err)
}
inputTopic := string(cfg.Kafka.Topics.InputRoomEvent)
outputTopic := string(cfg.Kafka.Topics.OutputRoomEvent)
exe.DeleteTopic(inputTopic)
if err := exe.CreateTopic(inputTopic); err != nil {
panic(err)
@ -181,7 +192,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R
panic(err)
}
if err := createDatabase(testDatabaseName); err != nil {
if err = createDatabase(testDatabaseName); err != nil {
panic(err)
}
@ -191,18 +202,11 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R
// We append to the environment rather than replacing so that any additional
// postgres and golang environment variables such as PGHOST are passed to
// the roomserver process.
cmd.Env = append(
os.Environ(),
fmt.Sprintf("DATABASE=%s", testDatabase),
fmt.Sprintf("KAFKA_URIS=%s", kafkaURI),
fmt.Sprintf("TOPIC_INPUT_ROOM_EVENT=%s", inputTopic),
fmt.Sprintf("TOPIC_OUTPUT_ROOM_EVENT=%s", outputTopic),
fmt.Sprintf("BIND_ADDRESS=%s", roomserverAddr),
)
cmd.Stderr = os.Stderr
cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)}
gotOutput, err := runAndReadFromTopic(cmd, outputTopic, len(wantOutput), func() {
queryAPI := api.NewRoomserverQueryAPIHTTP("http://"+roomserverAddr, nil)
queryAPI := api.NewRoomserverQueryAPIHTTP("http://"+string(cfg.Listen.RoomServer), nil)
checkQueries(queryAPI)
})
if err != nil {

33
src/github.com/matrix-org/dendrite/cmd/syncserver-integration-tests/main.go
View file

@ -17,12 +17,14 @@ package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"os"
"os/exec"
"path/filepath"
"time"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/common/test"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib"
@ -62,12 +64,6 @@ var exe = test.KafkaExecutor{
OutputWriter: os.Stderr,
}
var syncServerConfigFileContents = (`consumer_uris: ["` + kafkaURI + `"]
roomserver_topic: "` + inputTopic + `"
database: "` + testDatabase + `"
server_name: "localhost"
`)
var timeout time.Duration
var clientEventTestData []string
@ -126,11 +122,27 @@ func clientEventJSONForOutputRoomEvent(outputRoomEvent string) string {
// then starts the sync server. The Cmd being executed is returned. A channel is also returned,
// which will have any termination errors sent down it, followed immediately by the channel being closed.
func startSyncServer() (*exec.Cmd, chan error) {
const configFilename = "sync-api-server-config-test.yaml"
dir, err := ioutil.TempDir("", "syncapi-server-test")
if err != nil {
panic(err)
}
cfg, _, err := test.MakeConfig(dir, kafkaURI, testDatabase, "localhost", 10000)
if err != nil {
panic(err)
}
// TODO use the address assigned by the config generator rather than clobbering.
cfg.Matrix.ServerName = "localhost"
cfg.Listen.SyncAPI = config.Address(syncserverAddr)
cfg.Kafka.Topics.OutputRoomEvent = config.Topic(inputTopic)
if err := test.WriteConfig(cfg, dir); err != nil {
panic(err)
}
serverArgs := []string{
"--config", configFilename,
"--listen", syncserverAddr,
"--config", filepath.Join(dir, test.ConfigFile),
}
databases := []string{
@ -140,9 +152,6 @@ func startSyncServer() (*exec.Cmd, chan error) {
cmd, cmdChan := test.StartServer(
"sync-api",
serverArgs,
"",
configFilename,
syncServerConfigFileContents,
postgresDatabase,
postgresContainerName,
databases,

369
src/github.com/matrix-org/dendrite/common/config/config.go Normal file
View file

@ -0,0 +1,369 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"bytes"
"crypto/sha256"
"encoding/pem"
"fmt"
"github.com/matrix-org/gomatrixserverlib"
"golang.org/x/crypto/ed25519"
"gopkg.in/yaml.v2"
"io/ioutil"
"path/filepath"
"strings"
"time"
)
// Version is the current version of the config format.
// This will change whenever we make breaking changes to the config format.
const Version = "v0"
// Dendrite contains all the config used by a dendrite process.
// Relative paths are resolved relative to the current working directory
type Dendrite struct {
// The version of the configuration file.
// If the version in a file doesn't match the current dendrite config
// version then we can give a clear error message telling the user
// to update their config file to the current version.
// The version of the file should only be different if there has
// been a breaking change to the config file format.
Version string `yaml:"version"`
// The configuration required for a matrix server.
Matrix struct {
// The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
ServerName gomatrixserverlib.ServerName `yaml:"server_name"`
// Path to the private key which will be used to sign requests and events.
PrivateKeyPath Path `yaml:"private_key"`
// The private key which will be used to sign requests and events.
PrivateKey ed25519.PrivateKey `yaml:"-"`
// An arbitrary string used to uniquely identify the PrivateKey. Must start with the
// prefix "ed25519:".
KeyID gomatrixserverlib.KeyID `yaml:"-"`
// List of paths to X509 certificates used by the external federation listeners.
// These are used to calculate the TLS fingerprints to publish for this server.
// Other matrix servers talking to this server will expect the x509 certificate
// to match one of these certificates.
// The certificates should be in PEM format.
FederationCertificatePaths []Path `yaml:"federation_certificates"`
// A list of SHA256 TLS fingerprints for the X509 certificates used by the
// federation listener for this server.
TLSFingerPrints []gomatrixserverlib.TLSFingerprint `yaml:"-"`
// How long a remote server can cache our server key for before requesting it again.
// Increasing this number will reduce the number of requests made by remote servers
// for our key, but increases the period a compromised key will be considered valid
// by remote servers.
// Defaults to 24 hours.
KeyValidityPeriod time.Duration `yaml:"key_validity_period"`
} `yaml:"matrix"`
// The configuration specific to the media repostitory.
Media struct {
// The base path to where the media files will be stored. May be relative or absolute.
BasePath Path `yaml:"base_path"`
// The absolute base path to where media files will be stored.
AbsBasePath Path `yaml:"-"`
// The maximum file size in bytes that is allowed to be stored on this server.
// Note: if max_file_size_bytes is set to 0, the size is unlimited.
// Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB)
MaxFileSizeBytes *FileSizeBytes `yaml:"max_file_size_bytes,omitempty"`
// Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated
DynamicThumbnails bool `yaml:"dynamic_thumbnails"`
// The maximum number of simultaneous thumbnail generators. default: 10
MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"`
// A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content
ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"`
} `yaml:"media"`
// The configuration for talking to kafka.
Kafka struct {
// A list of kafka addresses to connect to.
Addresses []string `yaml:"addresses"`
// The names of the topics to use when reading and writing from kafka.
Topics struct {
// Topic for roomserver/api.InputRoomEvent events.
InputRoomEvent Topic `yaml:"input_room_event"`
// Topic for roomserver/api.OutputRoomEvent events.
OutputRoomEvent Topic `yaml:"output_room_event"`
}
} `yaml:"kafka"`
// Postgres Config
Database struct {
// The Account database stores the login details and account information
// for local users. It is accessed by the ClientAPI.
Account DataSource `yaml:"account"`
// The Device database stores session information for the devices of logged
// in local users. It is accessed by the ClientAPI, the MediaAPI and the SyncAPI.
Device DataSource `yaml:"device"`
// The MediaAPI database stores information about files uploaded and downloaded
// by local users. It is only accessed by the MediaAPI.
MediaAPI DataSource `yaml:"media_api"`
// The ServerKey database caches the public keys of remote servers.
// It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI.
ServerKey DataSource `yaml:"server_key"`
// The SyncAPI stores information used by the SyncAPI server.
// It is only accessed by the SyncAPI server.
SyncAPI DataSource `yaml:"sync_api"`
// The RoomServer database stores information about matrix rooms.
// It is only accessed by the RoomServer.
RoomServer DataSource `yaml:"room_server"`
} `yaml:"database"`
// The internal addresses the components will listen on.
// These should not be exposed externally as they expose metrics and debugging APIs.
Listen struct {
MediaAPI Address `yaml:"media_api"`
ClientAPI Address `yaml:"client_api"`
FederationAPI Address `yaml:"federation_api"`
SyncAPI Address `yaml:"sync_api"`
RoomServer Address `yaml:"room_server"`
} `yaml:"listen"`
}
// A Path on the filesystem.
type Path string
// A DataSource for opening a postgresql database using lib/pq.
type DataSource string
// A Topic in kafka.
type Topic string
// An Address to listen on.
type Address string
// FileSizeBytes is a file size in bytes
type FileSizeBytes int64
// ThumbnailSize contains a single thumbnail size configuration
type ThumbnailSize struct {
// Maximum width of the thumbnail image
Width int `yaml:"width"`
// Maximum height of the thumbnail image
Height int `yaml:"height"`
// ResizeMethod is one of crop or scale.
// crop scales to fill the requested dimensions and crops the excess.
// scale scales to fit the requested dimensions and one dimension may be smaller than requested.
ResizeMethod string `yaml:"method,omitempty"`
}
// Load a yaml config file
func Load(configPath string) (*Dendrite, error) {
configData, err := ioutil.ReadFile(configPath)
if err != nil {
return nil, err
}
basePath, err := filepath.Abs(".")
if err != nil {
return nil, err
}
// Pass the current working directory and ioutil.ReadFile so that they can
// be mocked in the tests
return loadConfig(basePath, configData, ioutil.ReadFile)
}
// An Error indicates a problem parsing the config.
type Error struct {
// List of problems encountered parsing the config.
Problems []string
}
func loadConfig(
basePath string,
configData []byte,
readFile func(string) ([]byte, error),
) (*Dendrite, error) {
var config Dendrite
var err error
if err = yaml.Unmarshal(configData, &config); err != nil {
return nil, err
}
config.setDefaults()
if err = config.check(); err != nil {
return nil, err
}
privateKeyPath := absPath(basePath, config.Matrix.PrivateKeyPath)
privateKeyData, err := readFile(privateKeyPath)
if err != nil {
return nil, err
}
if config.Matrix.KeyID, config.Matrix.PrivateKey, err = readKeyPEM(privateKeyPath, privateKeyData); err != nil {
return nil, err
}
for _, certPath := range config.Matrix.FederationCertificatePaths {
absCertPath := absPath(basePath, certPath)
pemData, err := readFile(absCertPath)
if err != nil {
return nil, err
}
fingerprint := fingerprintPEM(pemData)
if fingerprint == nil {
return nil, fmt.Errorf("no certificate PEM data in %q", absCertPath)
}
config.Matrix.TLSFingerPrints = append(config.Matrix.TLSFingerPrints, *fingerprint)
}
config.Media.AbsBasePath = Path(absPath(basePath, config.Media.BasePath))
return &config, nil
}
func (config *Dendrite) setDefaults() {
if config.Matrix.KeyValidityPeriod == 0 {
config.Matrix.KeyValidityPeriod = 24 * time.Hour
}
if config.Media.MaxThumbnailGenerators == 0 {
config.Media.MaxThumbnailGenerators = 10
}
if config.Media.MaxFileSizeBytes == nil {
defaultMaxFileSizeBytes := FileSizeBytes(10485760)
config.Media.MaxFileSizeBytes = &defaultMaxFileSizeBytes
}
}
func (e Error) Error() string {
if len(e.Problems) == 1 {
return e.Problems[0]
}
return fmt.Sprintf(
"%s (and %d other problems)", e.Problems[0], len(e.Problems)-1,
)
}
func (config *Dendrite) check() error {
var problems []string
if config.Version != Version {
return Error{[]string{fmt.Sprintf(
"unknown config version %q, expected %q", config.Version, Version,
)}}
}
checkNotEmpty := func(key string, value string) {
if value == "" {
problems = append(problems, fmt.Sprintf("missing config key %q", key))
}
}
checkNotZero := func(key string, value int64) {
if value == 0 {
problems = append(problems, fmt.Sprintf("missing config key %q", key))
}
}
checkPositive := func(key string, value int64) {
if value < 0 {
problems = append(problems, fmt.Sprintf("invalid value for config key %q: %d", key, value))
}
}
checkNotEmpty("matrix.server_name", string(config.Matrix.ServerName))
checkNotEmpty("matrix.private_key", string(config.Matrix.PrivateKeyPath))
checkNotZero("matrix.federation_certificates", int64(len(config.Matrix.FederationCertificatePaths)))
checkNotEmpty("media.base_path", string(config.Media.BasePath))
checkPositive("media.max_file_size_bytes", int64(*config.Media.MaxFileSizeBytes))
checkPositive("media.max_thumbnail_generators", int64(config.Media.MaxThumbnailGenerators))
for i, size := range config.Media.ThumbnailSizes {
checkPositive(fmt.Sprintf("media.thumbnail_sizes[%d].width", i), int64(size.Width))
checkPositive(fmt.Sprintf("media.thumbnail_sizes[%d].height", i), int64(size.Height))
}
checkNotZero("kafka.addresses", int64(len(config.Kafka.Addresses)))
checkNotEmpty("kafka.topics.input_room_event", string(config.Kafka.Topics.InputRoomEvent))
checkNotEmpty("kafka.topics.output_room_event", string(config.Kafka.Topics.OutputRoomEvent))
checkNotEmpty("database.account", string(config.Database.Account))
checkNotEmpty("database.device", string(config.Database.Device))
checkNotEmpty("database.server_key", string(config.Database.ServerKey))
checkNotEmpty("database.media_api", string(config.Database.MediaAPI))
checkNotEmpty("database.sync_api", string(config.Database.SyncAPI))
checkNotEmpty("database.room_server", string(config.Database.RoomServer))
checkNotEmpty("listen.media_api", string(config.Listen.MediaAPI))
checkNotEmpty("listen.client_api", string(config.Listen.ClientAPI))
checkNotEmpty("listen.federation_api", string(config.Listen.FederationAPI))
checkNotEmpty("listen.sync_api", string(config.Listen.SyncAPI))
checkNotEmpty("listen.room_server", string(config.Listen.RoomServer))
if problems != nil {
return Error{problems}
}
return nil
}
func absPath(dir string, path Path) string {
if filepath.IsAbs(string(path)) {
// filepath.Join cleans the path so we should clean the absolute paths as well for consistency.
return filepath.Clean(string(path))
}
return filepath.Join(dir, string(path))
}
func readKeyPEM(path string, data []byte) (gomatrixserverlib.KeyID, ed25519.PrivateKey, error) {
for {
var keyBlock *pem.Block
keyBlock, data = pem.Decode(data)
if data == nil {
return "", nil, fmt.Errorf("no matrix private key PEM data in %q", path)
}
if keyBlock.Type == "MATRIX PRIVATE KEY" {
keyID := keyBlock.Headers["Key-ID"]
if keyID == "" {
return "", nil, fmt.Errorf("missing key ID in PEM data in %q", path)
}
if !strings.HasPrefix(keyID, "ed25519:") {
return "", nil, fmt.Errorf("key ID %q doesn't start with \"ed25519:\" in %q", keyID, path)
}
_, privKey, err := ed25519.GenerateKey(bytes.NewReader(keyBlock.Bytes))
if err != nil {
return "", nil, err
}
return gomatrixserverlib.KeyID(keyID), privKey, nil
}
}
}
func fingerprintPEM(data []byte) *gomatrixserverlib.TLSFingerprint {
for {
var certDERBlock *pem.Block
certDERBlock, data = pem.Decode(data)
if data == nil {
return nil
}
if certDERBlock.Type == "CERTIFICATE" {
digest := sha256.Sum256(certDERBlock.Bytes)
return &gomatrixserverlib.TLSFingerprint{digest[:]}
}
}
}
// RoomServerURL returns an HTTP URL for where the roomserver is listening.
func (config *Dendrite) RoomServerURL() string {
// Hard code the roomserver to talk HTTP for now.
// If we support HTTPS we need to think of a practical way to do certificate validation.
// People setting up servers shouldn't need to get a certificate valid for the public
// internet for an internal API.
return "http://" + string(config.Listen.RoomServer)
}

134
src/github.com/matrix-org/dendrite/common/config/config_test.go Normal file
View file

@ -0,0 +1,134 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"fmt"
"testing"
)
func TestLoadConfigRelative(t *testing.T) {
_, err := loadConfig("/my/config/dir", []byte(testConfig),
mockReadFile{
"/my/config/dir/matrix_key.pem": testKey,
"/my/config/dir/tls_cert.pem": testCert,
}.readFile,
)
if err != nil {
t.Error("failed to load config:", err)
}
}
const testConfig = `
version: v0
matrix:
server_name: localhost
private_key: matrix_key.pem
federation_certificates: [tls_cert.pem]
media:
base_path: media_store
kafka:
addresses: ["localhost:9092"]
topics:
input_room_event: input.room
output_room_event: output.room
database:
media_api: "postgresql:///media_api"
account: "postgresql:///account"
device: "postgresql:///device"
server_key: "postgresql:///server_keys"
sync_api: "postgresql:///syn_api"
room_server: "postgresql:///room_server"
listen:
room_server: "localhost:7770"
client_api: "localhost:7771"
federation_api: "localhost:7772"
sync_api: "localhost:7773"
media_api: "localhost:7774"
`
type mockReadFile map[string]string
func (m mockReadFile) readFile(path string) ([]byte, error) {
data, ok := m[path]
if !ok {
return nil, fmt.Errorf("no such file %q", path)
}
return []byte(data), nil
}
func TestReadKey(t *testing.T) {
keyID, _, err := readKeyPEM("path/to/key", []byte(testKey))
if err != nil {
t.Error("failed to load private key:", err)
}
wantKeyID := testKeyID
if wantKeyID != string(keyID) {
t.Errorf("wanted key ID to be %q, got %q", wantKeyID, keyID)
}
}
const testKeyID = "ed25519:c8NsuQ"
const testKey = `
-----BEGIN MATRIX PRIVATE KEY-----
Key-ID: ` + testKeyID + `
7KRZiZ2sTyRR8uqqUjRwczuwRXXkUMYIUHq4Mc3t4bE=
-----END MATRIX PRIVATE KEY-----
`
func TestFingerprintPEM(t *testing.T) {
got := fingerprintPEM([]byte(testCert))
if got == nil {
t.Error("failed to calculate fingerprint")
}
if string(got.SHA256) != testCertFingerprint {
t.Errorf("bad fingerprint: wanted %q got %q", got, testCertFingerprint)
}
}
const testCertFingerprint = "56.\\SPQxE\xd4\x95\xfb\xf6\xd5\x04\x91\xcb/\x07\xb1^\x88\x08\xe3\xc1p\xdfY\x04\x19w\xcb"
const testCert = `
-----BEGIN CERTIFICATE-----
MIIE0zCCArugAwIBAgIJAPype3u24LJeMA0GCSqGSIb3DQEBCwUAMAAwHhcNMTcw
NjEzMTQyODU4WhcNMTgwNjEzMTQyODU4WjAAMIICIjANBgkqhkiG9w0BAQEFAAOC
Ag8AMIICCgKCAgEA3vNSr7lCh/alxPFqairp/PYohwdsqPvOD7zf7dJCNhy0gbdC
9/APwIbPAPL9nU+o9ud1ACNCKBCQin/9LnI5vd5pa/Ne+mmRADDLB/BBBoywSJWG
NSfKJ9n3XY1bjgtqi53uUh+RDdQ7sXudDqCUxiiJZmS7oqK/mp88XXAgCbuXUY29
GmzbbDz37vntuSxDgUOnJ8uPSvRp5YPKogA3JwW1SyrlLt4Z30CQ6nH3Y2Q5SVfJ
NIQyMrnwyjA9bCdXezv1cLXoTYn7U9BRyzXTZeXs3y3ldnRfISXN35CU04Az1F8j
lfj7nXMEqI/qAj/qhxZ8nVBB+rpNOZy9RJko3O+G5Qa/EvzkQYV1rW4TM2Yme88A
QyJspoV/0bXk6gG987PonK2Uk5djxSULhnGVIqswydyH0Nzb+slRp2bSoWbaNlee
+6TIeiyTQYc055pCHOp22gtLrC5LQGchksi02St2ZzRHdnlfqCJ8S9sS7x3trzds
cYueg1sGI+O8szpQ3eUM7OhJOBrx6OlR7+QYnQg1wr/V+JAz1qcyTC1URcwfeqtg
QjxFdBD9LfCtfK+AO51H9ugtsPJqOh33PmvfvUBEM05OHCA0lNaWJHROGpm4T4cc
YQI9JQk/0lB7itF1qK5RG74qgKdjkBkfZxi0OqkUgHk6YHtJlKfET8zfrtcCAwEA
AaNQME4wHQYDVR0OBBYEFGwb0NgH0Zr7Ga23njEJ85Ozf8M9MB8GA1UdIwQYMBaA
FGwb0NgH0Zr7Ga23njEJ85Ozf8M9MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEL
BQADggIBAKU3RHXggbq/pLhGinU5q/9QT0TB/0bBnF1wNFkKQC0FrNJ+ZnBNmusy
oqOn7DEohBCCDxT0kgOC05gLEsGLkSXlVyqCsPFfycCFhtu1QzSRtQNRxB3pW3Wq
4/RFVYv0PGBjVBKxImQlEmXJWEDwemGKqDQZPtqR/FTHTbJcaT0xQr5+1oG6lawt
I/2cW6GQ0kYW/Szps8FgNdSNgVqCjjNIzBYbWhRWMx/63qD1ReUbY7/Yw9KKT8nK
zXERpbTM9k+Pnm0g9Gep+9HJ1dBFJeuTPugKeSeyqg2OJbENw1hxGs/HjBXw7580
ioiMn/kMj6Tg/f3HCfKrdHHBFQw0/fJW6o17QImYIpPOPzc5RjXBrCJWb34kxqEd
NQdKgejWiV/LlVsguIF8hVZH2kRzvoyypkVUtSUYGmjvA5UXoORQZfJ+b41llq1B
GcSF6iaVbAFKnsUyyr1i9uHz/6Muqflphv/SfZxGheIn5u3PnhXrzDagvItjw0NS
n0Xq64k7fc42HXJpF8CGBkSaIhtlzcruO+vqR80B9r62+D0V7VmHOnP135MT6noU
8F0JQfEtP+I8NII5jHSF/khzSgP5g80LS9tEc2ILnIHK1StkInAoRQQ+/HsQsgbz
ANAf5kxmMsM0zlN2hkxl0H6o7wKlBSw3RI3cjfilXiMWRPJrzlc4
-----END CERTIFICATE-----
`

58
src/github.com/matrix-org/dendrite/common/readkey.go
View file

@ -1,58 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package common
import (
"bytes"
"encoding/base64"
"fmt"
"github.com/matrix-org/gomatrixserverlib"
"golang.org/x/crypto/ed25519"
"io"
"strings"
)
// ReadKey reads a server's private ed25519 key.
// If the key is the empty string then a random key is generated.
// Otherwise the key is the key ID and the base64 encoded private key
// separated by a single space character.
// E.g "ed25519:abcd ABCDEFGHIJKLMNOPabcdefghijklmnop01234567890"
func ReadKey(key string) (gomatrixserverlib.KeyID, ed25519.PrivateKey, error) {
var keyID gomatrixserverlib.KeyID
var seed io.Reader
if key == "" {
// TODO: We should fail if we don't have a private key rather than
// generating a throw away key.
keyID = gomatrixserverlib.KeyID("ed25519:something")
} else {
// TODO: We should be reading this from a PEM formatted file instead of
// reading from the environment directly.
parts := strings.SplitN(key, " ", 2)
keyID = gomatrixserverlib.KeyID(parts[0])
if len(parts) != 2 {
return "", nil, fmt.Errorf("Invalid server key: %q", key)
}
seedBytes, err := base64.RawStdEncoding.DecodeString(parts[1])
if err != nil {
return "", nil, err
}
seed = bytes.NewReader(seedBytes)
}
_, privKey, err := ed25519.GenerateKey(seed)
if err != nil {
return "", nil, err
}
return keyID, privKey, nil
}

193
src/github.com/matrix-org/dendrite/common/test/config.go Normal file
View file

@ -0,0 +1,193 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/base64"
"encoding/pem"
"fmt"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/gomatrixserverlib"
"gopkg.in/yaml.v2"
"io/ioutil"
"math/big"
"os"
"path/filepath"
"time"
)
const (
// ConfigFile is the name of the config file for a server.
ConfigFile = "dendrite.yaml"
// ServerKeyFile is the name of the file holding the matrix server private key.
ServerKeyFile = "server_key.pem"
// TLSCertFile is the name of the file holding the TLS certificate used for federation.
TLSCertFile = "tls_cert.pem"
// TLSKeyFile is the name of the file holding the TLS key used for federation.
TLSKeyFile = "tls_key.pem"
// MediaDir is the name of the directory used to store media.
MediaDir = "media"
)
// MakeConfig makes a config suitable for running integration tests.
// Generates new matrix and TLS keys for the server.
func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*config.Dendrite, int, error) {
var cfg config.Dendrite
port := startPort
assignAddress := func() config.Address {
result := config.Address(fmt.Sprintf("%s:%d", host, port))
port++
return result
}
serverKeyPath := filepath.Join(configDir, ServerKeyFile)
tlsCertPath := filepath.Join(configDir, TLSKeyFile)
tlsKeyPath := filepath.Join(configDir, TLSCertFile)
mediaBasePath := filepath.Join(configDir, MediaDir)
if err := newMatrixKey(serverKeyPath); err != nil {
return nil, 0, err
}
if err := newTLSKey(tlsKeyPath, tlsCertPath); err != nil {
return nil, 0, err
}
cfg.Version = config.Version
cfg.Matrix.ServerName = gomatrixserverlib.ServerName(assignAddress())
cfg.Matrix.PrivateKeyPath = config.Path(serverKeyPath)
cfg.Matrix.FederationCertificatePaths = []config.Path{config.Path(tlsCertPath)}
cfg.Media.BasePath = config.Path(mediaBasePath)
cfg.Kafka.Addresses = []string{kafkaURI}
// TODO: Different servers should be using different topics.
// Make this configurable somehow?
cfg.Kafka.Topics.InputRoomEvent = "test.room.input"
cfg.Kafka.Topics.OutputRoomEvent = "test.room.output"
// TODO: Use different databases for the different schemas.
// Using the same database for every schema currently works because
// the table names are globally unique. But we might not want to
// rely on that in the future.
cfg.Database.Account = config.DataSource(database)
cfg.Database.Device = config.DataSource(database)
cfg.Database.MediaAPI = config.DataSource(database)
cfg.Database.RoomServer = config.DataSource(database)
cfg.Database.ServerKey = config.DataSource(database)
cfg.Database.SyncAPI = config.DataSource(database)
cfg.Listen.ClientAPI = assignAddress()
cfg.Listen.FederationAPI = assignAddress()
cfg.Listen.MediaAPI = assignAddress()
cfg.Listen.RoomServer = assignAddress()
cfg.Listen.SyncAPI = assignAddress()
return &cfg, port, nil
}
// WriteConfig writes the config file to the directory.
func WriteConfig(cfg *config.Dendrite, configDir string) error {
data, err := yaml.Marshal(cfg)
if err != nil {
return err
}
if err = ioutil.WriteFile(filepath.Join(configDir, ConfigFile), data, 0666); err != nil {
return err
}
return nil
}
// newMatrixKey generates a new ed25519 matrix server key and writes it to a file.
func newMatrixKey(matrixKeyPath string) error {
var data [35]byte
if _, err := rand.Read(data[:]); err != nil {
return err
}
keyOut, err := os.OpenFile(matrixKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err
}
defer keyOut.Close()
if err = pem.Encode(keyOut, &pem.Block{
Type: "MATRIX PRIVATE KEY",
Headers: map[string]string{
"Key-ID": "ed25519:" + base64.RawStdEncoding.EncodeToString(data[:3]),
},
Bytes: data[3:],
}); err != nil {
return err
}
return nil
}
const certificateDuration = time.Hour * 24 * 365 * 10
// newTLSKey generates a new RSA TLS key and certificate and writes it to a file.
func newTLSKey(tlsKeyPath, tlsCertPath string) error {
priv, err := rsa.GenerateKey(rand.Reader, 4096)
if err != nil {
return err
}
notBefore := time.Now()
notAfter := notBefore.Add(certificateDuration)
serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
if err != nil {
return err
}
template := x509.Certificate{
SerialNumber: serialNumber,
NotBefore: notBefore,
NotAfter: notAfter,
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
}
derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
if err != nil {
return err
}
certOut, err := os.Create(tlsCertPath)
if err != nil {
return err
}
defer certOut.Close()
if err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
return err
}
keyOut, err := os.OpenFile(tlsKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err
}
defer keyOut.Close()
if err = pem.Encode(keyOut, &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(priv),
}); err != nil {
return err
}
return nil
}

19
src/github.com/matrix-org/dendrite/common/test/server.go
View file

@ -16,11 +16,12 @@ package test
import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/matrix-org/dendrite/common/config"
)
// Defaulting allows assignment of string variables with a fallback default value
@ -69,7 +70,7 @@ func CreateBackgroundCommand(command string, args []string) (*exec.Cmd, chan err
// which will have any termination errors sent down it, followed immediately by the channel being closed.
// If postgresContainerName is not an empty string, psql will be run from inside that container. If it is
// an empty string, psql will be assumed to be in PATH.
func StartServer(serverType string, serverArgs []string, suffix, configFilename, configFileContents, postgresDatabase, postgresContainerName string, databases []string) (*exec.Cmd, chan error) {
func StartServer(serverType string, serverArgs []string, postgresDatabase, postgresContainerName string, databases []string) (*exec.Cmd, chan error) {
if len(databases) > 0 {
var dbCmd string
var dbArgs []string
@ -89,12 +90,6 @@ func StartServer(serverType string, serverArgs []string, suffix, configFilename,
}
}
if configFilename != "" {
if err := ioutil.WriteFile(configFilename, []byte(configFileContents), 0644); err != nil {
panic(err)
}
}
return CreateBackgroundCommand(
filepath.Join(filepath.Dir(os.Args[0]), "dendrite-"+serverType+"-server"),
serverArgs,
@ -102,12 +97,12 @@ func StartServer(serverType string, serverArgs []string, suffix, configFilename,
}
// StartProxy creates a reverse proxy
func StartProxy(bindAddr, syncAddr, clientAddr, mediaAddr string) (*exec.Cmd, chan error) {
func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) {
proxyArgs := []string{
"--bind-address", bindAddr,
"--sync-api-server-url", syncAddr,
"--client-api-server-url", clientAddr,
"--media-api-server-url", mediaAddr,
"--sync-api-server-url", "http://" + string(cfg.Listen.SyncAPI),
"--client-api-server-url", "http://" + string(cfg.Listen.ClientAPI),
"--media-api-server-url", "http://" + string(cfg.Listen.MediaAPI),
}
return CreateBackgroundCommand(
filepath.Join(filepath.Dir(os.Args[0]), "client-api-proxy"),

39
src/github.com/matrix-org/dendrite/federationapi/config/config.go
View file

@ -1,39 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"github.com/matrix-org/gomatrixserverlib"
"golang.org/x/crypto/ed25519"
"time"
)
// FederationAPI contains the config information necessary to spin up a federationapi process.
type FederationAPI struct {
// The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
ServerName gomatrixserverlib.ServerName
// The private key which will be used to sign requests.
PrivateKey ed25519.PrivateKey
// An arbitrary string used to uniquely identify the PrivateKey. Must start with the
// prefix "ed25519:".
KeyID gomatrixserverlib.KeyID
// A list of SHA256 TLS fingerprints for this server.
TLSFingerPrints []gomatrixserverlib.TLSFingerprint
// How long a remote server can cache our server key for before requesting it again.
// Increasing this number will reduce the number of requests made by remote servers
// for our key, but increases the period a compromised key will be considered valid
// by remote servers.
ValidityPeriod time.Duration
}

22
src/github.com/matrix-org/dendrite/federationapi/readers/keys.go
View file

@ -16,7 +16,7 @@ package readers
import (
"encoding/json"
"github.com/matrix-org/dendrite/federationapi/config"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"golang.org/x/crypto/ed25519"
@ -26,29 +26,29 @@ import (
// LocalKeys returns the local keys for the server.
// See https://matrix.org/docs/spec/server_server/unstable.html#publishing-keys
func LocalKeys(req *http.Request, cfg config.FederationAPI) util.JSONResponse {
keys, err := localKeys(cfg, time.Now().Add(cfg.ValidityPeriod))
func LocalKeys(req *http.Request, cfg config.Dendrite) util.JSONResponse {
keys, err := localKeys(cfg, time.Now().Add(cfg.Matrix.KeyValidityPeriod))
if err != nil {
return util.ErrorResponse(err)
}
return util.JSONResponse{Code: 200, JSON: keys}
}
func localKeys(cfg config.FederationAPI, validUntil time.Time) (*gomatrixserverlib.ServerKeys, error) {
func localKeys(cfg config.Dendrite, validUntil time.Time) (*gomatrixserverlib.ServerKeys, error) {
var keys gomatrixserverlib.ServerKeys
keys.ServerName = cfg.ServerName
keys.FromServer = cfg.ServerName
keys.ServerName = cfg.Matrix.ServerName
keys.FromServer = cfg.Matrix.ServerName
publicKey := cfg.PrivateKey.Public().(ed25519.PublicKey)
publicKey := cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey)
keys.VerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.VerifyKey{
cfg.KeyID: {
cfg.Matrix.KeyID: {
gomatrixserverlib.Base64String(publicKey),
},
}
keys.TLSFingerprints = cfg.TLSFingerPrints
keys.TLSFingerprints = cfg.Matrix.TLSFingerPrints
keys.OldVerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.OldVerifyKey{}
keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(validUntil)
@ -57,7 +57,9 @@ func localKeys(cfg config.FederationAPI, validUntil time.Time) (*gomatrixserverl
return nil, err
}
keys.Raw, err = gomatrixserverlib.SignJSON(string(cfg.ServerName), cfg.KeyID, cfg.PrivateKey, toSign)
keys.Raw, err = gomatrixserverlib.SignJSON(
string(cfg.Matrix.ServerName), cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, toSign,
)
if err != nil {
return nil, err
}

4
src/github.com/matrix-org/dendrite/federationapi/routing/routing.go
View file

@ -17,7 +17,7 @@ package routing
import (
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/dendrite/federationapi/config"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/federationapi/readers"
"github.com/matrix-org/dendrite/federationapi/writers"
"github.com/matrix-org/dendrite/roomserver/api"
@ -36,7 +36,7 @@ const (
// Setup registers HTTP handlers with the given ServeMux.
func Setup(
servMux *http.ServeMux,
cfg config.FederationAPI,
cfg config.Dendrite,
query api.RoomserverQueryAPI,
producer *producers.RoomserverProducer,
keys gomatrixserverlib.KeyRing,

8
src/github.com/matrix-org/dendrite/federationapi/writers/send.go
View file

@ -6,7 +6,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/dendrite/federationapi/config"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
@ -19,13 +19,13 @@ func Send(
req *http.Request,
txnID gomatrixserverlib.TransactionID,
now time.Time,
cfg config.FederationAPI,
cfg config.Dendrite,
query api.RoomserverQueryAPI,
producer *producers.RoomserverProducer,
keys gomatrixserverlib.KeyRing,
federation *gomatrixserverlib.FederationClient,
) util.JSONResponse {
request, errResp := gomatrixserverlib.VerifyHTTPRequest(req, now, cfg.ServerName, keys)
request, errResp := gomatrixserverlib.VerifyHTTPRequest(req, now, cfg.Matrix.ServerName, keys)
if request == nil {
return errResp
}
@ -45,7 +45,7 @@ func Send(
t.Origin = request.Origin()
t.TransactionID = txnID
t.Destination = cfg.ServerName
t.Destination = cfg.Matrix.ServerName
resp, err := t.processTransaction()
if err != nil {

42
src/github.com/matrix-org/dendrite/mediaapi/config/config.go
View file

@ -1,42 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"github.com/matrix-org/dendrite/mediaapi/types"
"github.com/matrix-org/gomatrixserverlib"
)
// MediaAPI contains the config information necessary to spin up a mediaapi process.
type MediaAPI struct {
// The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'.
ServerName gomatrixserverlib.ServerName `yaml:"server_name"`
// The base path to where the media files will be stored. May be relative or absolute.
BasePath types.Path `yaml:"base_path"`
// The absolute base path to where media files will be stored.
AbsBasePath types.Path `yaml:"-"`
// The maximum file size in bytes that is allowed to be stored on this server.
// Note: if MaxFileSizeBytes is set to 0, the size is unlimited.
// Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB)
MaxFileSizeBytes *types.FileSizeBytes `yaml:"max_file_size_bytes,omitempty"`
// The postgres connection config for connecting to the database e.g a postgres:// URI
DataSource string `yaml:"database"`
// Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated
DynamicThumbnails bool `yaml:"dynamic_thumbnails"`
// The maximum number of simultaneous thumbnail generators. default: 10
MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"`
// A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content
ThumbnailSizes []types.ThumbnailSize `yaml:"thumbnail_sizes"`
}

11
src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go
View file

@ -26,13 +26,14 @@ import (
"strings"
log "github.com/Sirupsen/logrus"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/mediaapi/types"
)
// GetPathFromBase64Hash evaluates the path to a media file from its Base64Hash
// 3 subdirectories are created for more manageable browsing and use the remainder as the file name.
// For example, if Base64Hash is 'qwerty', the path will be 'q/w/erty/file'.
func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath types.Path) (string, error) {
func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath config.Path) (string, error) {
if len(base64Hash) < 3 {
return "", fmt.Errorf("Invalid filePath (Base64Hash too short - min 3 characters): %q", base64Hash)
}
@ -66,7 +67,7 @@ func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath types.Path)
// If the final path exists and the file size matches, the file does not need to be moved.
// In error cases where the file is not a duplicate, the caller may decide to remove the final path.
// Returns the final path of the file, whether it is a duplicate and an error.
func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata, absBasePath types.Path, logger *log.Entry) (types.Path, bool, error) {
func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata, absBasePath config.Path, logger *log.Entry) (types.Path, bool, error) {
// Note: in all error and success cases, we need to remove the temporary directory
defer RemoveDir(tmpDir, logger)
duplicate := false
@ -104,7 +105,7 @@ func RemoveDir(dir types.Path, logger *log.Entry) {
}
// WriteTempFile writes to a new temporary file
func WriteTempFile(reqReader io.Reader, maxFileSizeBytes types.FileSizeBytes, absBasePath types.Path) (types.Base64Hash, types.FileSizeBytes, types.Path, error) {
func WriteTempFile(reqReader io.Reader, maxFileSizeBytes config.FileSizeBytes, absBasePath config.Path) (types.Base64Hash, types.FileSizeBytes, types.Path, error) {
tmpFileWriter, tmpFile, tmpDir, err := createTempFileWriter(absBasePath)
if err != nil {
return "", -1, "", err
@ -144,7 +145,7 @@ func moveFile(src types.Path, dst types.Path) error {
return nil
}
func createTempFileWriter(absBasePath types.Path) (*bufio.Writer, *os.File, types.Path, error) {
func createTempFileWriter(absBasePath config.Path) (*bufio.Writer, *os.File, types.Path, error) {
tmpDir, err := createTempDir(absBasePath)
if err != nil {
return nil, nil, "", fmt.Errorf("Failed to create temp dir: %q", err)
@ -157,7 +158,7 @@ func createTempFileWriter(absBasePath types.Path) (*bufio.Writer, *os.File, type
}
// createTempDir creates a tmp/<random string> directory within baseDirectory and returns its path
func createTempDir(baseDirectory types.Path) (types.Path, error) {
func createTempDir(baseDirectory config.Path) (types.Path, error) {
baseTmpDir := filepath.Join(string(baseDirectory), "tmp")
if err := os.MkdirAll(baseTmpDir, 0770); err != nil {
return "", fmt.Errorf("Failed to create base temp dir: %v", err)

6
src/github.com/matrix-org/dendrite/mediaapi/routing/routing.go
View file

@ -19,7 +19,7 @@ import (
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/mediaapi/config"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/mediaapi/storage"
"github.com/matrix-org/dendrite/mediaapi/types"
"github.com/matrix-org/dendrite/mediaapi/writers"
@ -32,7 +32,7 @@ const pathPrefixR0 = "/_matrix/media/v1"
// Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
// to clients which need to make outbound HTTP requests.
func Setup(servMux *http.ServeMux, httpClient *http.Client, cfg *config.MediaAPI, db *storage.Database) {
func Setup(servMux *http.ServeMux, httpClient *http.Client, cfg *config.Dendrite, db *storage.Database) {
apiMux := mux.NewRouter()
r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
@ -59,7 +59,7 @@ func Setup(servMux *http.ServeMux, httpClient *http.Client, cfg *config.MediaAPI
servMux.Handle("/api/", http.StripPrefix("/api", apiMux))
}
func makeDownloadAPI(name string, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) http.HandlerFunc {
func makeDownloadAPI(name string, cfg *config.Dendrite, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) http.HandlerFunc {
return prometheus.InstrumentHandler(name, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
req = util.RequestWithLogging(req)

5
src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go
View file

@ -22,6 +22,7 @@ import (
"sync"
log "github.com/Sirupsen/logrus"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/mediaapi/storage"
"github.com/matrix-org/dendrite/mediaapi/types"
)
@ -56,7 +57,7 @@ func GetThumbnailPath(src types.Path, config types.ThumbnailSize) types.Path {
// * has a small file size
// If a pre-generated thumbnail size is the best match, but it has not been generated yet, the caller can use the returned size to generate it.
// Returns nil if no thumbnail matches the criteria
func SelectThumbnail(desired types.ThumbnailSize, thumbnails []*types.ThumbnailMetadata, thumbnailSizes []types.ThumbnailSize) (*types.ThumbnailMetadata, *types.ThumbnailSize) {
func SelectThumbnail(desired types.ThumbnailSize, thumbnails []*types.ThumbnailMetadata, thumbnailSizes []config.ThumbnailSize) (*types.ThumbnailMetadata, *types.ThumbnailSize) {
var chosenThumbnail *types.ThumbnailMetadata
var chosenThumbnailSize *types.ThumbnailSize
bestFit := newThumbnailFitness()
@ -76,7 +77,7 @@ func SelectThumbnail(desired types.ThumbnailSize, thumbnails []*types.ThumbnailM
if desired.ResizeMethod == "scale" && thumbnailSize.ResizeMethod != "scale" {
continue
}
fitness := calcThumbnailFitness(thumbnailSize, nil, desired)
fitness := calcThumbnailFitness(types.ThumbnailSize(thumbnailSize), nil, desired)
if isBetter := fitness.betterThan(bestFit, desired.ResizeMethod == "crop"); isBetter {
bestFit = fitness
chosenThumbnailSize = &types.ThumbnailSize{

3
src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go
View file

@ -21,13 +21,14 @@ import (
"time"
log "github.com/Sirupsen/logrus"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/mediaapi/storage"
"github.com/matrix-org/dendrite/mediaapi/types"
"gopkg.in/h2non/bimg.v1"
)
// GenerateThumbnails generates the configured thumbnail sizes for the source file
func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
func GenerateThumbnails(src types.Path, configs []config.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
buffer, err := bimg.Read(string(src))
if err != nil {
logger.WithError(err).WithField("src", src).Error("Failed to read src file")

5
src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go
View file

@ -28,13 +28,14 @@ import (
"time"
log "github.com/Sirupsen/logrus"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/mediaapi/storage"
"github.com/matrix-org/dendrite/mediaapi/types"
"github.com/nfnt/resize"
)
// GenerateThumbnails generates the configured thumbnail sizes for the source file
func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
func GenerateThumbnails(src types.Path, configs []config.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) {
img, err := readFile(string(src))
if err != nil {
logger.WithError(err).WithField("src", src).Error("Failed to read src file")
@ -42,7 +43,7 @@ func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMeta
}
for _, config := range configs {
// Note: createThumbnail does locking based on activeThumbnailGeneration
busy, err = createThumbnail(src, img, config, mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger)
busy, err = createThumbnail(src, img, types.ThumbnailSize(config), mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger)
if err != nil {
logger.WithError(err).WithField("src", src).Error("Failed to generate thumbnails")
return false, err

12
src/github.com/matrix-org/dendrite/mediaapi/types/types.go
View file

@ -17,6 +17,7 @@ package types
import (
"sync"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
@ -79,16 +80,7 @@ type ActiveRemoteRequests struct {
}
// ThumbnailSize contains a single thumbnail size configuration
type ThumbnailSize struct {
// Maximum width of the thumbnail image
Width int `yaml:"width"`
// Maximum height of the thumbnail image
Height int `yaml:"height"`
// ResizeMethod is one of crop or scale.
// crop scales to fill the requested dimensions and crops the excess.
// scale scales to fit the requested dimensions and one dimension may be smaller than requested.
ResizeMethod string `yaml:"method,omitempty"`
}
type ThumbnailSize config.ThumbnailSize
// ThumbnailMetadata contains the metadata about an individual thumbnail
type ThumbnailMetadata struct {

22
src/github.com/matrix-org/dendrite/mediaapi/writers/download.go
View file

@ -29,7 +29,7 @@ import (
log "github.com/Sirupsen/logrus"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/mediaapi/config"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/mediaapi/fileutils"
"github.com/matrix-org/dendrite/mediaapi/storage"
"github.com/matrix-org/dendrite/mediaapi/thumbnailer"
@ -59,7 +59,7 @@ type downloadRequest struct {
// If they are present in the cache, they are served directly.
// If they are not present in the cache, they are obtained from the remote server and
// simultaneously served back to the client and written into the cache.
func Download(w http.ResponseWriter, req *http.Request, origin gomatrixserverlib.ServerName, mediaID types.MediaID, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, isThumbnailRequest bool) {
func Download(w http.ResponseWriter, req *http.Request, origin gomatrixserverlib.ServerName, mediaID types.MediaID, cfg *config.Dendrite, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, isThumbnailRequest bool) {
r := &downloadRequest{
MediaMetadata: &types.MediaMetadata{
MediaID: mediaID,
@ -167,7 +167,7 @@ func (r *downloadRequest) Validate() *util.JSONResponse {
return nil
}
func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse {
func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.Dendrite, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse {
// check if we have a record of the media in our database
mediaMetadata, err := db.GetMediaMetadata(r.MediaMetadata.MediaID, r.MediaMetadata.Origin)
if err != nil {
@ -176,7 +176,7 @@ func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI
return &resErr
}
if mediaMetadata == nil {
if r.MediaMetadata.Origin == cfg.ServerName {
if r.MediaMetadata.Origin == cfg.Matrix.ServerName {
// If we do not have a record and the origin is local, the file is not found
return &util.JSONResponse{
Code: 404,
@ -192,12 +192,12 @@ func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI
// If we have a record, we can respond from the local file
r.MediaMetadata = mediaMetadata
}
return r.respondFromLocalFile(w, cfg.AbsBasePath, activeThumbnailGeneration, cfg.MaxThumbnailGenerators, db, cfg.DynamicThumbnails, cfg.ThumbnailSizes)
return r.respondFromLocalFile(w, cfg.Media.AbsBasePath, activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators, db, cfg.Media.DynamicThumbnails, cfg.Media.ThumbnailSizes)
}
// respondFromLocalFile reads a file from local storage and writes it to the http.ResponseWriter
// Returns a util.JSONResponse error in case of error
func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []types.ThumbnailSize) *util.JSONResponse {
func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePath config.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []config.ThumbnailSize) *util.JSONResponse {
filePath, err := fileutils.GetPathFromBase64Hash(r.MediaMetadata.Base64Hash, absBasePath)
if err != nil {
r.Logger.WithError(err).Error("Failed to get file path from metadata")
@ -284,7 +284,7 @@ func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePat
}
// Note: Thumbnail generation may be ongoing asynchronously.
func (r *downloadRequest) getThumbnailFile(filePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []types.ThumbnailSize) (*os.File, *types.ThumbnailMetadata, *util.JSONResponse) {
func (r *downloadRequest) getThumbnailFile(filePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []config.ThumbnailSize) (*os.File, *types.ThumbnailMetadata, *util.JSONResponse) {
var thumbnail *types.ThumbnailMetadata
var resErr *util.JSONResponse
@ -383,7 +383,7 @@ func (r *downloadRequest) generateThumbnail(filePath types.Path, thumbnailSize t
// regardless of how many download requests are received.
// Note: The named errorResponse return variable is used in a deferred broadcast of the metadata and error response to waiting goroutines.
// Returns a util.JSONResponse error in case of error
func (r *downloadRequest) getRemoteFile(cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) (errorResponse *util.JSONResponse) {
func (r *downloadRequest) getRemoteFile(cfg *config.Dendrite, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) (errorResponse *util.JSONResponse) {
// Note: getMediaMetadataFromActiveRequest uses mutexes and conditions from activeRemoteRequests
mediaMetadata, resErr := r.getMediaMetadataFromActiveRequest(activeRemoteRequests)
if resErr != nil {
@ -414,7 +414,7 @@ func (r *downloadRequest) getRemoteFile(cfg *config.MediaAPI, db *storage.Databa
if mediaMetadata == nil {
// If we do not have a record, we need to fetch the remote file first and then respond from the local file
resErr := r.fetchRemoteFileAndStoreMetadata(cfg.AbsBasePath, *cfg.MaxFileSizeBytes, db, cfg.ThumbnailSizes, activeThumbnailGeneration, cfg.MaxThumbnailGenerators)
resErr := r.fetchRemoteFileAndStoreMetadata(cfg.Media.AbsBasePath, *cfg.Media.MaxFileSizeBytes, db, cfg.Media.ThumbnailSizes, activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators)
if resErr != nil {
return resErr
}
@ -476,7 +476,7 @@ func (r *downloadRequest) broadcastMediaMetadata(activeRemoteRequests *types.Act
}
// fetchRemoteFileAndStoreMetadata fetches the file from the remote server and stores its metadata in the database
func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath types.Path, maxFileSizeBytes types.FileSizeBytes, db *storage.Database, thumbnailSizes []types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse {
func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath config.Path, maxFileSizeBytes config.FileSizeBytes, db *storage.Database, thumbnailSizes []config.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse {
finalPath, duplicate, resErr := r.fetchRemoteFile(absBasePath, maxFileSizeBytes)
if resErr != nil {
return resErr
@ -524,7 +524,7 @@ func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath types.Path
return nil
}
func (r *downloadRequest) fetchRemoteFile(absBasePath types.Path, maxFileSizeBytes types.FileSizeBytes) (types.Path, bool, *util.JSONResponse) {
func (r *downloadRequest) fetchRemoteFile(absBasePath config.Path, maxFileSizeBytes config.FileSizeBytes) (types.Path, bool, *util.JSONResponse) {
r.Logger.Info("Fetching remote file")
// create request for remote file

30
src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go
View file

@ -24,7 +24,7 @@ import (
log "github.com/Sirupsen/logrus"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/mediaapi/config"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/mediaapi/fileutils"
"github.com/matrix-org/dendrite/mediaapi/storage"
"github.com/matrix-org/dendrite/mediaapi/thumbnailer"
@ -51,7 +51,7 @@ type uploadResponse struct {
// This implementation supports a configurable maximum file size limit in bytes. If a user tries to upload more than this, they will receive an error that their upload is too large.
// Uploaded files are processed piece-wise to avoid DoS attacks which would starve the server of memory.
// TODO: We should time out requests if they have not received any data within a configured timeout period.
func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse {
func Upload(req *http.Request, cfg *config.Dendrite, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse {
r, resErr := parseAndValidateRequest(req, cfg)
if resErr != nil {
return *resErr
@ -64,7 +64,7 @@ func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database, activ
return util.JSONResponse{
Code: 200,
JSON: uploadResponse{
ContentURI: fmt.Sprintf("mxc://%s/%s", cfg.ServerName, r.MediaMetadata.MediaID),
ContentURI: fmt.Sprintf("mxc://%s/%s", cfg.Matrix.ServerName, r.MediaMetadata.MediaID),
},
}
}
@ -72,7 +72,7 @@ func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database, activ
// parseAndValidateRequest parses the incoming upload request to validate and extract
// all the metadata about the media being uploaded.
// Returns either an uploadRequest or an error formatted as a util.JSONResponse
func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRequest, *util.JSONResponse) {
func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRequest, *util.JSONResponse) {
if req.Method != "POST" {
return nil, &util.JSONResponse{
Code: 405,
@ -82,22 +82,22 @@ func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRe
r := &uploadRequest{
MediaMetadata: &types.MediaMetadata{
Origin: cfg.ServerName,
Origin: cfg.Matrix.ServerName,
FileSizeBytes: types.FileSizeBytes(req.ContentLength),
ContentType: types.ContentType(req.Header.Get("Content-Type")),
UploadName: types.Filename(url.PathEscape(req.FormValue("filename"))),
},
Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.ServerName),
Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.Matrix.ServerName),
}
if resErr := r.Validate(*cfg.MaxFileSizeBytes); resErr != nil {
if resErr := r.Validate(*cfg.Media.MaxFileSizeBytes); resErr != nil {
return nil, resErr
}
return r, nil
}
func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse {
func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.Dendrite, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse {
r.Logger.WithFields(log.Fields{
"UploadName": r.MediaMetadata.UploadName,
"FileSizeBytes": r.MediaMetadata.FileSizeBytes,
@ -108,10 +108,10 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db *
// method of deduplicating files to save storage, as well as a way to conduct
// integrity checks on the file data in the repository.
// Data is truncated to maxFileSizeBytes. Content-Length was reported as 0 < Content-Length <= maxFileSizeBytes so this is OK.
hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.MaxFileSizeBytes, cfg.AbsBasePath)
hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.Media.MaxFileSizeBytes, cfg.Media.AbsBasePath)
if err != nil {
r.Logger.WithError(err).WithFields(log.Fields{
"MaxFileSizeBytes": *cfg.MaxFileSizeBytes,
"MaxFileSizeBytes": *cfg.Media.MaxFileSizeBytes,
}).Warn("Error while transferring file")
fileutils.RemoveDir(tmpDir, r.Logger)
return &util.JSONResponse{
@ -147,12 +147,12 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db *
return &util.JSONResponse{
Code: 200,
JSON: uploadResponse{
ContentURI: fmt.Sprintf("mxc://%s/%s", cfg.ServerName, r.MediaMetadata.MediaID),
ContentURI: fmt.Sprintf("mxc://%s/%s", cfg.Matrix.ServerName, r.MediaMetadata.MediaID),
},
}
}
if resErr := r.storeFileAndMetadata(tmpDir, cfg.AbsBasePath, db, cfg.ThumbnailSizes, activeThumbnailGeneration, cfg.MaxThumbnailGenerators); resErr != nil {
if resErr := r.storeFileAndMetadata(tmpDir, cfg.Media.AbsBasePath, db, cfg.Media.ThumbnailSizes, activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators); resErr != nil {
return resErr
}
@ -160,14 +160,14 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db *
}
// Validate validates the uploadRequest fields
func (r *uploadRequest) Validate(maxFileSizeBytes types.FileSizeBytes) *util.JSONResponse {
func (r *uploadRequest) Validate(maxFileSizeBytes config.FileSizeBytes) *util.JSONResponse {
if r.MediaMetadata.FileSizeBytes < 1 {
return &util.JSONResponse{
Code: 411,
JSON: jsonerror.Unknown("HTTP Content-Length request header must be greater than zero."),
}
}
if maxFileSizeBytes > 0 && r.MediaMetadata.FileSizeBytes > maxFileSizeBytes {
if maxFileSizeBytes > 0 && r.MediaMetadata.FileSizeBytes > types.FileSizeBytes(maxFileSizeBytes) {
return &util.JSONResponse{
Code: 413,
JSON: jsonerror.Unknown(fmt.Sprintf("HTTP Content-Length is greater than the maximum allowed upload size (%v).", maxFileSizeBytes)),
@ -215,7 +215,7 @@ func (r *uploadRequest) Validate(maxFileSizeBytes types.FileSizeBytes) *util.JSO
// The order of operations is important as it avoids metadata entering the database before the file
// is ready, and if we fail to move the file, it never gets added to the database.
// Returns a util.JSONResponse error and cleans up directories in case of error.
func (r *uploadRequest) storeFileAndMetadata(tmpDir types.Path, absBasePath types.Path, db *storage.Database, thumbnailSizes []types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse {
func (r *uploadRequest) storeFileAndMetadata(tmpDir types.Path, absBasePath config.Path, db *storage.Database, thumbnailSizes []config.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse {
finalPath, duplicate, err := fileutils.MoveFileWithHashCheck(tmpDir, r.MediaMetadata, absBasePath, r.Logger)
if err != nil {
r.Logger.WithError(err).Error("Failed to move file.")

33
src/github.com/matrix-org/dendrite/syncapi/config/config.go
View file

@ -1,33 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"github.com/matrix-org/gomatrixserverlib"
)
// Sync contains the config information necessary to spin up a sync-server process.
type Sync struct {
// Where the room server is listening for queries.
RoomserverURL string `yaml:"roomserver_url"`
// The topic for events which are written by the room server output log.
RoomserverOutputTopic string `yaml:"roomserver_topic"`
// A list of URIs to consume events from. These kafka logs should be produced by a Room Server.
KafkaConsumerURIs []string `yaml:"consumer_uris"`
// The postgres connection config for connecting to the database e.g a postgres:// URI
DataSource string `yaml:"database"`
// The server_name of the running process e.g "localhost"
ServerName gomatrixserverlib.ServerName `yaml:"server_name"`
}

11
src/github.com/matrix-org/dendrite/syncapi/consumers/roomserver.go
View file

@ -20,8 +20,8 @@ import (
log "github.com/Sirupsen/logrus"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/syncapi/config"
"github.com/matrix-org/dendrite/syncapi/storage"
"github.com/matrix-org/dendrite/syncapi/sync"
"github.com/matrix-org/dendrite/syncapi/types"
@ -38,14 +38,15 @@ type OutputRoomEvent struct {
}
// NewOutputRoomEvent creates a new OutputRoomEvent consumer. Call Start() to begin consuming from room servers.
func NewOutputRoomEvent(cfg *config.Sync, n *sync.Notifier, store *storage.SyncServerDatabase) (*OutputRoomEvent, error) {
kafkaConsumer, err := sarama.NewConsumer(cfg.KafkaConsumerURIs, nil)
func NewOutputRoomEvent(cfg *config.Dendrite, n *sync.Notifier, store *storage.SyncServerDatabase) (*OutputRoomEvent, error) {
kafkaConsumer, err := sarama.NewConsumer(cfg.Kafka.Addresses, nil)
if err != nil {
return nil, err
}
roomServerURL := cfg.RoomServerURL()
consumer := common.ContinualConsumer{
Topic: cfg.RoomserverOutputTopic,
Topic: string(cfg.Kafka.Topics.OutputRoomEvent),
Consumer: kafkaConsumer,
PartitionStore: store,
}
@ -53,7 +54,7 @@ func NewOutputRoomEvent(cfg *config.Sync, n *sync.Notifier, store *storage.SyncS
roomServerConsumer: &consumer,
db: store,
notifier: n,
query: api.NewRoomserverQueryAPIHTTP(cfg.RoomserverURL, nil),
query: api.NewRoomserverQueryAPIHTTP(roomServerURL, nil),
}
consumer.ProcessMessage = s.onMessage

3
src/github.com/matrix-org/dendrite/syncapi/routing/routing.go
View file

@ -21,7 +21,6 @@ import (
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/syncapi/config"
"github.com/matrix-org/dendrite/syncapi/sync"
"github.com/matrix-org/util"
"github.com/prometheus/client_golang/prometheus"
@ -30,7 +29,7 @@ import (
const pathPrefixR0 = "/_matrix/client/r0"
// SetupSyncServerListeners configures the given mux with sync-server listeners
func SetupSyncServerListeners(servMux *http.ServeMux, httpClient *http.Client, cfg config.Sync, srp *sync.RequestPool, deviceDB *devices.Database) {
func SetupSyncServerListeners(servMux *http.ServeMux, httpClient *http.Client, srp *sync.RequestPool, deviceDB *devices.Database) {
apiMux := mux.NewRouter()
r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
r0mux.Handle("/sync", common.MakeAuthAPI("sync", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {