diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-room-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-room-server/main.go index b3c004d18..0a1686791 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-room-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-room-server/main.go @@ -41,6 +41,8 @@ var ( func main() { common.SetupLogging(logDir) + flag.Parse() + if *configPath == "" { log.Fatal("--config must be supplied") } diff --git a/src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/main.go b/src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/main.go index 9902f4ef8..29a79d762 100644 --- a/src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/main.go @@ -22,10 +22,11 @@ import ( "os/exec" "path" "path/filepath" - "strconv" "time" "github.com/matrix-org/dendrite/common/test" + "github.com/matrix-org/gomatrixserverlib" + "gopkg.in/yaml.v2" ) var ( @@ -41,10 +42,10 @@ var ( postgresContainerName = os.Getenv("POSTGRES_CONTAINER") // Test image to be uploaded/downloaded testJPEG = test.Defaulting(os.Getenv("TEST_JPEG_PATH"), "src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/totem.jpg") + kafkaURI = test.Defaulting(os.Getenv("KAFKA_URIS"), "localhost:9092") ) -var thumbnailPregenerationConfig = (` -thumbnail_sizes: +var thumbnailSizes = (` - width: 32 height: 32 method: crop @@ -68,65 +69,51 @@ var testDatabaseTemplate = "dbname=%s sslmode=disable binary_parameters=yes" var timeout time.Duration +var port = 10000 + func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error, string, *exec.Cmd, chan error, string, string) { dir, err := ioutil.TempDir("", serverType+"-server-test"+suffix) if err != nil { panic(err) } - serverAddr := "localhost:177" + suffix + "9" proxyAddr := "localhost:1800" + suffix - configFilename := serverType + "-server-test-config" + suffix + ".yaml" - configFileContents := makeConfig(proxyAddr, suffix, dir, dynamicThumbnails) + database := fmt.Sprintf(testDatabaseTemplate, testDatabaseName+suffix) + cfg, nextPort, err := test.MakeConfig(dir, kafkaURI, database, "localhost", port) + cfg.Matrix.ServerName = gomatrixserverlib.ServerName(proxyAddr) + if err != nil { + panic(err) + } + if err = yaml.Unmarshal([]byte(thumbnailSizes), &cfg.Media.ThumbnailSizes); err != nil { + panic(err) + } + + port = nextPort + if err = test.WriteConfig(cfg, dir); err != nil { + panic(err) + } serverArgs := []string{ - "--config", configFilename, - "--listen", serverAddr, + "--config", filepath.Join(dir, test.ConfigFile), } databases := []string{ testDatabaseName + suffix, } - proxyCmd, proxyCmdChan := test.StartProxy( - proxyAddr, - "http://localhost:177"+suffix+"6", - "http://localhost:177"+suffix+"8", - "http://"+serverAddr, - ) + proxyCmd, proxyCmdChan := test.StartProxy(proxyAddr, cfg) cmd, cmdChan := test.StartServer( serverType, serverArgs, - suffix, - configFilename, - configFileContents, postgresDatabase, postgresContainerName, databases, ) - fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, serverAddr, dir) - return cmd, cmdChan, serverAddr, proxyCmd, proxyCmdChan, proxyAddr, dir -} - -func makeConfig(serverAddr, suffix, basePath string, dynamicThumbnails bool) string { - return fmt.Sprintf( - ` -server_name: "%s" -base_path: %s -max_file_size_bytes: %s -database: "%s" -dynamic_thumbnails: %s -%s`, - serverAddr, - basePath, - "10485760", - fmt.Sprintf(testDatabaseTemplate, testDatabaseName+suffix), - strconv.FormatBool(dynamicThumbnails), - thumbnailPregenerationConfig, - ) + fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, cfg.Listen.MediaAPI, dir) + return cmd, cmdChan, string(cfg.Listen.MediaAPI), proxyCmd, proxyCmdChan, proxyAddr, dir } func cleanUpServer(cmd *exec.Cmd, dir string) { diff --git a/src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go b/src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go index f11c1c89c..940293f8e 100644 --- a/src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go @@ -16,6 +16,7 @@ package main import ( "fmt" + "io/ioutil" "os" "os/exec" "path/filepath" @@ -34,8 +35,6 @@ var ( zookeeperURI = defaulting(os.Getenv("ZOOKEEPER_URI"), "localhost:2181") // The URI the kafka server is listening on. kafkaURI = defaulting(os.Getenv("KAFKA_URIS"), "localhost:9092") - // The address the roomserver should listen on. - roomserverAddr = defaulting(os.Getenv("ROOMSERVER_URI"), "localhost:9876") // How long to wait for the roomserver to write the expected output messages. // This needs to be high enough to account for the time it takes to create // the postgres database tables which can take a while on travis. @@ -164,10 +163,22 @@ func runAndReadFromTopic(runCmd *exec.Cmd, topic string, count int, checkQueryAP // a api.RoomserverQueryAPI client. The caller can use this function to check the // behaviour of the query API. func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverQueryAPI)) { - const ( - inputTopic = "roomserverInput" - outputTopic = "roomserverOutput" - ) + dir, err := ioutil.TempDir("", "room-server-test") + if err != nil { + panic(err) + } + + cfg, _, err := test.MakeConfig(dir, kafkaURI, testDatabase, "localhost", 10000) + if err != nil { + panic(err) + } + if err := test.WriteConfig(cfg, dir); err != nil { + panic(err) + } + + inputTopic := string(cfg.Kafka.Topics.InputRoomEvent) + outputTopic := string(cfg.Kafka.Topics.OutputRoomEvent) + exe.DeleteTopic(inputTopic) if err := exe.CreateTopic(inputTopic); err != nil { panic(err) @@ -181,7 +192,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R panic(err) } - if err := createDatabase(testDatabaseName); err != nil { + if err = createDatabase(testDatabaseName); err != nil { panic(err) } @@ -191,18 +202,11 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R // We append to the environment rather than replacing so that any additional // postgres and golang environment variables such as PGHOST are passed to // the roomserver process. - cmd.Env = append( - os.Environ(), - fmt.Sprintf("DATABASE=%s", testDatabase), - fmt.Sprintf("KAFKA_URIS=%s", kafkaURI), - fmt.Sprintf("TOPIC_INPUT_ROOM_EVENT=%s", inputTopic), - fmt.Sprintf("TOPIC_OUTPUT_ROOM_EVENT=%s", outputTopic), - fmt.Sprintf("BIND_ADDRESS=%s", roomserverAddr), - ) cmd.Stderr = os.Stderr + cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)} gotOutput, err := runAndReadFromTopic(cmd, outputTopic, len(wantOutput), func() { - queryAPI := api.NewRoomserverQueryAPIHTTP("http://"+roomserverAddr, nil) + queryAPI := api.NewRoomserverQueryAPIHTTP("http://"+string(cfg.Listen.RoomServer), nil) checkQueries(queryAPI) }) if err != nil { diff --git a/src/github.com/matrix-org/dendrite/cmd/syncserver-integration-tests/main.go b/src/github.com/matrix-org/dendrite/cmd/syncserver-integration-tests/main.go index b73405583..d1cf9fd12 100644 --- a/src/github.com/matrix-org/dendrite/cmd/syncserver-integration-tests/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/syncserver-integration-tests/main.go @@ -17,12 +17,14 @@ package main import ( "encoding/json" "fmt" + "io/ioutil" "net/http" "os" "os/exec" "path/filepath" "time" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/common/test" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -62,12 +64,6 @@ var exe = test.KafkaExecutor{ OutputWriter: os.Stderr, } -var syncServerConfigFileContents = (`consumer_uris: ["` + kafkaURI + `"] -roomserver_topic: "` + inputTopic + `" -database: "` + testDatabase + `" -server_name: "localhost" -`) - var timeout time.Duration var clientEventTestData []string @@ -126,11 +122,27 @@ func clientEventJSONForOutputRoomEvent(outputRoomEvent string) string { // then starts the sync server. The Cmd being executed is returned. A channel is also returned, // which will have any termination errors sent down it, followed immediately by the channel being closed. func startSyncServer() (*exec.Cmd, chan error) { - const configFilename = "sync-api-server-config-test.yaml" + + dir, err := ioutil.TempDir("", "syncapi-server-test") + if err != nil { + panic(err) + } + + cfg, _, err := test.MakeConfig(dir, kafkaURI, testDatabase, "localhost", 10000) + if err != nil { + panic(err) + } + // TODO use the address assigned by the config generator rather than clobbering. + cfg.Matrix.ServerName = "localhost" + cfg.Listen.SyncAPI = config.Address(syncserverAddr) + cfg.Kafka.Topics.OutputRoomEvent = config.Topic(inputTopic) + + if err := test.WriteConfig(cfg, dir); err != nil { + panic(err) + } serverArgs := []string{ - "--config", configFilename, - "--listen", syncserverAddr, + "--config", filepath.Join(dir, test.ConfigFile), } databases := []string{ @@ -140,9 +152,6 @@ func startSyncServer() (*exec.Cmd, chan error) { cmd, cmdChan := test.StartServer( "sync-api", serverArgs, - "", - configFilename, - syncServerConfigFileContents, postgresDatabase, postgresContainerName, databases, diff --git a/src/github.com/matrix-org/dendrite/common/test/config.go b/src/github.com/matrix-org/dendrite/common/test/config.go new file mode 100644 index 000000000..dc81c29c8 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/common/test/config.go @@ -0,0 +1,193 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "fmt" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/gomatrixserverlib" + "gopkg.in/yaml.v2" + "io/ioutil" + "math/big" + "os" + "path/filepath" + "time" +) + +const ( + // ConfigFile is the name of the config file for a server. + ConfigFile = "dendrite.yaml" + // ServerKeyFile is the name of the file holding the matrix server private key. + ServerKeyFile = "server_key.pem" + // TLSCertFile is the name of the file holding the TLS certificate used for federation. + TLSCertFile = "tls_cert.pem" + // TLSKeyFile is the name of the file holding the TLS key used for federation. + TLSKeyFile = "tls_key.pem" + // MediaDir is the name of the directory used to store media. + MediaDir = "media" +) + +// MakeConfig makes a config suitable for running integration tests. +// Generates new matrix and TLS keys for the server. +func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*config.Dendrite, int, error) { + var cfg config.Dendrite + + port := startPort + assignAddress := func() config.Address { + result := config.Address(fmt.Sprintf("%s:%d", host, port)) + port++ + return result + } + + serverKeyPath := filepath.Join(configDir, ServerKeyFile) + tlsCertPath := filepath.Join(configDir, TLSKeyFile) + tlsKeyPath := filepath.Join(configDir, TLSCertFile) + mediaBasePath := filepath.Join(configDir, MediaDir) + + if err := newMatrixKey(serverKeyPath); err != nil { + return nil, 0, err + } + + if err := newTLSKey(tlsKeyPath, tlsCertPath); err != nil { + return nil, 0, err + } + + cfg.Version = config.Version + + cfg.Matrix.ServerName = gomatrixserverlib.ServerName(assignAddress()) + cfg.Matrix.PrivateKeyPath = config.Path(serverKeyPath) + cfg.Matrix.FederationCertificatePaths = []config.Path{config.Path(tlsCertPath)} + + cfg.Media.BasePath = config.Path(mediaBasePath) + + cfg.Kafka.Addresses = []string{kafkaURI} + // TODO: Different servers should be using different topics. + // Make this configurable somehow? + cfg.Kafka.Topics.InputRoomEvent = "test.room.input" + cfg.Kafka.Topics.OutputRoomEvent = "test.room.output" + + // TODO: Use different databases for the different schemas. + // Using the same database for every schema currently works because + // the table names are globally unique. But we might not want to + // rely on that in the future. + cfg.Database.Account = config.DataSource(database) + cfg.Database.Device = config.DataSource(database) + cfg.Database.MediaServer = config.DataSource(database) + cfg.Database.RoomServer = config.DataSource(database) + cfg.Database.ServerKey = config.DataSource(database) + cfg.Database.SyncServer = config.DataSource(database) + + cfg.Listen.ClientAPI = assignAddress() + cfg.Listen.FederationAPI = assignAddress() + cfg.Listen.MediaAPI = assignAddress() + cfg.Listen.RoomServer = assignAddress() + cfg.Listen.SyncAPI = assignAddress() + + return &cfg, port, nil +} + +// WriteConfig writes the config file to the directory. +func WriteConfig(cfg *config.Dendrite, configDir string) error { + data, err := yaml.Marshal(cfg) + if err != nil { + return err + } + if err = ioutil.WriteFile(filepath.Join(configDir, ConfigFile), data, 0666); err != nil { + return err + } + return nil +} + +// newMatrixKey generates a new ed25519 matrix server key and writes it to a file. +func newMatrixKey(matrixKeyPath string) error { + var data [35]byte + if _, err := rand.Read(data[:]); err != nil { + return err + } + keyOut, err := os.OpenFile(matrixKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return err + } + defer keyOut.Close() + if err = pem.Encode(keyOut, &pem.Block{ + Type: "MATRIX PRIVATE KEY", + Headers: map[string]string{ + "Key-ID": "ed25519:" + base64.RawStdEncoding.EncodeToString(data[:3]), + }, + Bytes: data[3:], + }); err != nil { + return err + } + + return nil +} + +const certificateDuration = time.Hour * 24 * 365 * 10 + +// newTLSKey generates a new RSA TLS key and certificate and writes it to a file. +func newTLSKey(tlsKeyPath, tlsCertPath string) error { + priv, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return err + } + + notBefore := time.Now() + notAfter := notBefore.Add(certificateDuration) + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return err + } + + template := x509.Certificate{ + SerialNumber: serialNumber, + NotBefore: notBefore, + NotAfter: notAfter, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + return err + } + certOut, err := os.Create(tlsCertPath) + if err != nil { + return err + } + defer certOut.Close() + if err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { + return err + } + + keyOut, err := os.OpenFile(tlsKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return err + } + defer keyOut.Close() + if err = pem.Encode(keyOut, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(priv), + }); err != nil { + return err + } + + return nil +} diff --git a/src/github.com/matrix-org/dendrite/common/test/server.go b/src/github.com/matrix-org/dendrite/common/test/server.go index dca0aa94c..2f089fe34 100644 --- a/src/github.com/matrix-org/dendrite/common/test/server.go +++ b/src/github.com/matrix-org/dendrite/common/test/server.go @@ -16,11 +16,12 @@ package test import ( "fmt" - "io/ioutil" "os" "os/exec" "path/filepath" "strings" + + "github.com/matrix-org/dendrite/common/config" ) // Defaulting allows assignment of string variables with a fallback default value @@ -69,7 +70,7 @@ func CreateBackgroundCommand(command string, args []string) (*exec.Cmd, chan err // which will have any termination errors sent down it, followed immediately by the channel being closed. // If postgresContainerName is not an empty string, psql will be run from inside that container. If it is // an empty string, psql will be assumed to be in PATH. -func StartServer(serverType string, serverArgs []string, suffix, configFilename, configFileContents, postgresDatabase, postgresContainerName string, databases []string) (*exec.Cmd, chan error) { +func StartServer(serverType string, serverArgs []string, postgresDatabase, postgresContainerName string, databases []string) (*exec.Cmd, chan error) { if len(databases) > 0 { var dbCmd string var dbArgs []string @@ -89,12 +90,6 @@ func StartServer(serverType string, serverArgs []string, suffix, configFilename, } } - if configFilename != "" { - if err := ioutil.WriteFile(configFilename, []byte(configFileContents), 0644); err != nil { - panic(err) - } - } - return CreateBackgroundCommand( filepath.Join(filepath.Dir(os.Args[0]), "dendrite-"+serverType+"-server"), serverArgs, @@ -102,12 +97,12 @@ func StartServer(serverType string, serverArgs []string, suffix, configFilename, } // StartProxy creates a reverse proxy -func StartProxy(bindAddr, syncAddr, clientAddr, mediaAddr string) (*exec.Cmd, chan error) { +func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) { proxyArgs := []string{ "--bind-address", bindAddr, - "--sync-api-server-url", syncAddr, - "--client-api-server-url", clientAddr, - "--media-api-server-url", mediaAddr, + "--sync-api-server-url", "http://" + string(cfg.Listen.SyncAPI), + "--client-api-server-url", "http://" + string(cfg.Listen.ClientAPI), + "--media-api-server-url", "http://" + string(cfg.Listen.MediaAPI), } return CreateBackgroundCommand( filepath.Join(filepath.Dir(os.Args[0]), "client-api-proxy"),