Fix the integration tests

src/github.com/matrix-org/dendrite/cmd/dendrite-room-server/main.go

@@ -41,6 +41,8 @@ var (
 func main() {
 	common.SetupLogging(logDir)
 
+	flag.Parse()
+
 	if *configPath == "" {
 		log.Fatal("--config must be supplied")
 	}

src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/main.go

@@ -22,10 +22,11 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
-	"strconv"
 	"time"
 
 	"github.com/matrix-org/dendrite/common/test"
+	"github.com/matrix-org/gomatrixserverlib"
+	"gopkg.in/yaml.v2"
 )
 
 var (
@@ -41,10 +42,10 @@ var (
 	postgresContainerName = os.Getenv("POSTGRES_CONTAINER")
 	// Test image to be uploaded/downloaded
 	testJPEG = test.Defaulting(os.Getenv("TEST_JPEG_PATH"), "src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/totem.jpg")
+	kafkaURI = test.Defaulting(os.Getenv("KAFKA_URIS"), "localhost:9092")
 )
 
-var thumbnailPregenerationConfig = (`
-thumbnail_sizes:
+var thumbnailSizes = (`
 - width: 32
   height: 32
   method: crop
@@ -68,65 +69,51 @@ var testDatabaseTemplate = "dbname=%s sslmode=disable binary_parameters=yes"
 
 var timeout time.Duration
 
+var port = 10000
+
 func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error, string, *exec.Cmd, chan error, string, string) {
 	dir, err := ioutil.TempDir("", serverType+"-server-test"+suffix)
 	if err != nil {
 		panic(err)
 	}
 
-	serverAddr := "localhost:177" + suffix + "9"
 	proxyAddr := "localhost:1800" + suffix
 
-	configFilename := serverType + "-server-test-config" + suffix + ".yaml"
-	configFileContents := makeConfig(proxyAddr, suffix, dir, dynamicThumbnails)
+	database := fmt.Sprintf(testDatabaseTemplate, testDatabaseName+suffix)
+	cfg, nextPort, err := test.MakeConfig(dir, kafkaURI, database, "localhost", port)
+	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(proxyAddr)
+	if err != nil {
+		panic(err)
+	}
+	if err = yaml.Unmarshal([]byte(thumbnailSizes), &cfg.Media.ThumbnailSizes); err != nil {
+		panic(err)
+	}
+
+	port = nextPort
+	if err = test.WriteConfig(cfg, dir); err != nil {
+		panic(err)
+	}
 
 	serverArgs := []string{
-		"--config", configFilename,
-		"--listen", serverAddr,
+		"--config", filepath.Join(dir, test.ConfigFile),
 	}
 
 	databases := []string{
 		testDatabaseName + suffix,
 	}
 
-	proxyCmd, proxyCmdChan := test.StartProxy(
-		proxyAddr,
-		"http://localhost:177"+suffix+"6",
-		"http://localhost:177"+suffix+"8",
-		"http://"+serverAddr,
-	)
+	proxyCmd, proxyCmdChan := test.StartProxy(proxyAddr, cfg)
 
 	cmd, cmdChan := test.StartServer(
 		serverType,
 		serverArgs,
-		suffix,
-		configFilename,
-		configFileContents,
 		postgresDatabase,
 		postgresContainerName,
 		databases,
 	)
 
-	fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, serverAddr, dir)
-	return cmd, cmdChan, serverAddr, proxyCmd, proxyCmdChan, proxyAddr, dir
-}
-
-func makeConfig(serverAddr, suffix, basePath string, dynamicThumbnails bool) string {
-	return fmt.Sprintf(
-		`
-server_name: "%s"
-base_path: %s
-max_file_size_bytes: %s
-database: "%s"
-dynamic_thumbnails: %s
-%s`,
-		serverAddr,
-		basePath,
-		"10485760",
-		fmt.Sprintf(testDatabaseTemplate, testDatabaseName+suffix),
-		strconv.FormatBool(dynamicThumbnails),
-		thumbnailPregenerationConfig,
-	)
+	fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, cfg.Listen.MediaAPI, dir)
+	return cmd, cmdChan, string(cfg.Listen.MediaAPI), proxyCmd, proxyCmdChan, proxyAddr, dir
 }
 
 func cleanUpServer(cmd *exec.Cmd, dir string) {

src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -34,8 +35,6 @@ var (
 	zookeeperURI = defaulting(os.Getenv("ZOOKEEPER_URI"), "localhost:2181")
 	// The URI the kafka server is listening on.
 	kafkaURI = defaulting(os.Getenv("KAFKA_URIS"), "localhost:9092")
-	// The address the roomserver should listen on.
-	roomserverAddr = defaulting(os.Getenv("ROOMSERVER_URI"), "localhost:9876")
 	// How long to wait for the roomserver to write the expected output messages.
 	// This needs to be high enough to account for the time it takes to create
 	// the postgres database tables which can take a while on travis.
@@ -164,10 +163,22 @@ func runAndReadFromTopic(runCmd *exec.Cmd, topic string, count int, checkQueryAP
 // a api.RoomserverQueryAPI client. The caller can use this function to check the
 // behaviour of the query API.
 func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverQueryAPI)) {
-	const (
-		inputTopic  = "roomserverInput"
-		outputTopic = "roomserverOutput"
-	)
+	dir, err := ioutil.TempDir("", "room-server-test")
+	if err != nil {
+		panic(err)
+	}
+
+	cfg, _, err := test.MakeConfig(dir, kafkaURI, testDatabase, "localhost", 10000)
+	if err != nil {
+		panic(err)
+	}
+	if err := test.WriteConfig(cfg, dir); err != nil {
+		panic(err)
+	}
+
+	inputTopic := string(cfg.Kafka.Topics.InputRoomEvent)
+	outputTopic := string(cfg.Kafka.Topics.OutputRoomEvent)
+
 	exe.DeleteTopic(inputTopic)
 	if err := exe.CreateTopic(inputTopic); err != nil {
 		panic(err)
@@ -181,7 +192,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R
 		panic(err)
 	}
 
-	if err := createDatabase(testDatabaseName); err != nil {
+	if err = createDatabase(testDatabaseName); err != nil {
 		panic(err)
 	}
 
@@ -191,18 +202,11 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R
 	// We append to the environment rather than replacing so that any additional
 	// postgres and golang environment variables such as PGHOST are passed to
 	// the roomserver process.
-	cmd.Env = append(
-		os.Environ(),
-		fmt.Sprintf("DATABASE=%s", testDatabase),
-		fmt.Sprintf("KAFKA_URIS=%s", kafkaURI),
-		fmt.Sprintf("TOPIC_INPUT_ROOM_EVENT=%s", inputTopic),
-		fmt.Sprintf("TOPIC_OUTPUT_ROOM_EVENT=%s", outputTopic),
-		fmt.Sprintf("BIND_ADDRESS=%s", roomserverAddr),
-	)
 	cmd.Stderr = os.Stderr
+	cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)}
 
 	gotOutput, err := runAndReadFromTopic(cmd, outputTopic, len(wantOutput), func() {
-		queryAPI := api.NewRoomserverQueryAPIHTTP("http://"+roomserverAddr, nil)
+		queryAPI := api.NewRoomserverQueryAPIHTTP("http://"+string(cfg.Listen.RoomServer), nil)
 		checkQueries(queryAPI)
 	})
 	if err != nil {

src/github.com/matrix-org/dendrite/cmd/syncserver-integration-tests/main.go

@@ -17,12 +17,14 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"time"
 
+	"github.com/matrix-org/dendrite/common/config"
 	"github.com/matrix-org/dendrite/common/test"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -62,12 +64,6 @@ var exe = test.KafkaExecutor{
 	OutputWriter: os.Stderr,
 }
 
-var syncServerConfigFileContents = (`consumer_uris: ["` + kafkaURI + `"]
-roomserver_topic: "` + inputTopic + `"
-database: "` + testDatabase + `"
-server_name: "localhost"
-`)
-
 var timeout time.Duration
 var clientEventTestData []string
 
@@ -126,11 +122,27 @@ func clientEventJSONForOutputRoomEvent(outputRoomEvent string) string {
 // then starts the sync server. The Cmd being executed is returned. A channel is also returned,
 // which will have any termination errors sent down it, followed immediately by the channel being closed.
 func startSyncServer() (*exec.Cmd, chan error) {
-	const configFilename = "sync-api-server-config-test.yaml"
+
+	dir, err := ioutil.TempDir("", "syncapi-server-test")
+	if err != nil {
+		panic(err)
+	}
+
+	cfg, _, err := test.MakeConfig(dir, kafkaURI, testDatabase, "localhost", 10000)
+	if err != nil {
+		panic(err)
+	}
+	// TODO use the address assigned by the config generator rather than clobbering.
+	cfg.Matrix.ServerName = "localhost"
+	cfg.Listen.SyncAPI = config.Address(syncserverAddr)
+	cfg.Kafka.Topics.OutputRoomEvent = config.Topic(inputTopic)
+
+	if err := test.WriteConfig(cfg, dir); err != nil {
+		panic(err)
+	}
 
 	serverArgs := []string{
-		"--config", configFilename,
-		"--listen", syncserverAddr,
+		"--config", filepath.Join(dir, test.ConfigFile),
 	}
 
 	databases := []string{
@@ -140,9 +152,6 @@ func startSyncServer() (*exec.Cmd, chan error) {
 	cmd, cmdChan := test.StartServer(
 		"sync-api",
 		serverArgs,
-		"",
-		configFilename,
-		syncServerConfigFileContents,
 		postgresDatabase,
 		postgresContainerName,
 		databases,

src/github.com/matrix-org/dendrite/common/test/config.go

@@ -0,0 +1,193 @@
+// Copyright 2017 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/pem"
+	"fmt"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/gomatrixserverlib"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+	"math/big"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+const (
+	// ConfigFile is the name of the config file for a server.
+	ConfigFile = "dendrite.yaml"
+	// ServerKeyFile is the name of the file holding the matrix server private key.
+	ServerKeyFile = "server_key.pem"
+	// TLSCertFile is the name of the file holding the TLS certificate used for federation.
+	TLSCertFile = "tls_cert.pem"
+	// TLSKeyFile is the name of the file holding the TLS key used for federation.
+	TLSKeyFile = "tls_key.pem"
+	// MediaDir is the name of the directory used to store media.
+	MediaDir = "media"
+)
+
+// MakeConfig makes a config suitable for running integration tests.
+// Generates new matrix and TLS keys for the server.
+func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*config.Dendrite, int, error) {
+	var cfg config.Dendrite
+
+	port := startPort
+	assignAddress := func() config.Address {
+		result := config.Address(fmt.Sprintf("%s:%d", host, port))
+		port++
+		return result
+	}
+
+	serverKeyPath := filepath.Join(configDir, ServerKeyFile)
+	tlsCertPath := filepath.Join(configDir, TLSKeyFile)
+	tlsKeyPath := filepath.Join(configDir, TLSCertFile)
+	mediaBasePath := filepath.Join(configDir, MediaDir)
+
+	if err := newMatrixKey(serverKeyPath); err != nil {
+		return nil, 0, err
+	}
+
+	if err := newTLSKey(tlsKeyPath, tlsCertPath); err != nil {
+		return nil, 0, err
+	}
+
+	cfg.Version = config.Version
+
+	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(assignAddress())
+	cfg.Matrix.PrivateKeyPath = config.Path(serverKeyPath)
+	cfg.Matrix.FederationCertificatePaths = []config.Path{config.Path(tlsCertPath)}
+
+	cfg.Media.BasePath = config.Path(mediaBasePath)
+
+	cfg.Kafka.Addresses = []string{kafkaURI}
+	// TODO: Different servers should be using different topics.
+	// Make this configurable somehow?
+	cfg.Kafka.Topics.InputRoomEvent = "test.room.input"
+	cfg.Kafka.Topics.OutputRoomEvent = "test.room.output"
+
+	// TODO: Use different databases for the different schemas.
+	// Using the same database for every schema currently works because
+	// the table names are globally unique. But we might not want to
+	// rely on that in the future.
+	cfg.Database.Account = config.DataSource(database)
+	cfg.Database.Device = config.DataSource(database)
+	cfg.Database.MediaServer = config.DataSource(database)
+	cfg.Database.RoomServer = config.DataSource(database)
+	cfg.Database.ServerKey = config.DataSource(database)
+	cfg.Database.SyncServer = config.DataSource(database)
+
+	cfg.Listen.ClientAPI = assignAddress()
+	cfg.Listen.FederationAPI = assignAddress()
+	cfg.Listen.MediaAPI = assignAddress()
+	cfg.Listen.RoomServer = assignAddress()
+	cfg.Listen.SyncAPI = assignAddress()
+
+	return &cfg, port, nil
+}
+
+// WriteConfig writes the config file to the directory.
+func WriteConfig(cfg *config.Dendrite, configDir string) error {
+	data, err := yaml.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+	if err = ioutil.WriteFile(filepath.Join(configDir, ConfigFile), data, 0666); err != nil {
+		return err
+	}
+	return nil
+}
+
+// newMatrixKey generates a new ed25519 matrix server key and writes it to a file.
+func newMatrixKey(matrixKeyPath string) error {
+	var data [35]byte
+	if _, err := rand.Read(data[:]); err != nil {
+		return err
+	}
+	keyOut, err := os.OpenFile(matrixKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	defer keyOut.Close()
+	if err = pem.Encode(keyOut, &pem.Block{
+		Type: "MATRIX PRIVATE KEY",
+		Headers: map[string]string{
+			"Key-ID": "ed25519:" + base64.RawStdEncoding.EncodeToString(data[:3]),
+		},
+		Bytes: data[3:],
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+const certificateDuration = time.Hour * 24 * 365 * 10
+
+// newTLSKey generates a new RSA TLS key and certificate and writes it to a file.
+func newTLSKey(tlsKeyPath, tlsCertPath string) error {
+	priv, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return err
+	}
+
+	notBefore := time.Now()
+	notAfter := notBefore.Add(certificateDuration)
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+	if err != nil {
+		return err
+	}
+
+	template := x509.Certificate{
+		SerialNumber:          serialNumber,
+		NotBefore:             notBefore,
+		NotAfter:              notAfter,
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
+	if err != nil {
+		return err
+	}
+	certOut, err := os.Create(tlsCertPath)
+	if err != nil {
+		return err
+	}
+	defer certOut.Close()
+	if err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
+		return err
+	}
+
+	keyOut, err := os.OpenFile(tlsKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	defer keyOut.Close()
+	if err = pem.Encode(keyOut, &pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(priv),
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}

src/github.com/matrix-org/dendrite/common/test/server.go

@@ -16,11 +16,12 @@ package test
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
+
+	"github.com/matrix-org/dendrite/common/config"
 )
 
 // Defaulting allows assignment of string variables with a fallback default value
@@ -69,7 +70,7 @@ func CreateBackgroundCommand(command string, args []string) (*exec.Cmd, chan err
 // which will have any termination errors sent down it, followed immediately by the channel being closed.
 // If postgresContainerName is not an empty string, psql will be run from inside that container. If it is
 // an empty string, psql will be assumed to be in PATH.
-func StartServer(serverType string, serverArgs []string, suffix, configFilename, configFileContents, postgresDatabase, postgresContainerName string, databases []string) (*exec.Cmd, chan error) {
+func StartServer(serverType string, serverArgs []string, postgresDatabase, postgresContainerName string, databases []string) (*exec.Cmd, chan error) {
 	if len(databases) > 0 {
 		var dbCmd string
 		var dbArgs []string
@@ -89,12 +90,6 @@ func StartServer(serverType string, serverArgs []string, suffix, configFilename,
 		}
 	}
 
-	if configFilename != "" {
-		if err := ioutil.WriteFile(configFilename, []byte(configFileContents), 0644); err != nil {
-			panic(err)
-		}
-	}
-
 	return CreateBackgroundCommand(
 		filepath.Join(filepath.Dir(os.Args[0]), "dendrite-"+serverType+"-server"),
 		serverArgs,
@@ -102,12 +97,12 @@ func StartServer(serverType string, serverArgs []string, suffix, configFilename,
 }
 
 // StartProxy creates a reverse proxy
-func StartProxy(bindAddr, syncAddr, clientAddr, mediaAddr string) (*exec.Cmd, chan error) {
+func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) {
 	proxyArgs := []string{
 		"--bind-address", bindAddr,
-		"--sync-api-server-url", syncAddr,
-		"--client-api-server-url", clientAddr,
-		"--media-api-server-url", mediaAddr,
+		"--sync-api-server-url", "http://" + string(cfg.Listen.SyncAPI),
+		"--client-api-server-url", "http://" + string(cfg.Listen.ClientAPI),
+		"--media-api-server-url", "http://" + string(cfg.Listen.MediaAPI),
 	}
 	return CreateBackgroundCommand(
 		filepath.Join(filepath.Dir(os.Args[0]), "client-api-proxy"),