From c6e59ca9679c4da50b59565f0f46c2bbe9d835aa Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 16 Jun 2017 16:52:03 +0100 Subject: [PATCH 1/5] Add a single config file format for dendrite. (#141) * Add a single config file format for dendrite. This config file combines all the configuration needed by the dendrite components into a single file. * Resolve relative paths against the current working directory * Review comments * more review comments! * Document what the purpose of the version field is --- .../dendrite/common/config/config.go | 360 ++++++++++++++++++ .../dendrite/common/config/config_test.go | 134 +++++++ 2 files changed, 494 insertions(+) create mode 100644 src/github.com/matrix-org/dendrite/common/config/config.go create mode 100644 src/github.com/matrix-org/dendrite/common/config/config_test.go diff --git a/src/github.com/matrix-org/dendrite/common/config/config.go b/src/github.com/matrix-org/dendrite/common/config/config.go new file mode 100644 index 000000000..e06f894a9 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/common/config/config.go @@ -0,0 +1,360 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "bytes" + "crypto/sha256" + "encoding/pem" + "fmt" + "github.com/matrix-org/gomatrixserverlib" + "golang.org/x/crypto/ed25519" + "gopkg.in/yaml.v2" + "io/ioutil" + "path/filepath" + "strings" + "time" +) + +// Version is the current version of the config format. +// This will change whenever we make breaking changes to the config format. +const Version = "v0" + +// Dendrite contains all the config used by a dendrite process. +// Relative paths are resolved relative to the current working directory +type Dendrite struct { + // The version of the configuration file. + // If the version in a file doesn't match the current dendrite config + // version then we can give a clear error message telling the user + // to update their config file to the current version. + // The version of the file should only be different if there has + // been a breaking change to the config file format. + Version string `yaml:"version"` + + // The configuration required for a matrix server. + Matrix struct { + // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. + ServerName gomatrixserverlib.ServerName `yaml:"server_name"` + // Path to the private key which will be used to sign requests and events. + PrivateKeyPath Path `yaml:"private_key"` + // The private key which will be used to sign requests and events. + PrivateKey ed25519.PrivateKey `yaml:"-"` + // An arbitrary string used to uniquely identify the PrivateKey. Must start with the + // prefix "ed25519:". + KeyID gomatrixserverlib.KeyID `yaml:"-"` + // List of paths to X509 certificates used by the external federation listeners. + // These are used to calculate the TLS fingerprints to publish for this server. + // Other matrix servers talking to this server will expect the x509 certificate + // to match one of these certificates. + // The certificates should be in PEM format. + FederationCertificatePaths []Path `yaml:"federation_certificates"` + // A list of SHA256 TLS fingerprints for the X509 certificates used by the + // federation listener for this server. + TLSFingerPrints []gomatrixserverlib.TLSFingerprint `yaml:"-"` + // How long a remote server can cache our server key for before requesting it again. + // Increasing this number will reduce the number of requests made by remote servers + // for our key, but increases the period a compromised key will be considered valid + // by remote servers. + // Defaults to 24 hours. + KeyValidityPeriod time.Duration `yaml:"key_validity_period"` + } `yaml:"matrix"` + + // The configuration specific to the media repostitory. + Media struct { + // The base path to where the media files will be stored. May be relative or absolute. + BasePath Path `yaml:"base_path"` + // The absolute base path to where media files will be stored. + AbsBasePath Path `yaml:"-"` + // The maximum file size in bytes that is allowed to be stored on this server. + // Note: if max_file_size_bytes is set to 0, the size is unlimited. + // Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) + MaxFileSizeBytes *FileSizeBytes `yaml:"max_file_size_bytes,omitempty"` + // Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated + DynamicThumbnails bool `yaml:"dynamic_thumbnails"` + // The maximum number of simultaneous thumbnail generators. default: 10 + MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"` + // A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content + ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"` + } `yaml:"media"` + + // The configuration for talking to kafka. + Kafka struct { + // A list of kafka addresses to connect to. + Addresses []string `yaml:"addresses"` + // The names of the topics to use when reading and writing from kafka. + Topics struct { + // Topic for roomserver/api.InputRoomEvent events. + InputRoomEvent Topic `yaml:"input_room_event"` + // Topic for roomserver/api.OutputRoomEvent events. + OutputRoomEvent Topic `yaml:"output_room_event"` + } + } `yaml:"kafka"` + + // Postgres Config + Database struct { + // The Account database stores the login details and account information + // for local users. It is accessed by the ClientAPI. + Account DataSource `yaml:"account"` + // The Device database stores session information for the devices of logged + // in local users. It is accessed by the ClientAPI, the MediaAPI and the SyncAPI. + Device DataSource `yaml:"device"` + // The MediaAPI database stores information about files uploaded and downloaded + // by local users. It is only accessed by the MediaAPI. + MediaAPI DataSource `yaml:"media_api"` + // The ServerKey database caches the public keys of remote servers. + // It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI. + ServerKey DataSource `yaml:"server_key"` + // The SyncAPI stores information used by the SyncAPI server. + // It is only accessed by the SyncAPI server. + SyncAPI DataSource `yaml:"sync_api"` + // The RoomServer database stores information about matrix rooms. + // It is only accessed by the RoomServer. + RoomServer DataSource `yaml:"room_server"` + } `yaml:"database"` + + // The internal addresses the components will listen on. + // These should not be exposed externally as they expose metrics and debugging APIs. + Listen struct { + MediaAPI Address `yaml:"media_api"` + ClientAPI Address `yaml:"client_api"` + FederationAPI Address `yaml:"federation_api"` + SyncAPI Address `yaml:"sync_api"` + RoomServer Address `yaml:"room_server"` + } `yaml:"listen"` +} + +// A Path on the filesystem. +type Path string + +// A DataSource for opening a postgresql database using lib/pq. +type DataSource string + +// A Topic in kafka. +type Topic string + +// An Address to listen on. +type Address string + +// FileSizeBytes is a file size in bytes +type FileSizeBytes int64 + +// ThumbnailSize contains a single thumbnail size configuration +type ThumbnailSize struct { + // Maximum width of the thumbnail image + Width int `yaml:"width"` + // Maximum height of the thumbnail image + Height int `yaml:"height"` + // ResizeMethod is one of crop or scale. + // crop scales to fill the requested dimensions and crops the excess. + // scale scales to fit the requested dimensions and one dimension may be smaller than requested. + ResizeMethod string `yaml:"method,omitempty"` +} + +// Load a yaml config file +func Load(configPath string) (*Dendrite, error) { + configData, err := ioutil.ReadFile(configPath) + if err != nil { + return nil, err + } + basePath, err := filepath.Abs(".") + if err != nil { + return nil, err + } + // Pass the current working directory and ioutil.ReadFile so that they can + // be mocked in the tests + return loadConfig(basePath, configData, ioutil.ReadFile) +} + +// An Error indicates a problem parsing the config. +type Error struct { + // List of problems encountered parsing the config. + Problems []string +} + +func loadConfig( + basePath string, + configData []byte, + readFile func(string) ([]byte, error), +) (*Dendrite, error) { + var config Dendrite + var err error + if err = yaml.Unmarshal(configData, &config); err != nil { + return nil, err + } + + config.setDefaults() + + if err = config.check(); err != nil { + return nil, err + } + + privateKeyPath := absPath(basePath, config.Matrix.PrivateKeyPath) + privateKeyData, err := readFile(privateKeyPath) + if err != nil { + return nil, err + } + + if config.Matrix.KeyID, config.Matrix.PrivateKey, err = readKeyPEM(privateKeyPath, privateKeyData); err != nil { + return nil, err + } + + for _, certPath := range config.Matrix.FederationCertificatePaths { + absCertPath := absPath(basePath, certPath) + pemData, err := readFile(absCertPath) + if err != nil { + return nil, err + } + fingerprint := fingerprintPEM(pemData) + if fingerprint == nil { + return nil, fmt.Errorf("no certificate PEM data in %q", absCertPath) + } + config.Matrix.TLSFingerPrints = append(config.Matrix.TLSFingerPrints, *fingerprint) + } + + config.Media.AbsBasePath = Path(absPath(basePath, config.Media.BasePath)) + + return &config, nil +} + +func (config *Dendrite) setDefaults() { + if config.Matrix.KeyValidityPeriod == 0 { + config.Matrix.KeyValidityPeriod = 24 * time.Hour + } + + if config.Media.MaxThumbnailGenerators == 0 { + config.Media.MaxThumbnailGenerators = 10 + } + + if config.Media.MaxFileSizeBytes == nil { + defaultMaxFileSizeBytes := FileSizeBytes(10485760) + config.Media.MaxFileSizeBytes = &defaultMaxFileSizeBytes + } +} + +func (e Error) Error() string { + if len(e.Problems) == 1 { + return e.Problems[0] + } + return fmt.Sprintf( + "%s (and %d other problems)", e.Problems[0], len(e.Problems)-1, + ) +} + +func (config *Dendrite) check() error { + var problems []string + + if config.Version != Version { + return Error{[]string{fmt.Sprintf( + "unknown config version %q, expected %q", config.Version, Version, + )}} + } + + checkNotEmpty := func(key string, value string) { + if value == "" { + problems = append(problems, fmt.Sprintf("missing config key %q", key)) + } + } + + checkNotZero := func(key string, value int64) { + if value == 0 { + problems = append(problems, fmt.Sprintf("missing config key %q", key)) + } + } + + checkPositive := func(key string, value int64) { + if value < 0 { + problems = append(problems, fmt.Sprintf("invalid value for config key %q: %d", key, value)) + } + } + + checkNotEmpty("matrix.server_name", string(config.Matrix.ServerName)) + checkNotEmpty("matrix.private_key", string(config.Matrix.PrivateKeyPath)) + checkNotZero("matrix.federation_certificates", int64(len(config.Matrix.FederationCertificatePaths))) + + checkNotEmpty("media.base_path", string(config.Media.BasePath)) + checkPositive("media.max_file_size_bytes", int64(*config.Media.MaxFileSizeBytes)) + checkPositive("media.max_thumbnail_generators", int64(config.Media.MaxThumbnailGenerators)) + for i, size := range config.Media.ThumbnailSizes { + checkPositive(fmt.Sprintf("media.thumbnail_sizes[%d].width", i), int64(size.Width)) + checkPositive(fmt.Sprintf("media.thumbnail_sizes[%d].height", i), int64(size.Height)) + } + + checkNotZero("kafka.addresses", int64(len(config.Kafka.Addresses))) + checkNotEmpty("kafka.topics.input_room_event", string(config.Kafka.Topics.InputRoomEvent)) + checkNotEmpty("kafka.topics.output_room_event", string(config.Kafka.Topics.OutputRoomEvent)) + checkNotEmpty("database.account", string(config.Database.Account)) + checkNotEmpty("database.device", string(config.Database.Device)) + checkNotEmpty("database.server_key", string(config.Database.ServerKey)) + checkNotEmpty("database.media_api", string(config.Database.MediaAPI)) + checkNotEmpty("database.sync_api", string(config.Database.SyncAPI)) + checkNotEmpty("database.room_server", string(config.Database.RoomServer)) + checkNotEmpty("listen.media_api", string(config.Listen.MediaAPI)) + checkNotEmpty("listen.client_api", string(config.Listen.ClientAPI)) + checkNotEmpty("listen.federation_api", string(config.Listen.FederationAPI)) + checkNotEmpty("listen.sync_api", string(config.Listen.SyncAPI)) + checkNotEmpty("listen.room_server", string(config.Listen.RoomServer)) + + if problems != nil { + return Error{problems} + } + + return nil +} + +func absPath(dir string, path Path) string { + if filepath.IsAbs(string(path)) { + // filepath.Join cleans the path so we should clean the absolute paths as well for consistency. + return filepath.Clean(string(path)) + } + return filepath.Join(dir, string(path)) +} + +func readKeyPEM(path string, data []byte) (gomatrixserverlib.KeyID, ed25519.PrivateKey, error) { + for { + var keyBlock *pem.Block + keyBlock, data = pem.Decode(data) + if data == nil { + return "", nil, fmt.Errorf("no matrix private key PEM data in %q", path) + } + if keyBlock.Type == "MATRIX PRIVATE KEY" { + keyID := keyBlock.Headers["Key-ID"] + if keyID == "" { + return "", nil, fmt.Errorf("missing key ID in PEM data in %q", path) + } + if !strings.HasPrefix(keyID, "ed25519:") { + return "", nil, fmt.Errorf("key ID %q doesn't start with \"ed25519:\" in %q", keyID, path) + } + _, privKey, err := ed25519.GenerateKey(bytes.NewReader(keyBlock.Bytes)) + if err != nil { + return "", nil, err + } + return gomatrixserverlib.KeyID(keyID), privKey, nil + } + } +} + +func fingerprintPEM(data []byte) *gomatrixserverlib.TLSFingerprint { + for { + var certDERBlock *pem.Block + certDERBlock, data = pem.Decode(data) + if data == nil { + return nil + } + if certDERBlock.Type == "CERTIFICATE" { + digest := sha256.Sum256(certDERBlock.Bytes) + return &gomatrixserverlib.TLSFingerprint{digest[:]} + } + } +} diff --git a/src/github.com/matrix-org/dendrite/common/config/config_test.go b/src/github.com/matrix-org/dendrite/common/config/config_test.go new file mode 100644 index 000000000..f690882ef --- /dev/null +++ b/src/github.com/matrix-org/dendrite/common/config/config_test.go @@ -0,0 +1,134 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "testing" +) + +func TestLoadConfigRelative(t *testing.T) { + _, err := loadConfig("/my/config/dir", []byte(testConfig), + mockReadFile{ + "/my/config/dir/matrix_key.pem": testKey, + "/my/config/dir/tls_cert.pem": testCert, + }.readFile, + ) + if err != nil { + t.Error("failed to load config:", err) + } +} + +const testConfig = ` +version: v0 +matrix: + server_name: localhost + private_key: matrix_key.pem + federation_certificates: [tls_cert.pem] +media: + base_path: media_store +kafka: + addresses: ["localhost:9092"] + topics: + input_room_event: input.room + output_room_event: output.room +database: + media_api: "postgresql:///media_api" + account: "postgresql:///account" + device: "postgresql:///device" + server_key: "postgresql:///server_keys" + sync_api: "postgresql:///syn_api" + room_server: "postgresql:///room_server" +listen: + room_server: "localhost:7770" + client_api: "localhost:7771" + federation_api: "localhost:7772" + sync_api: "localhost:7773" + media_api: "localhost:7774" +` + +type mockReadFile map[string]string + +func (m mockReadFile) readFile(path string) ([]byte, error) { + data, ok := m[path] + if !ok { + return nil, fmt.Errorf("no such file %q", path) + } + return []byte(data), nil +} + +func TestReadKey(t *testing.T) { + keyID, _, err := readKeyPEM("path/to/key", []byte(testKey)) + if err != nil { + t.Error("failed to load private key:", err) + } + wantKeyID := testKeyID + if wantKeyID != string(keyID) { + t.Errorf("wanted key ID to be %q, got %q", wantKeyID, keyID) + } +} + +const testKeyID = "ed25519:c8NsuQ" + +const testKey = ` +-----BEGIN MATRIX PRIVATE KEY----- +Key-ID: ` + testKeyID + ` +7KRZiZ2sTyRR8uqqUjRwczuwRXXkUMYIUHq4Mc3t4bE= +-----END MATRIX PRIVATE KEY----- +` + +func TestFingerprintPEM(t *testing.T) { + got := fingerprintPEM([]byte(testCert)) + if got == nil { + t.Error("failed to calculate fingerprint") + } + if string(got.SHA256) != testCertFingerprint { + t.Errorf("bad fingerprint: wanted %q got %q", got, testCertFingerprint) + } + +} + +const testCertFingerprint = "56.\\SPQxE\xd4\x95\xfb\xf6\xd5\x04\x91\xcb/\x07\xb1^\x88\x08\xe3\xc1p\xdfY\x04\x19w\xcb" + +const testCert = ` +-----BEGIN CERTIFICATE----- +MIIE0zCCArugAwIBAgIJAPype3u24LJeMA0GCSqGSIb3DQEBCwUAMAAwHhcNMTcw +NjEzMTQyODU4WhcNMTgwNjEzMTQyODU4WjAAMIICIjANBgkqhkiG9w0BAQEFAAOC +Ag8AMIICCgKCAgEA3vNSr7lCh/alxPFqairp/PYohwdsqPvOD7zf7dJCNhy0gbdC +9/APwIbPAPL9nU+o9ud1ACNCKBCQin/9LnI5vd5pa/Ne+mmRADDLB/BBBoywSJWG +NSfKJ9n3XY1bjgtqi53uUh+RDdQ7sXudDqCUxiiJZmS7oqK/mp88XXAgCbuXUY29 +GmzbbDz37vntuSxDgUOnJ8uPSvRp5YPKogA3JwW1SyrlLt4Z30CQ6nH3Y2Q5SVfJ +NIQyMrnwyjA9bCdXezv1cLXoTYn7U9BRyzXTZeXs3y3ldnRfISXN35CU04Az1F8j +lfj7nXMEqI/qAj/qhxZ8nVBB+rpNOZy9RJko3O+G5Qa/EvzkQYV1rW4TM2Yme88A +QyJspoV/0bXk6gG987PonK2Uk5djxSULhnGVIqswydyH0Nzb+slRp2bSoWbaNlee ++6TIeiyTQYc055pCHOp22gtLrC5LQGchksi02St2ZzRHdnlfqCJ8S9sS7x3trzds +cYueg1sGI+O8szpQ3eUM7OhJOBrx6OlR7+QYnQg1wr/V+JAz1qcyTC1URcwfeqtg +QjxFdBD9LfCtfK+AO51H9ugtsPJqOh33PmvfvUBEM05OHCA0lNaWJHROGpm4T4cc +YQI9JQk/0lB7itF1qK5RG74qgKdjkBkfZxi0OqkUgHk6YHtJlKfET8zfrtcCAwEA +AaNQME4wHQYDVR0OBBYEFGwb0NgH0Zr7Ga23njEJ85Ozf8M9MB8GA1UdIwQYMBaA +FGwb0NgH0Zr7Ga23njEJ85Ozf8M9MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggIBAKU3RHXggbq/pLhGinU5q/9QT0TB/0bBnF1wNFkKQC0FrNJ+ZnBNmusy +oqOn7DEohBCCDxT0kgOC05gLEsGLkSXlVyqCsPFfycCFhtu1QzSRtQNRxB3pW3Wq +4/RFVYv0PGBjVBKxImQlEmXJWEDwemGKqDQZPtqR/FTHTbJcaT0xQr5+1oG6lawt +I/2cW6GQ0kYW/Szps8FgNdSNgVqCjjNIzBYbWhRWMx/63qD1ReUbY7/Yw9KKT8nK +zXERpbTM9k+Pnm0g9Gep+9HJ1dBFJeuTPugKeSeyqg2OJbENw1hxGs/HjBXw7580 +ioiMn/kMj6Tg/f3HCfKrdHHBFQw0/fJW6o17QImYIpPOPzc5RjXBrCJWb34kxqEd +NQdKgejWiV/LlVsguIF8hVZH2kRzvoyypkVUtSUYGmjvA5UXoORQZfJ+b41llq1B +GcSF6iaVbAFKnsUyyr1i9uHz/6Muqflphv/SfZxGheIn5u3PnhXrzDagvItjw0NS +n0Xq64k7fc42HXJpF8CGBkSaIhtlzcruO+vqR80B9r62+D0V7VmHOnP135MT6noU +8F0JQfEtP+I8NII5jHSF/khzSgP5g80LS9tEc2ILnIHK1StkInAoRQQ+/HsQsgbz +ANAf5kxmMsM0zlN2hkxl0H6o7wKlBSw3RI3cjfilXiMWRPJrzlc4 +-----END CERTIFICATE----- +` From 3fbe728d9498ae59d0ee301fb9c11419566c57f8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 19 Jun 2017 12:56:55 +0100 Subject: [PATCH 2/5] Add utility functions for generating dendrite configs for integration tests (#142) --- .../matrix-org/dendrite/common/test/config.go | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 src/github.com/matrix-org/dendrite/common/test/config.go diff --git a/src/github.com/matrix-org/dendrite/common/test/config.go b/src/github.com/matrix-org/dendrite/common/test/config.go new file mode 100644 index 000000000..8c957e77a --- /dev/null +++ b/src/github.com/matrix-org/dendrite/common/test/config.go @@ -0,0 +1,193 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "fmt" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/gomatrixserverlib" + "gopkg.in/yaml.v2" + "io/ioutil" + "math/big" + "os" + "path/filepath" + "time" +) + +const ( + // ConfigFile is the name of the config file for a server. + ConfigFile = "dendrite.yaml" + // ServerKeyFile is the name of the file holding the matrix server private key. + ServerKeyFile = "server_key.pem" + // TLSCertFile is the name of the file holding the TLS certificate used for federation. + TLSCertFile = "tls_cert.pem" + // TLSKeyFile is the name of the file holding the TLS key used for federation. + TLSKeyFile = "tls_key.pem" + // MediaDir is the name of the directory used to store media. + MediaDir = "media" +) + +// MakeConfig makes a config suitable for running integration tests. +// Generates new matrix and TLS keys for the server. +func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*config.Dendrite, int, error) { + var cfg config.Dendrite + + port := startPort + assignAddress := func() config.Address { + result := config.Address(fmt.Sprintf("%s:%d", host, port)) + port++ + return result + } + + serverKeyPath := filepath.Join(configDir, ServerKeyFile) + tlsCertPath := filepath.Join(configDir, TLSKeyFile) + tlsKeyPath := filepath.Join(configDir, TLSCertFile) + mediaBasePath := filepath.Join(configDir, MediaDir) + + if err := newMatrixKey(serverKeyPath); err != nil { + return nil, 0, err + } + + if err := newTLSKey(tlsKeyPath, tlsCertPath); err != nil { + return nil, 0, err + } + + cfg.Version = config.Version + + cfg.Matrix.ServerName = gomatrixserverlib.ServerName(assignAddress()) + cfg.Matrix.PrivateKeyPath = config.Path(serverKeyPath) + cfg.Matrix.FederationCertificatePaths = []config.Path{config.Path(tlsCertPath)} + + cfg.Media.BasePath = config.Path(mediaBasePath) + + cfg.Kafka.Addresses = []string{kafkaURI} + // TODO: Different servers should be using different topics. + // Make this configurable somehow? + cfg.Kafka.Topics.InputRoomEvent = "test.room.input" + cfg.Kafka.Topics.OutputRoomEvent = "test.room.output" + + // TODO: Use different databases for the different schemas. + // Using the same database for every schema currently works because + // the table names are globally unique. But we might not want to + // rely on that in the future. + cfg.Database.Account = config.DataSource(database) + cfg.Database.Device = config.DataSource(database) + cfg.Database.MediaAPI = config.DataSource(database) + cfg.Database.RoomServer = config.DataSource(database) + cfg.Database.ServerKey = config.DataSource(database) + cfg.Database.SyncAPI = config.DataSource(database) + + cfg.Listen.ClientAPI = assignAddress() + cfg.Listen.FederationAPI = assignAddress() + cfg.Listen.MediaAPI = assignAddress() + cfg.Listen.RoomServer = assignAddress() + cfg.Listen.SyncAPI = assignAddress() + + return &cfg, port, nil +} + +// WriteConfig writes the config file to the directory. +func WriteConfig(cfg *config.Dendrite, configDir string) error { + data, err := yaml.Marshal(cfg) + if err != nil { + return err + } + if err = ioutil.WriteFile(filepath.Join(configDir, ConfigFile), data, 0666); err != nil { + return err + } + return nil +} + +// newMatrixKey generates a new ed25519 matrix server key and writes it to a file. +func newMatrixKey(matrixKeyPath string) error { + var data [35]byte + if _, err := rand.Read(data[:]); err != nil { + return err + } + keyOut, err := os.OpenFile(matrixKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return err + } + defer keyOut.Close() + if err = pem.Encode(keyOut, &pem.Block{ + Type: "MATRIX PRIVATE KEY", + Headers: map[string]string{ + "Key-ID": "ed25519:" + base64.RawStdEncoding.EncodeToString(data[:3]), + }, + Bytes: data[3:], + }); err != nil { + return err + } + + return nil +} + +const certificateDuration = time.Hour * 24 * 365 * 10 + +// newTLSKey generates a new RSA TLS key and certificate and writes it to a file. +func newTLSKey(tlsKeyPath, tlsCertPath string) error { + priv, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return err + } + + notBefore := time.Now() + notAfter := notBefore.Add(certificateDuration) + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return err + } + + template := x509.Certificate{ + SerialNumber: serialNumber, + NotBefore: notBefore, + NotAfter: notAfter, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + return err + } + certOut, err := os.Create(tlsCertPath) + if err != nil { + return err + } + defer certOut.Close() + if err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { + return err + } + + keyOut, err := os.OpenFile(tlsKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return err + } + defer keyOut.Close() + if err = pem.Encode(keyOut, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(priv), + }); err != nil { + return err + } + + return nil +} From e67f9401bec418e1e828931cd53e0b4eab528104 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 19 Jun 2017 15:21:04 +0100 Subject: [PATCH 3/5] Replace the cmd specific config with common config. (#144) * Move all the dendrite config in to a single place * Add tests for config parsing * replace syncserver config with common config * Replace client API config with common config * Replace federation API config with common config * Replace media api config with common config * Replace room server config with common config * Remove unused readKey function * Fix the integration tests * Comment on hardcoding roomserver to HTTP * Add a method for getting RoomServerURL This moves the hardcoding of HTTPs into one place. --- .../dendrite/clientapi/config/config.go | 37 --- .../dendrite/clientapi/readers/login.go | 9 +- .../dendrite/clientapi/routing/routing.go | 4 +- .../dendrite/clientapi/writers/createroom.go | 14 +- .../dendrite/clientapi/writers/joinroom.go | 12 +- .../dendrite/clientapi/writers/sendevent.go | 17 +- .../cmd/dendrite-client-api-server/main.go | 74 ++---- .../dendrite-federation-api-server/main.go | 87 ++----- .../cmd/dendrite-media-api-server/main.go | 222 +----------------- .../dendrite/cmd/dendrite-room-server/main.go | 40 ++-- .../cmd/dendrite-sync-api-server/main.go | 52 ++-- .../cmd/mediaapi-integration-tests/main.go | 61 ++--- .../cmd/roomserver-integration-tests/main.go | 36 +-- .../cmd/syncserver-integration-tests/main.go | 33 ++- .../dendrite/common/config/config.go | 9 + .../matrix-org/dendrite/common/readkey.go | 58 ----- .../matrix-org/dendrite/common/test/server.go | 19 +- .../dendrite/federationapi/config/config.go | 39 --- .../dendrite/federationapi/readers/keys.go | 22 +- .../dendrite/federationapi/routing/routing.go | 4 +- .../dendrite/federationapi/writers/send.go | 8 +- .../dendrite/mediaapi/config/config.go | 42 ---- .../dendrite/mediaapi/fileutils/fileutils.go | 11 +- .../dendrite/mediaapi/routing/routing.go | 6 +- .../mediaapi/thumbnailer/thumbnailer.go | 5 +- .../mediaapi/thumbnailer/thumbnailer_bimg.go | 3 +- .../mediaapi/thumbnailer/thumbnailer_nfnt.go | 5 +- .../dendrite/mediaapi/types/types.go | 12 +- .../dendrite/mediaapi/writers/download.go | 22 +- .../dendrite/mediaapi/writers/upload.go | 30 +-- .../dendrite/syncapi/config/config.go | 33 --- .../dendrite/syncapi/consumers/roomserver.go | 11 +- .../dendrite/syncapi/routing/routing.go | 3 +- 33 files changed, 282 insertions(+), 758 deletions(-) delete mode 100644 src/github.com/matrix-org/dendrite/clientapi/config/config.go delete mode 100644 src/github.com/matrix-org/dendrite/common/readkey.go delete mode 100644 src/github.com/matrix-org/dendrite/federationapi/config/config.go delete mode 100644 src/github.com/matrix-org/dendrite/mediaapi/config/config.go delete mode 100644 src/github.com/matrix-org/dendrite/syncapi/config/config.go diff --git a/src/github.com/matrix-org/dendrite/clientapi/config/config.go b/src/github.com/matrix-org/dendrite/clientapi/config/config.go deleted file mode 100644 index c5e846ec4..000000000 --- a/src/github.com/matrix-org/dendrite/clientapi/config/config.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "github.com/matrix-org/gomatrixserverlib" - "golang.org/x/crypto/ed25519" -) - -// ClientAPI contains the config information necessary to spin up a clientapi process. -type ClientAPI struct { - // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. - ServerName gomatrixserverlib.ServerName - // The private key which will be used to sign events. - PrivateKey ed25519.PrivateKey - // An arbitrary string used to uniquely identify the PrivateKey. Must start with the - // prefix "ed25519:". - KeyID gomatrixserverlib.KeyID - // A list of URIs to send events to. These kafka logs should be consumed by a Room Server. - KafkaProducerURIs []string - // The topic for events which are written to the logs. - ClientAPIOutputTopic string - // The URL of the roomserver which can service Query API requests - RoomserverURL string -} diff --git a/src/github.com/matrix-org/dendrite/clientapi/readers/login.go b/src/github.com/matrix-org/dendrite/clientapi/readers/login.go index 50ee0daa2..270b2e5ac 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/readers/login.go +++ b/src/github.com/matrix-org/dendrite/clientapi/readers/login.go @@ -21,9 +21,9 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/clientapi/config" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -56,7 +56,10 @@ func passwordLogin() loginFlows { } // Login implements GET and POST /login -func Login(req *http.Request, accountDB *accounts.Database, deviceDB *devices.Database, cfg config.ClientAPI) util.JSONResponse { +func Login( + req *http.Request, accountDB *accounts.Database, deviceDB *devices.Database, + cfg config.Dendrite, +) util.JSONResponse { if req.Method == "GET" { // TODO: support other forms of login other than password, depending on config options return util.JSONResponse{ Code: 200, @@ -109,7 +112,7 @@ func Login(req *http.Request, accountDB *accounts.Database, deviceDB *devices.Da JSON: loginResponse{ UserID: dev.UserID, AccessToken: dev.AccessToken, - HomeServer: cfg.ServerName, + HomeServer: cfg.Matrix.ServerName, }, } } diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go b/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go index 66f26a708..f6c0989fb 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go @@ -22,11 +22,11 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/clientapi/config" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/clientapi/readers" "github.com/matrix-org/dendrite/clientapi/writers" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -38,7 +38,7 @@ const pathPrefixR0 = "/_matrix/client/r0" // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. func Setup( - servMux *http.ServeMux, httpClient *http.Client, cfg config.ClientAPI, + servMux *http.ServeMux, httpClient *http.Client, cfg config.Dendrite, producer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI, accountDB *accounts.Database, deviceDB *devices.Database, diff --git a/src/github.com/matrix-org/dendrite/clientapi/writers/createroom.go b/src/github.com/matrix-org/dendrite/clientapi/writers/createroom.go index a25017001..290951900 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/writers/createroom.go +++ b/src/github.com/matrix-org/dendrite/clientapi/writers/createroom.go @@ -23,11 +23,11 @@ import ( log "github.com/Sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/config" "github.com/matrix-org/dendrite/clientapi/events" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -91,15 +91,15 @@ type fledglingEvent struct { } // CreateRoom implements /createRoom -func CreateRoom(req *http.Request, device *authtypes.Device, cfg config.ClientAPI, producer *producers.RoomserverProducer) util.JSONResponse { +func CreateRoom(req *http.Request, device *authtypes.Device, cfg config.Dendrite, producer *producers.RoomserverProducer) util.JSONResponse { // TODO: Check room ID doesn't clash with an existing one, and we // probably shouldn't be using pseudo-random strings, maybe GUIDs? - roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.ServerName) + roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.Matrix.ServerName) return createRoom(req, device, cfg, roomID, producer) } // createRoom implements /createRoom -func createRoom(req *http.Request, device *authtypes.Device, cfg config.ClientAPI, roomID string, producer *producers.RoomserverProducer) util.JSONResponse { +func createRoom(req *http.Request, device *authtypes.Device, cfg config.Dendrite, roomID string, producer *producers.RoomserverProducer) util.JSONResponse { logger := util.GetLogger(req.Context()) userID := device.UserID var r createRoomRequest @@ -201,7 +201,7 @@ func createRoom(req *http.Request, device *authtypes.Device, cfg config.ClientAP // buildEvent fills out auth_events for the builder then builds the event func buildEvent(builder *gomatrixserverlib.EventBuilder, provider gomatrixserverlib.AuthEventProvider, - cfg config.ClientAPI) (*gomatrixserverlib.Event, error) { + cfg config.Dendrite) (*gomatrixserverlib.Event, error) { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) if err != nil { @@ -212,9 +212,9 @@ func buildEvent(builder *gomatrixserverlib.EventBuilder, return nil, err } builder.AuthEvents = refs - eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), cfg.ServerName) + eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), cfg.Matrix.ServerName) now := time.Now() - event, err := builder.Build(eventID, now, cfg.ServerName, cfg.KeyID, cfg.PrivateKey) + event, err := builder.Build(eventID, now, cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey) if err != nil { return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %s", builder.Type, err) } diff --git a/src/github.com/matrix-org/dendrite/clientapi/writers/joinroom.go b/src/github.com/matrix-org/dendrite/clientapi/writers/joinroom.go index 2b679b56e..57bc3e86b 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/writers/joinroom.go +++ b/src/github.com/matrix-org/dendrite/clientapi/writers/joinroom.go @@ -17,10 +17,10 @@ package writers import ( "fmt" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/config" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" @@ -36,7 +36,7 @@ func JoinRoomByIDOrAlias( req *http.Request, device *authtypes.Device, roomIDOrAlias string, - cfg config.ClientAPI, + cfg config.Dendrite, federation *gomatrixserverlib.FederationClient, producer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI, @@ -67,7 +67,7 @@ type joinRoomReq struct { req *http.Request content map[string]interface{} userID string - cfg config.ClientAPI + cfg config.Dendrite federation *gomatrixserverlib.FederationClient producer *producers.RoomserverProducer queryAPI api.RoomserverQueryAPI @@ -95,7 +95,7 @@ func (r joinRoomReq) joinRoomByAlias(roomAlias string) util.JSONResponse { JSON: jsonerror.BadJSON("Room alias must be in the form '#localpart:domain'"), } } - if domain == r.cfg.ServerName { + if domain == r.cfg.Matrix.ServerName { // TODO: Implement joining local room aliases. panic(fmt.Errorf("Joining local room aliases is not implemented")) } else { @@ -212,9 +212,9 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib r.writeToBuilder(&respMakeJoin.JoinEvent, roomID) now := time.Now() - eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), r.cfg.ServerName) + eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), r.cfg.Matrix.ServerName) event, err := respMakeJoin.JoinEvent.Build( - eventID, now, r.cfg.ServerName, r.cfg.KeyID, r.cfg.PrivateKey, + eventID, now, r.cfg.Matrix.ServerName, r.cfg.Matrix.KeyID, r.cfg.Matrix.PrivateKey, ) if err != nil { res := httputil.LogThenError(r.req, err) diff --git a/src/github.com/matrix-org/dendrite/clientapi/writers/sendevent.go b/src/github.com/matrix-org/dendrite/clientapi/writers/sendevent.go index ec65fadb6..22efa35d5 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/writers/sendevent.go +++ b/src/github.com/matrix-org/dendrite/clientapi/writers/sendevent.go @@ -21,10 +21,10 @@ import ( "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/config" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -39,7 +39,14 @@ type sendEventResponse struct { // SendEvent implements: // /rooms/{roomID}/send/{eventType}/{txnID} // /rooms/{roomID}/state/{eventType}/{stateKey} -func SendEvent(req *http.Request, device *authtypes.Device, roomID, eventType, txnID string, stateKey *string, cfg config.ClientAPI, queryAPI api.RoomserverQueryAPI, producer *producers.RoomserverProducer) util.JSONResponse { +func SendEvent( + req *http.Request, + device *authtypes.Device, + roomID, eventType, txnID string, stateKey *string, + cfg config.Dendrite, + queryAPI api.RoomserverQueryAPI, + producer *producers.RoomserverProducer, +) util.JSONResponse { // parse the incoming http request userID := device.UserID var r map[string]interface{} // must be a JSON object @@ -86,8 +93,10 @@ func SendEvent(req *http.Request, device *authtypes.Device, roomID, eventType, t refs = append(refs, e.EventReference()) } builder.AuthEvents = refs - eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), cfg.ServerName) - e, err := builder.Build(eventID, time.Now(), cfg.ServerName, cfg.KeyID, cfg.PrivateKey) + eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), cfg.Matrix.ServerName) + e, err := builder.Build( + eventID, time.Now(), cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, + ) if err != nil { return httputil.LogThenError(req, err) } diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-client-api-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-client-api-server/main.go index aabfb0653..51472b759 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-client-api-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-client-api-server/main.go @@ -15,16 +15,16 @@ package main import ( + "flag" "net/http" "os" - "strings" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/clientapi/config" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/clientapi/routing" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/common/keydb" "github.com/matrix-org/dendrite/roomserver/api" @@ -34,70 +34,45 @@ import ( ) var ( - kafkaURIs = strings.Split(os.Getenv("KAFKA_URIS"), ",") - bindAddr = os.Getenv("BIND_ADDRESS") - logDir = os.Getenv("LOG_DIR") - roomserverURL = os.Getenv("ROOMSERVER_URL") - clientAPIOutputTopic = os.Getenv("CLIENTAPI_OUTPUT_TOPIC") - serverName = gomatrixserverlib.ServerName(os.Getenv("SERVER_NAME")) - serverKey = os.Getenv("SERVER_KEY") - accountDataSource = os.Getenv("ACCOUNT_DATABASE") - keyDataSource = os.Getenv("KEY_DATABASE") + logDir = os.Getenv("LOG_DIR") + configPath = flag.String("config", "dendrite.yaml", "The path to the config file, For more information see the config file in this repository") ) func main() { common.SetupLogging(logDir) - if bindAddr == "" { - log.Panic("No BIND_ADDRESS environment variable found.") - } - if len(kafkaURIs) == 0 { - // the kafka default is :9092 - kafkaURIs = []string{"localhost:9092"} - } - if roomserverURL == "" { - log.Panic("No ROOMSERVER_URL environment variable found.") - } - if clientAPIOutputTopic == "" { - log.Panic("No CLIENTAPI_OUTPUT_TOPIC environment variable found. This should match the roomserver input topic.") - } - if serverName == "" { - serverName = "localhost" - } - cfg := config.ClientAPI{ - ServerName: serverName, - KafkaProducerURIs: kafkaURIs, - ClientAPIOutputTopic: clientAPIOutputTopic, - RoomserverURL: roomserverURL, - } + flag.Parse() - var err error - cfg.KeyID, cfg.PrivateKey, err = common.ReadKey(serverKey) + cfg, err := config.Load(*configPath) if err != nil { - log.Panicf("Failed to load private key: %s", err) + log.Fatalf("Invalid config file: %s", err) } - log.Info("Starting clientapi") + log.Info("config: ", cfg) - roomserverProducer, err := producers.NewRoomserverProducer(cfg.KafkaProducerURIs, cfg.ClientAPIOutputTopic) + roomserverProducer, err := producers.NewRoomserverProducer( + cfg.Kafka.Addresses, string(cfg.Kafka.Topics.InputRoomEvent), + ) if err != nil { - log.Panicf("Failed to setup kafka producers(%q): %s", cfg.KafkaProducerURIs, err) + log.Panicf("Failed to setup kafka producers(%q): %s", cfg.Kafka.Addresses, err) } - federation := gomatrixserverlib.NewFederationClient(cfg.ServerName, cfg.KeyID, cfg.PrivateKey) + federation := gomatrixserverlib.NewFederationClient( + cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, + ) - queryAPI := api.NewRoomserverQueryAPIHTTP(cfg.RoomserverURL, nil) - accountDB, err := accounts.NewDatabase(accountDataSource, serverName) + queryAPI := api.NewRoomserverQueryAPIHTTP(cfg.RoomServerURL(), nil) + accountDB, err := accounts.NewDatabase(string(cfg.Database.Account), cfg.Matrix.ServerName) if err != nil { - log.Panicf("Failed to setup account database(%q): %s", accountDataSource, err.Error()) + log.Panicf("Failed to setup account database(%q): %s", cfg.Database.Account, err.Error()) } - deviceDB, err := devices.NewDatabase(accountDataSource, serverName) + deviceDB, err := devices.NewDatabase(string(cfg.Database.Device), cfg.Matrix.ServerName) if err != nil { - log.Panicf("Failed to setup device database(%q): %s", accountDataSource, err.Error()) + log.Panicf("Failed to setup device database(%q): %s", cfg.Database.Device, err.Error()) } - keyDB, err := keydb.NewDatabase(keyDataSource) + keyDB, err := keydb.NewDatabase(string(cfg.Database.ServerKey)) if err != nil { - log.Panicf("Failed to setup key database(%q): %s", keyDataSource, err.Error()) + log.Panicf("Failed to setup key database(%q): %s", cfg.Database.ServerKey, err.Error()) } keyRing := gomatrixserverlib.KeyRing{ @@ -108,9 +83,10 @@ func main() { KeyDatabase: keyDB, } + log.Info("Starting client API server on ", cfg.Listen.ClientAPI) routing.Setup( - http.DefaultServeMux, http.DefaultClient, cfg, roomserverProducer, + http.DefaultServeMux, http.DefaultClient, *cfg, roomserverProducer, queryAPI, accountDB, deviceDB, federation, keyRing, ) - log.Fatal(http.ListenAndServe(bindAddr, nil)) + log.Fatal(http.ListenAndServe(string(cfg.Listen.ClientAPI), nil)) } diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go index 1a7abd6a7..f9576ce90 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go @@ -15,16 +15,14 @@ package main import ( - "encoding/base64" + "flag" "net/http" "os" - "strings" - "time" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/common/keydb" - "github.com/matrix-org/dendrite/federationapi/config" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -33,73 +31,28 @@ import ( ) var ( - bindAddr = os.Getenv("BIND_ADDRESS") logDir = os.Getenv("LOG_DIR") - serverName = gomatrixserverlib.ServerName(os.Getenv("SERVER_NAME")) - serverKey = os.Getenv("SERVER_KEY") - // Base64 encoded SHA256 TLS fingerprint of the X509 certificate used by - // the public federation listener for this server. - // Can be generated from a PEM certificate called "server.crt" using: - // - // openssl x509 -noout -fingerprint -sha256 -inform pem -in server.crt |\ - // python -c 'print raw_input()[19:].replace(":","").decode("hex").encode("base64").rstrip("=\n")' - // - tlsFingerprint = os.Getenv("TLS_FINGERPRINT") - kafkaURIs = strings.Split(os.Getenv("KAFKA_URIS"), ",") - roomserverURL = os.Getenv("ROOMSERVER_URL") - roomserverInputTopic = os.Getenv("TOPIC_INPUT_ROOM_EVENT") - keyDataSource = os.Getenv("KEY_DATABASE") + configPath = flag.String("config", "dendrite.yaml", "The path to the config file. For more information, see the config file in this repository.") ) func main() { common.SetupLogging(logDir) - if bindAddr == "" { - log.Panic("No BIND_ADDRESS environment variable found.") - } - if serverName == "" { - serverName = "localhost" + if *configPath == "" { + log.Fatal("--config must be supplied") } - - if tlsFingerprint == "" { - log.Panic("No TLS_FINGERPRINT environment variable found.") - } - - if len(kafkaURIs) == 0 { - // the kafka default is :9092 - kafkaURIs = []string{"localhost:9092"} - } - - if roomserverURL == "" { - log.Panic("No ROOMSERVER_URL environment variable found.") - } - - if roomserverInputTopic == "" { - log.Panic("No TOPIC_INPUT_ROOM_EVENT environment variable found. This should match the roomserver input topic.") - } - cfg := config.FederationAPI{ - ServerName: serverName, - // TODO: make the validity period configurable. - ValidityPeriod: 24 * time.Hour, - } - - var err error - cfg.KeyID, cfg.PrivateKey, err = common.ReadKey(serverKey) + cfg, err := config.Load(*configPath) if err != nil { - log.Panicf("Failed to load private key: %s", err) + log.Fatalf("Invalid config file: %s", err) } - var fingerprintSHA256 []byte - if fingerprintSHA256, err = base64.RawStdEncoding.DecodeString(tlsFingerprint); err != nil { - log.Panicf("Failed to load TLS fingerprint: %s", err) - } - cfg.TLSFingerPrints = []gomatrixserverlib.TLSFingerprint{{fingerprintSHA256}} + federation := gomatrixserverlib.NewFederationClient( + cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, + ) - federation := gomatrixserverlib.NewFederationClient(cfg.ServerName, cfg.KeyID, cfg.PrivateKey) - - keyDB, err := keydb.NewDatabase(keyDataSource) + keyDB, err := keydb.NewDatabase(string(cfg.Database.ServerKey)) if err != nil { - log.Panicf("Failed to setup key database(%q): %s", keyDataSource, err.Error()) + log.Panicf("Failed to setup key database(%q): %s", cfg.Database.ServerKey, err.Error()) } keyRing := gomatrixserverlib.KeyRing{ @@ -109,13 +62,19 @@ func main() { }, KeyDatabase: keyDB, } - queryAPI := api.NewRoomserverQueryAPIHTTP(roomserverURL, nil) - roomserverProducer, err := producers.NewRoomserverProducer(kafkaURIs, roomserverInputTopic) + queryAPI := api.NewRoomserverQueryAPIHTTP(cfg.RoomServerURL(), nil) + + roomserverProducer, err := producers.NewRoomserverProducer( + cfg.Kafka.Addresses, string(cfg.Kafka.Topics.InputRoomEvent), + ) + if err != nil { - log.Panicf("Failed to setup kafka producers(%s): %s", kafkaURIs, err) + log.Panicf("Failed to setup kafka producers(%s): %s", cfg.Kafka.Addresses, err) } - routing.Setup(http.DefaultServeMux, cfg, queryAPI, roomserverProducer, keyRing, federation) - log.Fatal(http.ListenAndServe(bindAddr, nil)) + log.Info("Starting federation API server on ", cfg.Listen.FederationAPI) + + routing.Setup(http.DefaultServeMux, *cfg, queryAPI, roomserverProducer, keyRing, federation) + log.Fatal(http.ListenAndServe(string(cfg.Listen.FederationAPI), nil)) } diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go index fb39a03a6..2c62cdb97 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-media-api-server/main.go @@ -16,35 +16,20 @@ package main import ( "flag" - "fmt" - "io/ioutil" "net/http" "os" - "os/user" - "path/filepath" - "strconv" - "strings" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/mediaapi/config" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/routing" "github.com/matrix-org/dendrite/mediaapi/storage" - "github.com/matrix-org/dendrite/mediaapi/types" - "github.com/matrix-org/gomatrixserverlib" log "github.com/Sirupsen/logrus" - yaml "gopkg.in/yaml.v2" ) var ( - bindAddr = flag.String("listen", "", "The port to listen on.") - dataSource = os.Getenv("DATABASE") logDir = os.Getenv("LOG_DIR") - serverName = os.Getenv("SERVER_NAME") - basePath = os.Getenv("BASE_PATH") - // Note: if the MAX_FILE_SIZE_BYTES is set to 0, it will be unlimited - maxFileSizeBytesString = os.Getenv("MAX_FILE_SIZE_BYTES") - configPath = flag.String("config", "", "The path to the config file. For more information, see the config file in this repository.") + configPath = flag.String("config", "", "The path to the config file. For more information, see the config file in this repository.") ) func main() { @@ -52,206 +37,21 @@ func main() { flag.Parse() - log.WithFields(log.Fields{ - "listen": *bindAddr, - "DATABASE": dataSource, - "LOG_DIR": logDir, - "SERVER_NAME": serverName, - "BASE_PATH": basePath, - "MAX_FILE_SIZE_BYTES": maxFileSizeBytesString, - "config": *configPath, - }).Info("Loading configuration based on config file and environment variables") - - cfg, err := configureServer() + if *configPath == "" { + log.Fatal("--config must be supplied") + } + cfg, err := config.Load(*configPath) if err != nil { - log.WithError(err).Fatal("Invalid configuration") + log.Fatalf("Invalid config file: %s", err) } - log.WithFields(log.Fields{ - "listen": *bindAddr, - "LOG_DIR": logDir, - "CONFIG_PATH": *configPath, - "ServerName": cfg.ServerName, - "AbsBasePath": cfg.AbsBasePath, - "MaxFileSizeBytes": *cfg.MaxFileSizeBytes, - "DataSource": cfg.DataSource, - "DynamicThumbnails": cfg.DynamicThumbnails, - "MaxThumbnailGenerators": cfg.MaxThumbnailGenerators, - "ThumbnailSizes": cfg.ThumbnailSizes, - }).Info("Starting mediaapi server with configuration") - - db, err := storage.Open(cfg.DataSource) + db, err := storage.Open(string(cfg.Database.MediaAPI)) if err != nil { log.WithError(err).Panic("Failed to open database") } + log.Info("Starting media API server on ", cfg.Listen.MediaAPI) + routing.Setup(http.DefaultServeMux, http.DefaultClient, cfg, db) - log.Fatal(http.ListenAndServe(*bindAddr, nil)) -} - -// configureServer loads configuration from a yaml file and overrides with environment variables -func configureServer() (*config.MediaAPI, error) { - if *configPath == "" { - log.Fatal("--config must be supplied") - } - cfg, err := loadConfig(*configPath) - if err != nil { - log.WithError(err).Fatal("Invalid config file") - } - - // override values from environment variables - applyOverrides(cfg) - - if err := validateConfig(cfg); err != nil { - return nil, err - } - - return cfg, nil -} - -// FIXME: make common somehow? copied from sync api -func loadConfig(configPath string) (*config.MediaAPI, error) { - contents, err := ioutil.ReadFile(configPath) - if err != nil { - return nil, err - } - var cfg config.MediaAPI - if err = yaml.Unmarshal(contents, &cfg); err != nil { - return nil, err - } - return &cfg, nil -} - -func applyOverrides(cfg *config.MediaAPI) { - if serverName != "" { - if cfg.ServerName != "" { - log.WithFields(log.Fields{ - "server_name": cfg.ServerName, - "SERVER_NAME": serverName, - }).Info("Overriding server_name from config file with environment variable") - } - cfg.ServerName = gomatrixserverlib.ServerName(serverName) - } - if cfg.ServerName == "" { - log.Info("ServerName not set. Defaulting to 'localhost'.") - cfg.ServerName = "localhost" - } - - if basePath != "" { - if cfg.BasePath != "" { - log.WithFields(log.Fields{ - "base_path": cfg.BasePath, - "BASE_PATH": basePath, - }).Info("Overriding base_path from config file with environment variable") - } - cfg.BasePath = types.Path(basePath) - } - - if maxFileSizeBytesString != "" { - if cfg.MaxFileSizeBytes != nil { - log.WithFields(log.Fields{ - "max_file_size_bytes": *cfg.MaxFileSizeBytes, - "MAX_FILE_SIZE_BYTES": maxFileSizeBytesString, - }).Info("Overriding max_file_size_bytes from config file with environment variable") - } - maxFileSizeBytesInt, err := strconv.ParseInt(maxFileSizeBytesString, 10, 64) - if err != nil { - maxFileSizeBytesInt = 10 * 1024 * 1024 - log.WithError(err).WithField( - "MAX_FILE_SIZE_BYTES", maxFileSizeBytesString, - ).Infof("MAX_FILE_SIZE_BYTES not set? Defaulting to %v bytes.", maxFileSizeBytesInt) - } - maxFileSizeBytes := types.FileSizeBytes(maxFileSizeBytesInt) - cfg.MaxFileSizeBytes = &maxFileSizeBytes - } - - if dataSource != "" { - if cfg.DataSource != "" { - log.WithFields(log.Fields{ - "database": cfg.DataSource, - "DATABASE": dataSource, - }).Info("Overriding database from config file with environment variable") - } - cfg.DataSource = dataSource - } - - if cfg.MaxThumbnailGenerators == 0 { - log.WithField( - "max_thumbnail_generators", cfg.MaxThumbnailGenerators, - ).Info("Using default max_thumbnail_generators value of 10") - cfg.MaxThumbnailGenerators = 10 - } -} - -func validateConfig(cfg *config.MediaAPI) error { - if *bindAddr == "" { - log.Fatal("--listen must be supplied") - } - - absBasePath, err := getAbsolutePath(cfg.BasePath) - if err != nil { - return fmt.Errorf("invalid base path (%v): %q", cfg.BasePath, err) - } - cfg.AbsBasePath = types.Path(absBasePath) - - if *cfg.MaxFileSizeBytes < 0 { - return fmt.Errorf("invalid max file size bytes (%v)", *cfg.MaxFileSizeBytes) - } - - if cfg.DataSource == "" { - return fmt.Errorf("invalid database (%v)", cfg.DataSource) - } - - for _, config := range cfg.ThumbnailSizes { - if config.Width <= 0 || config.Height <= 0 { - return fmt.Errorf("invalid thumbnail size %vx%v", config.Width, config.Height) - } - } - - return nil -} - -func getAbsolutePath(basePath types.Path) (types.Path, error) { - var err error - if basePath == "" { - var wd string - wd, err = os.Getwd() - return types.Path(wd), err - } - // Note: If we got here len(basePath) >= 1 - if basePath[0] == '~' { - basePath, err = expandHomeDir(basePath) - if err != nil { - return "", err - } - } - absBasePath, err := filepath.Abs(string(basePath)) - return types.Path(absBasePath), err -} - -// expandHomeDir parses paths beginning with ~/path or ~user/path and replaces the home directory part -func expandHomeDir(basePath types.Path) (types.Path, error) { - slash := strings.Index(string(basePath), "/") - if slash == -1 { - // pretend the slash is after the path as none was found within the string - // simplifies code using slash below - slash = len(basePath) - } - var usr *user.User - var err error - if slash == 1 { - // basePath is ~ or ~/path - usr, err = user.Current() - if err != nil { - return "", fmt.Errorf("failed to get user's home directory: %q", err) - } - } else { - // slash > 1 - // basePath is ~user or ~user/path - usr, err = user.Lookup(string(basePath[1:slash])) - if err != nil { - return "", fmt.Errorf("failed to get user's home directory: %q", err) - } - } - return types.Path(filepath.Join(usr.HomeDir, string(basePath[slash:]))), nil + log.Fatal(http.ListenAndServe(string(cfg.Listen.MediaAPI), nil)) } diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-room-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-room-server/main.go index d06be276d..0a1686791 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-room-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-room-server/main.go @@ -15,14 +15,16 @@ package main import ( + "flag" "fmt" "net/http" _ "net/http/pprof" "os" "strconv" - "strings" + log "github.com/Sirupsen/logrus" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/input" "github.com/matrix-org/dendrite/roomserver/query" "github.com/matrix-org/dendrite/roomserver/storage" @@ -31,42 +33,48 @@ import ( ) var ( - database = os.Getenv("DATABASE") - kafkaURIs = strings.Split(os.Getenv("KAFKA_URIS"), ",") - inputRoomEventTopic = os.Getenv("TOPIC_INPUT_ROOM_EVENT") - outputRoomEventTopic = os.Getenv("TOPIC_OUTPUT_ROOM_EVENT") - bindAddr = os.Getenv("BIND_ADDRESS") - // Shuts the roomserver down after processing a given number of messages. - // This is useful for running benchmarks for seeing how quickly the server - // can process a given number of messages. + logDir = os.Getenv("LOG_DIR") + configPath = flag.String("config", "", "The path to the config file. For more information, see the config file in this repository.") stopProcessingAfter = os.Getenv("STOP_AFTER") ) func main() { - db, err := storage.Open(database) + common.SetupLogging(logDir) + + flag.Parse() + + if *configPath == "" { + log.Fatal("--config must be supplied") + } + cfg, err := config.Load(*configPath) + if err != nil { + log.Fatalf("Invalid config file: %s", err) + } + + db, err := storage.Open(string(cfg.Database.RoomServer)) if err != nil { panic(err) } - kafkaConsumer, err := sarama.NewConsumer(kafkaURIs, nil) + kafkaConsumer, err := sarama.NewConsumer(cfg.Kafka.Addresses, nil) if err != nil { panic(err) } - kafkaProducer, err := sarama.NewSyncProducer(kafkaURIs, nil) + kafkaProducer, err := sarama.NewSyncProducer(cfg.Kafka.Addresses, nil) if err != nil { panic(err) } consumer := input.Consumer{ ContinualConsumer: common.ContinualConsumer{ - Topic: inputRoomEventTopic, + Topic: string(cfg.Kafka.Topics.InputRoomEvent), Consumer: kafkaConsumer, PartitionStore: db, }, DB: db, Producer: kafkaProducer, - OutputRoomEventTopic: outputRoomEventTopic, + OutputRoomEventTopic: string(cfg.Kafka.Topics.OutputRoomEvent), } if stopProcessingAfter != "" { @@ -93,10 +101,10 @@ func main() { http.DefaultServeMux.Handle("/metrics", prometheus.Handler()) - fmt.Println("Started roomserver") + log.Info("Started room server on ", cfg.Listen.RoomServer) // TODO: Implement clean shutdown. - if err := http.ListenAndServe(bindAddr, nil); err != nil { + if err := http.ListenAndServe(string(cfg.Listen.RoomServer), nil); err != nil { panic(err) } } diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server/main.go index 638eba3cd..a8984c1d5 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-sync-api-server/main.go @@ -16,13 +16,12 @@ package main import ( "flag" - "io/ioutil" "net/http" "os" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/syncapi/config" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/syncapi/consumers" "github.com/matrix-org/dendrite/syncapi/routing" "github.com/matrix-org/dendrite/syncapi/storage" @@ -30,27 +29,9 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" log "github.com/Sirupsen/logrus" - yaml "gopkg.in/yaml.v2" ) -var configPath = flag.String("config", "sync-server-config.yaml", "The path to the config file. For more information, see the config file in this repository.") -var bindAddr = flag.String("listen", ":4200", "The port to listen on.") - -func loadConfig(configPath string) (*config.Sync, error) { - contents, err := ioutil.ReadFile(configPath) - if err != nil { - return nil, err - } - var cfg config.Sync - if err = yaml.Unmarshal(contents, &cfg); err != nil { - return nil, err - } - // check required fields - if cfg.ServerName == "" { - log.Fatalf("'server_name' must be supplied in %s", configPath) - } - return &cfg, nil -} +var configPath = flag.String("config", "dendrite.yaml", "The path to the config file. For more information, see the config file in this repository.") func main() { common.SetupLogging(os.Getenv("LOG_DIR")) @@ -60,26 +41,21 @@ func main() { if *configPath == "" { log.Fatal("--config must be supplied") } - cfg, err := loadConfig(*configPath) + cfg, err := config.Load(*configPath) if err != nil { log.Fatalf("Invalid config file: %s", err) } - if *bindAddr == "" { - log.Fatal("--listen must be supplied") + log.Info("config: ", cfg) + + db, err := storage.NewSyncServerDatabase(string(cfg.Database.SyncAPI)) + if err != nil { + log.Panicf("startup: failed to create sync server database with data source %s : %s", cfg.Database.SyncAPI, err) } - log.Info("sync server config: ", cfg) - - db, err := storage.NewSyncServerDatabase(cfg.DataSource) + deviceDB, err := devices.NewDatabase(string(cfg.Database.Device), cfg.Matrix.ServerName) if err != nil { - log.Panicf("startup: failed to create sync server database with data source %s : %s", cfg.DataSource, err) - } - - // TODO: DO NOT USE THIS DATA SOURCE (it's the sync one, not devices!) - deviceDB, err := devices.NewDatabase(cfg.DataSource, cfg.ServerName) - if err != nil { - log.Panicf("startup: failed to create device database with data source %s : %s", cfg.DataSource, err) + log.Panicf("startup: failed to create device database with data source %s : %s", cfg.Database.Device, err) } pos, err := db.SyncStreamPosition() @@ -88,7 +64,7 @@ func main() { } n := sync.NewNotifier(types.StreamPosition(pos)) - if err := n.Load(db); err != nil { + if err = n.Load(db); err != nil { log.Panicf("startup: failed to set up notifier: %s", err) } consumer, err := consumers.NewOutputRoomEvent(cfg, n, db) @@ -99,7 +75,7 @@ func main() { log.Panicf("startup: failed to start room server consumer") } - log.Info("Starting sync server on ", *bindAddr) - routing.SetupSyncServerListeners(http.DefaultServeMux, http.DefaultClient, *cfg, sync.NewRequestPool(db, n), deviceDB) - log.Fatal(http.ListenAndServe(*bindAddr, nil)) + log.Info("Starting sync server on ", cfg.Listen.SyncAPI) + routing.SetupSyncServerListeners(http.DefaultServeMux, http.DefaultClient, sync.NewRequestPool(db, n), deviceDB) + log.Fatal(http.ListenAndServe(string(cfg.Listen.SyncAPI), nil)) } diff --git a/src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/main.go b/src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/main.go index 9902f4ef8..29a79d762 100644 --- a/src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/main.go @@ -22,10 +22,11 @@ import ( "os/exec" "path" "path/filepath" - "strconv" "time" "github.com/matrix-org/dendrite/common/test" + "github.com/matrix-org/gomatrixserverlib" + "gopkg.in/yaml.v2" ) var ( @@ -41,10 +42,10 @@ var ( postgresContainerName = os.Getenv("POSTGRES_CONTAINER") // Test image to be uploaded/downloaded testJPEG = test.Defaulting(os.Getenv("TEST_JPEG_PATH"), "src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/totem.jpg") + kafkaURI = test.Defaulting(os.Getenv("KAFKA_URIS"), "localhost:9092") ) -var thumbnailPregenerationConfig = (` -thumbnail_sizes: +var thumbnailSizes = (` - width: 32 height: 32 method: crop @@ -68,65 +69,51 @@ var testDatabaseTemplate = "dbname=%s sslmode=disable binary_parameters=yes" var timeout time.Duration +var port = 10000 + func startMediaAPI(suffix string, dynamicThumbnails bool) (*exec.Cmd, chan error, string, *exec.Cmd, chan error, string, string) { dir, err := ioutil.TempDir("", serverType+"-server-test"+suffix) if err != nil { panic(err) } - serverAddr := "localhost:177" + suffix + "9" proxyAddr := "localhost:1800" + suffix - configFilename := serverType + "-server-test-config" + suffix + ".yaml" - configFileContents := makeConfig(proxyAddr, suffix, dir, dynamicThumbnails) + database := fmt.Sprintf(testDatabaseTemplate, testDatabaseName+suffix) + cfg, nextPort, err := test.MakeConfig(dir, kafkaURI, database, "localhost", port) + cfg.Matrix.ServerName = gomatrixserverlib.ServerName(proxyAddr) + if err != nil { + panic(err) + } + if err = yaml.Unmarshal([]byte(thumbnailSizes), &cfg.Media.ThumbnailSizes); err != nil { + panic(err) + } + + port = nextPort + if err = test.WriteConfig(cfg, dir); err != nil { + panic(err) + } serverArgs := []string{ - "--config", configFilename, - "--listen", serverAddr, + "--config", filepath.Join(dir, test.ConfigFile), } databases := []string{ testDatabaseName + suffix, } - proxyCmd, proxyCmdChan := test.StartProxy( - proxyAddr, - "http://localhost:177"+suffix+"6", - "http://localhost:177"+suffix+"8", - "http://"+serverAddr, - ) + proxyCmd, proxyCmdChan := test.StartProxy(proxyAddr, cfg) cmd, cmdChan := test.StartServer( serverType, serverArgs, - suffix, - configFilename, - configFileContents, postgresDatabase, postgresContainerName, databases, ) - fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, serverAddr, dir) - return cmd, cmdChan, serverAddr, proxyCmd, proxyCmdChan, proxyAddr, dir -} - -func makeConfig(serverAddr, suffix, basePath string, dynamicThumbnails bool) string { - return fmt.Sprintf( - ` -server_name: "%s" -base_path: %s -max_file_size_bytes: %s -database: "%s" -dynamic_thumbnails: %s -%s`, - serverAddr, - basePath, - "10485760", - fmt.Sprintf(testDatabaseTemplate, testDatabaseName+suffix), - strconv.FormatBool(dynamicThumbnails), - thumbnailPregenerationConfig, - ) + fmt.Printf("==TESTSERVER== STARTED %v -> %v : %v\n", proxyAddr, cfg.Listen.MediaAPI, dir) + return cmd, cmdChan, string(cfg.Listen.MediaAPI), proxyCmd, proxyCmdChan, proxyAddr, dir } func cleanUpServer(cmd *exec.Cmd, dir string) { diff --git a/src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go b/src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go index f11c1c89c..940293f8e 100644 --- a/src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go @@ -16,6 +16,7 @@ package main import ( "fmt" + "io/ioutil" "os" "os/exec" "path/filepath" @@ -34,8 +35,6 @@ var ( zookeeperURI = defaulting(os.Getenv("ZOOKEEPER_URI"), "localhost:2181") // The URI the kafka server is listening on. kafkaURI = defaulting(os.Getenv("KAFKA_URIS"), "localhost:9092") - // The address the roomserver should listen on. - roomserverAddr = defaulting(os.Getenv("ROOMSERVER_URI"), "localhost:9876") // How long to wait for the roomserver to write the expected output messages. // This needs to be high enough to account for the time it takes to create // the postgres database tables which can take a while on travis. @@ -164,10 +163,22 @@ func runAndReadFromTopic(runCmd *exec.Cmd, topic string, count int, checkQueryAP // a api.RoomserverQueryAPI client. The caller can use this function to check the // behaviour of the query API. func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverQueryAPI)) { - const ( - inputTopic = "roomserverInput" - outputTopic = "roomserverOutput" - ) + dir, err := ioutil.TempDir("", "room-server-test") + if err != nil { + panic(err) + } + + cfg, _, err := test.MakeConfig(dir, kafkaURI, testDatabase, "localhost", 10000) + if err != nil { + panic(err) + } + if err := test.WriteConfig(cfg, dir); err != nil { + panic(err) + } + + inputTopic := string(cfg.Kafka.Topics.InputRoomEvent) + outputTopic := string(cfg.Kafka.Topics.OutputRoomEvent) + exe.DeleteTopic(inputTopic) if err := exe.CreateTopic(inputTopic); err != nil { panic(err) @@ -181,7 +192,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R panic(err) } - if err := createDatabase(testDatabaseName); err != nil { + if err = createDatabase(testDatabaseName); err != nil { panic(err) } @@ -191,18 +202,11 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R // We append to the environment rather than replacing so that any additional // postgres and golang environment variables such as PGHOST are passed to // the roomserver process. - cmd.Env = append( - os.Environ(), - fmt.Sprintf("DATABASE=%s", testDatabase), - fmt.Sprintf("KAFKA_URIS=%s", kafkaURI), - fmt.Sprintf("TOPIC_INPUT_ROOM_EVENT=%s", inputTopic), - fmt.Sprintf("TOPIC_OUTPUT_ROOM_EVENT=%s", outputTopic), - fmt.Sprintf("BIND_ADDRESS=%s", roomserverAddr), - ) cmd.Stderr = os.Stderr + cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)} gotOutput, err := runAndReadFromTopic(cmd, outputTopic, len(wantOutput), func() { - queryAPI := api.NewRoomserverQueryAPIHTTP("http://"+roomserverAddr, nil) + queryAPI := api.NewRoomserverQueryAPIHTTP("http://"+string(cfg.Listen.RoomServer), nil) checkQueries(queryAPI) }) if err != nil { diff --git a/src/github.com/matrix-org/dendrite/cmd/syncserver-integration-tests/main.go b/src/github.com/matrix-org/dendrite/cmd/syncserver-integration-tests/main.go index b73405583..d1cf9fd12 100644 --- a/src/github.com/matrix-org/dendrite/cmd/syncserver-integration-tests/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/syncserver-integration-tests/main.go @@ -17,12 +17,14 @@ package main import ( "encoding/json" "fmt" + "io/ioutil" "net/http" "os" "os/exec" "path/filepath" "time" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/common/test" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -62,12 +64,6 @@ var exe = test.KafkaExecutor{ OutputWriter: os.Stderr, } -var syncServerConfigFileContents = (`consumer_uris: ["` + kafkaURI + `"] -roomserver_topic: "` + inputTopic + `" -database: "` + testDatabase + `" -server_name: "localhost" -`) - var timeout time.Duration var clientEventTestData []string @@ -126,11 +122,27 @@ func clientEventJSONForOutputRoomEvent(outputRoomEvent string) string { // then starts the sync server. The Cmd being executed is returned. A channel is also returned, // which will have any termination errors sent down it, followed immediately by the channel being closed. func startSyncServer() (*exec.Cmd, chan error) { - const configFilename = "sync-api-server-config-test.yaml" + + dir, err := ioutil.TempDir("", "syncapi-server-test") + if err != nil { + panic(err) + } + + cfg, _, err := test.MakeConfig(dir, kafkaURI, testDatabase, "localhost", 10000) + if err != nil { + panic(err) + } + // TODO use the address assigned by the config generator rather than clobbering. + cfg.Matrix.ServerName = "localhost" + cfg.Listen.SyncAPI = config.Address(syncserverAddr) + cfg.Kafka.Topics.OutputRoomEvent = config.Topic(inputTopic) + + if err := test.WriteConfig(cfg, dir); err != nil { + panic(err) + } serverArgs := []string{ - "--config", configFilename, - "--listen", syncserverAddr, + "--config", filepath.Join(dir, test.ConfigFile), } databases := []string{ @@ -140,9 +152,6 @@ func startSyncServer() (*exec.Cmd, chan error) { cmd, cmdChan := test.StartServer( "sync-api", serverArgs, - "", - configFilename, - syncServerConfigFileContents, postgresDatabase, postgresContainerName, databases, diff --git a/src/github.com/matrix-org/dendrite/common/config/config.go b/src/github.com/matrix-org/dendrite/common/config/config.go index e06f894a9..a4977731f 100644 --- a/src/github.com/matrix-org/dendrite/common/config/config.go +++ b/src/github.com/matrix-org/dendrite/common/config/config.go @@ -358,3 +358,12 @@ func fingerprintPEM(data []byte) *gomatrixserverlib.TLSFingerprint { } } } + +// RoomServerURL returns an HTTP URL for where the roomserver is listening. +func (config *Dendrite) RoomServerURL() string { + // Hard code the roomserver to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return "http://" + string(config.Listen.RoomServer) +} diff --git a/src/github.com/matrix-org/dendrite/common/readkey.go b/src/github.com/matrix-org/dendrite/common/readkey.go deleted file mode 100644 index 7c7d95dfd..000000000 --- a/src/github.com/matrix-org/dendrite/common/readkey.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package common - -import ( - "bytes" - "encoding/base64" - "fmt" - "github.com/matrix-org/gomatrixserverlib" - "golang.org/x/crypto/ed25519" - "io" - "strings" -) - -// ReadKey reads a server's private ed25519 key. -// If the key is the empty string then a random key is generated. -// Otherwise the key is the key ID and the base64 encoded private key -// separated by a single space character. -// E.g "ed25519:abcd ABCDEFGHIJKLMNOPabcdefghijklmnop01234567890" -func ReadKey(key string) (gomatrixserverlib.KeyID, ed25519.PrivateKey, error) { - var keyID gomatrixserverlib.KeyID - var seed io.Reader - if key == "" { - // TODO: We should fail if we don't have a private key rather than - // generating a throw away key. - keyID = gomatrixserverlib.KeyID("ed25519:something") - } else { - // TODO: We should be reading this from a PEM formatted file instead of - // reading from the environment directly. - parts := strings.SplitN(key, " ", 2) - keyID = gomatrixserverlib.KeyID(parts[0]) - if len(parts) != 2 { - return "", nil, fmt.Errorf("Invalid server key: %q", key) - } - seedBytes, err := base64.RawStdEncoding.DecodeString(parts[1]) - if err != nil { - return "", nil, err - } - seed = bytes.NewReader(seedBytes) - } - _, privKey, err := ed25519.GenerateKey(seed) - if err != nil { - return "", nil, err - } - return keyID, privKey, nil -} diff --git a/src/github.com/matrix-org/dendrite/common/test/server.go b/src/github.com/matrix-org/dendrite/common/test/server.go index dca0aa94c..2f089fe34 100644 --- a/src/github.com/matrix-org/dendrite/common/test/server.go +++ b/src/github.com/matrix-org/dendrite/common/test/server.go @@ -16,11 +16,12 @@ package test import ( "fmt" - "io/ioutil" "os" "os/exec" "path/filepath" "strings" + + "github.com/matrix-org/dendrite/common/config" ) // Defaulting allows assignment of string variables with a fallback default value @@ -69,7 +70,7 @@ func CreateBackgroundCommand(command string, args []string) (*exec.Cmd, chan err // which will have any termination errors sent down it, followed immediately by the channel being closed. // If postgresContainerName is not an empty string, psql will be run from inside that container. If it is // an empty string, psql will be assumed to be in PATH. -func StartServer(serverType string, serverArgs []string, suffix, configFilename, configFileContents, postgresDatabase, postgresContainerName string, databases []string) (*exec.Cmd, chan error) { +func StartServer(serverType string, serverArgs []string, postgresDatabase, postgresContainerName string, databases []string) (*exec.Cmd, chan error) { if len(databases) > 0 { var dbCmd string var dbArgs []string @@ -89,12 +90,6 @@ func StartServer(serverType string, serverArgs []string, suffix, configFilename, } } - if configFilename != "" { - if err := ioutil.WriteFile(configFilename, []byte(configFileContents), 0644); err != nil { - panic(err) - } - } - return CreateBackgroundCommand( filepath.Join(filepath.Dir(os.Args[0]), "dendrite-"+serverType+"-server"), serverArgs, @@ -102,12 +97,12 @@ func StartServer(serverType string, serverArgs []string, suffix, configFilename, } // StartProxy creates a reverse proxy -func StartProxy(bindAddr, syncAddr, clientAddr, mediaAddr string) (*exec.Cmd, chan error) { +func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) { proxyArgs := []string{ "--bind-address", bindAddr, - "--sync-api-server-url", syncAddr, - "--client-api-server-url", clientAddr, - "--media-api-server-url", mediaAddr, + "--sync-api-server-url", "http://" + string(cfg.Listen.SyncAPI), + "--client-api-server-url", "http://" + string(cfg.Listen.ClientAPI), + "--media-api-server-url", "http://" + string(cfg.Listen.MediaAPI), } return CreateBackgroundCommand( filepath.Join(filepath.Dir(os.Args[0]), "client-api-proxy"), diff --git a/src/github.com/matrix-org/dendrite/federationapi/config/config.go b/src/github.com/matrix-org/dendrite/federationapi/config/config.go deleted file mode 100644 index 0e2012ca6..000000000 --- a/src/github.com/matrix-org/dendrite/federationapi/config/config.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "github.com/matrix-org/gomatrixserverlib" - "golang.org/x/crypto/ed25519" - "time" -) - -// FederationAPI contains the config information necessary to spin up a federationapi process. -type FederationAPI struct { - // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. - ServerName gomatrixserverlib.ServerName - // The private key which will be used to sign requests. - PrivateKey ed25519.PrivateKey - // An arbitrary string used to uniquely identify the PrivateKey. Must start with the - // prefix "ed25519:". - KeyID gomatrixserverlib.KeyID - // A list of SHA256 TLS fingerprints for this server. - TLSFingerPrints []gomatrixserverlib.TLSFingerprint - // How long a remote server can cache our server key for before requesting it again. - // Increasing this number will reduce the number of requests made by remote servers - // for our key, but increases the period a compromised key will be considered valid - // by remote servers. - ValidityPeriod time.Duration -} diff --git a/src/github.com/matrix-org/dendrite/federationapi/readers/keys.go b/src/github.com/matrix-org/dendrite/federationapi/readers/keys.go index 572cfb52b..87c9f79bc 100644 --- a/src/github.com/matrix-org/dendrite/federationapi/readers/keys.go +++ b/src/github.com/matrix-org/dendrite/federationapi/readers/keys.go @@ -16,7 +16,7 @@ package readers import ( "encoding/json" - "github.com/matrix-org/dendrite/federationapi/config" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "golang.org/x/crypto/ed25519" @@ -26,29 +26,29 @@ import ( // LocalKeys returns the local keys for the server. // See https://matrix.org/docs/spec/server_server/unstable.html#publishing-keys -func LocalKeys(req *http.Request, cfg config.FederationAPI) util.JSONResponse { - keys, err := localKeys(cfg, time.Now().Add(cfg.ValidityPeriod)) +func LocalKeys(req *http.Request, cfg config.Dendrite) util.JSONResponse { + keys, err := localKeys(cfg, time.Now().Add(cfg.Matrix.KeyValidityPeriod)) if err != nil { return util.ErrorResponse(err) } return util.JSONResponse{Code: 200, JSON: keys} } -func localKeys(cfg config.FederationAPI, validUntil time.Time) (*gomatrixserverlib.ServerKeys, error) { +func localKeys(cfg config.Dendrite, validUntil time.Time) (*gomatrixserverlib.ServerKeys, error) { var keys gomatrixserverlib.ServerKeys - keys.ServerName = cfg.ServerName - keys.FromServer = cfg.ServerName + keys.ServerName = cfg.Matrix.ServerName + keys.FromServer = cfg.Matrix.ServerName - publicKey := cfg.PrivateKey.Public().(ed25519.PublicKey) + publicKey := cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey) keys.VerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.VerifyKey{ - cfg.KeyID: { + cfg.Matrix.KeyID: { gomatrixserverlib.Base64String(publicKey), }, } - keys.TLSFingerprints = cfg.TLSFingerPrints + keys.TLSFingerprints = cfg.Matrix.TLSFingerPrints keys.OldVerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.OldVerifyKey{} keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(validUntil) @@ -57,7 +57,9 @@ func localKeys(cfg config.FederationAPI, validUntil time.Time) (*gomatrixserverl return nil, err } - keys.Raw, err = gomatrixserverlib.SignJSON(string(cfg.ServerName), cfg.KeyID, cfg.PrivateKey, toSign) + keys.Raw, err = gomatrixserverlib.SignJSON( + string(cfg.Matrix.ServerName), cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, toSign, + ) if err != nil { return nil, err } diff --git a/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go b/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go index 0f20d0115..d09858ca9 100644 --- a/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go @@ -17,7 +17,7 @@ package routing import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/federationapi/config" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/federationapi/readers" "github.com/matrix-org/dendrite/federationapi/writers" "github.com/matrix-org/dendrite/roomserver/api" @@ -36,7 +36,7 @@ const ( // Setup registers HTTP handlers with the given ServeMux. func Setup( servMux *http.ServeMux, - cfg config.FederationAPI, + cfg config.Dendrite, query api.RoomserverQueryAPI, producer *producers.RoomserverProducer, keys gomatrixserverlib.KeyRing, diff --git a/src/github.com/matrix-org/dendrite/federationapi/writers/send.go b/src/github.com/matrix-org/dendrite/federationapi/writers/send.go index d799c94c9..cdaabb811 100644 --- a/src/github.com/matrix-org/dendrite/federationapi/writers/send.go +++ b/src/github.com/matrix-org/dendrite/federationapi/writers/send.go @@ -6,7 +6,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/federationapi/config" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -19,13 +19,13 @@ func Send( req *http.Request, txnID gomatrixserverlib.TransactionID, now time.Time, - cfg config.FederationAPI, + cfg config.Dendrite, query api.RoomserverQueryAPI, producer *producers.RoomserverProducer, keys gomatrixserverlib.KeyRing, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { - request, errResp := gomatrixserverlib.VerifyHTTPRequest(req, now, cfg.ServerName, keys) + request, errResp := gomatrixserverlib.VerifyHTTPRequest(req, now, cfg.Matrix.ServerName, keys) if request == nil { return errResp } @@ -45,7 +45,7 @@ func Send( t.Origin = request.Origin() t.TransactionID = txnID - t.Destination = cfg.ServerName + t.Destination = cfg.Matrix.ServerName resp, err := t.processTransaction() if err != nil { diff --git a/src/github.com/matrix-org/dendrite/mediaapi/config/config.go b/src/github.com/matrix-org/dendrite/mediaapi/config/config.go deleted file mode 100644 index 14c12e861..000000000 --- a/src/github.com/matrix-org/dendrite/mediaapi/config/config.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "github.com/matrix-org/dendrite/mediaapi/types" - "github.com/matrix-org/gomatrixserverlib" -) - -// MediaAPI contains the config information necessary to spin up a mediaapi process. -type MediaAPI struct { - // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. - ServerName gomatrixserverlib.ServerName `yaml:"server_name"` - // The base path to where the media files will be stored. May be relative or absolute. - BasePath types.Path `yaml:"base_path"` - // The absolute base path to where media files will be stored. - AbsBasePath types.Path `yaml:"-"` - // The maximum file size in bytes that is allowed to be stored on this server. - // Note: if MaxFileSizeBytes is set to 0, the size is unlimited. - // Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) - MaxFileSizeBytes *types.FileSizeBytes `yaml:"max_file_size_bytes,omitempty"` - // The postgres connection config for connecting to the database e.g a postgres:// URI - DataSource string `yaml:"database"` - // Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated - DynamicThumbnails bool `yaml:"dynamic_thumbnails"` - // The maximum number of simultaneous thumbnail generators. default: 10 - MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"` - // A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content - ThumbnailSizes []types.ThumbnailSize `yaml:"thumbnail_sizes"` -} diff --git a/src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go b/src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go index a166f4765..706afe0d5 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/fileutils/fileutils.go @@ -26,13 +26,14 @@ import ( "strings" log "github.com/Sirupsen/logrus" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/types" ) // GetPathFromBase64Hash evaluates the path to a media file from its Base64Hash // 3 subdirectories are created for more manageable browsing and use the remainder as the file name. // For example, if Base64Hash is 'qwerty', the path will be 'q/w/erty/file'. -func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath types.Path) (string, error) { +func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath config.Path) (string, error) { if len(base64Hash) < 3 { return "", fmt.Errorf("Invalid filePath (Base64Hash too short - min 3 characters): %q", base64Hash) } @@ -66,7 +67,7 @@ func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath types.Path) // If the final path exists and the file size matches, the file does not need to be moved. // In error cases where the file is not a duplicate, the caller may decide to remove the final path. // Returns the final path of the file, whether it is a duplicate and an error. -func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata, absBasePath types.Path, logger *log.Entry) (types.Path, bool, error) { +func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata, absBasePath config.Path, logger *log.Entry) (types.Path, bool, error) { // Note: in all error and success cases, we need to remove the temporary directory defer RemoveDir(tmpDir, logger) duplicate := false @@ -104,7 +105,7 @@ func RemoveDir(dir types.Path, logger *log.Entry) { } // WriteTempFile writes to a new temporary file -func WriteTempFile(reqReader io.Reader, maxFileSizeBytes types.FileSizeBytes, absBasePath types.Path) (types.Base64Hash, types.FileSizeBytes, types.Path, error) { +func WriteTempFile(reqReader io.Reader, maxFileSizeBytes config.FileSizeBytes, absBasePath config.Path) (types.Base64Hash, types.FileSizeBytes, types.Path, error) { tmpFileWriter, tmpFile, tmpDir, err := createTempFileWriter(absBasePath) if err != nil { return "", -1, "", err @@ -144,7 +145,7 @@ func moveFile(src types.Path, dst types.Path) error { return nil } -func createTempFileWriter(absBasePath types.Path) (*bufio.Writer, *os.File, types.Path, error) { +func createTempFileWriter(absBasePath config.Path) (*bufio.Writer, *os.File, types.Path, error) { tmpDir, err := createTempDir(absBasePath) if err != nil { return nil, nil, "", fmt.Errorf("Failed to create temp dir: %q", err) @@ -157,7 +158,7 @@ func createTempFileWriter(absBasePath types.Path) (*bufio.Writer, *os.File, type } // createTempDir creates a tmp/ directory within baseDirectory and returns its path -func createTempDir(baseDirectory types.Path) (types.Path, error) { +func createTempDir(baseDirectory config.Path) (types.Path, error) { baseTmpDir := filepath.Join(string(baseDirectory), "tmp") if err := os.MkdirAll(baseTmpDir, 0770); err != nil { return "", fmt.Errorf("Failed to create base temp dir: %v", err) diff --git a/src/github.com/matrix-org/dendrite/mediaapi/routing/routing.go b/src/github.com/matrix-org/dendrite/mediaapi/routing/routing.go index 47985fc16..1241bd0ee 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/routing/routing.go @@ -19,7 +19,7 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/mediaapi/config" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/dendrite/mediaapi/writers" @@ -32,7 +32,7 @@ const pathPrefixR0 = "/_matrix/media/v1" // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. -func Setup(servMux *http.ServeMux, httpClient *http.Client, cfg *config.MediaAPI, db *storage.Database) { +func Setup(servMux *http.ServeMux, httpClient *http.Client, cfg *config.Dendrite, db *storage.Database) { apiMux := mux.NewRouter() r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() @@ -59,7 +59,7 @@ func Setup(servMux *http.ServeMux, httpClient *http.Client, cfg *config.MediaAPI servMux.Handle("/api/", http.StripPrefix("/api", apiMux)) } -func makeDownloadAPI(name string, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) http.HandlerFunc { +func makeDownloadAPI(name string, cfg *config.Dendrite, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) http.HandlerFunc { return prometheus.InstrumentHandler(name, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { req = util.RequestWithLogging(req) diff --git a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go index 16268d13b..ded71b651 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer.go @@ -22,6 +22,7 @@ import ( "sync" log "github.com/Sirupsen/logrus" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" ) @@ -56,7 +57,7 @@ func GetThumbnailPath(src types.Path, config types.ThumbnailSize) types.Path { // * has a small file size // If a pre-generated thumbnail size is the best match, but it has not been generated yet, the caller can use the returned size to generate it. // Returns nil if no thumbnail matches the criteria -func SelectThumbnail(desired types.ThumbnailSize, thumbnails []*types.ThumbnailMetadata, thumbnailSizes []types.ThumbnailSize) (*types.ThumbnailMetadata, *types.ThumbnailSize) { +func SelectThumbnail(desired types.ThumbnailSize, thumbnails []*types.ThumbnailMetadata, thumbnailSizes []config.ThumbnailSize) (*types.ThumbnailMetadata, *types.ThumbnailSize) { var chosenThumbnail *types.ThumbnailMetadata var chosenThumbnailSize *types.ThumbnailSize bestFit := newThumbnailFitness() @@ -76,7 +77,7 @@ func SelectThumbnail(desired types.ThumbnailSize, thumbnails []*types.ThumbnailM if desired.ResizeMethod == "scale" && thumbnailSize.ResizeMethod != "scale" { continue } - fitness := calcThumbnailFitness(thumbnailSize, nil, desired) + fitness := calcThumbnailFitness(types.ThumbnailSize(thumbnailSize), nil, desired) if isBetter := fitness.betterThan(bestFit, desired.ResizeMethod == "crop"); isBetter { bestFit = fitness chosenThumbnailSize = &types.ThumbnailSize{ diff --git a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go index 41edce7cb..ad3b8b00d 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_bimg.go @@ -21,13 +21,14 @@ import ( "time" log "github.com/Sirupsen/logrus" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" "gopkg.in/h2non/bimg.v1" ) // GenerateThumbnails generates the configured thumbnail sizes for the source file -func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) { +func GenerateThumbnails(src types.Path, configs []config.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) { buffer, err := bimg.Read(string(src)) if err != nil { logger.WithError(err).WithField("src", src).Error("Failed to read src file") diff --git a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go index 0b75ea52a..3e822dd02 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/thumbnailer/thumbnailer_nfnt.go @@ -28,13 +28,14 @@ import ( "time" log "github.com/Sirupsen/logrus" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/nfnt/resize" ) // GenerateThumbnails generates the configured thumbnail sizes for the source file -func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) { +func GenerateThumbnails(src types.Path, configs []config.ThumbnailSize, mediaMetadata *types.MediaMetadata, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, logger *log.Entry) (busy bool, errorReturn error) { img, err := readFile(string(src)) if err != nil { logger.WithError(err).WithField("src", src).Error("Failed to read src file") @@ -42,7 +43,7 @@ func GenerateThumbnails(src types.Path, configs []types.ThumbnailSize, mediaMeta } for _, config := range configs { // Note: createThumbnail does locking based on activeThumbnailGeneration - busy, err = createThumbnail(src, img, config, mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger) + busy, err = createThumbnail(src, img, types.ThumbnailSize(config), mediaMetadata, activeThumbnailGeneration, maxThumbnailGenerators, db, logger) if err != nil { logger.WithError(err).WithField("src", src).Error("Failed to generate thumbnails") return false, err diff --git a/src/github.com/matrix-org/dendrite/mediaapi/types/types.go b/src/github.com/matrix-org/dendrite/mediaapi/types/types.go index 3a0888f88..26b09e4f5 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/types/types.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/types/types.go @@ -17,6 +17,7 @@ package types import ( "sync" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -79,16 +80,7 @@ type ActiveRemoteRequests struct { } // ThumbnailSize contains a single thumbnail size configuration -type ThumbnailSize struct { - // Maximum width of the thumbnail image - Width int `yaml:"width"` - // Maximum height of the thumbnail image - Height int `yaml:"height"` - // ResizeMethod is one of crop or scale. - // crop scales to fill the requested dimensions and crops the excess. - // scale scales to fit the requested dimensions and one dimension may be smaller than requested. - ResizeMethod string `yaml:"method,omitempty"` -} +type ThumbnailSize config.ThumbnailSize // ThumbnailMetadata contains the metadata about an individual thumbnail type ThumbnailMetadata struct { diff --git a/src/github.com/matrix-org/dendrite/mediaapi/writers/download.go b/src/github.com/matrix-org/dendrite/mediaapi/writers/download.go index 90f6ca35b..8066327ca 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/writers/download.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/writers/download.go @@ -29,7 +29,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/mediaapi/config" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/fileutils" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/thumbnailer" @@ -59,7 +59,7 @@ type downloadRequest struct { // If they are present in the cache, they are served directly. // If they are not present in the cache, they are obtained from the remote server and // simultaneously served back to the client and written into the cache. -func Download(w http.ResponseWriter, req *http.Request, origin gomatrixserverlib.ServerName, mediaID types.MediaID, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, isThumbnailRequest bool) { +func Download(w http.ResponseWriter, req *http.Request, origin gomatrixserverlib.ServerName, mediaID types.MediaID, cfg *config.Dendrite, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, isThumbnailRequest bool) { r := &downloadRequest{ MediaMetadata: &types.MediaMetadata{ MediaID: mediaID, @@ -167,7 +167,7 @@ func (r *downloadRequest) Validate() *util.JSONResponse { return nil } -func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse { +func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.Dendrite, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse { // check if we have a record of the media in our database mediaMetadata, err := db.GetMediaMetadata(r.MediaMetadata.MediaID, r.MediaMetadata.Origin) if err != nil { @@ -176,7 +176,7 @@ func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI return &resErr } if mediaMetadata == nil { - if r.MediaMetadata.Origin == cfg.ServerName { + if r.MediaMetadata.Origin == cfg.Matrix.ServerName { // If we do not have a record and the origin is local, the file is not found return &util.JSONResponse{ Code: 404, @@ -192,12 +192,12 @@ func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI // If we have a record, we can respond from the local file r.MediaMetadata = mediaMetadata } - return r.respondFromLocalFile(w, cfg.AbsBasePath, activeThumbnailGeneration, cfg.MaxThumbnailGenerators, db, cfg.DynamicThumbnails, cfg.ThumbnailSizes) + return r.respondFromLocalFile(w, cfg.Media.AbsBasePath, activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators, db, cfg.Media.DynamicThumbnails, cfg.Media.ThumbnailSizes) } // respondFromLocalFile reads a file from local storage and writes it to the http.ResponseWriter // Returns a util.JSONResponse error in case of error -func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []types.ThumbnailSize) *util.JSONResponse { +func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePath config.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []config.ThumbnailSize) *util.JSONResponse { filePath, err := fileutils.GetPathFromBase64Hash(r.MediaMetadata.Base64Hash, absBasePath) if err != nil { r.Logger.WithError(err).Error("Failed to get file path from metadata") @@ -284,7 +284,7 @@ func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePat } // Note: Thumbnail generation may be ongoing asynchronously. -func (r *downloadRequest) getThumbnailFile(filePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []types.ThumbnailSize) (*os.File, *types.ThumbnailMetadata, *util.JSONResponse) { +func (r *downloadRequest) getThumbnailFile(filePath types.Path, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int, db *storage.Database, dynamicThumbnails bool, thumbnailSizes []config.ThumbnailSize) (*os.File, *types.ThumbnailMetadata, *util.JSONResponse) { var thumbnail *types.ThumbnailMetadata var resErr *util.JSONResponse @@ -383,7 +383,7 @@ func (r *downloadRequest) generateThumbnail(filePath types.Path, thumbnailSize t // regardless of how many download requests are received. // Note: The named errorResponse return variable is used in a deferred broadcast of the metadata and error response to waiting goroutines. // Returns a util.JSONResponse error in case of error -func (r *downloadRequest) getRemoteFile(cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) (errorResponse *util.JSONResponse) { +func (r *downloadRequest) getRemoteFile(cfg *config.Dendrite, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration) (errorResponse *util.JSONResponse) { // Note: getMediaMetadataFromActiveRequest uses mutexes and conditions from activeRemoteRequests mediaMetadata, resErr := r.getMediaMetadataFromActiveRequest(activeRemoteRequests) if resErr != nil { @@ -414,7 +414,7 @@ func (r *downloadRequest) getRemoteFile(cfg *config.MediaAPI, db *storage.Databa if mediaMetadata == nil { // If we do not have a record, we need to fetch the remote file first and then respond from the local file - resErr := r.fetchRemoteFileAndStoreMetadata(cfg.AbsBasePath, *cfg.MaxFileSizeBytes, db, cfg.ThumbnailSizes, activeThumbnailGeneration, cfg.MaxThumbnailGenerators) + resErr := r.fetchRemoteFileAndStoreMetadata(cfg.Media.AbsBasePath, *cfg.Media.MaxFileSizeBytes, db, cfg.Media.ThumbnailSizes, activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators) if resErr != nil { return resErr } @@ -476,7 +476,7 @@ func (r *downloadRequest) broadcastMediaMetadata(activeRemoteRequests *types.Act } // fetchRemoteFileAndStoreMetadata fetches the file from the remote server and stores its metadata in the database -func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath types.Path, maxFileSizeBytes types.FileSizeBytes, db *storage.Database, thumbnailSizes []types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse { +func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath config.Path, maxFileSizeBytes config.FileSizeBytes, db *storage.Database, thumbnailSizes []config.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse { finalPath, duplicate, resErr := r.fetchRemoteFile(absBasePath, maxFileSizeBytes) if resErr != nil { return resErr @@ -524,7 +524,7 @@ func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath types.Path return nil } -func (r *downloadRequest) fetchRemoteFile(absBasePath types.Path, maxFileSizeBytes types.FileSizeBytes) (types.Path, bool, *util.JSONResponse) { +func (r *downloadRequest) fetchRemoteFile(absBasePath config.Path, maxFileSizeBytes config.FileSizeBytes) (types.Path, bool, *util.JSONResponse) { r.Logger.Info("Fetching remote file") // create request for remote file diff --git a/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go b/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go index 7071b4179..163ea7b68 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/writers/upload.go @@ -24,7 +24,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/mediaapi/config" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/mediaapi/fileutils" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/thumbnailer" @@ -51,7 +51,7 @@ type uploadResponse struct { // This implementation supports a configurable maximum file size limit in bytes. If a user tries to upload more than this, they will receive an error that their upload is too large. // Uploaded files are processed piece-wise to avoid DoS attacks which would starve the server of memory. // TODO: We should time out requests if they have not received any data within a configured timeout period. -func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse { +func Upload(req *http.Request, cfg *config.Dendrite, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) util.JSONResponse { r, resErr := parseAndValidateRequest(req, cfg) if resErr != nil { return *resErr @@ -64,7 +64,7 @@ func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database, activ return util.JSONResponse{ Code: 200, JSON: uploadResponse{ - ContentURI: fmt.Sprintf("mxc://%s/%s", cfg.ServerName, r.MediaMetadata.MediaID), + ContentURI: fmt.Sprintf("mxc://%s/%s", cfg.Matrix.ServerName, r.MediaMetadata.MediaID), }, } } @@ -72,7 +72,7 @@ func Upload(req *http.Request, cfg *config.MediaAPI, db *storage.Database, activ // parseAndValidateRequest parses the incoming upload request to validate and extract // all the metadata about the media being uploaded. // Returns either an uploadRequest or an error formatted as a util.JSONResponse -func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRequest, *util.JSONResponse) { +func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRequest, *util.JSONResponse) { if req.Method != "POST" { return nil, &util.JSONResponse{ Code: 405, @@ -82,22 +82,22 @@ func parseAndValidateRequest(req *http.Request, cfg *config.MediaAPI) (*uploadRe r := &uploadRequest{ MediaMetadata: &types.MediaMetadata{ - Origin: cfg.ServerName, + Origin: cfg.Matrix.ServerName, FileSizeBytes: types.FileSizeBytes(req.ContentLength), ContentType: types.ContentType(req.Header.Get("Content-Type")), UploadName: types.Filename(url.PathEscape(req.FormValue("filename"))), }, - Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.ServerName), + Logger: util.GetLogger(req.Context()).WithField("Origin", cfg.Matrix.ServerName), } - if resErr := r.Validate(*cfg.MaxFileSizeBytes); resErr != nil { + if resErr := r.Validate(*cfg.Media.MaxFileSizeBytes); resErr != nil { return nil, resErr } return r, nil } -func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse { +func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.Dendrite, db *storage.Database, activeThumbnailGeneration *types.ActiveThumbnailGeneration) *util.JSONResponse { r.Logger.WithFields(log.Fields{ "UploadName": r.MediaMetadata.UploadName, "FileSizeBytes": r.MediaMetadata.FileSizeBytes, @@ -108,10 +108,10 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db * // method of deduplicating files to save storage, as well as a way to conduct // integrity checks on the file data in the repository. // Data is truncated to maxFileSizeBytes. Content-Length was reported as 0 < Content-Length <= maxFileSizeBytes so this is OK. - hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.MaxFileSizeBytes, cfg.AbsBasePath) + hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(reqReader, *cfg.Media.MaxFileSizeBytes, cfg.Media.AbsBasePath) if err != nil { r.Logger.WithError(err).WithFields(log.Fields{ - "MaxFileSizeBytes": *cfg.MaxFileSizeBytes, + "MaxFileSizeBytes": *cfg.Media.MaxFileSizeBytes, }).Warn("Error while transferring file") fileutils.RemoveDir(tmpDir, r.Logger) return &util.JSONResponse{ @@ -147,12 +147,12 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db * return &util.JSONResponse{ Code: 200, JSON: uploadResponse{ - ContentURI: fmt.Sprintf("mxc://%s/%s", cfg.ServerName, r.MediaMetadata.MediaID), + ContentURI: fmt.Sprintf("mxc://%s/%s", cfg.Matrix.ServerName, r.MediaMetadata.MediaID), }, } } - if resErr := r.storeFileAndMetadata(tmpDir, cfg.AbsBasePath, db, cfg.ThumbnailSizes, activeThumbnailGeneration, cfg.MaxThumbnailGenerators); resErr != nil { + if resErr := r.storeFileAndMetadata(tmpDir, cfg.Media.AbsBasePath, db, cfg.Media.ThumbnailSizes, activeThumbnailGeneration, cfg.Media.MaxThumbnailGenerators); resErr != nil { return resErr } @@ -160,14 +160,14 @@ func (r *uploadRequest) doUpload(reqReader io.Reader, cfg *config.MediaAPI, db * } // Validate validates the uploadRequest fields -func (r *uploadRequest) Validate(maxFileSizeBytes types.FileSizeBytes) *util.JSONResponse { +func (r *uploadRequest) Validate(maxFileSizeBytes config.FileSizeBytes) *util.JSONResponse { if r.MediaMetadata.FileSizeBytes < 1 { return &util.JSONResponse{ Code: 411, JSON: jsonerror.Unknown("HTTP Content-Length request header must be greater than zero."), } } - if maxFileSizeBytes > 0 && r.MediaMetadata.FileSizeBytes > maxFileSizeBytes { + if maxFileSizeBytes > 0 && r.MediaMetadata.FileSizeBytes > types.FileSizeBytes(maxFileSizeBytes) { return &util.JSONResponse{ Code: 413, JSON: jsonerror.Unknown(fmt.Sprintf("HTTP Content-Length is greater than the maximum allowed upload size (%v).", maxFileSizeBytes)), @@ -215,7 +215,7 @@ func (r *uploadRequest) Validate(maxFileSizeBytes types.FileSizeBytes) *util.JSO // The order of operations is important as it avoids metadata entering the database before the file // is ready, and if we fail to move the file, it never gets added to the database. // Returns a util.JSONResponse error and cleans up directories in case of error. -func (r *uploadRequest) storeFileAndMetadata(tmpDir types.Path, absBasePath types.Path, db *storage.Database, thumbnailSizes []types.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse { +func (r *uploadRequest) storeFileAndMetadata(tmpDir types.Path, absBasePath config.Path, db *storage.Database, thumbnailSizes []config.ThumbnailSize, activeThumbnailGeneration *types.ActiveThumbnailGeneration, maxThumbnailGenerators int) *util.JSONResponse { finalPath, duplicate, err := fileutils.MoveFileWithHashCheck(tmpDir, r.MediaMetadata, absBasePath, r.Logger) if err != nil { r.Logger.WithError(err).Error("Failed to move file.") diff --git a/src/github.com/matrix-org/dendrite/syncapi/config/config.go b/src/github.com/matrix-org/dendrite/syncapi/config/config.go deleted file mode 100644 index 32f047229..000000000 --- a/src/github.com/matrix-org/dendrite/syncapi/config/config.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "github.com/matrix-org/gomatrixserverlib" -) - -// Sync contains the config information necessary to spin up a sync-server process. -type Sync struct { - // Where the room server is listening for queries. - RoomserverURL string `yaml:"roomserver_url"` - // The topic for events which are written by the room server output log. - RoomserverOutputTopic string `yaml:"roomserver_topic"` - // A list of URIs to consume events from. These kafka logs should be produced by a Room Server. - KafkaConsumerURIs []string `yaml:"consumer_uris"` - // The postgres connection config for connecting to the database e.g a postgres:// URI - DataSource string `yaml:"database"` - // The server_name of the running process e.g "localhost" - ServerName gomatrixserverlib.ServerName `yaml:"server_name"` -} diff --git a/src/github.com/matrix-org/dendrite/syncapi/consumers/roomserver.go b/src/github.com/matrix-org/dendrite/syncapi/consumers/roomserver.go index 31c48ac57..e5a971c85 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/consumers/roomserver.go +++ b/src/github.com/matrix-org/dendrite/syncapi/consumers/roomserver.go @@ -20,8 +20,8 @@ import ( log "github.com/Sirupsen/logrus" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/syncapi/config" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" "github.com/matrix-org/dendrite/syncapi/types" @@ -38,14 +38,15 @@ type OutputRoomEvent struct { } // NewOutputRoomEvent creates a new OutputRoomEvent consumer. Call Start() to begin consuming from room servers. -func NewOutputRoomEvent(cfg *config.Sync, n *sync.Notifier, store *storage.SyncServerDatabase) (*OutputRoomEvent, error) { - kafkaConsumer, err := sarama.NewConsumer(cfg.KafkaConsumerURIs, nil) +func NewOutputRoomEvent(cfg *config.Dendrite, n *sync.Notifier, store *storage.SyncServerDatabase) (*OutputRoomEvent, error) { + kafkaConsumer, err := sarama.NewConsumer(cfg.Kafka.Addresses, nil) if err != nil { return nil, err } + roomServerURL := cfg.RoomServerURL() consumer := common.ContinualConsumer{ - Topic: cfg.RoomserverOutputTopic, + Topic: string(cfg.Kafka.Topics.OutputRoomEvent), Consumer: kafkaConsumer, PartitionStore: store, } @@ -53,7 +54,7 @@ func NewOutputRoomEvent(cfg *config.Sync, n *sync.Notifier, store *storage.SyncS roomServerConsumer: &consumer, db: store, notifier: n, - query: api.NewRoomserverQueryAPIHTTP(cfg.RoomserverURL, nil), + query: api.NewRoomserverQueryAPIHTTP(roomServerURL, nil), } consumer.ProcessMessage = s.onMessage diff --git a/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go b/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go index ad09cd172..0b65a08ac 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/syncapi/routing/routing.go @@ -21,7 +21,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/syncapi/config" "github.com/matrix-org/dendrite/syncapi/sync" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" @@ -30,7 +29,7 @@ import ( const pathPrefixR0 = "/_matrix/client/r0" // SetupSyncServerListeners configures the given mux with sync-server listeners -func SetupSyncServerListeners(servMux *http.ServeMux, httpClient *http.Client, cfg config.Sync, srp *sync.RequestPool, deviceDB *devices.Database) { +func SetupSyncServerListeners(servMux *http.ServeMux, httpClient *http.Client, srp *sync.RequestPool, deviceDB *devices.Database) { apiMux := mux.NewRouter() r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() r0mux.Handle("/sync", common.MakeAuthAPI("sync", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { From cdd1fdcd3d791cfb965506ac28d96cf111043085 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 25 Jun 2017 01:06:07 +0100 Subject: [PATCH 4/5] trivial typo in usage output --- src/github.com/matrix-org/dendrite/cmd/client-api-proxy/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/github.com/matrix-org/dendrite/cmd/client-api-proxy/main.go b/src/github.com/matrix-org/dendrite/cmd/client-api-proxy/main.go index 8aab6575d..f2b64d44a 100644 --- a/src/github.com/matrix-org/dendrite/cmd/client-api-proxy/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/client-api-proxy/main.go @@ -106,7 +106,7 @@ func main() { if *syncServerURL == "" { flag.Usage() - fmt.Fprintln(os.Stderr, "no --sync-server-url specified.") + fmt.Fprintln(os.Stderr, "no --sync-api-server-url specified.") os.Exit(1) } From eb029116b0a0bf15523b43a3eacf99d339546962 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 25 Jun 2017 01:20:04 +0100 Subject: [PATCH 5/5] trivial fix to actually parse fed-api-server's CLI args --- .../dendrite/cmd/dendrite-federation-api-server/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go index f9576ce90..f4f19cdcd 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go @@ -38,6 +38,8 @@ var ( func main() { common.SetupLogging(logDir) + flag.Parse() + if *configPath == "" { log.Fatal("--config must be supplied") }