From c6e59ca9679c4da50b59565f0f46c2bbe9d835aa Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 16 Jun 2017 16:52:03 +0100 Subject: [PATCH] Add a single config file format for dendrite. (#141) * Add a single config file format for dendrite. This config file combines all the configuration needed by the dendrite components into a single file. * Resolve relative paths against the current working directory * Review comments * more review comments! * Document what the purpose of the version field is --- .../dendrite/common/config/config.go | 360 ++++++++++++++++++ .../dendrite/common/config/config_test.go | 134 +++++++ 2 files changed, 494 insertions(+) create mode 100644 src/github.com/matrix-org/dendrite/common/config/config.go create mode 100644 src/github.com/matrix-org/dendrite/common/config/config_test.go diff --git a/src/github.com/matrix-org/dendrite/common/config/config.go b/src/github.com/matrix-org/dendrite/common/config/config.go new file mode 100644 index 000000000..e06f894a9 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/common/config/config.go @@ -0,0 +1,360 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "bytes" + "crypto/sha256" + "encoding/pem" + "fmt" + "github.com/matrix-org/gomatrixserverlib" + "golang.org/x/crypto/ed25519" + "gopkg.in/yaml.v2" + "io/ioutil" + "path/filepath" + "strings" + "time" +) + +// Version is the current version of the config format. +// This will change whenever we make breaking changes to the config format. +const Version = "v0" + +// Dendrite contains all the config used by a dendrite process. +// Relative paths are resolved relative to the current working directory +type Dendrite struct { + // The version of the configuration file. + // If the version in a file doesn't match the current dendrite config + // version then we can give a clear error message telling the user + // to update their config file to the current version. + // The version of the file should only be different if there has + // been a breaking change to the config file format. + Version string `yaml:"version"` + + // The configuration required for a matrix server. + Matrix struct { + // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. + ServerName gomatrixserverlib.ServerName `yaml:"server_name"` + // Path to the private key which will be used to sign requests and events. + PrivateKeyPath Path `yaml:"private_key"` + // The private key which will be used to sign requests and events. + PrivateKey ed25519.PrivateKey `yaml:"-"` + // An arbitrary string used to uniquely identify the PrivateKey. Must start with the + // prefix "ed25519:". + KeyID gomatrixserverlib.KeyID `yaml:"-"` + // List of paths to X509 certificates used by the external federation listeners. + // These are used to calculate the TLS fingerprints to publish for this server. + // Other matrix servers talking to this server will expect the x509 certificate + // to match one of these certificates. + // The certificates should be in PEM format. + FederationCertificatePaths []Path `yaml:"federation_certificates"` + // A list of SHA256 TLS fingerprints for the X509 certificates used by the + // federation listener for this server. + TLSFingerPrints []gomatrixserverlib.TLSFingerprint `yaml:"-"` + // How long a remote server can cache our server key for before requesting it again. + // Increasing this number will reduce the number of requests made by remote servers + // for our key, but increases the period a compromised key will be considered valid + // by remote servers. + // Defaults to 24 hours. + KeyValidityPeriod time.Duration `yaml:"key_validity_period"` + } `yaml:"matrix"` + + // The configuration specific to the media repostitory. + Media struct { + // The base path to where the media files will be stored. May be relative or absolute. + BasePath Path `yaml:"base_path"` + // The absolute base path to where media files will be stored. + AbsBasePath Path `yaml:"-"` + // The maximum file size in bytes that is allowed to be stored on this server. + // Note: if max_file_size_bytes is set to 0, the size is unlimited. + // Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) + MaxFileSizeBytes *FileSizeBytes `yaml:"max_file_size_bytes,omitempty"` + // Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated + DynamicThumbnails bool `yaml:"dynamic_thumbnails"` + // The maximum number of simultaneous thumbnail generators. default: 10 + MaxThumbnailGenerators int `yaml:"max_thumbnail_generators"` + // A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content + ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"` + } `yaml:"media"` + + // The configuration for talking to kafka. + Kafka struct { + // A list of kafka addresses to connect to. + Addresses []string `yaml:"addresses"` + // The names of the topics to use when reading and writing from kafka. + Topics struct { + // Topic for roomserver/api.InputRoomEvent events. + InputRoomEvent Topic `yaml:"input_room_event"` + // Topic for roomserver/api.OutputRoomEvent events. + OutputRoomEvent Topic `yaml:"output_room_event"` + } + } `yaml:"kafka"` + + // Postgres Config + Database struct { + // The Account database stores the login details and account information + // for local users. It is accessed by the ClientAPI. + Account DataSource `yaml:"account"` + // The Device database stores session information for the devices of logged + // in local users. It is accessed by the ClientAPI, the MediaAPI and the SyncAPI. + Device DataSource `yaml:"device"` + // The MediaAPI database stores information about files uploaded and downloaded + // by local users. It is only accessed by the MediaAPI. + MediaAPI DataSource `yaml:"media_api"` + // The ServerKey database caches the public keys of remote servers. + // It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI. + ServerKey DataSource `yaml:"server_key"` + // The SyncAPI stores information used by the SyncAPI server. + // It is only accessed by the SyncAPI server. + SyncAPI DataSource `yaml:"sync_api"` + // The RoomServer database stores information about matrix rooms. + // It is only accessed by the RoomServer. + RoomServer DataSource `yaml:"room_server"` + } `yaml:"database"` + + // The internal addresses the components will listen on. + // These should not be exposed externally as they expose metrics and debugging APIs. + Listen struct { + MediaAPI Address `yaml:"media_api"` + ClientAPI Address `yaml:"client_api"` + FederationAPI Address `yaml:"federation_api"` + SyncAPI Address `yaml:"sync_api"` + RoomServer Address `yaml:"room_server"` + } `yaml:"listen"` +} + +// A Path on the filesystem. +type Path string + +// A DataSource for opening a postgresql database using lib/pq. +type DataSource string + +// A Topic in kafka. +type Topic string + +// An Address to listen on. +type Address string + +// FileSizeBytes is a file size in bytes +type FileSizeBytes int64 + +// ThumbnailSize contains a single thumbnail size configuration +type ThumbnailSize struct { + // Maximum width of the thumbnail image + Width int `yaml:"width"` + // Maximum height of the thumbnail image + Height int `yaml:"height"` + // ResizeMethod is one of crop or scale. + // crop scales to fill the requested dimensions and crops the excess. + // scale scales to fit the requested dimensions and one dimension may be smaller than requested. + ResizeMethod string `yaml:"method,omitempty"` +} + +// Load a yaml config file +func Load(configPath string) (*Dendrite, error) { + configData, err := ioutil.ReadFile(configPath) + if err != nil { + return nil, err + } + basePath, err := filepath.Abs(".") + if err != nil { + return nil, err + } + // Pass the current working directory and ioutil.ReadFile so that they can + // be mocked in the tests + return loadConfig(basePath, configData, ioutil.ReadFile) +} + +// An Error indicates a problem parsing the config. +type Error struct { + // List of problems encountered parsing the config. + Problems []string +} + +func loadConfig( + basePath string, + configData []byte, + readFile func(string) ([]byte, error), +) (*Dendrite, error) { + var config Dendrite + var err error + if err = yaml.Unmarshal(configData, &config); err != nil { + return nil, err + } + + config.setDefaults() + + if err = config.check(); err != nil { + return nil, err + } + + privateKeyPath := absPath(basePath, config.Matrix.PrivateKeyPath) + privateKeyData, err := readFile(privateKeyPath) + if err != nil { + return nil, err + } + + if config.Matrix.KeyID, config.Matrix.PrivateKey, err = readKeyPEM(privateKeyPath, privateKeyData); err != nil { + return nil, err + } + + for _, certPath := range config.Matrix.FederationCertificatePaths { + absCertPath := absPath(basePath, certPath) + pemData, err := readFile(absCertPath) + if err != nil { + return nil, err + } + fingerprint := fingerprintPEM(pemData) + if fingerprint == nil { + return nil, fmt.Errorf("no certificate PEM data in %q", absCertPath) + } + config.Matrix.TLSFingerPrints = append(config.Matrix.TLSFingerPrints, *fingerprint) + } + + config.Media.AbsBasePath = Path(absPath(basePath, config.Media.BasePath)) + + return &config, nil +} + +func (config *Dendrite) setDefaults() { + if config.Matrix.KeyValidityPeriod == 0 { + config.Matrix.KeyValidityPeriod = 24 * time.Hour + } + + if config.Media.MaxThumbnailGenerators == 0 { + config.Media.MaxThumbnailGenerators = 10 + } + + if config.Media.MaxFileSizeBytes == nil { + defaultMaxFileSizeBytes := FileSizeBytes(10485760) + config.Media.MaxFileSizeBytes = &defaultMaxFileSizeBytes + } +} + +func (e Error) Error() string { + if len(e.Problems) == 1 { + return e.Problems[0] + } + return fmt.Sprintf( + "%s (and %d other problems)", e.Problems[0], len(e.Problems)-1, + ) +} + +func (config *Dendrite) check() error { + var problems []string + + if config.Version != Version { + return Error{[]string{fmt.Sprintf( + "unknown config version %q, expected %q", config.Version, Version, + )}} + } + + checkNotEmpty := func(key string, value string) { + if value == "" { + problems = append(problems, fmt.Sprintf("missing config key %q", key)) + } + } + + checkNotZero := func(key string, value int64) { + if value == 0 { + problems = append(problems, fmt.Sprintf("missing config key %q", key)) + } + } + + checkPositive := func(key string, value int64) { + if value < 0 { + problems = append(problems, fmt.Sprintf("invalid value for config key %q: %d", key, value)) + } + } + + checkNotEmpty("matrix.server_name", string(config.Matrix.ServerName)) + checkNotEmpty("matrix.private_key", string(config.Matrix.PrivateKeyPath)) + checkNotZero("matrix.federation_certificates", int64(len(config.Matrix.FederationCertificatePaths))) + + checkNotEmpty("media.base_path", string(config.Media.BasePath)) + checkPositive("media.max_file_size_bytes", int64(*config.Media.MaxFileSizeBytes)) + checkPositive("media.max_thumbnail_generators", int64(config.Media.MaxThumbnailGenerators)) + for i, size := range config.Media.ThumbnailSizes { + checkPositive(fmt.Sprintf("media.thumbnail_sizes[%d].width", i), int64(size.Width)) + checkPositive(fmt.Sprintf("media.thumbnail_sizes[%d].height", i), int64(size.Height)) + } + + checkNotZero("kafka.addresses", int64(len(config.Kafka.Addresses))) + checkNotEmpty("kafka.topics.input_room_event", string(config.Kafka.Topics.InputRoomEvent)) + checkNotEmpty("kafka.topics.output_room_event", string(config.Kafka.Topics.OutputRoomEvent)) + checkNotEmpty("database.account", string(config.Database.Account)) + checkNotEmpty("database.device", string(config.Database.Device)) + checkNotEmpty("database.server_key", string(config.Database.ServerKey)) + checkNotEmpty("database.media_api", string(config.Database.MediaAPI)) + checkNotEmpty("database.sync_api", string(config.Database.SyncAPI)) + checkNotEmpty("database.room_server", string(config.Database.RoomServer)) + checkNotEmpty("listen.media_api", string(config.Listen.MediaAPI)) + checkNotEmpty("listen.client_api", string(config.Listen.ClientAPI)) + checkNotEmpty("listen.federation_api", string(config.Listen.FederationAPI)) + checkNotEmpty("listen.sync_api", string(config.Listen.SyncAPI)) + checkNotEmpty("listen.room_server", string(config.Listen.RoomServer)) + + if problems != nil { + return Error{problems} + } + + return nil +} + +func absPath(dir string, path Path) string { + if filepath.IsAbs(string(path)) { + // filepath.Join cleans the path so we should clean the absolute paths as well for consistency. + return filepath.Clean(string(path)) + } + return filepath.Join(dir, string(path)) +} + +func readKeyPEM(path string, data []byte) (gomatrixserverlib.KeyID, ed25519.PrivateKey, error) { + for { + var keyBlock *pem.Block + keyBlock, data = pem.Decode(data) + if data == nil { + return "", nil, fmt.Errorf("no matrix private key PEM data in %q", path) + } + if keyBlock.Type == "MATRIX PRIVATE KEY" { + keyID := keyBlock.Headers["Key-ID"] + if keyID == "" { + return "", nil, fmt.Errorf("missing key ID in PEM data in %q", path) + } + if !strings.HasPrefix(keyID, "ed25519:") { + return "", nil, fmt.Errorf("key ID %q doesn't start with \"ed25519:\" in %q", keyID, path) + } + _, privKey, err := ed25519.GenerateKey(bytes.NewReader(keyBlock.Bytes)) + if err != nil { + return "", nil, err + } + return gomatrixserverlib.KeyID(keyID), privKey, nil + } + } +} + +func fingerprintPEM(data []byte) *gomatrixserverlib.TLSFingerprint { + for { + var certDERBlock *pem.Block + certDERBlock, data = pem.Decode(data) + if data == nil { + return nil + } + if certDERBlock.Type == "CERTIFICATE" { + digest := sha256.Sum256(certDERBlock.Bytes) + return &gomatrixserverlib.TLSFingerprint{digest[:]} + } + } +} diff --git a/src/github.com/matrix-org/dendrite/common/config/config_test.go b/src/github.com/matrix-org/dendrite/common/config/config_test.go new file mode 100644 index 000000000..f690882ef --- /dev/null +++ b/src/github.com/matrix-org/dendrite/common/config/config_test.go @@ -0,0 +1,134 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "testing" +) + +func TestLoadConfigRelative(t *testing.T) { + _, err := loadConfig("/my/config/dir", []byte(testConfig), + mockReadFile{ + "/my/config/dir/matrix_key.pem": testKey, + "/my/config/dir/tls_cert.pem": testCert, + }.readFile, + ) + if err != nil { + t.Error("failed to load config:", err) + } +} + +const testConfig = ` +version: v0 +matrix: + server_name: localhost + private_key: matrix_key.pem + federation_certificates: [tls_cert.pem] +media: + base_path: media_store +kafka: + addresses: ["localhost:9092"] + topics: + input_room_event: input.room + output_room_event: output.room +database: + media_api: "postgresql:///media_api" + account: "postgresql:///account" + device: "postgresql:///device" + server_key: "postgresql:///server_keys" + sync_api: "postgresql:///syn_api" + room_server: "postgresql:///room_server" +listen: + room_server: "localhost:7770" + client_api: "localhost:7771" + federation_api: "localhost:7772" + sync_api: "localhost:7773" + media_api: "localhost:7774" +` + +type mockReadFile map[string]string + +func (m mockReadFile) readFile(path string) ([]byte, error) { + data, ok := m[path] + if !ok { + return nil, fmt.Errorf("no such file %q", path) + } + return []byte(data), nil +} + +func TestReadKey(t *testing.T) { + keyID, _, err := readKeyPEM("path/to/key", []byte(testKey)) + if err != nil { + t.Error("failed to load private key:", err) + } + wantKeyID := testKeyID + if wantKeyID != string(keyID) { + t.Errorf("wanted key ID to be %q, got %q", wantKeyID, keyID) + } +} + +const testKeyID = "ed25519:c8NsuQ" + +const testKey = ` +-----BEGIN MATRIX PRIVATE KEY----- +Key-ID: ` + testKeyID + ` +7KRZiZ2sTyRR8uqqUjRwczuwRXXkUMYIUHq4Mc3t4bE= +-----END MATRIX PRIVATE KEY----- +` + +func TestFingerprintPEM(t *testing.T) { + got := fingerprintPEM([]byte(testCert)) + if got == nil { + t.Error("failed to calculate fingerprint") + } + if string(got.SHA256) != testCertFingerprint { + t.Errorf("bad fingerprint: wanted %q got %q", got, testCertFingerprint) + } + +} + +const testCertFingerprint = "56.\\SPQxE\xd4\x95\xfb\xf6\xd5\x04\x91\xcb/\x07\xb1^\x88\x08\xe3\xc1p\xdfY\x04\x19w\xcb" + +const testCert = ` +-----BEGIN CERTIFICATE----- +MIIE0zCCArugAwIBAgIJAPype3u24LJeMA0GCSqGSIb3DQEBCwUAMAAwHhcNMTcw +NjEzMTQyODU4WhcNMTgwNjEzMTQyODU4WjAAMIICIjANBgkqhkiG9w0BAQEFAAOC +Ag8AMIICCgKCAgEA3vNSr7lCh/alxPFqairp/PYohwdsqPvOD7zf7dJCNhy0gbdC +9/APwIbPAPL9nU+o9ud1ACNCKBCQin/9LnI5vd5pa/Ne+mmRADDLB/BBBoywSJWG +NSfKJ9n3XY1bjgtqi53uUh+RDdQ7sXudDqCUxiiJZmS7oqK/mp88XXAgCbuXUY29 +GmzbbDz37vntuSxDgUOnJ8uPSvRp5YPKogA3JwW1SyrlLt4Z30CQ6nH3Y2Q5SVfJ +NIQyMrnwyjA9bCdXezv1cLXoTYn7U9BRyzXTZeXs3y3ldnRfISXN35CU04Az1F8j +lfj7nXMEqI/qAj/qhxZ8nVBB+rpNOZy9RJko3O+G5Qa/EvzkQYV1rW4TM2Yme88A +QyJspoV/0bXk6gG987PonK2Uk5djxSULhnGVIqswydyH0Nzb+slRp2bSoWbaNlee ++6TIeiyTQYc055pCHOp22gtLrC5LQGchksi02St2ZzRHdnlfqCJ8S9sS7x3trzds +cYueg1sGI+O8szpQ3eUM7OhJOBrx6OlR7+QYnQg1wr/V+JAz1qcyTC1URcwfeqtg +QjxFdBD9LfCtfK+AO51H9ugtsPJqOh33PmvfvUBEM05OHCA0lNaWJHROGpm4T4cc +YQI9JQk/0lB7itF1qK5RG74qgKdjkBkfZxi0OqkUgHk6YHtJlKfET8zfrtcCAwEA +AaNQME4wHQYDVR0OBBYEFGwb0NgH0Zr7Ga23njEJ85Ozf8M9MB8GA1UdIwQYMBaA +FGwb0NgH0Zr7Ga23njEJ85Ozf8M9MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggIBAKU3RHXggbq/pLhGinU5q/9QT0TB/0bBnF1wNFkKQC0FrNJ+ZnBNmusy +oqOn7DEohBCCDxT0kgOC05gLEsGLkSXlVyqCsPFfycCFhtu1QzSRtQNRxB3pW3Wq +4/RFVYv0PGBjVBKxImQlEmXJWEDwemGKqDQZPtqR/FTHTbJcaT0xQr5+1oG6lawt +I/2cW6GQ0kYW/Szps8FgNdSNgVqCjjNIzBYbWhRWMx/63qD1ReUbY7/Yw9KKT8nK +zXERpbTM9k+Pnm0g9Gep+9HJ1dBFJeuTPugKeSeyqg2OJbENw1hxGs/HjBXw7580 +ioiMn/kMj6Tg/f3HCfKrdHHBFQw0/fJW6o17QImYIpPOPzc5RjXBrCJWb34kxqEd +NQdKgejWiV/LlVsguIF8hVZH2kRzvoyypkVUtSUYGmjvA5UXoORQZfJ+b41llq1B +GcSF6iaVbAFKnsUyyr1i9uHz/6Muqflphv/SfZxGheIn5u3PnhXrzDagvItjw0NS +n0Xq64k7fc42HXJpF8CGBkSaIhtlzcruO+vqR80B9r62+D0V7VmHOnP135MT6noU +8F0JQfEtP+I8NII5jHSF/khzSgP5g80LS9tEc2ILnIHK1StkInAoRQQ+/HsQsgbz +ANAf5kxmMsM0zlN2hkxl0H6o7wKlBSw3RI3cjfilXiMWRPJrzlc4 +-----END CERTIFICATE----- +`