From 9a0905fe2fec98985ac025cee1069f2b9fe425f1 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 5 Jun 2020 16:35:33 +0100 Subject: [PATCH] WIP base refactor --- appservice/appservice.go | 21 ++- clientapi/clientapi.go | 32 ++--- cmd/dendrite-appservice-server/main.go | 16 +-- cmd/dendrite-client-api-server/main.go | 24 +--- cmd/dendrite-demo-libp2p/p2pdendrite.go | 8 +- cmd/dendrite-edu-server/main.go | 11 +- cmd/dendrite-federation-api-server/main.go | 23 +-- cmd/dendrite-federation-sender-server/main.go | 17 +-- cmd/dendrite-key-server/main.go | 12 +- cmd/dendrite-media-api-server/main.go | 11 +- cmd/dendrite-monolith-server/main.go | 87 ++++------- cmd/dendrite-public-rooms-api-server/main.go | 19 +-- cmd/dendrite-room-server/main.go | 16 +-- cmd/dendrite-server-key-api-server/main.go | 10 +- cmd/dendrite-sync-api-server/main.go | 14 +- cmd/dendritejs/keyfetcher.go | 24 ++++ cmd/dendritejs/main.go | 67 +++++---- eduserver/eduserver.go | 8 +- federationapi/federationapi.go | 25 +--- federationsender/federationsender.go | 21 ++- internal/{basecomponent => setup}/base.go | 136 ++++++++++++++---- internal/{basecomponent => setup}/flags.go | 2 +- keyserver/keyserver.go | 12 +- mediaapi/mediaapi.go | 10 +- publicroomsapi/publicroomsapi.go | 23 ++- publicroomsapi/storage/storage_wasm.go | 3 +- roomserver/roomserver.go | 11 +- serverkeyapi/serverkeyapi.go | 13 +- syncapi/syncapi.go | 22 +-- 29 files changed, 315 insertions(+), 383 deletions(-) rename internal/{basecomponent => setup}/base.go (68%) rename internal/{basecomponent => setup}/flags.go (98%) diff --git a/appservice/appservice.go b/appservice/appservice.go index b5ffba5eb..7a68e71ae 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -32,24 +32,19 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/internal/transactions" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) // SetupAppServiceAPIComponent sets up and registers HTTP handlers for the AppServices // component. func SetupAppServiceAPIComponent( - base *basecomponent.BaseDendrite, - accountsDB accounts.Database, - deviceDB devices.Database, - federation *gomatrixserverlib.FederationClient, - rsAPI roomserverAPI.RoomserverInternalAPI, + base *setup.Base, transactionsCache *transactions.Cache, ) appserviceAPI.AppServiceQueryAPI { + // Create a connection to the appservice postgres DB appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService), base.Cfg.DbProperties()) if err != nil { @@ -69,7 +64,7 @@ func SetupAppServiceAPIComponent( workerStates[i] = ws // Create bot account for this AS if it doesn't already exist - if err = generateAppServiceAccount(accountsDB, deviceDB, appservice); err != nil { + if err = generateAppServiceAccount(base.AccountDB, base.DeviceDB, appservice); err != nil { logrus.WithFields(logrus.Fields{ "appservice": appservice.ID, }).WithError(err).Panicf("failed to generate bot account for appservice") @@ -88,8 +83,8 @@ func SetupAppServiceAPIComponent( inthttp.AddRoutes(appserviceQueryAPI, base.InternalAPIMux) consumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB, - rsAPI, workerStates, + base.Cfg, base.KafkaConsumer, base.AccountDB, appserviceDB, + base.RoomserverAPI(), workerStates, ) if err := consumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start appservice roomserver consumer") @@ -102,8 +97,8 @@ func SetupAppServiceAPIComponent( // Set up HTTP Endpoints routing.Setup( - base.PublicAPIMux, base.Cfg, rsAPI, - accountsDB, federation, transactionsCache, + base.PublicAPIMux, base.Cfg, base.RoomserverAPI(), + base.AccountDB, base.FederationClient, transactionsCache, ) return appserviceQueryAPI diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 0ce44c211..b30c0b236 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -15,37 +15,22 @@ package clientapi import ( - appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/consumers" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/clientapi/routing" - eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" - federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/internal/transactions" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) // SetupClientAPIComponent sets up and registers HTTP handlers for the ClientAPI // component. func SetupClientAPIComponent( - base *basecomponent.BaseDendrite, - deviceDB devices.Database, - accountsDB accounts.Database, - federation *gomatrixserverlib.FederationClient, - keyRing *gomatrixserverlib.KeyRing, - rsAPI roomserverAPI.RoomserverInternalAPI, - eduInputAPI eduServerAPI.EDUServerInputAPI, - asAPI appserviceAPI.AppServiceQueryAPI, + base *setup.Base, transactionsCache *transactions.Cache, - fsAPI federationSenderAPI.FederationSenderInternalAPI, ) { - roomserverProducer := producers.NewRoomserverProducer(rsAPI) - eduProducer := producers.NewEDUServerProducer(eduInputAPI) + roomserverProducer := producers.NewRoomserverProducer(base.RoomserverAPI()) + eduProducer := producers.NewEDUServerProducer(base.EDUServer()) userUpdateProducer := &producers.UserUpdateProducer{ Producer: base.KafkaProducer, @@ -58,15 +43,16 @@ func SetupClientAPIComponent( } consumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, accountsDB, rsAPI, + base.Cfg, base.KafkaConsumer, base.AccountDB, base.RoomserverAPI(), ) if err := consumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start room server consumer") } + keyRing := base.ServerKeyAPI().KeyRing() routing.Setup( - base.PublicAPIMux, base.Cfg, roomserverProducer, rsAPI, asAPI, - accountsDB, deviceDB, federation, *keyRing, userUpdateProducer, - syncProducer, eduProducer, transactionsCache, fsAPI, + base.PublicAPIMux, base.Cfg, roomserverProducer, base.RoomserverAPI(), base.AppserviceAPI(), + base.AccountDB, base.DeviceDB, base.FederationClient, *keyRing, userUpdateProducer, + syncProducer, eduProducer, transactionsCache, base.FederationSender(), ) } diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index 4fc2a173d..d1e211e04 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -16,24 +16,16 @@ package main import ( "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/internal/transactions" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "AppServiceAPI", true) - + cfg := setup.ParseFlags(false) + base := setup.NewBase(cfg, "AppServiceAPI", true) defer base.Close() // nolint: errcheck - accountDB := base.CreateAccountsDB() - deviceDB := base.CreateDeviceDB() - federation := base.CreateFederationClient() - rsAPI := base.RoomserverHTTPClient() - cache := transactions.New() - appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, rsAPI, cache, - ) + appservice.SetupAppServiceAPIComponent(base, transactions.New()) base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI)) diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 8ad4231d1..f440aef01 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -16,32 +16,16 @@ package main import ( "github.com/matrix-org/dendrite/clientapi" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/internal/transactions" ) func main() { - cfg := basecomponent.ParseFlags(false) + cfg := setup.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "ClientAPI", true) + base := setup.NewBase(cfg, "ClientAPI", true) defer base.Close() // nolint: errcheck - - accountDB := base.CreateAccountsDB() - deviceDB := base.CreateDeviceDB() - federation := base.CreateFederationClient() - - serverKeyAPI := base.ServerKeyAPIClient() - keyRing := serverKeyAPI.KeyRing() - - asQuery := base.AppserviceHTTPClient() - rsAPI := base.RoomserverHTTPClient() - fsAPI := base.FederationSenderHTTPClient() - eduInputAPI := base.EDUServerClient() - - clientapi.SetupClientAPIComponent( - base, deviceDB, accountDB, federation, keyRing, - rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, - ) + clientapi.SetupClientAPIComponent(base, transactions.New()) base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI)) diff --git a/cmd/dendrite-demo-libp2p/p2pdendrite.go b/cmd/dendrite-demo-libp2p/p2pdendrite.go index b7c5c66b0..4d2948c0a 100644 --- a/cmd/dendrite-demo-libp2p/p2pdendrite.go +++ b/cmd/dendrite-demo-libp2p/p2pdendrite.go @@ -22,7 +22,7 @@ import ( pstore "github.com/libp2p/go-libp2p-core/peerstore" record "github.com/libp2p/go-libp2p-record" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/libp2p/go-libp2p" circuit "github.com/libp2p/go-libp2p-circuit" @@ -37,9 +37,9 @@ import ( "github.com/matrix-org/dendrite/internal/config" ) -// P2PDendrite is a Peer-to-Peer variant of BaseDendrite. +// P2PDendrite is a Peer-to-Peer variant of Base. type P2PDendrite struct { - Base basecomponent.BaseDendrite + Base setup.Base // Store our libp2p object so that we can make outgoing connections from it // later @@ -54,7 +54,7 @@ type P2PDendrite struct { // The componentName is used for logging purposes, and should be a friendly name // of the component running, e.g. SyncAPI. func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite { - baseDendrite := basecomponent.NewBaseDendrite(cfg, componentName, false) + baseDendrite := setup.NewBase(cfg, componentName, false) ctx, cancel := context.WithCancel(context.Background()) diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-edu-server/main.go index ceeb13658..5fdf577c5 100644 --- a/cmd/dendrite-edu-server/main.go +++ b/cmd/dendrite-edu-server/main.go @@ -17,21 +17,20 @@ import ( "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/sirupsen/logrus" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "EDUServerAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBase(cfg, "EDUServerAPI", true) defer func() { if err := base.Close(); err != nil { - logrus.WithError(err).Warn("BaseDendrite close failed") + logrus.WithError(err).Warn("Base close failed") } }() - deviceDB := base.CreateDeviceDB() - eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB) + eduserver.SetupEDUServerComponent(base, cache.New()) base.SetupAndServeHTTP(string(base.Cfg.Bind.EDUServer), string(base.Cfg.Listen.EDUServer)) diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index 0c4d8b854..f43f4eb18 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -17,30 +17,17 @@ package main import ( "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/federationapi" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "FederationAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBase(cfg, "FederationAPI", true) defer base.Close() // nolint: errcheck - - accountDB := base.CreateAccountsDB() - deviceDB := base.CreateDeviceDB() - federation := base.CreateFederationClient() - serverKeyAPI := base.ServerKeyAPIClient() - keyRing := serverKeyAPI.KeyRing() - fsAPI := base.FederationSenderHTTPClient() - rsAPI := base.RoomserverHTTPClient() - asAPI := base.AppserviceHTTPClient() // TODO: this isn't a producer - eduProducer := producers.NewEDUServerProducer(base.EDUServerClient()) - - federationapi.SetupFederationAPIComponent( - base, accountDB, deviceDB, federation, keyRing, - rsAPI, asAPI, fsAPI, eduProducer, - ) + eduProducer := producers.NewEDUServerProducer(base.EDUServer()) + federationapi.SetupFederationAPIComponent(base, eduProducer) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) } diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 651eb257f..503638046 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -16,23 +16,14 @@ package main import ( "github.com/matrix-org/dendrite/federationsender" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "FederationSender", true) + cfg := setup.ParseFlags(false) + base := setup.NewBase(cfg, "FederationSender", true) defer base.Close() // nolint: errcheck - - federation := base.CreateFederationClient() - - serverKeyAPI := base.ServerKeyAPIClient() - keyRing := serverKeyAPI.KeyRing() - - rsAPI := base.RoomserverHTTPClient() - federationsender.SetupFederationSenderComponent( - base, federation, rsAPI, keyRing, - ) + federationsender.SetupFederationSenderComponent(base) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender)) diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-key-server/main.go index a8accbcb3..b558ad9ac 100644 --- a/cmd/dendrite-key-server/main.go +++ b/cmd/dendrite-key-server/main.go @@ -15,19 +15,15 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/keyserver" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "KeyServer", true) + cfg := setup.ParseFlags(false) + base := setup.NewBase(cfg, "KeyServer", true) defer base.Close() // nolint: errcheck - - accountDB := base.CreateAccountsDB() - deviceDB := base.CreateDeviceDB() - - keyserver.SetupKeyServerComponent(base, deviceDB, accountDB) + keyserver.SetupKeyServerComponent(base) base.SetupAndServeHTTP(string(base.Cfg.Bind.KeyServer), string(base.Cfg.Listen.KeyServer)) diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go index 018559383..bf088b82b 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-media-api-server/main.go @@ -15,18 +15,15 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/mediaapi" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "MediaAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBase(cfg, "MediaAPI", true) defer base.Close() // nolint: errcheck - - deviceDB := base.CreateDeviceDB() - - mediaapi.SetupMediaAPIComponent(base, deviceDB) + mediaapi.SetupMediaAPIComponent(base) base.SetupAndServeHTTP(string(base.Cfg.Bind.MediaAPI), string(base.Cfg.Listen.MediaAPI)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 57e8f66f5..84c1ee283 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -26,13 +26,12 @@ import ( "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/mediaapi" "github.com/matrix-org/dendrite/publicroomsapi" - "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/serverkeyapi" "github.com/matrix-org/dendrite/syncapi" @@ -49,7 +48,7 @@ var ( ) func main() { - cfg := basecomponent.ParseFlags(true) + cfg := setup.ParseFlags(true) if *enableHTTPAPIs { // If the HTTP APIs are enabled then we need to update the Listen // statements in the configuration so that we know where to find @@ -63,69 +62,43 @@ func main() { cfg.Listen.ServerKeyAPI = addr } - base := basecomponent.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs) + base := setup.NewBase(cfg, "Monolith", *enableHTTPAPIs) defer base.Close() // nolint: errcheck - accountDB := base.CreateAccountsDB() - deviceDB := base.CreateDeviceDB() - federation := base.CreateFederationClient() - - serverKeyAPI := serverkeyapi.SetupServerKeyAPIComponent( - base, federation, - ) - if base.UseHTTPAPIs { - serverKeyAPI = base.ServerKeyAPIClient() - } - keyRing := serverKeyAPI.KeyRing() - - rsComponent := roomserver.SetupRoomServerComponent( - base, keyRing, federation, - ) - rsAPI := rsComponent - if base.UseHTTPAPIs { - rsAPI = base.RoomserverHTTPClient() + serverKeyAPI := serverkeyapi.SetupServerKeyAPIComponent(base) + if !base.UseHTTPAPIs { + base.SetServerKeyAPI(serverKeyAPI) } - eduInputAPI := eduserver.SetupEDUServerComponent( - base, cache.New(), deviceDB, - ) - if base.UseHTTPAPIs { - eduInputAPI = base.EDUServerClient() + rsAPI := roomserver.SetupRoomServerComponent(base) + if !base.UseHTTPAPIs { + base.SetRoomserverAPI(rsAPI) } - asAPI := appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, rsAPI, transactions.New(), - ) - if base.UseHTTPAPIs { - asAPI = base.AppserviceHTTPClient() + eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) + if !base.UseHTTPAPIs { + base.SetEDUServer(eduInputAPI) } - fsAPI := federationsender.SetupFederationSenderComponent( - base, federation, rsAPI, keyRing, - ) - if base.UseHTTPAPIs { - fsAPI = base.FederationSenderHTTPClient() + asAPI := appservice.SetupAppServiceAPIComponent(base, transactions.New()) + if !base.UseHTTPAPIs { + base.SetAppserviceAPI(asAPI) } - rsComponent.SetFederationSenderAPI(fsAPI) - clientapi.SetupClientAPIComponent( - base, deviceDB, accountDB, - federation, keyRing, rsAPI, - eduInputAPI, asAPI, transactions.New(), fsAPI, - ) - - keyserver.SetupKeyServerComponent( - base, deviceDB, accountDB, - ) - eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer) - mediaapi.SetupMediaAPIComponent(base, deviceDB) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to public rooms db") + fsAPI := federationsender.SetupFederationSenderComponent(base) + if !base.UseHTTPAPIs { + base.SetFederationSender(fsAPI) } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, nil) - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) + rsAPI.SetFederationSenderAPI(fsAPI) + + clientapi.SetupClientAPIComponent(base, transactions.New()) + + keyserver.SetupKeyServerComponent(base) + eduProducer := producers.NewEDUServerProducer(base.EDUServer()) + federationapi.SetupFederationAPIComponent(base, eduProducer) + mediaapi.SetupMediaAPIComponent(base) + publicroomsapi.SetupPublicRoomsAPIComponent(base, nil) + syncapi.SetupSyncAPIComponent(base) internal.SetupHTTPAPI( http.DefaultServeMux, @@ -139,7 +112,7 @@ func main() { go func() { serv := http.Server{ Addr: *httpBindAddr, - WriteTimeout: basecomponent.HTTPServerTimeout, + WriteTimeout: setup.HTTPServerTimeout, } logrus.Info("Listening on ", serv.Addr) @@ -150,7 +123,7 @@ func main() { go func() { serv := http.Server{ Addr: *httpsBindAddr, - WriteTimeout: basecomponent.HTTPServerTimeout, + WriteTimeout: setup.HTTPServerTimeout, } logrus.Info("Listening on ", serv.Addr) diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index 0cad3670c..9590e2a66 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -15,26 +15,15 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/publicroomsapi" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/sirupsen/logrus" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "PublicRoomsAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBase(cfg, "PublicRoomsAPI", true) defer base.Close() // nolint: errcheck - - deviceDB := base.CreateDeviceDB() - - rsAPI := base.RoomserverHTTPClient() - - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to public rooms db") - } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, nil, nil) + publicroomsapi.SetupPublicRoomsAPIComponent(base, nil) base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI)) diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index 79745cded..82d525337 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -15,22 +15,16 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/roomserver" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "RoomServerAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBase(cfg, "RoomServerAPI", true) defer base.Close() // nolint: errcheck - federation := base.CreateFederationClient() - - serverKeyAPI := base.ServerKeyAPIClient() - keyRing := serverKeyAPI.KeyRing() - - fsAPI := base.FederationSenderHTTPClient() - rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation) - rsAPI.SetFederationSenderAPI(fsAPI) + rsAPI := roomserver.SetupRoomServerComponent(base) + rsAPI.SetFederationSenderAPI(base.FederationSender()) base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer)) diff --git a/cmd/dendrite-server-key-api-server/main.go b/cmd/dendrite-server-key-api-server/main.go index e206978cd..4b273298d 100644 --- a/cmd/dendrite-server-key-api-server/main.go +++ b/cmd/dendrite-server-key-api-server/main.go @@ -15,18 +15,16 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/serverkeyapi" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "ServerKeyAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBase(cfg, "ServerKeyAPI", true) defer base.Close() // nolint: errcheck - federation := base.CreateFederationClient() - - serverkeyapi.SetupServerKeyAPIComponent(base, federation) + serverkeyapi.SetupServerKeyAPIComponent(base) base.SetupAndServeHTTP(string(base.Cfg.Bind.ServerKeyAPI), string(base.Cfg.Listen.ServerKeyAPI)) } diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index bb56b4ab0..9661bcd4f 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -15,22 +15,16 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/syncapi" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "SyncAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBase(cfg, "SyncAPI", true) defer base.Close() // nolint: errcheck - deviceDB := base.CreateDeviceDB() - accountDB := base.CreateAccountsDB() - federation := base.CreateFederationClient() - - rsAPI := base.RoomserverHTTPClient() - - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) + syncapi.SetupSyncAPIComponent(base) base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI)) diff --git a/cmd/dendritejs/keyfetcher.go b/cmd/dendritejs/keyfetcher.go index cef045372..ae447e004 100644 --- a/cmd/dendritejs/keyfetcher.go +++ b/cmd/dendritejs/keyfetcher.go @@ -84,3 +84,27 @@ func (f *libp2pKeyFetcher) FetcherName() string { func (f *libp2pKeyFetcher) StoreKeys(ctx context.Context, results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) error { return nil } + +type libp2pServerKeyAPI struct { + keyRing *gomatrixserverlib.KeyRing +} + +func (a *libp2pServerKeyAPI) KeyRing() *gomatrixserverlib.KeyRing { + return a.keyRing +} + +func (a *libp2pServerKeyAPI) InputPublicKeys( + ctx context.Context, + request *InputPublicKeysRequest, + response *InputPublicKeysResponse, +) error { + return nil +} + +func (a *libp2pServerKeyAPI) QueryPublicKeys( + ctx context.Context, + request *QueryPublicKeysRequest, + response *QueryPublicKeysResponse, +) error { + return nil +} diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 4c1b81979..51443ba79 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -30,13 +30,14 @@ import ( "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/internal/transactions" + "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/mediaapi" "github.com/matrix-org/dendrite/publicroomsapi" - "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/dendrite/syncapi" go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p" @@ -160,6 +161,19 @@ func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http return } +func serverKeyAPI() api.ServerKeyInternalAPI { + fetcher := &libp2pKeyFetcher{} + keyRing := gomatrixserverlib.KeyRing{ + KeyFetchers: []gomatrixserverlib.KeyFetcher{ + fetcher, + }, + KeyDatabase: fetcher, + } + return libp2pServerKeyAPI{ + keyRing: keyRing, + } +} + func main() { cfg := &config.Dendrite{} cfg.SetDefaults() @@ -191,44 +205,27 @@ func main() { if err := cfg.Derive(); err != nil { logrus.Fatalf("Failed to derive values from config: %s", err) } - base := basecomponent.NewBaseDendrite(cfg, "Monolith", false) + base := setup.NewBase(cfg, "Monolith", false) defer base.Close() // nolint: errcheck - accountDB := base.CreateAccountsDB() - deviceDB := base.CreateDeviceDB() - federation := createFederationClient(cfg, node) + base.FederationClient = createFederationClient(cfg, node) + base.SetServerKeyAPI(serverKeyAPI()) - fetcher := &libp2pKeyFetcher{} - keyRing := gomatrixserverlib.KeyRing{ - KeyFetchers: []gomatrixserverlib.KeyFetcher{ - fetcher, - }, - KeyDatabase: fetcher, - } + base.SetRoomserverAPI(roomserver.SetupRoomServerComponent(base)) + base.SetEDUServer(eduserver.SetupEDUServerComponent(base, cache.New())) + base.SetAppserviceAPI(appservice.SetupAppServiceAPIComponent(base, transactions.New())) + base.SetFederationSender(federationsender.SetupFederationSenderComponent(base)) + rsAPI.SetFederationSenderAPI(base.FederationSender()) - rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation) - eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB) - asQuery := appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, rsAPI, transactions.New(), - ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, rsAPI, &keyRing) - rsAPI.SetFederationSenderAPI(fedSenderAPI) - p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI) + clientapi.SetupClientAPIComponent(base, transactions.New()) - clientapi.SetupClientAPIComponent( - base, deviceDB, accountDB, - federation, &keyRing, rsAPI, - eduInputAPI, asQuery, transactions.New(), fedSenderAPI, - ) - eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asQuery, fedSenderAPI, eduProducer) - mediaapi.SetupMediaAPIComponent(base, deviceDB) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), cfg.Matrix.ServerName) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to public rooms db") - } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, p2pPublicRoomProvider) - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) + keyserver.SetupKeyServerComponent(base) + eduProducer := producers.NewEDUServerProducer(base.EDUServer()) + federationapi.SetupFederationAPIComponent(base, eduProducer) + mediaapi.SetupMediaAPIComponent(base) + p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, base.FederationSender()) + publicroomsapi.SetupPublicRoomsAPIComponent(base, p2pPublicRoomProvider) + syncapi.SetupSyncAPIComponent(base) internal.SetupHTTPAPI( http.DefaultServeMux, diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go index 59decc8fb..ddb00fbc4 100644 --- a/eduserver/eduserver.go +++ b/eduserver/eduserver.go @@ -17,12 +17,11 @@ package eduserver import ( - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/eduserver/input" "github.com/matrix-org/dendrite/eduserver/inthttp" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" ) // SetupEDUServerComponent sets up and registers HTTP handlers for the @@ -30,13 +29,12 @@ import ( // allowing other components running in the same process to hit the query the // APIs directly instead of having to use HTTP. func SetupEDUServerComponent( - base *basecomponent.BaseDendrite, + base *setup.Base, eduCache *cache.EDUCache, - deviceDB devices.Database, ) api.EDUServerInputAPI { inputAPI := &input.EDUServerInputAPI{ Cache: eduCache, - DeviceDB: deviceDB, + DeviceDB: base.DeviceDB, Producer: base.KafkaProducer, OutputTypingEventTopic: string(base.Cfg.Kafka.Topics.OutputTypingEvent), OutputSendToDeviceEventTopic: string(base.Cfg.Kafka.Topics.OutputSendToDeviceEvent), diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index baeaa36bc..2835c47e3 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -15,37 +15,24 @@ package federationapi import ( - appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/internal/basecomponent" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/internal/setup" // TODO: Are we really wanting to pull in the producer from clientapi "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/federationapi/routing" - "github.com/matrix-org/gomatrixserverlib" ) // SetupFederationAPIComponent sets up and registers HTTP handlers for the // FederationAPI component. func SetupFederationAPIComponent( - base *basecomponent.BaseDendrite, - accountsDB accounts.Database, - deviceDB devices.Database, - federation *gomatrixserverlib.FederationClient, - keyRing *gomatrixserverlib.KeyRing, - rsAPI roomserverAPI.RoomserverInternalAPI, - asAPI appserviceAPI.AppServiceQueryAPI, - federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, + base *setup.Base, eduProducer *producers.EDUServerProducer, ) { - roomserverProducer := producers.NewRoomserverProducer(rsAPI) + roomserverProducer := producers.NewRoomserverProducer(base.RoomserverAPI()) routing.Setup( - base.PublicAPIMux, base.Cfg, rsAPI, asAPI, roomserverProducer, - eduProducer, federationSenderAPI, *keyRing, - federation, accountsDB, deviceDB, + base.PublicAPIMux, base.Cfg, base.RoomserverAPI(), base.AppserviceAPI(), roomserverProducer, + eduProducer, base.FederationSender(), *base.ServerKeyAPI().KeyRing(), + base.FederationClient, base.AccountDB, base.DeviceDB, ) } diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index e25c27237..e3030fc76 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -23,37 +23,30 @@ import ( "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/dendrite/federationsender/types" - "github.com/matrix-org/dendrite/internal/basecomponent" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/internal/setup" "github.com/sirupsen/logrus" ) // SetupFederationSenderComponent sets up and registers HTTP handlers for the // FederationSender component. -func SetupFederationSenderComponent( - base *basecomponent.BaseDendrite, - federation *gomatrixserverlib.FederationClient, - rsAPI roomserverAPI.RoomserverInternalAPI, - keyRing *gomatrixserverlib.KeyRing, -) api.FederationSenderInternalAPI { +func SetupFederationSenderComponent(base *setup.Base) api.FederationSenderInternalAPI { federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panic("failed to connect to federation sender db") } roomserverProducer := producers.NewRoomserverProducer( - rsAPI, base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, base.Cfg.Matrix.PrivateKey, + base.RoomserverAPI(), base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, base.Cfg.Matrix.PrivateKey, ) statistics := &types.Statistics{} queues := queue.NewOutgoingQueues( - base.Cfg.Matrix.ServerName, federation, roomserverProducer, statistics, + base.Cfg.Matrix.ServerName, base.FederationClient, roomserverProducer, statistics, ) rsConsumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, queues, - federationSenderDB, rsAPI, + federationSenderDB, base.RoomserverAPI(), ) if err = rsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start room server consumer") @@ -66,7 +59,9 @@ func SetupFederationSenderComponent( logrus.WithError(err).Panic("failed to start typing server consumer") } - queryAPI := internal.NewFederationSenderInternalAPI(federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing, statistics, queues) + queryAPI := internal.NewFederationSenderInternalAPI( + federationSenderDB, base.Cfg, roomserverProducer, base.FederationClient, base.ServerKeyAPI().KeyRing(), statistics, queues, + ) inthttp.AddRoutes(queryAPI, base.InternalAPIMux) return queryAPI diff --git a/internal/basecomponent/base.go b/internal/setup/base.go similarity index 68% rename from internal/basecomponent/base.go rename to internal/setup/base.go index 620b12d6a..de26ae40b 100644 --- a/internal/basecomponent/base.go +++ b/internal/setup/base.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package basecomponent +package setup import ( "database/sql" @@ -51,12 +51,12 @@ import ( _ "net/http/pprof" ) -// BaseDendrite is a base for creating new instances of dendrite. It parses +// Base is a base for creating new instances of dendrite. It parses // command line flags and config, and exposes methods for creating various // resources. All errors are handled by logging then exiting, so all methods // should only be used during start up. // Must be closed when shutting down. -type BaseDendrite struct { +type Base struct { componentName string tracerCloser io.Closer @@ -69,15 +69,27 @@ type BaseDendrite struct { ImmutableCache caching.ImmutableCache KafkaConsumer sarama.Consumer KafkaProducer sarama.SyncProducer + + // internal APIs + appserviceAPI appserviceAPI.AppServiceQueryAPI + roomserverAPI roomserverAPI.RoomserverInternalAPI + eduServer eduServerAPI.EDUServerInputAPI + federationSender federationSenderAPI.FederationSenderInternalAPI + serverKeyAPI serverKeyAPI.ServerKeyInternalAPI + + DeviceDB devices.Database + AccountDB accounts.Database + FederationClient *gomatrixserverlib.FederationClient } const HTTPServerTimeout = time.Minute * 5 const HTTPClientTimeout = time.Second * 30 -// NewBaseDendrite creates a new instance to be used by a component. +// NewBase creates a new instance to be used by a component. // The componentName is used for logging purposes, and should be a friendly name // of the compontent running, e.g. "SyncAPI" -func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs bool) *BaseDendrite { +// If `useHTTPAPIs` is true, HTTP clients will be made for all internal APIs. +func NewBase(cfg *config.Dendrite, componentName string, useHTTPAPIs bool) *Base { internal.SetupStdLogging() internal.SetupHookLogging(cfg.Logging, componentName) internal.SetupPprof() @@ -121,7 +133,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo // directory traversal attack e.g /../../../etc/passwd httpmux := mux.NewRouter().SkipClean(true) - return &BaseDendrite{ + b := &Base{ componentName: componentName, UseHTTPAPIs: useHTTPAPIs, tracerCloser: closer, @@ -133,15 +145,85 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo KafkaConsumer: kafkaConsumer, KafkaProducer: kafkaProducer, } + + if useHTTPAPIs { + b.appserviceAPI = b.createAppserviceHTTPClient() + b.roomserverAPI = b.createRoomserverHTTPClient() + b.eduServer = b.createEDUServerClient() + b.federationSender = b.createFederationSenderHTTPClient() + b.serverKeyAPI = b.createServerKeyAPIClient() + } + + b.DeviceDB = b.createDeviceDB() + b.AccountDB = b.createAccountsDB() + b.FederationClient = b.createFederationClient() + + return b } // Close implements io.Closer -func (b *BaseDendrite) Close() error { +func (b *Base) Close() error { return b.tracerCloser.Close() } -// AppserviceHTTPClient returns the AppServiceQueryAPI for hitting the appservice component over HTTP. -func (b *BaseDendrite) AppserviceHTTPClient() appserviceAPI.AppServiceQueryAPI { +// AppserviceAPI return the API or panics if one is not set. +func (b *Base) AppserviceAPI() appserviceAPI.AppServiceQueryAPI { + if b.appserviceAPI == nil { + logrus.Panic("AppserviceAPI is unset") + } + return b.appserviceAPI +} + +// RoomserverAPI return the API or panics if one is not set. +func (b *Base) RoomserverAPI() roomserverAPI.RoomserverInternalAPI { + if b.roomserverAPI == nil { + logrus.Panic("RoomserverAPI is unset") + } + return b.roomserverAPI +} + +// EDUServer return the API or panics if one is not set. +func (b *Base) EDUServer() eduServerAPI.EDUServerInputAPI { + if b.eduServer == nil { + logrus.Panic("EDUServer is unset") + } + return b.eduServer +} + +// FederationSender return the API or panics if one is not set. +func (b *Base) FederationSender() federationSenderAPI.FederationSenderInternalAPI { + if b.federationSender == nil { + logrus.Panic("FederationSender is unset") + } + return b.federationSender +} + +// ServerKeyAPI return the API or panics if one is not set. +func (b *Base) ServerKeyAPI() serverKeyAPI.ServerKeyInternalAPI { + if b.serverKeyAPI == nil { + logrus.Panic("ServerKeyAPI is unset") + } + return b.serverKeyAPI +} + +func (b *Base) SetServerKeyAPI(a serverKeyAPI.ServerKeyInternalAPI) { + b.serverKeyAPI = a +} +func (b *Base) SetFederationSender(a federationSenderAPI.FederationSenderInternalAPI) { + b.federationSender = a +} +func (b *Base) SetEDUServer(a eduServerAPI.EDUServerInputAPI) { + b.eduServer = a +} +func (b *Base) SetRoomserverAPI(a roomserverAPI.RoomserverInternalAPI) { + b.roomserverAPI = a +} +func (b *Base) SetAppserviceAPI(a appserviceAPI.AppServiceQueryAPI) { + b.appserviceAPI = a +} + +// createAppserviceHTTPClient returns the AppServiceQueryAPI for hitting the appservice component over HTTP. +func (b *Base) createAppserviceHTTPClient() appserviceAPI.AppServiceQueryAPI { a, err := asinthttp.NewAppserviceClient(b.Cfg.AppServiceURL(), b.httpClient) if err != nil { logrus.WithError(err).Panic("CreateHTTPAppServiceAPIs failed") @@ -149,8 +231,8 @@ func (b *BaseDendrite) AppserviceHTTPClient() appserviceAPI.AppServiceQueryAPI { return a } -// RoomserverHTTPClient returns RoomserverInternalAPI for hitting the roomserver over HTTP. -func (b *BaseDendrite) RoomserverHTTPClient() roomserverAPI.RoomserverInternalAPI { +// createRoomserverHTTPClient returns RoomserverInternalAPI for hitting the roomserver over HTTP. +func (b *Base) createRoomserverHTTPClient() roomserverAPI.RoomserverInternalAPI { rsAPI, err := rsinthttp.NewRoomserverClient(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache) if err != nil { logrus.WithError(err).Panic("RoomserverHTTPClient failed", b.httpClient) @@ -158,8 +240,8 @@ func (b *BaseDendrite) RoomserverHTTPClient() roomserverAPI.RoomserverInternalAP return rsAPI } -// EDUServerClient returns EDUServerInputAPI for hitting the EDU server over HTTP -func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI { +// createEDUServerClient returns EDUServerInputAPI for hitting the EDU server over HTTP +func (b *Base) createEDUServerClient() eduServerAPI.EDUServerInputAPI { e, err := eduinthttp.NewEDUServerClient(b.Cfg.EDUServerURL(), b.httpClient) if err != nil { logrus.WithError(err).Panic("EDUServerClient failed", b.httpClient) @@ -167,9 +249,9 @@ func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI { return e } -// FederationSenderHTTPClient returns FederationSenderInternalAPI for hitting +// createFederationSenderHTTPClient returns FederationSenderInternalAPI for hitting // the federation sender over HTTP -func (b *BaseDendrite) FederationSenderHTTPClient() federationSenderAPI.FederationSenderInternalAPI { +func (b *Base) createFederationSenderHTTPClient() federationSenderAPI.FederationSenderInternalAPI { f, err := fsinthttp.NewFederationSenderClient(b.Cfg.FederationSenderURL(), b.httpClient) if err != nil { logrus.WithError(err).Panic("FederationSenderHTTPClient failed", b.httpClient) @@ -177,8 +259,8 @@ func (b *BaseDendrite) FederationSenderHTTPClient() federationSenderAPI.Federati return f } -// ServerKeyAPIClient returns ServerKeyInternalAPI for hitting the server key API over HTTP -func (b *BaseDendrite) ServerKeyAPIClient() serverKeyAPI.ServerKeyInternalAPI { +// createServerKeyAPIClient returns ServerKeyInternalAPI for hitting the server key API over HTTP +func (b *Base) createServerKeyAPIClient() serverKeyAPI.ServerKeyInternalAPI { f, err := skinthttp.NewServerKeyClient( b.Cfg.ServerKeyAPIURL(), b.httpClient, @@ -190,9 +272,12 @@ func (b *BaseDendrite) ServerKeyAPIClient() serverKeyAPI.ServerKeyInternalAPI { return f } -// CreateDeviceDB creates a new instance of the device database. Should only be +// createDeviceDB creates a new instance of the device database. Should only be // called once per component. -func (b *BaseDendrite) CreateDeviceDB() devices.Database { +func (b *Base) createDeviceDB() devices.Database { + if b.Cfg.Database.Device == "" { + return nil + } db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to devices db") @@ -201,9 +286,12 @@ func (b *BaseDendrite) CreateDeviceDB() devices.Database { return db } -// CreateAccountsDB creates a new instance of the accounts database. Should only +// createAccountsDB creates a new instance of the accounts database. Should only // be called once per component. -func (b *BaseDendrite) CreateAccountsDB() accounts.Database { +func (b *Base) createAccountsDB() accounts.Database { + if b.Cfg.Database.Account == "" { + return nil + } db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to accounts db") @@ -212,9 +300,9 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database { return db } -// CreateFederationClient creates a new federation client. Should only be called +// createFederationClient creates a new federation client. Should only be called // once per component. -func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationClient { +func (b *Base) createFederationClient() *gomatrixserverlib.FederationClient { return gomatrixserverlib.NewFederationClient( b.Cfg.Matrix.ServerName, b.Cfg.Matrix.KeyID, b.Cfg.Matrix.PrivateKey, ) @@ -222,7 +310,7 @@ func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationCli // SetupAndServeHTTP sets up the HTTP server to serve endpoints registered on // ApiMux under /api/ and adds a prometheus handler under /metrics. -func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { +func (b *Base) SetupAndServeHTTP(bindaddr string, listenaddr string) { // If a separate bind address is defined, listen on that. Otherwise use // the listen address var addr string diff --git a/internal/basecomponent/flags.go b/internal/setup/flags.go similarity index 98% rename from internal/basecomponent/flags.go rename to internal/setup/flags.go index 117df0796..a1d10effc 100644 --- a/internal/basecomponent/flags.go +++ b/internal/setup/flags.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package basecomponent +package setup import ( "flag" diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index 4e785cbc7..fb6f15735 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -15,18 +15,12 @@ package keyserver import ( - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/keyserver/routing" ) // SetupFederationSenderComponent sets up and registers HTTP handlers for the // FederationSender component. -func SetupKeyServerComponent( - base *basecomponent.BaseDendrite, - deviceDB devices.Database, - accountsDB accounts.Database, -) { - routing.Setup(base.PublicAPIMux, base.Cfg, accountsDB, deviceDB) +func SetupKeyServerComponent(base *setup.Base) { + routing.Setup(base.PublicAPIMux, base.Cfg, base.AccountDB, base.DeviceDB) } diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index b5bec3907..2e63ce89f 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -15,8 +15,7 @@ package mediaapi import ( - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/mediaapi/routing" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/gomatrixserverlib" @@ -25,16 +24,13 @@ import ( // SetupMediaAPIComponent sets up and registers HTTP handlers for the MediaAPI // component. -func SetupMediaAPIComponent( - base *basecomponent.BaseDendrite, - deviceDB devices.Database, -) { +func SetupMediaAPIComponent(base *setup.Base) { mediaDB, err := storage.Open(string(base.Cfg.Database.MediaAPI), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to media db") } routing.Setup( - base.PublicAPIMux, base.Cfg, mediaDB, deviceDB, gomatrixserverlib.NewClient(), + base.PublicAPIMux, base.Cfg, mediaDB, base.DeviceDB, gomatrixserverlib.NewClient(), ) } diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index b53351ffd..bfd54431b 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -15,33 +15,28 @@ package publicroomsapi import ( - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/publicroomsapi/consumers" "github.com/matrix-org/dendrite/publicroomsapi/routing" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/publicroomsapi/types" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) // SetupPublicRoomsAPIComponent sets up and registers HTTP handlers for the PublicRoomsAPI // component. -func SetupPublicRoomsAPIComponent( - base *basecomponent.BaseDendrite, - deviceDB devices.Database, - publicRoomsDB storage.Database, - rsAPI roomserverAPI.RoomserverInternalAPI, - fedClient *gomatrixserverlib.FederationClient, - extRoomsProvider types.ExternalPublicRoomsProvider, -) { +func SetupPublicRoomsAPIComponent(base *setup.Base, extRoomsProvider types.ExternalPublicRoomsProvider) { + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), base.Cfg.Matrix.ServerName) + if err != nil { + logrus.WithError(err).Panicf("failed to connect to public rooms db") + } + rsConsumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, publicRoomsDB, rsAPI, + base.Cfg, base.KafkaConsumer, publicRoomsDB, base.RoomserverAPI(), ) if err := rsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start public rooms server consumer") } - routing.Setup(base.PublicAPIMux, deviceDB, publicRoomsDB, rsAPI, fedClient, extRoomsProvider) + routing.Setup(base.PublicAPIMux, base.DeviceDB, publicRoomsDB, base.RoomserverAPI(), base.FederationClient, extRoomsProvider) } diff --git a/publicroomsapi/storage/storage_wasm.go b/publicroomsapi/storage/storage_wasm.go index 70ceeaf85..3c96322a5 100644 --- a/publicroomsapi/storage/storage_wasm.go +++ b/publicroomsapi/storage/storage_wasm.go @@ -18,12 +18,13 @@ import ( "fmt" "net/url" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) // NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatrixserverlib.ServerName) (Database, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties internal.DbProperties, localServerName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, err diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index ae0b0794a..8ecb673ba 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -17,9 +17,8 @@ package roomserver import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/inthttp" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/roomserver/internal" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/sirupsen/logrus" @@ -30,9 +29,7 @@ import ( // allowing other components running in the same process to hit the query the // APIs directly instead of having to use HTTP. func SetupRoomServerComponent( - base *basecomponent.BaseDendrite, - keyRing gomatrixserverlib.JSONVerifier, - fedClient *gomatrixserverlib.FederationClient, + base *setup.Base, ) api.RoomserverInternalAPI { roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer), base.Cfg.DbProperties()) if err != nil { @@ -46,8 +43,8 @@ func SetupRoomServerComponent( OutputRoomEventTopic: string(base.Cfg.Kafka.Topics.OutputRoomEvent), ImmutableCache: base.ImmutableCache, ServerName: base.Cfg.Matrix.ServerName, - FedClient: fedClient, - KeyRing: keyRing, + FedClient: base.FederationClient, + KeyRing: base.ServerKeyAPI().KeyRing(), } inthttp.AddRoutes(internalAPI, base.InternalAPIMux) diff --git a/serverkeyapi/serverkeyapi.go b/serverkeyapi/serverkeyapi.go index 5bf8f67d7..3873f634c 100644 --- a/serverkeyapi/serverkeyapi.go +++ b/serverkeyapi/serverkeyapi.go @@ -4,7 +4,7 @@ import ( "crypto/ed25519" "encoding/base64" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/dendrite/serverkeyapi/internal" "github.com/matrix-org/dendrite/serverkeyapi/inthttp" @@ -14,10 +14,7 @@ import ( "github.com/sirupsen/logrus" ) -func SetupServerKeyAPIComponent( - base *basecomponent.BaseDendrite, - fedClient *gomatrixserverlib.FederationClient, -) api.ServerKeyInternalAPI { +func SetupServerKeyAPIComponent(base *setup.Base) api.ServerKeyInternalAPI { innerDB, err := storage.NewDatabase( string(base.Cfg.Database.ServerKey), base.Cfg.DbProperties(), @@ -35,11 +32,11 @@ func SetupServerKeyAPIComponent( } internalAPI := internal.ServerKeyAPI{ - FedClient: fedClient, + FedClient: base.FederationClient, OurKeyRing: gomatrixserverlib.KeyRing{ KeyFetchers: []gomatrixserverlib.KeyFetcher{ &gomatrixserverlib.DirectKeyFetcher{ - Client: fedClient.Client, + Client: base.FederationClient.Client, }, }, KeyDatabase: serverKeyDB, @@ -51,7 +48,7 @@ func SetupServerKeyAPIComponent( perspective := &gomatrixserverlib.PerspectiveKeyFetcher{ PerspectiveServerName: ps.ServerName, PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{}, - Client: fedClient.Client, + Client: base.FederationClient.Client, } for _, key := range ps.Keys { diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 762f4e9d3..2d9a2dbb9 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -19,13 +19,8 @@ import ( "github.com/sirupsen/logrus" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/internal/basecomponent" - "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/internal/setup" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/syncapi/consumers" "github.com/matrix-org/dendrite/syncapi/routing" "github.com/matrix-org/dendrite/syncapi/storage" @@ -34,14 +29,7 @@ import ( // SetupSyncAPIComponent sets up and registers HTTP handlers for the SyncAPI // component. -func SetupSyncAPIComponent( - base *basecomponent.BaseDendrite, - deviceDB devices.Database, - accountsDB accounts.Database, - rsAPI api.RoomserverInternalAPI, - federation *gomatrixserverlib.FederationClient, - cfg *config.Dendrite, -) { +func SetupSyncAPIComponent(base *setup.Base) { syncDB, err := storage.NewSyncServerDatasource(string(base.Cfg.Database.SyncAPI), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to sync db") @@ -58,10 +46,10 @@ func SetupSyncAPIComponent( logrus.WithError(err).Panicf("failed to start notifier") } - requestPool := sync.NewRequestPool(syncDB, notifier, accountsDB) + requestPool := sync.NewRequestPool(syncDB, notifier, base.AccountDB) roomConsumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, notifier, syncDB, rsAPI, + base.Cfg, base.KafkaConsumer, notifier, syncDB, base.RoomserverAPI(), ) if err = roomConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start room server consumer") @@ -88,5 +76,5 @@ func SetupSyncAPIComponent( logrus.WithError(err).Panicf("failed to start send-to-device consumer") } - routing.Setup(base.PublicAPIMux, requestPool, syncDB, deviceDB, federation, rsAPI, cfg) + routing.Setup(base.PublicAPIMux, requestPool, syncDB, base.DeviceDB, base.FederationClient, base.RoomserverAPI(), base.Cfg) }