diff --git a/.gitignore b/.gitignore index 6a13ed376..dbc84edb1 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,6 @@ cmd/dendrite-demo-yggdrasil/embed/fs*.go # Test dependencies test/wasm/node_modules + +media_store/ + diff --git a/appservice/appservice.go b/appservice/appservice.go index 782e090a1..fd903b980 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -30,7 +30,7 @@ import ( "github.com/matrix-org/dendrite/appservice/types" "github.com/matrix-org/dendrite/appservice/workers" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/kafka" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -45,7 +45,7 @@ func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceQuer // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *setup.BaseDendrite, + base *base.BaseDendrite, userAPI userapi.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, ) appserviceAPI.AppServiceQueryAPI { diff --git a/build/docker/config/dendrite-config.yaml b/build/docker/config/dendrite-config.yaml index bc5d66994..d6357747b 100644 --- a/build/docker/config/dendrite-config.yaml +++ b/build/docker/config/dendrite-config.yaml @@ -174,6 +174,11 @@ federation_api: connect: http://federation_api:7772 external_api: listen: http://0.0.0.0:8072 + database: + connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_federationapi?sslmode=disable + max_open_conns: 10 + max_idle_conns: 2 + conn_max_lifetime: -1 # List of paths to X.509 certificates to be used by the external federation listeners. # These certificates will be used to calculate the TLS fingerprints and other servers @@ -181,17 +186,6 @@ federation_api: # format. federation_certificates: [] -# Configuration for the Federation Sender. -federation_sender: - internal_api: - listen: http://0.0.0.0:7775 - connect: http://federation_sender:7775 - database: - connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_federationsender?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - # How many times we will try to resend a failed transaction to a specific server. The # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. send_max_retries: 16 @@ -207,6 +201,22 @@ federation_sender: host: localhost port: 8080 + # Perspective keyservers to use as a backup when direct key fetches fail. This may + # be required to satisfy key requests for servers that are no longer online when + # joining some rooms. + key_perspectives: + - server_name: matrix.org + keys: + - key_id: ed25519:auto + public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw + - key_id: ed25519:a_RXGa + public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ + + # This option will control whether Dendrite will prefer to look up keys directly + # or whether it should try perspective servers first, using direct fetches as a + # last resort. + prefer_direct_fetch: false + # Configuration for the Key Server (for end-to-end encryption). key_server: internal_api: @@ -267,33 +277,6 @@ room_server: max_idle_conns: 2 conn_max_lifetime: -1 -# Configuration for the Server Key API (for server signing keys). -signing_key_server: - internal_api: - listen: http://0.0.0.0:7780 - connect: http://signing_key_server:7780 - database: - connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_signingkeyserver?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - - # Perspective keyservers to use as a backup when direct key fetches fail. This may - # be required to satisfy key requests for servers that are no longer online when - # joining some rooms. - key_perspectives: - - server_name: matrix.org - keys: - - key_id: ed25519:auto - public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw - - key_id: ed25519:a_RXGa - public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ - - # This option will control whether Dendrite will prefer to look up keys directly - # or whether it should try perspective servers first, using direct fetches as a - # last resort. - prefer_direct_fetch: false - # Configuration for the Sync API. sync_api: internal_api: diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index 5174830b5..9bbd6a8f7 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -54,15 +54,6 @@ services: - ./config:/etc/dendrite networks: - internal - - federation_sender: - hostname: federation_sender - image: matrixdotorg/dendrite-polylith:latest - command: federationsender - volumes: - - ./config:/etc/dendrite - networks: - - internal key_server: hostname: key_server @@ -72,16 +63,7 @@ services: - ./config:/etc/dendrite networks: - internal - - signing_key_server: - hostname: signing_key_server - image: matrixdotorg/dendrite-polylith:latest - command: signingkeyserver - volumes: - - ./config:/etc/dendrite - networks: - - internal - + user_api: hostname: user_api image: matrixdotorg/dendrite-polylith:latest diff --git a/build/docker/postgres/create_db.sh b/build/docker/postgres/create_db.sh index eb77bb26f..a7107e272 100755 --- a/build/docker/postgres/create_db.sh +++ b/build/docker/postgres/create_db.sh @@ -1,5 +1,5 @@ #!/bin/sh -for db in userapi_accounts userapi_devices mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice naffka; do +for db in userapi_accounts userapi_devices mediaapi syncapi roomserver keyserver federationapi appservice naffka; do createdb -U dendrite -O dendrite dendrite_$db done diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 4bd043196..95176fff7 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -13,6 +13,7 @@ import ( "net" "net/http" "os" + "strings" "sync" "time" @@ -24,12 +25,13 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/federationsender" - "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/userapi" @@ -86,15 +88,15 @@ func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) { m.PineconeMulticast.Start() } else { m.PineconeMulticast.Stop() - m.DisconnectType(pineconeRouter.PeerTypeMulticast) + m.DisconnectType(int(pineconeRouter.PeerTypeMulticast)) } } func (m *DendriteMonolith) SetStaticPeer(uri string) { m.staticPeerMutex.Lock() - m.staticPeerURI = uri + m.staticPeerURI = strings.TrimSpace(uri) m.staticPeerMutex.Unlock() - m.DisconnectType(pineconeRouter.PeerTypeRemote) + m.DisconnectType(int(pineconeRouter.PeerTypeRemote)) if uri != "" { go func() { m.staticPeerAttempt <- struct{}{} @@ -104,7 +106,7 @@ func (m *DendriteMonolith) SetStaticPeer(uri string) { func (m *DendriteMonolith) DisconnectType(peertype int) { for _, p := range m.PineconeRouter.Peers() { - if peertype == p.PeerType { + if int(peertype) == p.PeerType { m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil) } } @@ -132,7 +134,11 @@ func (m *DendriteMonolith) Conduit(zone string, peertype int) (*Conduit, error) for i := 1; i <= 10; i++ { logrus.Errorf("Attempting authenticated connect (attempt %d)", i) var err error - conduit.port, err = m.PineconeRouter.AuthenticatedConnect(l, zone, peertype, true) + conduit.port, err = m.PineconeRouter.Connect( + l, + pineconeRouter.ConnectionZone(zone), + pineconeRouter.ConnectionPeerType(peertype), + ) switch err { case io.ErrClosedPipe: logrus.Errorf("Authenticated connect failed due to closed pipe (attempt %d)", i) @@ -194,16 +200,28 @@ func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, e } func (m *DendriteMonolith) staticPeerConnect() { + connected := map[string]bool{} // URI -> connected? attempt := func() { - if m.PineconeRouter.PeerCount(pineconeRouter.PeerTypeRemote) == 0 { - m.staticPeerMutex.RLock() - uri := m.staticPeerURI - m.staticPeerMutex.RUnlock() - if uri == "" { - return - } - if err := conn.ConnectToPeer(m.PineconeRouter, uri); err != nil { - logrus.WithError(err).Error("Failed to connect to static peer") + m.staticPeerMutex.RLock() + uri := m.staticPeerURI + m.staticPeerMutex.RUnlock() + if uri == "" { + return + } + for k := range connected { + delete(connected, k) + } + for _, uri := range strings.Split(uri, ",") { + connected[strings.TrimSpace(uri)] = false + } + for _, info := range m.PineconeRouter.Peers() { + connected[info.URI] = true + } + for k, online := range connected { + if !online { + if err := conn.ConnectToPeer(m.PineconeRouter, k); err != nil { + logrus.WithError(err).Error("Failed to connect to static peer") + } } } } @@ -259,7 +277,7 @@ func (m *DendriteMonolith) Start() { prefix := hex.EncodeToString(pk) cfg := &config.Dendrite{} - cfg.Defaults() + cfg.Defaults(true) cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk)) cfg.Global.PrivateKey = sk cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) @@ -270,9 +288,8 @@ func (m *DendriteMonolith) Start() { cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-mediaapi.db", m.CacheDirectory, prefix)) cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-syncapi.db", m.StorageDirectory, prefix)) cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-roomserver.db", m.StorageDirectory, prefix)) - cfg.SigningKeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-signingkeyserver.db", m.StorageDirectory, prefix)) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-keyserver.db", m.StorageDirectory, prefix)) - cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-federationsender.db", m.StorageDirectory, prefix)) + cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-federationsender.db", m.StorageDirectory, prefix)) cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-appservice.db", m.StorageDirectory, prefix)) cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory)) cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory)) @@ -281,7 +298,7 @@ func (m *DendriteMonolith) Start() { panic(err) } - base := setup.NewBaseDendrite(cfg, "Monolith", false) + base := base.NewBaseDendrite(cfg, "Monolith") defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() @@ -290,12 +307,10 @@ func (m *DendriteMonolith) Start() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - rsAPI := roomserver.NewInternalAPI( - base, keyRing, - ) + rsAPI := roomserver.NewInternalAPI(base) - fsAPI := federationsender.NewInternalAPI( - base, federation, rsAPI, m.userAPI, keyRing, true, + fsAPI := federationapi.NewInternalAPI( + base, federation, rsAPI, m.userAPI, base.Caches, true, ) keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI) @@ -310,7 +325,8 @@ func (m *DendriteMonolith) Start() { // The underlying roomserver implementation needs to be able to call the fedsender. // This is different to rsAPI which can be the http client which doesn't need this dependency - rsAPI.SetFederationSenderAPI(fsAPI) + rsAPI.SetFederationAPI(fsAPI) + rsAPI.SetKeyring(keyRing) monolith := setup.Monolith{ Config: base.Cfg, @@ -321,7 +337,7 @@ func (m *DendriteMonolith) Start() { AppserviceAPI: asAPI, EDUInternalAPI: eduInputAPI, - FederationSenderAPI: fsAPI, + FederationAPI: fsAPI, RoomserverAPI: rsAPI, UserAPI: m.userAPI, KeyAPI: keyAPI, diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 558fbaee9..b0b955580 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -15,12 +15,13 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggrooms" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/federationsender" - "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" @@ -81,7 +82,7 @@ func (m *DendriteMonolith) Start() { m.YggdrasilNode = ygg cfg := &config.Dendrite{} - cfg.Defaults() + cfg.Defaults(true) cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName()) cfg.Global.PrivateKey = ygg.PrivateKey() cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) @@ -92,9 +93,8 @@ func (m *DendriteMonolith) Start() { cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-mediaapi.db", m.StorageDirectory)) cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-syncapi.db", m.StorageDirectory)) cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-roomserver.db", m.StorageDirectory)) - cfg.SigningKeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-signingkeyserver.db", m.StorageDirectory)) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-keyserver.db", m.StorageDirectory)) - cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-federationsender.db", m.StorageDirectory)) + cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-federationsender.db", m.StorageDirectory)) cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-appservice.db", m.StorageDirectory)) cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory)) cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory)) @@ -102,7 +102,7 @@ func (m *DendriteMonolith) Start() { panic(err) } - base := setup.NewBaseDendrite(cfg, "Monolith", false) + base := base.NewBaseDendrite(cfg, "Monolith") defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() @@ -111,16 +111,14 @@ func (m *DendriteMonolith) Start() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - rsAPI := roomserver.NewInternalAPI( - base, keyRing, - ) + rsAPI := roomserver.NewInternalAPI(base) keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation) userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI) keyAPI.SetUserAPI(userAPI) - fsAPI := federationsender.NewInternalAPI( - base, federation, rsAPI, userAPI, keyRing, true, + fsAPI := federationapi.NewInternalAPI( + base, federation, rsAPI, userAPI, base.Caches, true, ) eduInputAPI := eduserver.NewInternalAPI( @@ -132,7 +130,8 @@ func (m *DendriteMonolith) Start() { // The underlying roomserver implementation needs to be able to call the fedsender. // This is different to rsAPI which can be the http client which doesn't need this dependency - rsAPI.SetFederationSenderAPI(fsAPI) + rsAPI.SetFederationAPI(fsAPI) + rsAPI.SetKeyring(keyRing) monolith := setup.Monolith{ Config: base.Cfg, @@ -141,12 +140,12 @@ func (m *DendriteMonolith) Start() { FedClient: federation, KeyRing: keyRing, - AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, - FederationSenderAPI: fsAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, + AppserviceAPI: asAPI, + EDUInternalAPI: eduInputAPI, + FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + KeyAPI: keyAPI, ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider( ygg, fsAPI, federation, ), diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 562d89d28..64a7aa5e2 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -21,7 +21,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/clientapi/routing" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" - federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/transactions" keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" @@ -43,7 +43,7 @@ func AddPublicRoutes( eduInputAPI eduServerAPI.EDUServerInputAPI, asAPI appserviceAPI.AppServiceQueryAPI, transactionsCache *transactions.Cache, - fsAPI federationSenderAPI.FederationSenderInternalAPI, + fsAPI federationAPI.FederationInternalAPI, userAPI userapi.UserInternalAPI, keyAPI keyserverAPI.KeyInternalAPI, extRoomsProvider api.ExtraPublicRoomsProvider, diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 96cb0262e..e408c264f 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -20,7 +20,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + federationAPI "github.com/matrix-org/dendrite/federationapi/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -47,7 +47,7 @@ func DirectoryRoom( federation *gomatrixserverlib.FederationClient, cfg *config.ClientAPI, rsAPI roomserverAPI.RoomserverInternalAPI, - fedSenderAPI federationSenderAPI.FederationSenderInternalAPI, + fedSenderAPI federationAPI.FederationInternalAPI, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('#', roomAlias) if err != nil { @@ -96,8 +96,8 @@ func DirectoryRoom( } } } else { - joinedHostsReq := federationSenderAPI.QueryJoinedHostServerNamesInRoomRequest{RoomID: res.RoomID} - var joinedHostsRes federationSenderAPI.QueryJoinedHostServerNamesInRoomResponse + joinedHostsReq := federationAPI.QueryJoinedHostServerNamesInRoomRequest{RoomID: res.RoomID} + var joinedHostsRes federationAPI.QueryJoinedHostServerNamesInRoomResponse if err = fedSenderAPI.QueryJoinedHostServerNamesInRoom(req.Context(), &joinedHostsReq, &joinedHostsRes); err != nil { util.GetLogger(req.Context()).WithError(err).Error("fedSenderAPI.QueryJoinedHostServerNamesInRoom failed") return jsonerror.InternalServerError() diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index ea07f30be..1f615dc26 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -180,7 +180,7 @@ func TestValidationOfApplicationServices(t *testing.T) { // Set up a config fakeConfig := &config.Dendrite{} - fakeConfig.Defaults() + fakeConfig.Defaults(true) fakeConfig.Global.ServerName = "localhost" fakeConfig.ClientAPI.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index a9babc8aa..13d9efb6d 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -27,7 +27,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" - federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/transactions" keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" @@ -56,12 +56,12 @@ func Setup( federation *gomatrixserverlib.FederationClient, syncProducer *producers.SyncAPIProducer, transactionsCache *transactions.Cache, - federationSender federationSenderAPI.FederationSenderInternalAPI, + federationSender federationAPI.FederationInternalAPI, keyAPI keyserverAPI.KeyInternalAPI, extRoomsProvider api.ExtraPublicRoomsProvider, mscCfg *config.MSCs, ) { - rateLimits := newRateLimits(&cfg.RateLimiting) + rateLimits := httputil.NewRateLimits(&cfg.RateLimiting) userInteractiveAuth := auth.NewUserInteractive(accountDB.GetAccountByPassword, cfg) unstableFeatures := map[string]bool{ @@ -127,7 +127,7 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -143,7 +143,7 @@ func Setup( if mscCfg.Enabled("msc2753") { r0mux.Handle("/peek/{roomIDOrAlias}", httputil.MakeAuthAPI(gomatrixserverlib.Peek, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -163,7 +163,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/join", httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -177,7 +177,7 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/leave", httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -211,7 +211,7 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/invite", httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -329,14 +329,14 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } return Register(req, userAPI, accountDB, cfg) })).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } return RegisterAvailable(req, cfg, accountDB) @@ -410,7 +410,7 @@ func Setup( r0mux.Handle("/rooms/{roomID}/typing/{userID}", httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -466,7 +466,7 @@ func Setup( r0mux.Handle("/account/whoami", httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } return Whoami(req, device) @@ -475,7 +475,7 @@ func Setup( r0mux.Handle("/account/password", httputil.MakeAuthAPI("password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } return Password(req, userAPI, accountDB, device, cfg) @@ -484,7 +484,7 @@ func Setup( r0mux.Handle("/account/deactivate", httputil.MakeAuthAPI("deactivate", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } return Deactivate(req, userInteractiveAuth, userAPI, device) @@ -495,7 +495,7 @@ func Setup( r0mux.Handle("/login", httputil.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } return Login(req, accountDB, userAPI, cfg) @@ -552,7 +552,7 @@ func Setup( r0mux.Handle("/profile/{userID}/avatar_url", httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -577,7 +577,7 @@ func Setup( r0mux.Handle("/profile/{userID}/displayname", httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -614,9 +614,23 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) + // Element logs get flooded unless this is handled + r0mux.Handle("/presence/{userID}/status", + httputil.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse { + if r := rateLimits.Limit(req); r != nil { + return *r + } + // TODO: Set presence (probably the responsibility of a presence server not clientapi) + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } + }), + ).Methods(http.MethodPut, http.MethodOptions) + r0mux.Handle("/voip/turnServer", httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } return RequestTurnServer(req, device, cfg) @@ -695,7 +709,7 @@ func Setup( r0mux.Handle("/user/{userID}/openid/request_token", httputil.MakeAuthAPI("openid_request_token", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -708,7 +722,7 @@ func Setup( r0mux.Handle("/user_directory/search", httputil.MakeAuthAPI("userdirectory_search", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } postContent := struct { @@ -753,7 +767,7 @@ func Setup( r0mux.Handle("/rooms/{roomID}/read_markers", httputil.MakeAuthAPI("rooms_read_markers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -766,7 +780,7 @@ func Setup( r0mux.Handle("/rooms/{roomID}/forget", httputil.MakeAuthAPI("rooms_forget", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -870,7 +884,7 @@ func Setup( r0mux.Handle("/capabilities", httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } return GetCapabilities(req, rsAPI) @@ -1085,8 +1099,8 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}", - httputil.MakeAuthAPI("set_receipt", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + if r := rateLimits.Limit(req); r != nil { return *r } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -1099,7 +1113,7 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/presence/{userId}/status", httputil.MakeAuthAPI("set_presence", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - if r := rateLimits.rateLimit(req); r != nil { + if r := rateLimits.Limit(req); r != nil { return *r } if !cfg.Matrix.PresenceEnabled { diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 715cf8ce6..cdb106c23 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -30,14 +30,13 @@ import ( "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed" "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/mscs" - "github.com/matrix-org/dendrite/signingkeyserver" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" @@ -50,7 +49,7 @@ import ( func createKeyDB( base *P2PDendrite, - db gomatrixserverlib.KeyDatabase, + db *gomatrixserverlib.KeyRing, ) { mdns := mDNSListener{ host: base.LibP2P, @@ -120,19 +119,18 @@ func main() { } cfg := config.Dendrite{} - cfg.Defaults() + cfg.Defaults(true) cfg.Global.ServerName = "p2p" cfg.Global.PrivateKey = privKey cfg.Global.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName)) cfg.Global.Kafka.UseNaffka = true - cfg.FederationSender.FederationMaxRetries = 6 + cfg.FederationAPI.FederationMaxRetries = 6 cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName)) cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) - cfg.SigningKeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-signingkeyserver.db", *instanceName)) - cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) + cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName)) cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName)) @@ -151,32 +149,29 @@ func main() { userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, nil, keyAPI) keyAPI.SetUserAPI(userAPI) - serverKeyAPI := signingkeyserver.NewInternalAPI( - &base.Base.Cfg.SigningKeyServer, federation, base.Base.Caches, - ) - keyRing := serverKeyAPI.KeyRing() - createKeyDB( - base, serverKeyAPI, - ) - rsAPI := roomserver.NewInternalAPI( - &base.Base, keyRing, + &base.Base, ) eduInputAPI := eduserver.NewInternalAPI( &base.Base, cache.New(), userAPI, ) asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) - fsAPI := federationsender.NewInternalAPI( - &base.Base, federation, rsAPI, userAPI, keyRing, true, + fsAPI := federationapi.NewInternalAPI( + &base.Base, federation, rsAPI, userAPI, base.Base.Caches, true, ) - rsAPI.SetFederationSenderAPI(fsAPI) + keyRing := fsAPI.KeyRing() + rsAPI.SetFederationAPI(fsAPI) provider := newPublicRoomsProvider(base.LibP2PPubsub, rsAPI) err = provider.Start() if err != nil { panic("failed to create new public rooms provider: " + err.Error()) } + createKeyDB( + base, keyRing, + ) + monolith := setup.Monolith{ Config: base.Base.Cfg, AccountDB: accountDB, @@ -186,9 +181,8 @@ func main() { AppserviceAPI: asAPI, EDUInternalAPI: eduInputAPI, - FederationSenderAPI: fsAPI, + FederationAPI: fsAPI, RoomserverAPI: rsAPI, - ServerKeyAPI: serverKeyAPI, UserAPI: userAPI, KeyAPI: keyAPI, ExtPublicRoomsProvider: provider, diff --git a/cmd/dendrite-demo-libp2p/mdnslistener.go b/cmd/dendrite-demo-libp2p/mdnslistener.go index c30aaa331..c6105e52c 100644 --- a/cmd/dendrite-demo-libp2p/mdnslistener.go +++ b/cmd/dendrite-demo-libp2p/mdnslistener.go @@ -25,7 +25,7 @@ import ( ) type mDNSListener struct { - keydb gomatrixserverlib.KeyDatabase + keydb *gomatrixserverlib.KeyRing host host.Host } @@ -35,7 +35,7 @@ func (n *mDNSListener) HandlePeerFound(p peer.AddrInfo) { } if pubkey, err := p.ID.ExtractPublicKey(); err == nil { raw, _ := pubkey.Raw() - if err := n.keydb.StoreKeys( + if err := n.keydb.KeyDatabase.StoreKeys( context.Background(), map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{ { diff --git a/cmd/dendrite-demo-libp2p/p2pdendrite.go b/cmd/dendrite-demo-libp2p/p2pdendrite.go index 45eb42a9c..ba1868b27 100644 --- a/cmd/dendrite-demo-libp2p/p2pdendrite.go +++ b/cmd/dendrite-demo-libp2p/p2pdendrite.go @@ -22,7 +22,6 @@ import ( pstore "github.com/libp2p/go-libp2p-core/peerstore" record "github.com/libp2p/go-libp2p-record" - "github.com/matrix-org/dendrite/setup" "github.com/libp2p/go-libp2p" circuit "github.com/libp2p/go-libp2p-circuit" @@ -34,12 +33,13 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) // P2PDendrite is a Peer-to-Peer variant of BaseDendrite. type P2PDendrite struct { - Base setup.BaseDendrite + Base base.BaseDendrite // Store our libp2p object so that we can make outgoing connections from it // later @@ -54,7 +54,7 @@ type P2PDendrite struct { // The componentName is used for logging purposes, and should be a friendly name // of the component running, e.g. SyncAPI. func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite { - baseDendrite := setup.NewBaseDendrite(cfg, componentName, false) + baseDendrite := base.NewBaseDendrite(cfg, componentName) ctx, cancel := context.WithCancel(context.Background()) diff --git a/cmd/dendrite-demo-pinecone/conn/client.go b/cmd/dendrite-demo-pinecone/conn/client.go index 29436fda9..e3cc0468c 100644 --- a/cmd/dendrite-demo-pinecone/conn/client.go +++ b/cmd/dendrite-demo-pinecone/conn/client.go @@ -7,7 +7,7 @@ import ( "net/http" "strings" - "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/gomatrixserverlib" "nhooyr.io/websocket" @@ -34,7 +34,12 @@ func ConnectToPeer(pRouter *pineconeRouter.Router, peer string) error { if parent == nil { return fmt.Errorf("failed to wrap connection") } - _, err := pRouter.AuthenticatedConnect(parent, "static", pineconeRouter.PeerTypeRemote, true) + _, err := pRouter.Connect( + parent, + pineconeRouter.ConnectionZone("static"), + pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote), + pineconeRouter.ConnectionURI(peer), + ) return err } @@ -70,7 +75,7 @@ func createTransport(s *pineconeSessions.Sessions) *http.Transport { } func CreateClient( - base *setup.BaseDendrite, s *pineconeSessions.Sessions, + base *base.BaseDendrite, s *pineconeSessions.Sessions, ) *gomatrixserverlib.Client { return gomatrixserverlib.NewClient( gomatrixserverlib.WithTransport(createTransport(s)), @@ -78,7 +83,7 @@ func CreateClient( } func CreateFederationClient( - base *setup.BaseDendrite, s *pineconeSessions.Sessions, + base *base.BaseDendrite, s *pineconeSessions.Sessions, ) *gomatrixserverlib.FederationClient { return gomatrixserverlib.NewFederationClient( base.Cfg.Global.ServerName, diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index a4f886199..e912b8420 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -26,6 +26,7 @@ import ( "net" "net/http" "os" + "strings" "time" "github.com/gorilla/mux" @@ -37,13 +38,14 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/federationsender" - "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" @@ -60,7 +62,7 @@ import ( var ( instanceName = flag.String("name", "dendrite-p2p-pinecone", "the name of this P2P demo instance") instancePort = flag.Int("port", 8008, "the port that the client API will listen on") - instancePeer = flag.String("peer", "", "the static Pinecone peer to connect to") + instancePeer = flag.String("peer", "", "the static Pinecone peers to connect to, comma separated-list") instanceListen = flag.String("listen", ":0", "the port Pinecone peers can connect to") ) @@ -108,9 +110,12 @@ func main() { continue } - port, err := pRouter.AuthenticatedConnect(conn, "", pineconeRouter.PeerTypeRemote, true) + port, err := pRouter.Connect( + conn, + pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote), + ) if err != nil { - logrus.WithError(err).Error("pSwitch.AuthenticatedConnect failed") + logrus.WithError(err).Error("pSwitch.Connect failed") continue } @@ -123,14 +128,22 @@ func main() { pMulticast.Start() connectToStaticPeer := func() { + connected := map[string]bool{} // URI -> connected? + for _, uri := range strings.Split(*instancePeer, ",") { + connected[strings.TrimSpace(uri)] = false + } attempt := func() { - if pRouter.PeerCount(pineconeRouter.PeerTypeRemote) == 0 { - uri := *instancePeer - if uri == "" { - return - } - if err := conn.ConnectToPeer(pRouter, uri); err != nil { - logrus.WithError(err).Error("Failed to connect to static peer") + for k := range connected { + connected[k] = false + } + for _, info := range pRouter.Peers() { + connected[info.URI] = true + } + for k, online := range connected { + if !online { + if err := conn.ConnectToPeer(pRouter, k); err != nil { + logrus.WithError(err).Error("Failed to connect to static peer") + } } } } @@ -141,7 +154,7 @@ func main() { } cfg := &config.Dendrite{} - cfg.Defaults() + cfg.Defaults(true) cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk)) cfg.Global.PrivateKey = sk cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) @@ -151,9 +164,8 @@ func main() { cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) - cfg.SigningKeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-signingkeyserver.db", *instanceName)) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName)) - cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) + cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName)) cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} @@ -161,7 +173,7 @@ func main() { panic(err) } - base := setup.NewBaseDendrite(cfg, "Monolith", false) + base := base.NewBaseDendrite(cfg, "Monolith") defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() @@ -170,12 +182,10 @@ func main() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - rsComponent := roomserver.NewInternalAPI( - base, keyRing, - ) + rsComponent := roomserver.NewInternalAPI(base) rsAPI := rsComponent - fsAPI := federationsender.NewInternalAPI( - base, federation, rsAPI, nil, keyRing, true, + fsAPI := federationapi.NewInternalAPI( + base, federation, rsAPI, nil, base.Caches, true, ) keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI) @@ -189,7 +199,8 @@ func main() { asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) - rsComponent.SetFederationSenderAPI(fsAPI) + rsComponent.SetFederationAPI(fsAPI) + rsComponent.SetKeyring(keyRing) monolith := setup.Monolith{ Config: base.Cfg, @@ -200,7 +211,7 @@ func main() { AppserviceAPI: asAPI, EDUInternalAPI: eduInputAPI, - FederationSenderAPI: fsAPI, + FederationAPI: fsAPI, RoomserverAPI: rsAPI, UserAPI: userAPI, KeyAPI: keyAPI, @@ -232,7 +243,11 @@ func main() { return } conn := conn.WrapWebSocketConn(c) - if _, err = pRouter.AuthenticatedConnect(conn, "websocket", pineconeRouter.PeerTypeRemote, true); err != nil { + if _, err = pRouter.Connect( + conn, + pineconeRouter.ConnectionZone("websocket"), + pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote), + ); err != nil { logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch") } }) diff --git a/cmd/dendrite-demo-pinecone/rooms/rooms.go b/cmd/dendrite-demo-pinecone/rooms/rooms.go index fc387a174..5972d129f 100644 --- a/cmd/dendrite-demo-pinecone/rooms/rooms.go +++ b/cmd/dendrite-demo-pinecone/rooms/rooms.go @@ -19,7 +19,7 @@ import ( "sync" "time" - "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -30,14 +30,14 @@ import ( type PineconeRoomProvider struct { r *pineconeRouter.Router s *pineconeSessions.Sessions - fedSender api.FederationSenderInternalAPI + fedSender api.FederationInternalAPI fedClient *gomatrixserverlib.FederationClient } func NewPineconeRoomProvider( r *pineconeRouter.Router, s *pineconeSessions.Sessions, - fedSender api.FederationSenderInternalAPI, + fedSender api.FederationInternalAPI, fedClient *gomatrixserverlib.FederationClient, ) *PineconeRoomProvider { p := &PineconeRoomProvider{ diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index af7997f62..219254c42 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -31,13 +31,14 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggrooms" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/federationsender" - "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/mscs" "github.com/matrix-org/dendrite/userapi" @@ -72,7 +73,7 @@ func main() { */ cfg := &config.Dendrite{} - cfg.Defaults() + cfg.Defaults(true) cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName()) cfg.Global.PrivateKey = ygg.PrivateKey() cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) @@ -82,9 +83,8 @@ func main() { cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) - cfg.SigningKeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-signingkeyserver.db", *instanceName)) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName)) - cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) + cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName)) cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) cfg.MSCs.MSCs = []string{"msc2836"} @@ -93,7 +93,7 @@ func main() { panic(err) } - base := setup.NewBaseDendrite(cfg, "Monolith", false) + base := base.NewBaseDendrite(cfg, "Monolith") defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() @@ -107,7 +107,7 @@ func main() { keyAPI.SetUserAPI(userAPI) rsComponent := roomserver.NewInternalAPI( - base, keyRing, + base, ) rsAPI := rsComponent @@ -117,11 +117,12 @@ func main() { asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) - fsAPI := federationsender.NewInternalAPI( - base, federation, rsAPI, userAPI, keyRing, true, + fsAPI := federationapi.NewInternalAPI( + base, federation, rsAPI, userAPI, base.Caches, true, ) - rsComponent.SetFederationSenderAPI(fsAPI) + rsComponent.SetFederationAPI(fsAPI) + rsComponent.SetKeyring(keyRing) monolith := setup.Monolith{ Config: base.Cfg, @@ -130,12 +131,12 @@ func main() { FedClient: federation, KeyRing: keyRing, - AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, - FederationSenderAPI: fsAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, + AppserviceAPI: asAPI, + EDUInternalAPI: eduInputAPI, + FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + KeyAPI: keyAPI, ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider( ygg, fsAPI, federation, ), diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/client.go b/cmd/dendrite-demo-yggdrasil/yggconn/client.go index c7409c210..358d3725e 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/client.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/client.go @@ -4,7 +4,7 @@ import ( "net/http" "time" - "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/gomatrixserverlib" ) @@ -18,7 +18,7 @@ func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) { } func (n *Node) CreateClient( - base *setup.BaseDendrite, + base *base.BaseDendrite, ) *gomatrixserverlib.Client { tr := &http.Transport{} tr.RegisterProtocol( @@ -39,7 +39,7 @@ func (n *Node) CreateClient( } func (n *Node) CreateFederationClient( - base *setup.BaseDendrite, + base *base.BaseDendrite, ) *gomatrixserverlib.FederationClient { tr := &http.Transport{} tr.RegisterProtocol( diff --git a/cmd/dendrite-demo-yggdrasil/yggrooms/yggrooms.go b/cmd/dendrite-demo-yggdrasil/yggrooms/yggrooms.go index 0174e84d7..402b86ed3 100644 --- a/cmd/dendrite-demo-yggdrasil/yggrooms/yggrooms.go +++ b/cmd/dendrite-demo-yggdrasil/yggrooms/yggrooms.go @@ -20,19 +20,19 @@ import ( "time" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn" - "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) type YggdrasilRoomProvider struct { node *yggconn.Node - fedSender api.FederationSenderInternalAPI + fedSender api.FederationInternalAPI fedClient *gomatrixserverlib.FederationClient } func NewYggdrasilRoomProvider( - node *yggconn.Node, fedSender api.FederationSenderInternalAPI, fedClient *gomatrixserverlib.FederationClient, + node *yggconn.Node, fedSender api.FederationInternalAPI, fedClient *gomatrixserverlib.FederationClient, ) *YggdrasilRoomProvider { p := &YggdrasilRoomProvider{ node: node, diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index cec0a8af6..ab18179fd 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -21,14 +21,14 @@ import ( "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup" + basepkg "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/mscs" - "github.com/matrix-org/dendrite/signingkeyserver" "github.com/matrix-org/dendrite/userapi" uapi "github.com/matrix-org/dendrite/userapi/api" "github.com/sirupsen/logrus" @@ -51,7 +51,7 @@ func main() { httpAddr := config.HTTPAddress("http://" + *httpBindAddr) httpsAddr := config.HTTPAddress("https://" + *httpsBindAddr) httpAPIAddr := httpAddr - + options := []basepkg.BaseDendriteOptions{} if *enableHTTPAPIs { logrus.Warnf("DANGER! The -api option is enabled, exposing internal APIs on %q!", *apiBindAddr) httpAPIAddr = config.HTTPAddress("http://" + *apiBindAddr) @@ -63,32 +63,20 @@ func main() { cfg.ClientAPI.InternalAPI.Connect = httpAPIAddr cfg.EDUServer.InternalAPI.Connect = httpAPIAddr cfg.FederationAPI.InternalAPI.Connect = httpAPIAddr - cfg.FederationSender.InternalAPI.Connect = httpAPIAddr cfg.KeyServer.InternalAPI.Connect = httpAPIAddr cfg.MediaAPI.InternalAPI.Connect = httpAPIAddr cfg.RoomServer.InternalAPI.Connect = httpAPIAddr - cfg.SigningKeyServer.InternalAPI.Connect = httpAPIAddr cfg.SyncAPI.InternalAPI.Connect = httpAPIAddr + options = append(options, basepkg.UseHTTPAPIs) } - base := setup.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs) + base := basepkg.NewBaseDendrite(cfg, "Monolith", options...) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() federation := base.CreateFederationClient() - skAPI := signingkeyserver.NewInternalAPI( - &base.Cfg.SigningKeyServer, federation, base.Caches, - ) - if base.UseHTTPAPIs { - signingkeyserver.AddInternalRoutes(base.InternalAPIMux, skAPI, base.Caches) - skAPI = base.SigningKeyServerHTTPClient() - } - keyRing := skAPI.KeyRing() - - rsImpl := roomserver.NewInternalAPI( - base, keyRing, - ) + rsImpl := roomserver.NewInternalAPI(base) // call functions directly on the impl unless running in HTTP mode rsAPI := rsImpl if base.UseHTTPAPIs { @@ -101,16 +89,18 @@ func main() { } } - fsAPI := federationsender.NewInternalAPI( - base, federation, rsAPI, nil, keyRing, false, + fsAPI := federationapi.NewInternalAPI( + base, federation, rsAPI, nil, base.Caches, false, ) if base.UseHTTPAPIs { - federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI) - fsAPI = base.FederationSenderHTTPClient() + federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI) + fsAPI = base.FederationAPIHTTPClient() } + keyRing := fsAPI.KeyRing() + // The underlying roomserver implementation needs to be able to call the fedsender. // This is different to rsAPI which can be the http client which doesn't need this dependency - rsImpl.SetFederationSenderAPI(fsAPI) + rsImpl.SetFederationAPI(fsAPI) keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI) userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI) @@ -150,13 +140,12 @@ func main() { FedClient: federation, KeyRing: keyRing, - AppserviceAPI: asAPI, - EDUInternalAPI: eduInputAPI, - FederationSenderAPI: fsAPI, - RoomserverAPI: rsAPI, - ServerKeyAPI: skAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, + AppserviceAPI: asAPI, + EDUInternalAPI: eduInputAPI, + FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + KeyAPI: keyAPI, } monolith.AddAllPublicRoutes( base.ProcessContext, @@ -186,9 +175,9 @@ func main() { if *certFile != "" && *keyFile != "" { go func() { base.SetupAndServeHTTP( - setup.NoListener, // internal API - httpsAddr, // external API - certFile, keyFile, // TLS settings + basepkg.NoListener, // internal API + httpsAddr, // external API + certFile, keyFile, // TLS settings ) }() } diff --git a/cmd/dendrite-polylith-multi/main.go b/cmd/dendrite-polylith-multi/main.go index c2208ca24..edfe6cdb0 100644 --- a/cmd/dendrite-polylith-multi/main.go +++ b/cmd/dendrite-polylith-multi/main.go @@ -21,13 +21,14 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities" "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/sirupsen/logrus" _ "github.com/mattn/go-sqlite3" ) -type entrypoint func(base *setup.BaseDendrite, cfg *config.Dendrite) +type entrypoint func(base *base.BaseDendrite, cfg *config.Dendrite) func main() { cfg := setup.ParseFlags(true) @@ -40,17 +41,15 @@ func main() { } components := map[string]entrypoint{ - "appservice": personalities.Appservice, - "clientapi": personalities.ClientAPI, - "eduserver": personalities.EDUServer, - "federationapi": personalities.FederationAPI, - "federationsender": personalities.FederationSender, - "keyserver": personalities.KeyServer, - "mediaapi": personalities.MediaAPI, - "roomserver": personalities.RoomServer, - "signingkeyserver": personalities.SigningKeyServer, - "syncapi": personalities.SyncAPI, - "userapi": personalities.UserAPI, + "appservice": personalities.Appservice, + "clientapi": personalities.ClientAPI, + "eduserver": personalities.EDUServer, + "federationapi": personalities.FederationAPI, + "keyserver": personalities.KeyServer, + "mediaapi": personalities.MediaAPI, + "roomserver": personalities.RoomServer, + "syncapi": personalities.SyncAPI, + "userapi": personalities.UserAPI, } start, ok := components[component] @@ -73,8 +72,8 @@ func main() { logrus.Infof("Starting %q component", component) - base := setup.NewBaseDendrite(cfg, component, false) // TODO - defer base.Close() // nolint: errcheck + base := base.NewBaseDendrite(cfg, component) // TODO + defer base.Close() // nolint: errcheck go start(base, cfg) base.WaitForShutdown() diff --git a/cmd/dendrite-polylith-multi/personalities/appservice.go b/cmd/dendrite-polylith-multi/personalities/appservice.go index d269b15d4..4f74434a4 100644 --- a/cmd/dendrite-polylith-multi/personalities/appservice.go +++ b/cmd/dendrite-polylith-multi/personalities/appservice.go @@ -16,11 +16,12 @@ package personalities import ( "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" + basepkg "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) -func Appservice(base *setup.BaseDendrite, cfg *config.Dendrite) { +func Appservice(base *base.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() rsAPI := base.RoomserverHTTPClient() @@ -29,7 +30,7 @@ func Appservice(base *setup.BaseDendrite, cfg *config.Dendrite) { base.SetupAndServeHTTP( base.Cfg.AppServiceAPI.InternalAPI.Listen, // internal listener - setup.NoListener, // external listener + basepkg.NoListener, // external listener nil, nil, ) } diff --git a/cmd/dendrite-polylith-multi/personalities/clientapi.go b/cmd/dendrite-polylith-multi/personalities/clientapi.go index 5e0c43548..bd9f7a109 100644 --- a/cmd/dendrite-polylith-multi/personalities/clientapi.go +++ b/cmd/dendrite-polylith-multi/personalities/clientapi.go @@ -17,17 +17,17 @@ package personalities import ( "github.com/matrix-org/dendrite/clientapi" "github.com/matrix-org/dendrite/internal/transactions" - "github.com/matrix-org/dendrite/setup" + basepkg "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) -func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { +func ClientAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { accountDB := base.CreateAccountsDB() federation := base.CreateFederationClient() asQuery := base.AppserviceHTTPClient() rsAPI := base.RoomserverHTTPClient() - fsAPI := base.FederationSenderHTTPClient() + fsAPI := base.FederationAPIHTTPClient() eduInputAPI := base.EDUServerClient() userAPI := base.UserAPIClient() keyAPI := base.KeyServerHTTPClient() diff --git a/cmd/dendrite-polylith-multi/personalities/eduserver.go b/cmd/dendrite-polylith-multi/personalities/eduserver.go index 55b986e8f..8719facb3 100644 --- a/cmd/dendrite-polylith-multi/personalities/eduserver.go +++ b/cmd/dendrite-polylith-multi/personalities/eduserver.go @@ -17,17 +17,17 @@ package personalities import ( "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/setup" + basepkg "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) -func EDUServer(base *setup.BaseDendrite, cfg *config.Dendrite) { +func EDUServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) { intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient()) eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI) base.SetupAndServeHTTP( base.Cfg.EDUServer.InternalAPI.Listen, // internal listener - setup.NoListener, // external listener + basepkg.NoListener, // external listener nil, nil, ) } diff --git a/cmd/dendrite-polylith-multi/personalities/federationapi.go b/cmd/dendrite-polylith-multi/personalities/federationapi.go index 5488fbf32..9b59cf45b 100644 --- a/cmd/dendrite-polylith-multi/personalities/federationapi.go +++ b/cmd/dendrite-polylith-multi/personalities/federationapi.go @@ -16,16 +16,15 @@ package personalities import ( "github.com/matrix-org/dendrite/federationapi" - "github.com/matrix-org/dendrite/setup" + basepkg "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) -func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { +func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() federation := base.CreateFederationClient() - serverKeyAPI := base.SigningKeyServerHTTPClient() - keyRing := serverKeyAPI.KeyRing() - fsAPI := base.FederationSenderHTTPClient() + fsAPI := base.FederationAPIHTTPClient() + keyRing := fsAPI.KeyRing() rsAPI := base.RoomserverHTTPClient() keyAPI := base.KeyServerHTTPClient() diff --git a/cmd/dendrite-polylith-multi/personalities/federationsender.go b/cmd/dendrite-polylith-multi/personalities/federationsender.go deleted file mode 100644 index 1b25480e9..000000000 --- a/cmd/dendrite-polylith-multi/personalities/federationsender.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - "github.com/matrix-org/dendrite/federationsender" - "github.com/matrix-org/dendrite/setup" - "github.com/matrix-org/dendrite/setup/config" -) - -func FederationSender(base *setup.BaseDendrite, cfg *config.Dendrite) { - federation := base.CreateFederationClient() - - serverKeyAPI := base.SigningKeyServerHTTPClient() - keyRing := serverKeyAPI.KeyRing() - - rsAPI := base.RoomserverHTTPClient() - fsAPI := federationsender.NewInternalAPI( - base, federation, rsAPI, nil, keyRing, false, - ) - federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI) - - base.SetupAndServeHTTP( - base.Cfg.FederationSender.InternalAPI.Listen, // internal listener - setup.NoListener, // external listener - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/keyserver.go b/cmd/dendrite-polylith-multi/personalities/keyserver.go index 8a99d7797..f8aa57b86 100644 --- a/cmd/dendrite-polylith-multi/personalities/keyserver.go +++ b/cmd/dendrite-polylith-multi/personalities/keyserver.go @@ -16,12 +16,12 @@ package personalities import ( "github.com/matrix-org/dendrite/keyserver" - "github.com/matrix-org/dendrite/setup" + basepkg "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) -func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) { - fsAPI := base.FederationSenderHTTPClient() +func KeyServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) { + fsAPI := base.FederationAPIHTTPClient() intAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI) intAPI.SetUserAPI(base.UserAPIClient()) @@ -29,7 +29,7 @@ func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) { base.SetupAndServeHTTP( base.Cfg.KeyServer.InternalAPI.Listen, // internal listener - setup.NoListener, // external listener + basepkg.NoListener, // external listener nil, nil, ) } diff --git a/cmd/dendrite-polylith-multi/personalities/mediaapi.go b/cmd/dendrite-polylith-multi/personalities/mediaapi.go index cf3e6882b..fa9d36a38 100644 --- a/cmd/dendrite-polylith-multi/personalities/mediaapi.go +++ b/cmd/dendrite-polylith-multi/personalities/mediaapi.go @@ -16,15 +16,15 @@ package personalities import ( "github.com/matrix-org/dendrite/mediaapi" - "github.com/matrix-org/dendrite/setup" + basepkg "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) -func MediaAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { +func MediaAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() client := base.CreateClient() - mediaapi.AddPublicRoutes(base.PublicMediaAPIMux, &base.Cfg.MediaAPI, userAPI, client) + mediaapi.AddPublicRoutes(base.PublicMediaAPIMux, &base.Cfg.MediaAPI, &base.Cfg.ClientAPI.RateLimiting, userAPI, client) base.SetupAndServeHTTP( base.Cfg.MediaAPI.InternalAPI.Listen, diff --git a/cmd/dendrite-polylith-multi/personalities/roomserver.go b/cmd/dendrite-polylith-multi/personalities/roomserver.go index 72f0f6d12..23514dbed 100644 --- a/cmd/dendrite-polylith-multi/personalities/roomserver.go +++ b/cmd/dendrite-polylith-multi/personalities/roomserver.go @@ -16,24 +16,21 @@ package personalities import ( "github.com/matrix-org/dendrite/roomserver" - "github.com/matrix-org/dendrite/setup" + basepkg "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" ) -func RoomServer(base *setup.BaseDendrite, cfg *config.Dendrite) { - serverKeyAPI := base.SigningKeyServerHTTPClient() - keyRing := serverKeyAPI.KeyRing() - +func RoomServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) { asAPI := base.AppserviceHTTPClient() - fsAPI := base.FederationSenderHTTPClient() - rsAPI := roomserver.NewInternalAPI(base, keyRing) - rsAPI.SetFederationSenderAPI(fsAPI) + fsAPI := base.FederationAPIHTTPClient() + rsAPI := roomserver.NewInternalAPI(base) + rsAPI.SetFederationAPI(fsAPI) rsAPI.SetAppserviceAPI(asAPI) roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI) base.SetupAndServeHTTP( base.Cfg.RoomServer.InternalAPI.Listen, // internal listener - setup.NoListener, // external listener + basepkg.NoListener, // external listener nil, nil, ) } diff --git a/cmd/dendrite-polylith-multi/personalities/signingkeyserver.go b/cmd/dendrite-polylith-multi/personalities/signingkeyserver.go deleted file mode 100644 index 0a7fc502a..000000000 --- a/cmd/dendrite-polylith-multi/personalities/signingkeyserver.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - "github.com/matrix-org/dendrite/setup" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/signingkeyserver" -) - -func SigningKeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) { - federation := base.CreateFederationClient() - - intAPI := signingkeyserver.NewInternalAPI(&base.Cfg.SigningKeyServer, federation, base.Caches) - signingkeyserver.AddInternalRoutes(base.InternalAPIMux, intAPI, base.Caches) - - base.SetupAndServeHTTP( - base.Cfg.SigningKeyServer.InternalAPI.Listen, - setup.NoListener, - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/syncapi.go b/cmd/dendrite-polylith-multi/personalities/syncapi.go index b9b202294..6fee8419b 100644 --- a/cmd/dendrite-polylith-multi/personalities/syncapi.go +++ b/cmd/dendrite-polylith-multi/personalities/syncapi.go @@ -15,12 +15,12 @@ package personalities import ( - "github.com/matrix-org/dendrite/setup" + basepkg "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/syncapi" ) -func SyncAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { +func SyncAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() federation := base.CreateFederationClient() diff --git a/cmd/dendrite-polylith-multi/personalities/userapi.go b/cmd/dendrite-polylith-multi/personalities/userapi.go index 6feb906d3..f147cda14 100644 --- a/cmd/dendrite-polylith-multi/personalities/userapi.go +++ b/cmd/dendrite-polylith-multi/personalities/userapi.go @@ -15,12 +15,12 @@ package personalities import ( - "github.com/matrix-org/dendrite/setup" + basepkg "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi" ) -func UserAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { +func UserAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { accountDB := base.CreateAccountsDB() userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient()) @@ -29,7 +29,7 @@ func UserAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { base.SetupAndServeHTTP( base.Cfg.UserAPI.InternalAPI.Listen, // internal listener - setup.NoListener, // external listener + basepkg.NoListener, // external listener nil, nil, ) } diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go index 9047a6513..8665afbf6 100644 --- a/cmd/dendritejs-pinecone/main.go +++ b/cmd/dendritejs-pinecone/main.go @@ -33,11 +33,12 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi" @@ -160,14 +161,13 @@ func startup() { pSessions := pineconeSessions.NewSessions(logger, pRouter) cfg := &config.Dendrite{} - cfg.Defaults() + cfg.Defaults(true) cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db" cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db" cfg.UserAPI.DeviceDatabase.ConnectionString = "file:/idb/dendritejs_device.db" - cfg.FederationSender.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db" + cfg.FederationAPI.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db" cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db" cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db" - cfg.SigningKeyServer.Database.ConnectionString = "file:/idb/dendritejs_signingkeyserver.db" cfg.SyncAPI.Database.ConnectionString = "file:/idb/dendritejs_syncapi.db" cfg.KeyServer.Database.ConnectionString = "file:/idb/dendritejs_e2ekey.db" cfg.Global.Kafka.UseNaffka = true @@ -180,7 +180,7 @@ func startup() { if err := cfg.Derive(); err != nil { logrus.Fatalf("Failed to derive values from config: %s", err) } - base := setup.NewBaseDendrite(cfg, "Monolith", false) + base := base.NewBaseDendrite(cfg, "Monolith") defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() @@ -192,14 +192,15 @@ func startup() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - rsAPI := roomserver.NewInternalAPI(base, keyRing) + rsAPI := roomserver.NewInternalAPI(base) eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), userAPI) asQuery := appservice.NewInternalAPI( base, userAPI, rsAPI, ) rsAPI.SetAppserviceAPI(asQuery) - fedSenderAPI := federationsender.NewInternalAPI(base, federation, rsAPI, userAPI, keyRing, true) - rsAPI.SetFederationSenderAPI(fedSenderAPI) + fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, userAPI, base.Caches, true) + rsAPI.SetFederationAPI(fedSenderAPI) + rsAPI.SetKeyring(keyRing) monolith := setup.Monolith{ Config: base.Cfg, @@ -208,12 +209,12 @@ func startup() { FedClient: federation, KeyRing: keyRing, - AppserviceAPI: asQuery, - EDUInternalAPI: eduInputAPI, - FederationSenderAPI: fedSenderAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, + AppserviceAPI: asQuery, + EDUInternalAPI: eduInputAPI, + FederationAPI: fedSenderAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + KeyAPI: keyAPI, //ServerKeyAPI: serverKeyAPI, ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pSessions, fedSenderAPI, federation), } diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 10aadb6e5..969581168 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -26,7 +26,7 @@ import ( "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" @@ -164,14 +164,13 @@ func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http func main() { cfg := &config.Dendrite{} - cfg.Defaults() + cfg.Defaults(true) cfg.UserAPI.AccountDatabase.ConnectionString = "file:/idb/dendritejs_account.db" cfg.AppServiceAPI.Database.ConnectionString = "file:/idb/dendritejs_appservice.db" cfg.UserAPI.DeviceDatabase.ConnectionString = "file:/idb/dendritejs_device.db" - cfg.FederationSender.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db" + cfg.FederationAPI.Database.ConnectionString = "file:/idb/dendritejs_fedsender.db" cfg.MediaAPI.Database.ConnectionString = "file:/idb/dendritejs_mediaapi.db" cfg.RoomServer.Database.ConnectionString = "file:/idb/dendritejs_roomserver.db" - cfg.SigningKeyServer.Database.ConnectionString = "file:/idb/dendritejs_signingkeyserver.db" cfg.SyncAPI.Database.ConnectionString = "file:/idb/dendritejs_syncapi.db" cfg.KeyServer.Database.ConnectionString = "file:/idb/dendritejs_e2ekey.db" cfg.Global.Kafka.UseNaffka = true @@ -188,7 +187,7 @@ func main() { if err := cfg.Derive(); err != nil { logrus.Fatalf("Failed to derive values from config: %s", err) } - base := setup.NewBaseDendrite(cfg, "Monolith", false) + base := setup.NewBaseDendrite(cfg, "Monolith") defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() @@ -205,14 +204,15 @@ func main() { KeyDatabase: fetcher, } - rsAPI := roomserver.NewInternalAPI(base, keyRing) + rsAPI := roomserver.NewInternalAPI(base) eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), userAPI) asQuery := appservice.NewInternalAPI( base, userAPI, rsAPI, ) rsAPI.SetAppserviceAPI(asQuery) - fedSenderAPI := federationsender.NewInternalAPI(base, federation, rsAPI, &keyRing, true) - rsAPI.SetFederationSenderAPI(fedSenderAPI) + fedSenderAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, true) + rsAPI.SetFederationAPI(fedSenderAPI) + rsAPI.SetKeyring(keyRing) p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI, federation) monolith := setup.Monolith{ diff --git a/cmd/dendritejs/publicrooms.go b/cmd/dendritejs/publicrooms.go index 19afc5bcf..2e3339a45 100644 --- a/cmd/dendritejs/publicrooms.go +++ b/cmd/dendritejs/publicrooms.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationapi/api" go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -31,12 +31,12 @@ import ( type libp2pPublicRoomsProvider struct { node *go_http_js_libp2p.P2pLocalNode providers []go_http_js_libp2p.PeerInfo - fedSender api.FederationSenderInternalAPI + fedSender api.FederationInternalAPI fedClient *gomatrixserverlib.FederationClient } func NewLibP2PPublicRoomsProvider( - node *go_http_js_libp2p.P2pLocalNode, fedSender api.FederationSenderInternalAPI, fedClient *gomatrixserverlib.FederationClient, + node *go_http_js_libp2p.P2pLocalNode, fedSender api.FederationInternalAPI, fedClient *gomatrixserverlib.FederationClient, ) *libp2pPublicRoomsProvider { p := &libp2pPublicRoomsProvider{ node: node, diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index e300a1caf..ff91e753f 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -17,19 +17,18 @@ func main() { flag.Parse() cfg := &config.Dendrite{} - cfg.Defaults() + cfg.Defaults(true) if *serverName != "" { cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName) } if *dbURI != "" { cfg.Global.Kafka.Database.ConnectionString = config.DataSource(*dbURI) cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(*dbURI) - cfg.FederationSender.Database.ConnectionString = config.DataSource(*dbURI) + cfg.FederationAPI.Database.ConnectionString = config.DataSource(*dbURI) cfg.KeyServer.Database.ConnectionString = config.DataSource(*dbURI) cfg.MSCs.Database.ConnectionString = config.DataSource(*dbURI) cfg.MediaAPI.Database.ConnectionString = config.DataSource(*dbURI) cfg.RoomServer.Database.ConnectionString = config.DataSource(*dbURI) - cfg.SigningKeyServer.Database.ConnectionString = config.DataSource(*dbURI) cfg.SyncAPI.Database.ConnectionString = config.DataSource(*dbURI) cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(*dbURI) cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(*dbURI) @@ -47,7 +46,7 @@ func main() { }, }, } - cfg.SigningKeyServer.KeyPerspectives = config.KeyPerspectives{ + cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{ { ServerName: "matrix.org", Keys: []config.KeyPerspectiveTrustKey{ @@ -83,11 +82,11 @@ func main() { if *defaultsForCI { cfg.AppServiceAPI.DisableTLSValidation = true cfg.ClientAPI.RateLimiting.Enabled = false - cfg.FederationSender.DisableTLSValidation = true + cfg.FederationAPI.DisableTLSValidation = true + // don't hit matrix.org when running tests!!! + cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{} cfg.MSCs.MSCs = []string{"msc2836", "msc2946", "msc2444", "msc2753"} cfg.Logging[0].Level = "trace" - // don't hit matrix.org when running tests!!! - cfg.SigningKeyServer.KeyPerspectives = config.KeyPerspectives{} cfg.UserAPI.BCryptCost = bcrypt.MinCost } diff --git a/cmd/goose/main.go b/cmd/goose/main.go index 83c97a729..8ed5cbd91 100644 --- a/cmd/goose/main.go +++ b/cmd/goose/main.go @@ -20,7 +20,7 @@ import ( const ( AppService = "appservice" - FederationSender = "federationsender" + FederationSender = "federationapi" KeyServer = "keyserver" MediaAPI = "mediaapi" RoomServer = "roomserver" diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 0c772fdd1..3e15f4b8e 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -194,6 +194,11 @@ federation_api: connect: http://localhost:7772 external_api: listen: http://[::]:8072 + database: + connection_string: file:federationapi.db + max_open_conns: 10 + max_idle_conns: 2 + conn_max_lifetime: -1 # List of paths to X.509 certificates to be used by the external federation listeners. # These certificates will be used to calculate the TLS fingerprints and other servers @@ -201,17 +206,6 @@ federation_api: # format. federation_certificates: [] -# Configuration for the Federation Sender. -federation_sender: - internal_api: - listen: http://localhost:7775 - connect: http://localhost:7775 - database: - connection_string: file:federationsender.db - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - # How many times we will try to resend a failed transaction to a specific server. The # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. send_max_retries: 16 @@ -227,6 +221,22 @@ federation_sender: host: localhost port: 8080 + # Perspective keyservers to use as a backup when direct key fetches fail. This may + # be required to satisfy key requests for servers that are no longer online when + # joining some rooms. + key_perspectives: + - server_name: matrix.org + keys: + - key_id: ed25519:auto + public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw + - key_id: ed25519:a_RXGa + public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ + + # This option will control whether Dendrite will prefer to look up keys directly + # or whether it should try perspective servers first, using direct fetches as a + # last resort. + prefer_direct_fetch: false + # Configuration for the Key Server (for end-to-end encryption). key_server: internal_api: @@ -301,33 +311,6 @@ room_server: max_idle_conns: 2 conn_max_lifetime: -1 -# Configuration for the Signing Key Server (for server signing keys). -signing_key_server: - internal_api: - listen: http://localhost:7780 - connect: http://localhost:7780 - database: - connection_string: file:signingkeyserver.db - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - - # Perspective keyservers to use as a backup when direct key fetches fail. This may - # be required to satisfy key requests for servers that are no longer online when - # joining some rooms. - key_perspectives: - - server_name: matrix.org - keys: - - key_id: ed25519:auto - public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw - - key_id: ed25519:a_RXGa - public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ - - # This option will control whether Dendrite will prefer to look up keys directly - # or whether it should try perspective servers first, using direct fetches as a - # last resort. - prefer_direct_fetch: false - # Configuration for the Sync API. sync_api: internal_api: diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go index 1137eed17..e8d2a4eb7 100644 --- a/eduserver/eduserver.go +++ b/eduserver/eduserver.go @@ -22,7 +22,7 @@ import ( "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/eduserver/input" "github.com/matrix-org/dendrite/eduserver/inthttp" - "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/kafka" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -37,7 +37,7 @@ func AddInternalRoutes(internalMux *mux.Router, inputAPI api.EDUServerInputAPI) // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *setup.BaseDendrite, + base *base.BaseDendrite, eduCache *cache.EDUCache, userAPI userapi.UserInternalAPI, ) api.EDUServerInputAPI { diff --git a/federationsender/api/api.go b/federationapi/api/api.go similarity index 90% rename from federationsender/api/api.go rename to federationapi/api/api.go index 4ebbb3abb..32a700276 100644 --- a/federationsender/api/api.go +++ b/federationapi/api/api.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/federationapi/types" userAPI "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" @@ -37,9 +37,12 @@ func (e *FederationClientError) Error() string { return fmt.Sprintf("%s - (retry_after=%s, blacklisted=%v)", e.Err, e.RetryAfter.String(), e.Blacklisted) } -// FederationSenderInternalAPI is used to query information from the federation sender. -type FederationSenderInternalAPI interface { +// FederationInternalAPI is used to query information from the federation sender. +type FederationInternalAPI interface { FederationClient + gomatrixserverlib.KeyDatabase + + KeyRing() *gomatrixserverlib.KeyRing SetUserAPI(api userAPI.UserInternalAPI) @@ -117,6 +120,14 @@ type QueryServerKeysResponse struct { ServerKeys []gomatrixserverlib.ServerKeys } +type QueryPublicKeysRequest struct { + Requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp `json:"requests"` +} + +type QueryPublicKeysResponse struct { + Results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult `json:"results"` +} + type PerformDirectoryLookupRequest struct { RoomAlias string `json:"room_alias"` ServerName gomatrixserverlib.ServerName `json:"server_name"` @@ -191,3 +202,10 @@ type PerformBroadcastEDURequest struct { type PerformBroadcastEDUResponse struct { } + +type InputPublicKeysRequest struct { + Keys map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult `json:"keys"` +} + +type InputPublicKeysResponse struct { +} diff --git a/federationsender/consumers/eduserver.go b/federationapi/consumers/eduserver.go similarity index 98% rename from federationsender/consumers/eduserver.go rename to federationapi/consumers/eduserver.go index f8c9e9301..1d0d56e98 100644 --- a/federationsender/consumers/eduserver.go +++ b/federationapi/consumers/eduserver.go @@ -22,8 +22,8 @@ import ( "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/federationsender/queue" - "github.com/matrix-org/dendrite/federationsender/storage" + "github.com/matrix-org/dendrite/federationapi/queue" + "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" @@ -46,7 +46,7 @@ type OutputEDUConsumer struct { // NewOutputEDUConsumer creates a new OutputEDUConsumer. Call Start() to begin consuming from EDU servers. func NewOutputEDUConsumer( process *process.ProcessContext, - cfg *config.FederationSender, + cfg *config.FederationAPI, kafkaConsumer sarama.Consumer, queues *queue.OutgoingQueues, store storage.Database, diff --git a/federationsender/consumers/keychange.go b/federationapi/consumers/keychange.go similarity index 97% rename from federationsender/consumers/keychange.go rename to federationapi/consumers/keychange.go index d5dc595c6..a8ae0894a 100644 --- a/federationsender/consumers/keychange.go +++ b/federationapi/consumers/keychange.go @@ -21,8 +21,8 @@ import ( "github.com/Shopify/sarama" eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/federationsender/queue" - "github.com/matrix-org/dendrite/federationsender/storage" + "github.com/matrix-org/dendrite/federationapi/queue" + "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/keyserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" @@ -53,7 +53,7 @@ func NewKeyChangeConsumer( c := &KeyChangeConsumer{ consumer: &internal.ContinualConsumer{ Process: process, - ComponentName: "federationsender/keychange", + ComponentName: "federationapi/keychange", Topic: string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputKeyChangeEvent)), Consumer: kafkaConsumer, PartitionStore: store, diff --git a/federationsender/consumers/roomserver.go b/federationapi/consumers/roomserver.go similarity index 97% rename from federationsender/consumers/roomserver.go rename to federationapi/consumers/roomserver.go index f9c4a5c27..20b1bacbc 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationapi/consumers/roomserver.go @@ -20,9 +20,9 @@ import ( "fmt" "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/federationsender/queue" - "github.com/matrix-org/dendrite/federationsender/storage" - "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/federationapi/queue" + "github.com/matrix-org/dendrite/federationapi/storage" + "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" @@ -33,7 +33,7 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - cfg *config.FederationSender + cfg *config.FederationAPI rsAPI api.RoomserverInternalAPI rsConsumer *internal.ContinualConsumer db storage.Database @@ -43,7 +43,7 @@ type OutputRoomEventConsumer struct { // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. func NewOutputRoomEventConsumer( process *process.ProcessContext, - cfg *config.FederationSender, + cfg *config.FederationAPI, kafkaConsumer sarama.Consumer, queues *queue.OutgoingQueues, store storage.Database, @@ -51,7 +51,7 @@ func NewOutputRoomEventConsumer( ) *OutputRoomEventConsumer { consumer := internal.ContinualConsumer{ Process: process, - ComponentName: "federationsender/roomserver", + ComponentName: "federationapi/roomserver", Topic: string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputRoomEvent)), Consumer: kafkaConsumer, PartitionStore: store, @@ -133,7 +133,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { } // processInboundPeek starts tracking a new federated inbound peek (replacing the existing one if any) -// causing the federationsender to start sending messages to the peeking server +// causing the federationapi to start sending messages to the peeking server func (s *OutputRoomEventConsumer) processInboundPeek(orp api.OutputNewInboundPeek) error { // FIXME: there's a race here - we should start /sending new peeked events diff --git a/federationsender/consumers/roomserver_test.go b/federationapi/consumers/roomserver_test.go similarity index 100% rename from federationsender/consumers/roomserver_test.go rename to federationapi/consumers/roomserver_test.go diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index c40d77a62..a9c5f68d1 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -17,26 +17,42 @@ package federationapi import ( "github.com/gorilla/mux" eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" + "github.com/matrix-org/dendrite/federationapi/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" - federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationapi/consumers" + "github.com/matrix-org/dendrite/federationapi/internal" + "github.com/matrix-org/dendrite/federationapi/inthttp" + "github.com/matrix-org/dendrite/federationapi/queue" + "github.com/matrix-org/dendrite/federationapi/statistics" + "github.com/matrix-org/dendrite/federationapi/storage" + "github.com/matrix-org/dendrite/internal/caching" keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" - userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/setup/kafka" + userAPI "github.com/matrix-org/dendrite/userapi/api" + "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/gomatrixserverlib" ) +// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions +// on the given input API. +func AddInternalRoutes(router *mux.Router, intAPI api.FederationInternalAPI) { + inthttp.AddRoutes(intAPI, router) +} + // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component. func AddPublicRoutes( fedRouter, keyRouter, wellKnownRouter *mux.Router, cfg *config.FederationAPI, - userAPI userapi.UserInternalAPI, + userAPI userAPI.UserInternalAPI, federation *gomatrixserverlib.FederationClient, keyRing gomatrixserverlib.JSONVerifier, rsAPI roomserverAPI.RoomserverInternalAPI, - federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, + federationAPI federationAPI.FederationInternalAPI, eduAPI eduserverAPI.EDUServerInputAPI, keyAPI keyserverAPI.KeyInternalAPI, mscCfg *config.MSCs, @@ -44,8 +60,71 @@ func AddPublicRoutes( ) { routing.Setup( fedRouter, keyRouter, wellKnownRouter, cfg, rsAPI, - eduAPI, federationSenderAPI, keyRing, + eduAPI, federationAPI, keyRing, federation, userAPI, keyAPI, mscCfg, servers, ) } + +// NewInternalAPI returns a concerete implementation of the internal API. Callers +// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. +func NewInternalAPI( + base *base.BaseDendrite, + federation *gomatrixserverlib.FederationClient, + rsAPI roomserverAPI.RoomserverInternalAPI, + userAPI userAPI.UserInternalAPI, + caches *caching.Caches, + resetBlacklist bool, +) api.FederationInternalAPI { + cfg := &base.Cfg.FederationAPI + + federationDB, err := storage.NewDatabase(&cfg.Database, base.Caches) + if err != nil { + logrus.WithError(err).Panic("failed to connect to federation sender db") + } + + if resetBlacklist { + _ = federationDB.RemoveAllServersFromBlacklist() + } + + stats := &statistics.Statistics{ + DB: federationDB, + FailuresUntilBlacklist: cfg.FederationMaxRetries, + } + + consumer, _ := kafka.SetupConsumerProducer(&cfg.Matrix.Kafka) + + queues := queue.NewOutgoingQueues( + federationDB, base.ProcessContext, + cfg.Matrix.DisableFederation, + cfg.Matrix.ServerName, federation, rsAPI, stats, + &queue.SigningInfo{ + KeyID: cfg.Matrix.KeyID, + PrivateKey: cfg.Matrix.PrivateKey, + ServerName: cfg.Matrix.ServerName, + }, + ) + + rsConsumer := consumers.NewOutputRoomEventConsumer( + base.ProcessContext, cfg, consumer, queues, + federationDB, rsAPI, + ) + if err = rsConsumer.Start(); err != nil { + logrus.WithError(err).Panic("failed to start room server consumer") + } + + tsConsumer := consumers.NewOutputEDUConsumer( + base.ProcessContext, cfg, consumer, queues, federationDB, + ) + if err := tsConsumer.Start(); err != nil { + logrus.WithError(err).Panic("failed to start typing server consumer") + } + keyConsumer := consumers.NewKeyChangeConsumer( + base.ProcessContext, &base.Cfg.KeyServer, consumer, queues, federationDB, rsAPI, + ) + if err := keyConsumer.Start(); err != nil { + logrus.WithError(err).Panic("failed to start key server consumer") + } + + return internal.NewFederationInternalAPI(federationDB, cfg, rsAPI, userAPI, federation, stats, caches, queues) +} diff --git a/signingkeyserver/serverkeyapi_test.go b/federationapi/federationapi_keys_test.go similarity index 94% rename from signingkeyserver/serverkeyapi_test.go rename to federationapi/federationapi_keys_test.go index bd6119aae..f9af9c76b 100644 --- a/signingkeyserver/serverkeyapi_test.go +++ b/federationapi/federationapi_keys_test.go @@ -1,4 +1,4 @@ -package signingkeyserver +package federationapi import ( "bytes" @@ -13,21 +13,21 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/signingkeyserver/api" "github.com/matrix-org/gomatrixserverlib" ) type server struct { name gomatrixserverlib.ServerName // server name validity time.Duration // key validity duration from now - config *config.SigningKeyServer // skeleton config, from TestMain - fedconfig *config.FederationAPI // + config *config.FederationAPI // skeleton config, from TestMain fedclient *gomatrixserverlib.FederationClient // uses MockRoundTripper cache *caching.Caches // server-specific cache - api api.SigningKeyServerAPI // server-specific server key API + api api.FederationInternalAPI // server-specific server key API } func (s *server) renew() { @@ -71,14 +71,14 @@ func TestMain(m *testing.M) { // Draw up just enough Dendrite config for the server key // API to work. cfg := &config.Dendrite{} - cfg.Defaults() + cfg.Defaults(true) cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name) cfg.Global.PrivateKey = testPriv + cfg.Global.Kafka.UseNaffka = true cfg.Global.KeyID = serverKeyID cfg.Global.KeyValidityPeriod = s.validity - cfg.SigningKeyServer.Database.ConnectionString = config.DataSource("file::memory:") - s.config = &cfg.SigningKeyServer - s.fedconfig = &cfg.FederationAPI + cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:") + s.config = &cfg.FederationAPI // Create a transport which redirects federation requests to // the mock round tripper. Since we're not *really* listening for @@ -93,7 +93,8 @@ func TestMain(m *testing.M) { ) // Finally, build the server key APIs. - s.api = NewInternalAPI(s.config, s.fedclient, s.cache) + sbase := base.NewBaseDendrite(cfg, "Monolith", base.NoCacheMetrics) + s.api = NewInternalAPI(sbase, s.fedclient, nil, nil, s.cache, true) } // Now that we have built our server key APIs, start the @@ -119,7 +120,7 @@ func (m *MockRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err } // Get the keys and JSON-ify them. - keys := routing.LocalKeys(s.fedconfig) + keys := routing.LocalKeys(s.config) body, err := json.MarshalIndent(keys.JSON, "", " ") if err != nil { return nil, err diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index cb4d81032..8b5bdd034 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -8,7 +8,7 @@ import ( "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/test" - "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" @@ -19,16 +19,16 @@ import ( func TestRoomsV3URLEscapeDoNot404(t *testing.T) { _, privKey, _ := ed25519.GenerateKey(nil) cfg := &config.Dendrite{} - cfg.Defaults() + cfg.Defaults(true) cfg.Global.KeyID = gomatrixserverlib.KeyID("ed25519:auto") cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost") cfg.Global.PrivateKey = privKey cfg.Global.Kafka.UseNaffka = true cfg.Global.Kafka.Database.ConnectionString = config.DataSource("file::memory:") - cfg.FederationSender.Database.ConnectionString = config.DataSource("file::memory:") - base := setup.NewBaseDendrite(cfg, "Monolith", false) + cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:") + base := base.NewBaseDendrite(cfg, "Monolith", base.NoCacheMetrics) keyRing := &test.NopJSONVerifier{} - fsAPI := base.FederationSenderHTTPClient() + fsAPI := base.FederationAPIHTTPClient() // TODO: This is pretty fragile, as if anything calls anything on these nils this test will break. // Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing. federationapi.AddPublicRoutes(base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicWellKnownAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, nil, &cfg.MSCs, nil) diff --git a/federationsender/internal/api.go b/federationapi/internal/api.go similarity index 70% rename from federationsender/internal/api.go rename to federationapi/internal/api.go index d8cf4afcf..42ff2068a 100644 --- a/federationsender/internal/api.go +++ b/federationapi/internal/api.go @@ -2,24 +2,29 @@ package internal import ( "context" + "crypto/ed25519" + "encoding/base64" "sync" "time" - "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/federationsender/queue" - "github.com/matrix-org/dendrite/federationsender/statistics" - "github.com/matrix-org/dendrite/federationsender/storage" + "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/queue" + "github.com/matrix-org/dendrite/federationapi/statistics" + "github.com/matrix-org/dendrite/federationapi/storage" + "github.com/matrix-org/dendrite/federationapi/storage/cache" + "github.com/matrix-org/dendrite/internal/caching" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" userAPI "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" ) -// FederationSenderInternalAPI is an implementation of api.FederationSenderInternalAPI -type FederationSenderInternalAPI struct { +// FederationInternalAPI is an implementation of api.FederationInternalAPI +type FederationInternalAPI struct { db storage.Database - cfg *config.FederationSender + cfg *config.FederationAPI statistics *statistics.Statistics rsAPI roomserverAPI.RoomserverInternalAPI userAPI userAPI.UserInternalAPI @@ -29,32 +34,85 @@ type FederationSenderInternalAPI struct { joins sync.Map // joins currently in progress } -func NewFederationSenderInternalAPI( - db storage.Database, cfg *config.FederationSender, +func NewFederationInternalAPI( + db storage.Database, cfg *config.FederationAPI, rsAPI roomserverAPI.RoomserverInternalAPI, userAPI userAPI.UserInternalAPI, federation *gomatrixserverlib.FederationClient, - keyRing *gomatrixserverlib.KeyRing, statistics *statistics.Statistics, + caches *caching.Caches, queues *queue.OutgoingQueues, -) *FederationSenderInternalAPI { - return &FederationSenderInternalAPI{ +) *FederationInternalAPI { + serverKeyDB, err := cache.NewKeyDatabase(db, caches) + if err != nil { + logrus.WithError(err).Panicf("failed to set up caching wrapper for server key database") + } + + keyRing := &gomatrixserverlib.KeyRing{ + KeyFetchers: []gomatrixserverlib.KeyFetcher{}, + KeyDatabase: serverKeyDB, + } + + addDirectFetcher := func() { + keyRing.KeyFetchers = append( + keyRing.KeyFetchers, + &gomatrixserverlib.DirectKeyFetcher{ + Client: federation, + }, + ) + } + + if cfg.PreferDirectFetch { + addDirectFetcher() + } else { + defer addDirectFetcher() + } + + var b64e = base64.StdEncoding.WithPadding(base64.NoPadding) + for _, ps := range cfg.KeyPerspectives { + perspective := &gomatrixserverlib.PerspectiveKeyFetcher{ + PerspectiveServerName: ps.ServerName, + PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{}, + Client: federation, + } + + for _, key := range ps.Keys { + rawkey, err := b64e.DecodeString(key.PublicKey) + if err != nil { + logrus.WithError(err).WithFields(logrus.Fields{ + "server_name": ps.ServerName, + "public_key": key.PublicKey, + }).Warn("Couldn't parse perspective key") + continue + } + perspective.PerspectiveServerKeys[key.KeyID] = rawkey + } + + keyRing.KeyFetchers = append(keyRing.KeyFetchers, perspective) + + logrus.WithFields(logrus.Fields{ + "server_name": ps.ServerName, + "num_public_keys": len(ps.Keys), + }).Info("Enabled perspective key fetcher") + } + + return &FederationInternalAPI{ db: db, cfg: cfg, rsAPI: rsAPI, userAPI: userAPI, - federation: federation, keyRing: keyRing, + federation: federation, statistics: statistics, queues: queues, } } -func (f *FederationSenderInternalAPI) SetUserAPI(userAPI userAPI.UserInternalAPI) { +func (f *FederationInternalAPI) SetUserAPI(userAPI userAPI.UserInternalAPI) { f.userAPI = userAPI } -func (a *FederationSenderInternalAPI) isBlacklistedOrBackingOff(s gomatrixserverlib.ServerName) (*statistics.ServerStatistics, error) { +func (a *FederationInternalAPI) isBlacklistedOrBackingOff(s gomatrixserverlib.ServerName) (*statistics.ServerStatistics, error) { stats := a.statistics.ForServer(s) until, blacklisted := stats.BackoffInfo() if blacklisted { @@ -89,7 +147,7 @@ func failBlacklistableError(err error, stats *statistics.ServerStatistics) (unti return } -func (a *FederationSenderInternalAPI) doRequest( +func (a *FederationInternalAPI) doRequest( s gomatrixserverlib.ServerName, request func() (interface{}, error), ) (interface{}, error) { stats, err := a.isBlacklistedOrBackingOff(s) @@ -114,7 +172,7 @@ func (a *FederationSenderInternalAPI) doRequest( return res, nil } -func (a *FederationSenderInternalAPI) GetUserDevices( +func (a *FederationInternalAPI) GetUserDevices( ctx context.Context, s gomatrixserverlib.ServerName, userID string, ) (gomatrixserverlib.RespUserDevices, error) { ctx, cancel := context.WithTimeout(ctx, time.Second*30) @@ -128,7 +186,7 @@ func (a *FederationSenderInternalAPI) GetUserDevices( return ires.(gomatrixserverlib.RespUserDevices), nil } -func (a *FederationSenderInternalAPI) ClaimKeys( +func (a *FederationInternalAPI) ClaimKeys( ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string, ) (gomatrixserverlib.RespClaimKeys, error) { ctx, cancel := context.WithTimeout(ctx, time.Second*30) @@ -142,7 +200,7 @@ func (a *FederationSenderInternalAPI) ClaimKeys( return ires.(gomatrixserverlib.RespClaimKeys), nil } -func (a *FederationSenderInternalAPI) QueryKeys( +func (a *FederationInternalAPI) QueryKeys( ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string, ) (gomatrixserverlib.RespQueryKeys, error) { ires, err := a.doRequest(s, func() (interface{}, error) { @@ -154,7 +212,7 @@ func (a *FederationSenderInternalAPI) QueryKeys( return ires.(gomatrixserverlib.RespQueryKeys), nil } -func (a *FederationSenderInternalAPI) Backfill( +func (a *FederationInternalAPI) Backfill( ctx context.Context, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string, ) (res gomatrixserverlib.Transaction, err error) { ctx, cancel := context.WithTimeout(ctx, time.Second*30) @@ -168,7 +226,7 @@ func (a *FederationSenderInternalAPI) Backfill( return ires.(gomatrixserverlib.Transaction), nil } -func (a *FederationSenderInternalAPI) LookupState( +func (a *FederationInternalAPI) LookupState( ctx context.Context, s gomatrixserverlib.ServerName, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion, ) (res gomatrixserverlib.RespState, err error) { ctx, cancel := context.WithTimeout(ctx, time.Second*30) @@ -182,7 +240,7 @@ func (a *FederationSenderInternalAPI) LookupState( return ires.(gomatrixserverlib.RespState), nil } -func (a *FederationSenderInternalAPI) LookupStateIDs( +func (a *FederationInternalAPI) LookupStateIDs( ctx context.Context, s gomatrixserverlib.ServerName, roomID, eventID string, ) (res gomatrixserverlib.RespStateIDs, err error) { ctx, cancel := context.WithTimeout(ctx, time.Second*30) @@ -196,7 +254,7 @@ func (a *FederationSenderInternalAPI) LookupStateIDs( return ires.(gomatrixserverlib.RespStateIDs), nil } -func (a *FederationSenderInternalAPI) GetEvent( +func (a *FederationInternalAPI) GetEvent( ctx context.Context, s gomatrixserverlib.ServerName, eventID string, ) (res gomatrixserverlib.Transaction, err error) { ctx, cancel := context.WithTimeout(ctx, time.Second*30) @@ -210,7 +268,7 @@ func (a *FederationSenderInternalAPI) GetEvent( return ires.(gomatrixserverlib.Transaction), nil } -func (a *FederationSenderInternalAPI) LookupServerKeys( +func (a *FederationInternalAPI) LookupServerKeys( ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, ) ([]gomatrixserverlib.ServerKeys, error) { ctx, cancel := context.WithTimeout(ctx, time.Minute) @@ -224,7 +282,7 @@ func (a *FederationSenderInternalAPI) LookupServerKeys( return ires.([]gomatrixserverlib.ServerKeys), nil } -func (a *FederationSenderInternalAPI) MSC2836EventRelationships( +func (a *FederationInternalAPI) MSC2836EventRelationships( ctx context.Context, s gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion, ) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) { @@ -239,7 +297,7 @@ func (a *FederationSenderInternalAPI) MSC2836EventRelationships( return ires.(gomatrixserverlib.MSC2836EventRelationshipsResponse), nil } -func (a *FederationSenderInternalAPI) MSC2946Spaces( +func (a *FederationInternalAPI) MSC2946Spaces( ctx context.Context, s gomatrixserverlib.ServerName, roomID string, r gomatrixserverlib.MSC2946SpacesRequest, ) (res gomatrixserverlib.MSC2946SpacesResponse, err error) { ctx, cancel := context.WithTimeout(ctx, time.Minute) diff --git a/signingkeyserver/internal/api.go b/federationapi/internal/keys.go similarity index 84% rename from signingkeyserver/internal/api.go rename to federationapi/internal/keys.go index f9a04a74f..2b7a8219a 100644 --- a/signingkeyserver/internal/api.go +++ b/federationapi/internal/keys.go @@ -6,36 +6,18 @@ import ( "fmt" "time" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/signingkeyserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) -type ServerKeyAPI struct { - api.SigningKeyServerAPI - - ServerName gomatrixserverlib.ServerName - ServerPublicKey ed25519.PublicKey - ServerKeyID gomatrixserverlib.KeyID - ServerKeyValidity time.Duration - OldServerKeys []config.OldVerifyKeys - - OurKeyRing gomatrixserverlib.KeyRing - FedClient gomatrixserverlib.KeyClient -} - -func (s *ServerKeyAPI) KeyRing() *gomatrixserverlib.KeyRing { +func (s *FederationInternalAPI) KeyRing() *gomatrixserverlib.KeyRing { // Return a keyring that forces requests to be proxied through the // below functions. That way we can enforce things like validity // and keeping the cache up-to-date. - return &gomatrixserverlib.KeyRing{ - KeyDatabase: s, - KeyFetchers: []gomatrixserverlib.KeyFetcher{}, - } + return s.keyRing } -func (s *ServerKeyAPI) StoreKeys( +func (s *FederationInternalAPI) StoreKeys( _ context.Context, results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, ) error { @@ -44,10 +26,10 @@ func (s *ServerKeyAPI) StoreKeys( ctx := context.Background() // Store any keys that we were given in our database. - return s.OurKeyRing.KeyDatabase.StoreKeys(ctx, results) + return s.keyRing.KeyDatabase.StoreKeys(ctx, results) } -func (s *ServerKeyAPI) FetchKeys( +func (s *FederationInternalAPI) FetchKeys( _ context.Context, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, ) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { @@ -75,7 +57,7 @@ func (s *ServerKeyAPI) FetchKeys( // For any key requests that we still have outstanding, next try to // fetch them directly. We'll go through each of the key fetchers to // ask for the remaining keys - for _, fetcher := range s.OurKeyRing.KeyFetchers { + for _, fetcher := range s.keyRing.KeyFetchers { // If there are no more keys to look up then stop. if len(requests) == 0 { break @@ -105,22 +87,22 @@ func (s *ServerKeyAPI) FetchKeys( return results, nil } -func (s *ServerKeyAPI) FetcherName() string { - return fmt.Sprintf("ServerKeyAPI (wrapping %q)", s.OurKeyRing.KeyDatabase.FetcherName()) +func (s *FederationInternalAPI) FetcherName() string { + return fmt.Sprintf("FederationInternalAPI (wrapping %q)", s.keyRing.KeyDatabase.FetcherName()) } // handleLocalKeys handles cases where the key request contains // a request for our own server keys, either current or old. -func (s *ServerKeyAPI) handleLocalKeys( +func (s *FederationInternalAPI) handleLocalKeys( _ context.Context, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, ) { for req := range requests { - if req.ServerName != s.ServerName { + if req.ServerName != s.cfg.Matrix.ServerName { continue } - if req.KeyID == s.ServerKeyID { + if req.KeyID == s.cfg.Matrix.KeyID { // We found a key request that is supposed to be for our own // keys. Remove it from the request list so we don't hit the // database or the fetchers for it. @@ -129,15 +111,15 @@ func (s *ServerKeyAPI) handleLocalKeys( // Insert our own key into the response. results[req] = gomatrixserverlib.PublicKeyLookupResult{ VerifyKey: gomatrixserverlib.VerifyKey{ - Key: gomatrixserverlib.Base64Bytes(s.ServerPublicKey), + Key: gomatrixserverlib.Base64Bytes(s.cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey)), }, ExpiredTS: gomatrixserverlib.PublicKeyNotExpired, - ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(s.ServerKeyValidity)), + ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(s.cfg.Matrix.KeyValidityPeriod)), } } else { // The key request doesn't match our current key. Let's see // if it matches any of our old verify keys. - for _, oldVerifyKey := range s.OldServerKeys { + for _, oldVerifyKey := range s.cfg.Matrix.OldVerifyKeys { if req.KeyID == oldVerifyKey.KeyID { // We found a key request that is supposed to be an expired // key. @@ -162,14 +144,14 @@ func (s *ServerKeyAPI) handleLocalKeys( // handleDatabaseKeys handles cases where the key requests can be // satisfied from our local database/cache. -func (s *ServerKeyAPI) handleDatabaseKeys( +func (s *FederationInternalAPI) handleDatabaseKeys( ctx context.Context, now gomatrixserverlib.Timestamp, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, ) error { // Ask the database/cache for the keys. - dbResults, err := s.OurKeyRing.KeyDatabase.FetchKeys(ctx, requests) + dbResults, err := s.keyRing.KeyDatabase.FetchKeys(ctx, requests) if err != nil { return err } @@ -196,7 +178,7 @@ func (s *ServerKeyAPI) handleDatabaseKeys( // handleFetcherKeys handles cases where a fetcher can satisfy // the remaining requests. -func (s *ServerKeyAPI) handleFetcherKeys( +func (s *FederationInternalAPI) handleFetcherKeys( ctx context.Context, _ gomatrixserverlib.Timestamp, fetcher gomatrixserverlib.KeyFetcher, @@ -248,10 +230,10 @@ func (s *ServerKeyAPI) handleFetcherKeys( } // Store the keys from our store map. - if err = s.OurKeyRing.KeyDatabase.StoreKeys(context.Background(), storeResults); err != nil { + if err = s.keyRing.KeyDatabase.StoreKeys(context.Background(), storeResults); err != nil { logrus.WithError(err).WithFields(logrus.Fields{ "fetcher_name": fetcher.FetcherName(), - "database_name": s.OurKeyRing.KeyDatabase.FetcherName(), + "database_name": s.keyRing.KeyDatabase.FetcherName(), }).Errorf("Failed to store keys in the database") return fmt.Errorf("server key API failed to store retrieved keys: %w", err) } diff --git a/federationsender/internal/perform.go b/federationapi/internal/perform.go similarity index 95% rename from federationsender/internal/perform.go rename to federationapi/internal/perform.go index afe4517b6..81a5e6e87 100644 --- a/federationsender/internal/perform.go +++ b/federationapi/internal/perform.go @@ -8,7 +8,7 @@ import ( "time" eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationapi/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/version" userAPI "github.com/matrix-org/dendrite/userapi/api" @@ -18,8 +18,8 @@ import ( "github.com/sirupsen/logrus" ) -// PerformLeaveRequest implements api.FederationSenderInternalAPI -func (r *FederationSenderInternalAPI) PerformDirectoryLookup( +// PerformLeaveRequest implements api.FederationInternalAPI +func (r *FederationInternalAPI) PerformDirectoryLookup( ctx context.Context, request *api.PerformDirectoryLookupRequest, response *api.PerformDirectoryLookupResponse, @@ -44,8 +44,8 @@ type federatedJoin struct { RoomID string } -// PerformJoin implements api.FederationSenderInternalAPI -func (r *FederationSenderInternalAPI) PerformJoin( +// PerformJoin implements api.FederationInternalAPI +func (r *FederationInternalAPI) PerformJoin( ctx context.Context, request *api.PerformJoinRequest, response *api.PerformJoinResponse, @@ -140,7 +140,7 @@ func (r *FederationSenderInternalAPI) PerformJoin( } // sendPresenceData sends presence data for a given roomID -func (r *FederationSenderInternalAPI) sendPresenceData( +func (r *FederationInternalAPI) sendPresenceData( ctx context.Context, roomID string, serverName gomatrixserverlib.ServerName, ) (err error) { // query current presence for users @@ -195,7 +195,7 @@ func (r *FederationSenderInternalAPI) sendPresenceData( return r.queues.SendEDU(edu, r.cfg.Matrix.ServerName, []gomatrixserverlib.ServerName{serverName}) } -func (r *FederationSenderInternalAPI) performJoinUsingServer( +func (r *FederationInternalAPI) performJoinUsingServer( ctx context.Context, roomID, userID string, content map[string]interface{}, @@ -326,8 +326,8 @@ func (r *FederationSenderInternalAPI) performJoinUsingServer( return nil } -// PerformOutboundPeekRequest implements api.FederationSenderInternalAPI -func (r *FederationSenderInternalAPI) PerformOutboundPeek( +// PerformOutboundPeekRequest implements api.FederationInternalAPI +func (r *FederationInternalAPI) PerformOutboundPeek( ctx context.Context, request *api.PerformOutboundPeekRequest, response *api.PerformOutboundPeekResponse, @@ -407,7 +407,7 @@ func (r *FederationSenderInternalAPI) PerformOutboundPeek( return lastErr } -func (r *FederationSenderInternalAPI) performOutboundPeekUsingServer( +func (r *FederationInternalAPI) performOutboundPeekUsingServer( ctx context.Context, roomID string, serverName gomatrixserverlib.ServerName, @@ -501,8 +501,8 @@ func (r *FederationSenderInternalAPI) performOutboundPeekUsingServer( return nil } -// PerformLeaveRequest implements api.FederationSenderInternalAPI -func (r *FederationSenderInternalAPI) PerformLeave( +// PerformLeaveRequest implements api.FederationInternalAPI +func (r *FederationInternalAPI) PerformLeave( ctx context.Context, request *api.PerformLeaveRequest, response *api.PerformLeaveResponse, @@ -591,8 +591,8 @@ func (r *FederationSenderInternalAPI) PerformLeave( ) } -// PerformLeaveRequest implements api.FederationSenderInternalAPI -func (r *FederationSenderInternalAPI) PerformInvite( +// PerformLeaveRequest implements api.FederationInternalAPI +func (r *FederationInternalAPI) PerformInvite( ctx context.Context, request *api.PerformInviteRequest, response *api.PerformInviteResponse, @@ -628,8 +628,8 @@ func (r *FederationSenderInternalAPI) PerformInvite( return nil } -// PerformServersAlive implements api.FederationSenderInternalAPI -func (r *FederationSenderInternalAPI) PerformServersAlive( +// PerformServersAlive implements api.FederationInternalAPI +func (r *FederationInternalAPI) PerformServersAlive( ctx context.Context, request *api.PerformServersAliveRequest, response *api.PerformServersAliveResponse, @@ -642,8 +642,8 @@ func (r *FederationSenderInternalAPI) PerformServersAlive( return nil } -// PerformServersAlive implements api.FederationSenderInternalAPI -func (r *FederationSenderInternalAPI) PerformBroadcastEDU( +// PerformServersAlive implements api.FederationInternalAPI +func (r *FederationInternalAPI) PerformBroadcastEDU( ctx context.Context, request *api.PerformBroadcastEDURequest, response *api.PerformBroadcastEDUResponse, diff --git a/federationsender/internal/query.go b/federationapi/internal/query.go similarity index 86% rename from federationsender/internal/query.go rename to federationapi/internal/query.go index af531f7d7..bac813331 100644 --- a/federationsender/internal/query.go +++ b/federationapi/internal/query.go @@ -5,13 +5,13 @@ import ( "fmt" "time" - "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) -// QueryJoinedHostServerNamesInRoom implements api.FederationSenderInternalAPI -func (f *FederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( +// QueryJoinedHostServerNamesInRoom implements api.FederationInternalAPI +func (f *FederationInternalAPI) QueryJoinedHostServerNamesInRoom( ctx context.Context, request *api.QueryJoinedHostServerNamesInRoomRequest, response *api.QueryJoinedHostServerNamesInRoomResponse, @@ -25,7 +25,7 @@ func (f *FederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( return } -func (a *FederationSenderInternalAPI) fetchServerKeysDirectly(ctx context.Context, serverName gomatrixserverlib.ServerName) (*gomatrixserverlib.ServerKeys, error) { +func (a *FederationInternalAPI) fetchServerKeysDirectly(ctx context.Context, serverName gomatrixserverlib.ServerName) (*gomatrixserverlib.ServerKeys, error) { ctx, cancel := context.WithTimeout(ctx, time.Second*30) defer cancel() ires, err := a.doRequest(serverName, func() (interface{}, error) { @@ -38,7 +38,7 @@ func (a *FederationSenderInternalAPI) fetchServerKeysDirectly(ctx context.Contex return &sks, nil } -func (a *FederationSenderInternalAPI) fetchServerKeysFromCache( +func (a *FederationInternalAPI) fetchServerKeysFromCache( ctx context.Context, req *api.QueryServerKeysRequest, ) ([]gomatrixserverlib.ServerKeys, error) { var results []gomatrixserverlib.ServerKeys @@ -64,7 +64,7 @@ func (a *FederationSenderInternalAPI) fetchServerKeysFromCache( return results, nil } -func (a *FederationSenderInternalAPI) QueryServerKeys( +func (a *FederationInternalAPI) QueryServerKeys( ctx context.Context, req *api.QueryServerKeysRequest, res *api.QueryServerKeysResponse, ) error { // attempt to satisfy the entire request from the cache first diff --git a/federationsender/inthttp/client.go b/federationapi/inthttp/client.go similarity index 59% rename from federationsender/inthttp/client.go rename to federationapi/inthttp/client.go index 6420909ce..0ee99e8bc 100644 --- a/federationsender/inthttp/client.go +++ b/federationapi/inthttp/client.go @@ -5,7 +5,8 @@ import ( "errors" "net/http" - "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" userAPI "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrix" @@ -15,47 +16,51 @@ import ( // HTTP paths for the internal HTTP API const ( - FederationSenderQueryJoinedHostServerNamesInRoomPath = "/federationsender/queryJoinedHostServerNamesInRoom" - FederationSenderQueryServerKeysPath = "/federationsender/queryServerKeys" + FederationAPIQueryJoinedHostServerNamesInRoomPath = "/federationapi/queryJoinedHostServerNamesInRoom" + FederationAPIQueryServerKeysPath = "/federationapi/queryServerKeys" - FederationSenderPerformDirectoryLookupRequestPath = "/federationsender/performDirectoryLookup" - FederationSenderPerformJoinRequestPath = "/federationsender/performJoinRequest" - FederationSenderPerformLeaveRequestPath = "/federationsender/performLeaveRequest" - FederationSenderPerformInviteRequestPath = "/federationsender/performInviteRequest" - FederationSenderPerformOutboundPeekRequestPath = "/federationsender/performOutboundPeekRequest" - FederationSenderPerformServersAlivePath = "/federationsender/performServersAlive" - FederationSenderPerformBroadcastEDUPath = "/federationsender/performBroadcastEDU" + FederationAPIPerformDirectoryLookupRequestPath = "/federationapi/performDirectoryLookup" + FederationAPIPerformJoinRequestPath = "/federationapi/performJoinRequest" + FederationAPIPerformLeaveRequestPath = "/federationapi/performLeaveRequest" + FederationAPIPerformInviteRequestPath = "/federationapi/performInviteRequest" + FederationAPIPerformOutboundPeekRequestPath = "/federationapi/performOutboundPeekRequest" + FederationAPIPerformServersAlivePath = "/federationapi/performServersAlive" + FederationAPIPerformBroadcastEDUPath = "/federationapi/performBroadcastEDU" - FederationSenderGetUserDevicesPath = "/federationsender/client/getUserDevices" - FederationSenderClaimKeysPath = "/federationsender/client/claimKeys" - FederationSenderQueryKeysPath = "/federationsender/client/queryKeys" - FederationSenderBackfillPath = "/federationsender/client/backfill" - FederationSenderLookupStatePath = "/federationsender/client/lookupState" - FederationSenderLookupStateIDsPath = "/federationsender/client/lookupStateIDs" - FederationSenderGetEventPath = "/federationsender/client/getEvent" - FederationSenderLookupServerKeysPath = "/federationsender/client/lookupServerKeys" - FederationSenderEventRelationshipsPath = "/federationsender/client/msc2836eventRelationships" - FederationSenderSpacesSummaryPath = "/federationsender/client/msc2946spacesSummary" + FederationAPIGetUserDevicesPath = "/federationapi/client/getUserDevices" + FederationAPIClaimKeysPath = "/federationapi/client/claimKeys" + FederationAPIQueryKeysPath = "/federationapi/client/queryKeys" + FederationAPIBackfillPath = "/federationapi/client/backfill" + FederationAPILookupStatePath = "/federationapi/client/lookupState" + FederationAPILookupStateIDsPath = "/federationapi/client/lookupStateIDs" + FederationAPIGetEventPath = "/federationapi/client/getEvent" + FederationAPILookupServerKeysPath = "/federationapi/client/lookupServerKeys" + FederationAPIEventRelationshipsPath = "/federationapi/client/msc2836eventRelationships" + FederationAPISpacesSummaryPath = "/federationapi/client/msc2946spacesSummary" + + FederationAPIInputPublicKeyPath = "/federationapi/inputPublicKey" + FederationAPIQueryPublicKeyPath = "/federationapi/queryPublicKey" ) -// NewFederationSenderClient creates a FederationSenderInternalAPI implemented by talking to a HTTP POST API. +// NewFederationAPIClient creates a FederationInternalAPI implemented by talking to a HTTP POST API. // If httpClient is nil an error is returned -func NewFederationSenderClient(federationSenderURL string, httpClient *http.Client) (api.FederationSenderInternalAPI, error) { +func NewFederationAPIClient(federationSenderURL string, httpClient *http.Client, cache caching.ServerKeyCache) (api.FederationInternalAPI, error) { if httpClient == nil { - return nil, errors.New("NewFederationSenderInternalAPIHTTP: httpClient is ") + return nil, errors.New("NewFederationInternalAPIHTTP: httpClient is ") } - return &httpFederationSenderInternalAPI{federationSenderURL, httpClient}, nil + return &httpFederationInternalAPI{federationSenderURL, httpClient, cache}, nil } -type httpFederationSenderInternalAPI struct { - federationSenderURL string - httpClient *http.Client +type httpFederationInternalAPI struct { + federationAPIURL string + httpClient *http.Client + cache caching.ServerKeyCache } -func (f *httpFederationSenderInternalAPI) SetUserAPI(userAPI userAPI.UserInternalAPI) {} +func (f *httpFederationInternalAPI) SetUserAPI(userAPI userAPI.UserInternalAPI) {} // Handle an instruction to make_leave & send_leave with a remote server. -func (h *httpFederationSenderInternalAPI) PerformLeave( +func (h *httpFederationInternalAPI) PerformLeave( ctx context.Context, request *api.PerformLeaveRequest, response *api.PerformLeaveResponse, @@ -63,12 +68,12 @@ func (h *httpFederationSenderInternalAPI) PerformLeave( span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLeaveRequest") defer span.Finish() - apiURL := h.federationSenderURL + FederationSenderPerformLeaveRequestPath + apiURL := h.federationAPIURL + FederationAPIPerformLeaveRequestPath return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // Handle sending an invite to a remote server. -func (h *httpFederationSenderInternalAPI) PerformInvite( +func (h *httpFederationInternalAPI) PerformInvite( ctx context.Context, request *api.PerformInviteRequest, response *api.PerformInviteResponse, @@ -76,12 +81,12 @@ func (h *httpFederationSenderInternalAPI) PerformInvite( span, ctx := opentracing.StartSpanFromContext(ctx, "PerformInviteRequest") defer span.Finish() - apiURL := h.federationSenderURL + FederationSenderPerformInviteRequestPath + apiURL := h.federationAPIURL + FederationAPIPerformInviteRequestPath return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // Handle starting a peek on a remote server. -func (h *httpFederationSenderInternalAPI) PerformOutboundPeek( +func (h *httpFederationInternalAPI) PerformOutboundPeek( ctx context.Context, request *api.PerformOutboundPeekRequest, response *api.PerformOutboundPeekResponse, @@ -89,11 +94,11 @@ func (h *httpFederationSenderInternalAPI) PerformOutboundPeek( span, ctx := opentracing.StartSpanFromContext(ctx, "PerformOutboundPeekRequest") defer span.Finish() - apiURL := h.federationSenderURL + FederationSenderPerformOutboundPeekRequestPath + apiURL := h.federationAPIURL + FederationAPIPerformOutboundPeekRequestPath return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } -func (h *httpFederationSenderInternalAPI) PerformServersAlive( +func (h *httpFederationInternalAPI) PerformServersAlive( ctx context.Context, request *api.PerformServersAliveRequest, response *api.PerformServersAliveResponse, @@ -101,12 +106,12 @@ func (h *httpFederationSenderInternalAPI) PerformServersAlive( span, ctx := opentracing.StartSpanFromContext(ctx, "PerformServersAlive") defer span.Finish() - apiURL := h.federationSenderURL + FederationSenderPerformServersAlivePath + apiURL := h.federationAPIURL + FederationAPIPerformServersAlivePath return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } -// QueryJoinedHostServerNamesInRoom implements FederationSenderInternalAPI -func (h *httpFederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( +// QueryJoinedHostServerNamesInRoom implements FederationInternalAPI +func (h *httpFederationInternalAPI) QueryJoinedHostServerNamesInRoom( ctx context.Context, request *api.QueryJoinedHostServerNamesInRoomRequest, response *api.QueryJoinedHostServerNamesInRoomResponse, @@ -114,12 +119,12 @@ func (h *httpFederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostServerNamesInRoom") defer span.Finish() - apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostServerNamesInRoomPath + apiURL := h.federationAPIURL + FederationAPIQueryJoinedHostServerNamesInRoomPath return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // Handle an instruction to make_join & send_join with a remote server. -func (h *httpFederationSenderInternalAPI) PerformJoin( +func (h *httpFederationInternalAPI) PerformJoin( ctx context.Context, request *api.PerformJoinRequest, response *api.PerformJoinResponse, @@ -127,7 +132,7 @@ func (h *httpFederationSenderInternalAPI) PerformJoin( span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoinRequest") defer span.Finish() - apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath + apiURL := h.federationAPIURL + FederationAPIPerformJoinRequestPath err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) if err != nil { response.LastError = &gomatrix.HTTPError{ @@ -139,7 +144,7 @@ func (h *httpFederationSenderInternalAPI) PerformJoin( } // Handle an instruction to make_join & send_join with a remote server. -func (h *httpFederationSenderInternalAPI) PerformDirectoryLookup( +func (h *httpFederationInternalAPI) PerformDirectoryLookup( ctx context.Context, request *api.PerformDirectoryLookupRequest, response *api.PerformDirectoryLookupResponse, @@ -147,12 +152,12 @@ func (h *httpFederationSenderInternalAPI) PerformDirectoryLookup( span, ctx := opentracing.StartSpanFromContext(ctx, "PerformDirectoryLookup") defer span.Finish() - apiURL := h.federationSenderURL + FederationSenderPerformDirectoryLookupRequestPath + apiURL := h.federationAPIURL + FederationAPIPerformDirectoryLookupRequestPath return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // Handle an instruction to broadcast an EDU to all servers in rooms we are joined to. -func (h *httpFederationSenderInternalAPI) PerformBroadcastEDU( +func (h *httpFederationInternalAPI) PerformBroadcastEDU( ctx context.Context, request *api.PerformBroadcastEDURequest, response *api.PerformBroadcastEDUResponse, @@ -160,7 +165,7 @@ func (h *httpFederationSenderInternalAPI) PerformBroadcastEDU( span, ctx := opentracing.StartSpanFromContext(ctx, "PerformBroadcastEDU") defer span.Finish() - apiURL := h.federationSenderURL + FederationSenderPerformBroadcastEDUPath + apiURL := h.federationAPIURL + FederationAPIPerformBroadcastEDUPath return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } @@ -171,7 +176,7 @@ type getUserDevices struct { Err *api.FederationClientError } -func (h *httpFederationSenderInternalAPI) GetUserDevices( +func (h *httpFederationInternalAPI) GetUserDevices( ctx context.Context, s gomatrixserverlib.ServerName, userID string, ) (gomatrixserverlib.RespUserDevices, error) { span, ctx := opentracing.StartSpanFromContext(ctx, "GetUserDevices") @@ -183,7 +188,7 @@ func (h *httpFederationSenderInternalAPI) GetUserDevices( UserID: userID, } var response getUserDevices - apiURL := h.federationSenderURL + FederationSenderGetUserDevicesPath + apiURL := h.federationAPIURL + FederationAPIGetUserDevicesPath err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response) if err != nil { return result, err @@ -201,7 +206,7 @@ type claimKeys struct { Err *api.FederationClientError } -func (h *httpFederationSenderInternalAPI) ClaimKeys( +func (h *httpFederationInternalAPI) ClaimKeys( ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string, ) (gomatrixserverlib.RespClaimKeys, error) { span, ctx := opentracing.StartSpanFromContext(ctx, "ClaimKeys") @@ -213,7 +218,7 @@ func (h *httpFederationSenderInternalAPI) ClaimKeys( OneTimeKeys: oneTimeKeys, } var response claimKeys - apiURL := h.federationSenderURL + FederationSenderClaimKeysPath + apiURL := h.federationAPIURL + FederationAPIClaimKeysPath err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response) if err != nil { return result, err @@ -231,7 +236,7 @@ type queryKeys struct { Err *api.FederationClientError } -func (h *httpFederationSenderInternalAPI) QueryKeys( +func (h *httpFederationInternalAPI) QueryKeys( ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string, ) (gomatrixserverlib.RespQueryKeys, error) { span, ctx := opentracing.StartSpanFromContext(ctx, "QueryKeys") @@ -243,7 +248,7 @@ func (h *httpFederationSenderInternalAPI) QueryKeys( Keys: keys, } var response queryKeys - apiURL := h.federationSenderURL + FederationSenderQueryKeysPath + apiURL := h.federationAPIURL + FederationAPIQueryKeysPath err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response) if err != nil { return result, err @@ -263,7 +268,7 @@ type backfill struct { Err *api.FederationClientError } -func (h *httpFederationSenderInternalAPI) Backfill( +func (h *httpFederationInternalAPI) Backfill( ctx context.Context, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string, ) (gomatrixserverlib.Transaction, error) { span, ctx := opentracing.StartSpanFromContext(ctx, "Backfill") @@ -276,7 +281,7 @@ func (h *httpFederationSenderInternalAPI) Backfill( EventIDs: eventIDs, } var response backfill - apiURL := h.federationSenderURL + FederationSenderBackfillPath + apiURL := h.federationAPIURL + FederationAPIBackfillPath err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response) if err != nil { return gomatrixserverlib.Transaction{}, err @@ -296,7 +301,7 @@ type lookupState struct { Err *api.FederationClientError } -func (h *httpFederationSenderInternalAPI) LookupState( +func (h *httpFederationInternalAPI) LookupState( ctx context.Context, s gomatrixserverlib.ServerName, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion, ) (gomatrixserverlib.RespState, error) { span, ctx := opentracing.StartSpanFromContext(ctx, "LookupState") @@ -309,7 +314,7 @@ func (h *httpFederationSenderInternalAPI) LookupState( RoomVersion: roomVersion, } var response lookupState - apiURL := h.federationSenderURL + FederationSenderLookupStatePath + apiURL := h.federationAPIURL + FederationAPILookupStatePath err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response) if err != nil { return gomatrixserverlib.RespState{}, err @@ -328,7 +333,7 @@ type lookupStateIDs struct { Err *api.FederationClientError } -func (h *httpFederationSenderInternalAPI) LookupStateIDs( +func (h *httpFederationInternalAPI) LookupStateIDs( ctx context.Context, s gomatrixserverlib.ServerName, roomID, eventID string, ) (gomatrixserverlib.RespStateIDs, error) { span, ctx := opentracing.StartSpanFromContext(ctx, "LookupStateIDs") @@ -340,7 +345,7 @@ func (h *httpFederationSenderInternalAPI) LookupStateIDs( EventID: eventID, } var response lookupStateIDs - apiURL := h.federationSenderURL + FederationSenderLookupStateIDsPath + apiURL := h.federationAPIURL + FederationAPILookupStateIDsPath err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response) if err != nil { return gomatrixserverlib.RespStateIDs{}, err @@ -358,7 +363,7 @@ type getEvent struct { Err *api.FederationClientError } -func (h *httpFederationSenderInternalAPI) GetEvent( +func (h *httpFederationInternalAPI) GetEvent( ctx context.Context, s gomatrixserverlib.ServerName, eventID string, ) (gomatrixserverlib.Transaction, error) { span, ctx := opentracing.StartSpanFromContext(ctx, "GetEvent") @@ -369,7 +374,7 @@ func (h *httpFederationSenderInternalAPI) GetEvent( EventID: eventID, } var response getEvent - apiURL := h.federationSenderURL + FederationSenderGetEventPath + apiURL := h.federationAPIURL + FederationAPIGetEventPath err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response) if err != nil { return gomatrixserverlib.Transaction{}, err @@ -380,13 +385,13 @@ func (h *httpFederationSenderInternalAPI) GetEvent( return *response.Res, nil } -func (h *httpFederationSenderInternalAPI) QueryServerKeys( +func (h *httpFederationInternalAPI) QueryServerKeys( ctx context.Context, req *api.QueryServerKeysRequest, res *api.QueryServerKeysResponse, ) error { span, ctx := opentracing.StartSpanFromContext(ctx, "QueryServerKeys") defer span.Finish() - apiURL := h.federationSenderURL + FederationSenderQueryServerKeysPath + apiURL := h.federationAPIURL + FederationAPIQueryServerKeysPath return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) } @@ -397,7 +402,7 @@ type lookupServerKeys struct { Err *api.FederationClientError } -func (h *httpFederationSenderInternalAPI) LookupServerKeys( +func (h *httpFederationInternalAPI) LookupServerKeys( ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, ) ([]gomatrixserverlib.ServerKeys, error) { span, ctx := opentracing.StartSpanFromContext(ctx, "LookupServerKeys") @@ -408,7 +413,7 @@ func (h *httpFederationSenderInternalAPI) LookupServerKeys( KeyRequests: keyRequests, } var response lookupServerKeys - apiURL := h.federationSenderURL + FederationSenderLookupServerKeysPath + apiURL := h.federationAPIURL + FederationAPILookupServerKeysPath err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response) if err != nil { return []gomatrixserverlib.ServerKeys{}, err @@ -427,7 +432,7 @@ type eventRelationships struct { Err *api.FederationClientError } -func (h *httpFederationSenderInternalAPI) MSC2836EventRelationships( +func (h *httpFederationInternalAPI) MSC2836EventRelationships( ctx context.Context, s gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion, ) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) { @@ -440,7 +445,7 @@ func (h *httpFederationSenderInternalAPI) MSC2836EventRelationships( RoomVer: roomVersion, } var response eventRelationships - apiURL := h.federationSenderURL + FederationSenderEventRelationshipsPath + apiURL := h.federationAPIURL + FederationAPIEventRelationshipsPath err = httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response) if err != nil { return res, err @@ -459,7 +464,7 @@ type spacesReq struct { Err *api.FederationClientError } -func (h *httpFederationSenderInternalAPI) MSC2946Spaces( +func (h *httpFederationInternalAPI) MSC2946Spaces( ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, r gomatrixserverlib.MSC2946SpacesRequest, ) (res gomatrixserverlib.MSC2946SpacesResponse, err error) { span, ctx := opentracing.StartSpanFromContext(ctx, "MSC2946Spaces") @@ -471,7 +476,7 @@ func (h *httpFederationSenderInternalAPI) MSC2946Spaces( RoomID: roomID, } var response spacesReq - apiURL := h.federationSenderURL + FederationSenderSpacesSummaryPath + apiURL := h.federationAPIURL + FederationAPISpacesSummaryPath err = httputil.PostJSON(ctx, span, h.httpClient, apiURL, &request, &response) if err != nil { return res, err @@ -481,3 +486,93 @@ func (h *httpFederationSenderInternalAPI) MSC2946Spaces( } return response.Res, nil } + +func (s *httpFederationInternalAPI) KeyRing() *gomatrixserverlib.KeyRing { + // This is a bit of a cheat - we tell gomatrixserverlib that this API is + // both the key database and the key fetcher. While this does have the + // rather unfortunate effect of preventing gomatrixserverlib from handling + // key fetchers directly, we can at least reimplement this behaviour on + // the other end of the API. + return &gomatrixserverlib.KeyRing{ + KeyDatabase: s, + KeyFetchers: []gomatrixserverlib.KeyFetcher{}, + } +} + +func (s *httpFederationInternalAPI) FetcherName() string { + return "httpServerKeyInternalAPI" +} + +func (s *httpFederationInternalAPI) StoreKeys( + _ context.Context, + results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, +) error { + // Run in a background context - we don't want to stop this work just + // because the caller gives up waiting. + ctx := context.Background() + request := api.InputPublicKeysRequest{ + Keys: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult), + } + response := api.InputPublicKeysResponse{} + for req, res := range results { + request.Keys[req] = res + s.cache.StoreServerKey(req, res) + } + return s.InputPublicKeys(ctx, &request, &response) +} + +func (s *httpFederationInternalAPI) FetchKeys( + _ context.Context, + requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, +) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { + // Run in a background context - we don't want to stop this work just + // because the caller gives up waiting. + ctx := context.Background() + result := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) + request := api.QueryPublicKeysRequest{ + Requests: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp), + } + response := api.QueryPublicKeysResponse{ + Results: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult), + } + for req, ts := range requests { + if res, ok := s.cache.GetServerKey(req, ts); ok { + result[req] = res + continue + } + request.Requests[req] = ts + } + err := s.QueryPublicKeys(ctx, &request, &response) + if err != nil { + return nil, err + } + for req, res := range response.Results { + result[req] = res + s.cache.StoreServerKey(req, res) + } + return result, nil +} + +func (h *httpFederationInternalAPI) InputPublicKeys( + ctx context.Context, + request *api.InputPublicKeysRequest, + response *api.InputPublicKeysResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "InputPublicKey") + defer span.Finish() + + apiURL := h.federationAPIURL + FederationAPIInputPublicKeyPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +func (h *httpFederationInternalAPI) QueryPublicKeys( + ctx context.Context, + request *api.QueryPublicKeysRequest, + response *api.QueryPublicKeysResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryPublicKey") + defer span.Finish() + + apiURL := h.federationAPIURL + FederationAPIQueryPublicKeyPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/federationsender/inthttp/server.go b/federationapi/inthttp/server.go similarity index 84% rename from federationsender/inthttp/server.go rename to federationapi/inthttp/server.go index a7fbc4eda..7133eddd0 100644 --- a/federationsender/inthttp/server.go +++ b/federationapi/inthttp/server.go @@ -5,16 +5,16 @@ import ( "net/http" "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/util" ) -// AddRoutes adds the FederationSenderInternalAPI handlers to the http.ServeMux. +// AddRoutes adds the FederationInternalAPI handlers to the http.ServeMux. // nolint:gocyclo -func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Router) { +func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router) { internalAPIMux.Handle( - FederationSenderQueryJoinedHostServerNamesInRoomPath, + FederationAPIQueryJoinedHostServerNamesInRoomPath, httputil.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { var request api.QueryJoinedHostServerNamesInRoomRequest var response api.QueryJoinedHostServerNamesInRoomResponse @@ -28,7 +28,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderPerformJoinRequestPath, + FederationAPIPerformJoinRequestPath, httputil.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse { var request api.PerformJoinRequest var response api.PerformJoinResponse @@ -40,7 +40,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderPerformLeaveRequestPath, + FederationAPIPerformLeaveRequestPath, httputil.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse { var request api.PerformLeaveRequest var response api.PerformLeaveResponse @@ -54,7 +54,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderPerformInviteRequestPath, + FederationAPIPerformInviteRequestPath, httputil.MakeInternalAPI("PerformInviteRequest", func(req *http.Request) util.JSONResponse { var request api.PerformInviteRequest var response api.PerformInviteResponse @@ -68,7 +68,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderPerformDirectoryLookupRequestPath, + FederationAPIPerformDirectoryLookupRequestPath, httputil.MakeInternalAPI("PerformDirectoryLookupRequest", func(req *http.Request) util.JSONResponse { var request api.PerformDirectoryLookupRequest var response api.PerformDirectoryLookupResponse @@ -82,7 +82,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderPerformServersAlivePath, + FederationAPIPerformServersAlivePath, httputil.MakeInternalAPI("PerformServersAliveRequest", func(req *http.Request) util.JSONResponse { var request api.PerformServersAliveRequest var response api.PerformServersAliveResponse @@ -96,7 +96,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderPerformBroadcastEDUPath, + FederationAPIPerformBroadcastEDUPath, httputil.MakeInternalAPI("PerformBroadcastEDU", func(req *http.Request) util.JSONResponse { var request api.PerformBroadcastEDURequest var response api.PerformBroadcastEDUResponse @@ -110,7 +110,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderGetUserDevicesPath, + FederationAPIGetUserDevicesPath, httputil.MakeInternalAPI("GetUserDevices", func(req *http.Request) util.JSONResponse { var request getUserDevices if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -132,7 +132,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderClaimKeysPath, + FederationAPIClaimKeysPath, httputil.MakeInternalAPI("ClaimKeys", func(req *http.Request) util.JSONResponse { var request claimKeys if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -154,7 +154,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderQueryKeysPath, + FederationAPIQueryKeysPath, httputil.MakeInternalAPI("QueryKeys", func(req *http.Request) util.JSONResponse { var request queryKeys if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -176,7 +176,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderBackfillPath, + FederationAPIBackfillPath, httputil.MakeInternalAPI("Backfill", func(req *http.Request) util.JSONResponse { var request backfill if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -198,7 +198,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderLookupStatePath, + FederationAPILookupStatePath, httputil.MakeInternalAPI("LookupState", func(req *http.Request) util.JSONResponse { var request lookupState if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -220,7 +220,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderLookupStateIDsPath, + FederationAPILookupStateIDsPath, httputil.MakeInternalAPI("LookupStateIDs", func(req *http.Request) util.JSONResponse { var request lookupStateIDs if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -242,7 +242,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderGetEventPath, + FederationAPIGetEventPath, httputil.MakeInternalAPI("GetEvent", func(req *http.Request) util.JSONResponse { var request getEvent if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -264,7 +264,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderQueryServerKeysPath, + FederationAPIQueryServerKeysPath, httputil.MakeInternalAPI("QueryServerKeys", func(req *http.Request) util.JSONResponse { var request api.QueryServerKeysRequest var response api.QueryServerKeysResponse @@ -278,7 +278,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderLookupServerKeysPath, + FederationAPILookupServerKeysPath, httputil.MakeInternalAPI("LookupServerKeys", func(req *http.Request) util.JSONResponse { var request lookupServerKeys if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -300,7 +300,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderEventRelationshipsPath, + FederationAPIEventRelationshipsPath, httputil.MakeInternalAPI("MSC2836EventRelationships", func(req *http.Request) util.JSONResponse { var request eventRelationships if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -322,7 +322,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle( - FederationSenderSpacesSummaryPath, + FederationAPISpacesSummaryPath, httputil.MakeInternalAPI("MSC2946SpacesSummary", func(req *http.Request) util.JSONResponse { var request spacesReq if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -343,4 +343,32 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route return util.JSONResponse{Code: http.StatusOK, JSON: request} }), ) + internalAPIMux.Handle(FederationAPIQueryPublicKeyPath, + httputil.MakeInternalAPI("queryPublicKeys", func(req *http.Request) util.JSONResponse { + request := api.QueryPublicKeysRequest{} + response := api.QueryPublicKeysResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + keys, err := intAPI.FetchKeys(req.Context(), request.Requests) + if err != nil { + return util.ErrorResponse(err) + } + response.Results = keys + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(FederationAPIInputPublicKeyPath, + httputil.MakeInternalAPI("inputPublicKeys", func(req *http.Request) util.JSONResponse { + request := api.InputPublicKeysRequest{} + response := api.InputPublicKeysResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := intAPI.StoreKeys(req.Context(), request.Keys); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) } diff --git a/federationsender/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go similarity index 98% rename from federationsender/queue/destinationqueue.go rename to federationapi/queue/destinationqueue.go index 0123fa24c..1306e8588 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationapi/queue/destinationqueue.go @@ -21,9 +21,9 @@ import ( "sync" "time" - "github.com/matrix-org/dendrite/federationsender/statistics" - "github.com/matrix-org/dendrite/federationsender/storage" - "github.com/matrix-org/dendrite/federationsender/storage/shared" + "github.com/matrix-org/dendrite/federationapi/statistics" + "github.com/matrix-org/dendrite/federationapi/storage" + "github.com/matrix-org/dendrite/federationapi/storage/shared" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrix" @@ -81,7 +81,7 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re context.TODO(), "", // TODO: remove this, as we don't need to persist the transaction ID oq.destination, // the destination server name - receipt, // NIDs from federationsender_queue_json table + receipt, // NIDs from federationapi_queue_json table ); err != nil { logrus.WithError(err).Errorf("failed to associate PDU %q with destination %q", event.EventID(), oq.destination) return @@ -124,7 +124,7 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share if err := oq.db.AssociateEDUWithDestination( context.TODO(), oq.destination, // the destination server name - receipt, // NIDs from federationsender_queue_json table + receipt, // NIDs from federationapi_queue_json table ); err != nil { logrus.WithError(err).Errorf("failed to associate EDU with destination %q", oq.destination) return diff --git a/federationsender/queue/queue.go b/federationapi/queue/queue.go similarity index 96% rename from federationsender/queue/queue.go rename to federationapi/queue/queue.go index a2ef6e5f8..8a6ad1555 100644 --- a/federationsender/queue/queue.go +++ b/federationapi/queue/queue.go @@ -22,9 +22,9 @@ import ( "sync" "time" - "github.com/matrix-org/dendrite/federationsender/statistics" - "github.com/matrix-org/dendrite/federationsender/storage" - "github.com/matrix-org/dendrite/federationsender/storage/shared" + "github.com/matrix-org/dendrite/federationapi/statistics" + "github.com/matrix-org/dendrite/federationapi/storage" + "github.com/matrix-org/dendrite/federationapi/storage/shared" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" @@ -58,7 +58,7 @@ func init() { var destinationQueueTotal = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "dendrite", - Subsystem: "federationsender", + Subsystem: "federationapi", Name: "destination_queues_total", }, ) @@ -66,7 +66,7 @@ var destinationQueueTotal = prometheus.NewGauge( var destinationQueueRunning = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "dendrite", - Subsystem: "federationsender", + Subsystem: "federationapi", Name: "destination_queues_running", }, ) @@ -74,7 +74,7 @@ var destinationQueueRunning = prometheus.NewGauge( var destinationQueueBackingOff = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "dendrite", - Subsystem: "federationsender", + Subsystem: "federationapi", Name: "destination_queues_backing_off", }, ) diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go index bba3272b9..49a6c558f 100644 --- a/federationapi/routing/keys.go +++ b/federationapi/routing/keys.go @@ -21,7 +21,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" @@ -179,7 +179,7 @@ func localKeys(cfg *config.FederationAPI, validUntil time.Time) (*gomatrixserver func NotaryKeys( httpReq *http.Request, cfg *config.FederationAPI, - fsAPI federationSenderAPI.FederationSenderInternalAPI, + fsAPI federationAPI.FederationInternalAPI, req *gomatrixserverlib.PublicKeyNotaryLookupRequest, ) util.JSONResponse { if req == nil { @@ -203,8 +203,8 @@ func NotaryKeys( return util.ErrorResponse(err) } } else { - var resp federationSenderAPI.QueryServerKeysResponse - err := fsAPI.QueryServerKeys(httpReq.Context(), &federationSenderAPI.QueryServerKeysRequest{ + var resp federationAPI.QueryServerKeysResponse + err := fsAPI.QueryServerKeys(httpReq.Context(), &federationAPI.QueryServerKeysRequest{ ServerName: serverName, KeyIDToCriteria: kidToCriteria, }, &resp) diff --git a/federationapi/routing/peek.go b/federationapi/routing/peek.go index 8f83cb157..511329997 100644 --- a/federationapi/routing/peek.go +++ b/federationapi/routing/peek.go @@ -33,7 +33,7 @@ func Peek( roomID, peekID string, remoteVersions []gomatrixserverlib.RoomVersion, ) util.JSONResponse { - // TODO: check if we're just refreshing an existing peek by querying the federationsender + // TODO: check if we're just refreshing an existing peek by querying the federationapi verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index b4158f0cd..47d3b2df9 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -19,7 +19,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/jsonerror" - federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + federationAPI "github.com/matrix-org/dendrite/federationapi/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrix" @@ -33,7 +33,7 @@ func RoomAliasToID( federation *gomatrixserverlib.FederationClient, cfg *config.FederationAPI, rsAPI roomserverAPI.RoomserverInternalAPI, - senderAPI federationSenderAPI.FederationSenderInternalAPI, + senderAPI federationAPI.FederationInternalAPI, ) util.JSONResponse { roomAlias := httpReq.FormValue("room_alias") if roomAlias == "" { @@ -64,8 +64,8 @@ func RoomAliasToID( } if queryRes.RoomID != "" { - serverQueryReq := federationSenderAPI.QueryJoinedHostServerNamesInRoomRequest{RoomID: queryRes.RoomID} - var serverQueryRes federationSenderAPI.QueryJoinedHostServerNamesInRoomResponse + serverQueryReq := federationAPI.QueryJoinedHostServerNamesInRoomRequest{RoomID: queryRes.RoomID} + var serverQueryRes federationAPI.QueryJoinedHostServerNamesInRoomResponse if err = senderAPI.QueryJoinedHostServerNamesInRoom(httpReq.Context(), &serverQueryReq, &serverQueryRes); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("senderAPI.QueryJoinedHostServerNamesInRoom failed") return jsonerror.InternalServerError() diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index e409e2ef0..791e044a6 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -21,7 +21,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" - federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/httputil" keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" @@ -46,7 +45,7 @@ func Setup( cfg *config.FederationAPI, rsAPI roomserverAPI.RoomserverInternalAPI, eduAPI eduserverAPI.EDUServerInputAPI, - fsAPI federationSenderAPI.FederationSenderInternalAPI, + fsAPI federationAPI.FederationInternalAPI, keys gomatrixserverlib.JSONVerifier, federation *gomatrixserverlib.FederationClient, userAPI userapi.UserInternalAPI, diff --git a/federationsender/statistics/statistics.go b/federationapi/statistics/statistics.go similarity index 99% rename from federationsender/statistics/statistics.go rename to federationapi/statistics/statistics.go index b5fe7513d..8bac99cbc 100644 --- a/federationsender/statistics/statistics.go +++ b/federationapi/statistics/statistics.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "github.com/matrix-org/dendrite/federationsender/storage" + "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" "go.uber.org/atomic" diff --git a/federationsender/statistics/statistics_test.go b/federationapi/statistics/statistics_test.go similarity index 100% rename from federationsender/statistics/statistics_test.go rename to federationapi/statistics/statistics_test.go diff --git a/signingkeyserver/storage/cache/keydb.go b/federationapi/storage/cache/keydb.go similarity index 100% rename from signingkeyserver/storage/cache/keydb.go rename to federationapi/storage/cache/keydb.go diff --git a/federationsender/storage/interface.go b/federationapi/storage/interface.go similarity index 96% rename from federationsender/storage/interface.go rename to federationapi/storage/interface.go index 58c8a7cfa..a36f51528 100644 --- a/federationsender/storage/interface.go +++ b/federationapi/storage/interface.go @@ -17,14 +17,15 @@ package storage import ( "context" - "github.com/matrix-org/dendrite/federationsender/storage/shared" - "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/federationapi/storage/shared" + "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) type Database interface { internal.PartitionStorer + gomatrixserverlib.KeyDatabase UpdateRoom(ctx context.Context, roomID, oldEventID, newEventID string, addHosts []types.JoinedHost, removeHosts []string) (joinedHosts []types.JoinedHost, err error) diff --git a/federationsender/storage/postgres/blacklist_table.go b/federationapi/storage/postgres/blacklist_table.go similarity index 100% rename from federationsender/storage/postgres/blacklist_table.go rename to federationapi/storage/postgres/blacklist_table.go diff --git a/federationsender/storage/postgres/deltas/2021020411080000_rooms.go b/federationapi/storage/postgres/deltas/2021020411080000_rooms.go similarity index 100% rename from federationsender/storage/postgres/deltas/2021020411080000_rooms.go rename to federationapi/storage/postgres/deltas/2021020411080000_rooms.go diff --git a/federationsender/storage/postgres/inbound_peeks_table.go b/federationapi/storage/postgres/inbound_peeks_table.go similarity index 99% rename from federationsender/storage/postgres/inbound_peeks_table.go rename to federationapi/storage/postgres/inbound_peeks_table.go index fe35ce44c..df5c60761 100644 --- a/federationsender/storage/postgres/inbound_peeks_table.go +++ b/federationapi/storage/postgres/inbound_peeks_table.go @@ -19,7 +19,7 @@ import ( "database/sql" "time" - "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" diff --git a/federationsender/storage/postgres/joined_hosts_table.go b/federationapi/storage/postgres/joined_hosts_table.go similarity index 99% rename from federationsender/storage/postgres/joined_hosts_table.go rename to federationapi/storage/postgres/joined_hosts_table.go index 0c1e91eeb..5c95b72a8 100644 --- a/federationsender/storage/postgres/joined_hosts_table.go +++ b/federationapi/storage/postgres/joined_hosts_table.go @@ -20,7 +20,7 @@ import ( "database/sql" "github.com/lib/pq" - "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" diff --git a/federationsender/storage/postgres/notary_server_keys_json_table.go b/federationapi/storage/postgres/notary_server_keys_json_table.go similarity index 97% rename from federationsender/storage/postgres/notary_server_keys_json_table.go rename to federationapi/storage/postgres/notary_server_keys_json_table.go index 42e58ba79..65221c088 100644 --- a/federationsender/storage/postgres/notary_server_keys_json_table.go +++ b/federationapi/storage/postgres/notary_server_keys_json_table.go @@ -18,7 +18,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/federationsender/storage/tables" + "github.com/matrix-org/dendrite/federationapi/storage/tables" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/federationsender/storage/postgres/notary_server_keys_metadata_table.go b/federationapi/storage/postgres/notary_server_keys_metadata_table.go similarity index 98% rename from federationsender/storage/postgres/notary_server_keys_metadata_table.go rename to federationapi/storage/postgres/notary_server_keys_metadata_table.go index b460dcd88..72faf4809 100644 --- a/federationsender/storage/postgres/notary_server_keys_metadata_table.go +++ b/federationapi/storage/postgres/notary_server_keys_metadata_table.go @@ -20,7 +20,7 @@ import ( "encoding/json" "github.com/lib/pq" - "github.com/matrix-org/dendrite/federationsender/storage/tables" + "github.com/matrix-org/dendrite/federationapi/storage/tables" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/federationsender/storage/postgres/outbound_peeks_table.go b/federationapi/storage/postgres/outbound_peeks_table.go similarity index 99% rename from federationsender/storage/postgres/outbound_peeks_table.go rename to federationapi/storage/postgres/outbound_peeks_table.go index 596b4bcc7..c22d893f7 100644 --- a/federationsender/storage/postgres/outbound_peeks_table.go +++ b/federationapi/storage/postgres/outbound_peeks_table.go @@ -19,7 +19,7 @@ import ( "database/sql" "time" - "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" diff --git a/federationsender/storage/postgres/queue_edus_table.go b/federationapi/storage/postgres/queue_edus_table.go similarity index 100% rename from federationsender/storage/postgres/queue_edus_table.go rename to federationapi/storage/postgres/queue_edus_table.go diff --git a/federationsender/storage/postgres/queue_json_table.go b/federationapi/storage/postgres/queue_json_table.go similarity index 100% rename from federationsender/storage/postgres/queue_json_table.go rename to federationapi/storage/postgres/queue_json_table.go diff --git a/federationsender/storage/postgres/queue_pdus_table.go b/federationapi/storage/postgres/queue_pdus_table.go similarity index 100% rename from federationsender/storage/postgres/queue_pdus_table.go rename to federationapi/storage/postgres/queue_pdus_table.go diff --git a/signingkeyserver/storage/postgres/server_key_table.go b/federationapi/storage/postgres/server_key_table.go similarity index 84% rename from signingkeyserver/storage/postgres/server_key_table.go rename to federationapi/storage/postgres/server_key_table.go index 87f1c211c..16e294e05 100644 --- a/signingkeyserver/storage/postgres/server_key_table.go +++ b/federationapi/storage/postgres/server_key_table.go @@ -21,10 +21,11 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) -const serverKeysSchema = ` +const serverSigningKeysSchema = ` -- A cache of signing keys downloaded from remote servers. CREATE TABLE IF NOT EXISTS keydb_server_keys ( -- The name of the matrix server the key is for. @@ -48,39 +49,40 @@ CREATE TABLE IF NOT EXISTS keydb_server_keys ( CREATE INDEX IF NOT EXISTS keydb_server_name_and_key_id ON keydb_server_keys (server_name_and_key_id); ` -const bulkSelectServerKeysSQL = "" + +const bulkSelectServerSigningKeysSQL = "" + "SELECT server_name, server_key_id, valid_until_ts, expired_ts, " + " server_key FROM keydb_server_keys" + " WHERE server_name_and_key_id = ANY($1)" -const upsertServerKeysSQL = "" + +const upsertServerSigningKeysSQL = "" + "INSERT INTO keydb_server_keys (server_name, server_key_id," + " server_name_and_key_id, valid_until_ts, expired_ts, server_key)" + " VALUES ($1, $2, $3, $4, $5, $6)" + " ON CONFLICT ON CONSTRAINT keydb_server_keys_unique" + " DO UPDATE SET valid_until_ts = $4, expired_ts = $5, server_key = $6" -type serverKeyStatements struct { +type serverSigningKeyStatements struct { bulkSelectServerKeysStmt *sql.Stmt upsertServerKeysStmt *sql.Stmt } -func (s *serverKeyStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(serverKeysSchema) +func NewPostgresServerSigningKeysTable(db *sql.DB) (s *serverSigningKeyStatements, err error) { + s = &serverSigningKeyStatements{} + _, err = db.Exec(serverSigningKeysSchema) if err != nil { return } - if s.bulkSelectServerKeysStmt, err = db.Prepare(bulkSelectServerKeysSQL); err != nil { + if s.bulkSelectServerKeysStmt, err = db.Prepare(bulkSelectServerSigningKeysSQL); err != nil { return } - if s.upsertServerKeysStmt, err = db.Prepare(upsertServerKeysSQL); err != nil { + if s.upsertServerKeysStmt, err = db.Prepare(upsertServerSigningKeysSQL); err != nil { return } - return + return s, nil } -func (s *serverKeyStatements) bulkSelectServerKeys( - ctx context.Context, +func (s *serverSigningKeyStatements) BulkSelectServerKeys( + ctx context.Context, txn *sql.Tx, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, ) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { var nameAndKeyIDs []string @@ -121,12 +123,13 @@ func (s *serverKeyStatements) bulkSelectServerKeys( return results, rows.Err() } -func (s *serverKeyStatements) upsertServerKeys( - ctx context.Context, +func (s *serverSigningKeyStatements) UpsertServerKeys( + ctx context.Context, txn *sql.Tx, request gomatrixserverlib.PublicKeyLookupRequest, key gomatrixserverlib.PublicKeyLookupResult, ) error { - _, err := s.upsertServerKeysStmt.ExecContext( + stmt := sqlutil.TxStmt(txn, s.upsertServerKeysStmt) + _, err := stmt.ExecContext( ctx, string(request.ServerName), string(request.KeyID), diff --git a/federationsender/storage/postgres/storage.go b/federationapi/storage/postgres/storage.go similarity index 76% rename from federationsender/storage/postgres/storage.go rename to federationapi/storage/postgres/storage.go index 5507bad78..1f6afe37c 100644 --- a/federationsender/storage/postgres/storage.go +++ b/federationapi/storage/postgres/storage.go @@ -19,8 +19,8 @@ import ( "database/sql" "fmt" - "github.com/matrix-org/dendrite/federationsender/storage/postgres/deltas" - "github.com/matrix-org/dendrite/federationsender/storage/shared" + "github.com/matrix-org/dendrite/federationapi/storage/postgres/deltas" + "github.com/matrix-org/dendrite/federationapi/storage/shared" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" @@ -35,7 +35,7 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationSenderCache) (*Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationCache) (*Database, error) { var d Database var err error if d.db, err = sqlutil.Open(dbProperties); err != nil { @@ -78,24 +78,29 @@ func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationS if err != nil { return nil, fmt.Errorf("NewPostgresNotaryServerKeysMetadataTable: %s", err) } + serverSigningKeys, err := NewPostgresServerSigningKeysTable(d.db) + if err != nil { + return nil, err + } m := sqlutil.NewMigrations() deltas.LoadRemoveRoomsTable(m) if err = m.RunDeltas(d.db, dbProperties); err != nil { return nil, err } d.Database = shared.Database{ - DB: d.db, - Cache: cache, - Writer: d.writer, - FederationSenderJoinedHosts: joinedHosts, - FederationSenderQueuePDUs: queuePDUs, - FederationSenderQueueEDUs: queueEDUs, - FederationSenderQueueJSON: queueJSON, - FederationSenderBlacklist: blacklist, - FederationSenderInboundPeeks: inboundPeeks, - FederationSenderOutboundPeeks: outboundPeeks, - NotaryServerKeysJSON: notaryJSON, - NotaryServerKeysMetadata: notaryMetadata, + DB: d.db, + Cache: cache, + Writer: d.writer, + FederationJoinedHosts: joinedHosts, + FederationQueuePDUs: queuePDUs, + FederationQueueEDUs: queueEDUs, + FederationQueueJSON: queueJSON, + FederationBlacklist: blacklist, + FederationInboundPeeks: inboundPeeks, + FederationOutboundPeeks: outboundPeeks, + NotaryServerKeysJSON: notaryJSON, + NotaryServerKeysMetadata: notaryMetadata, + ServerSigningKeys: serverSigningKeys, } if err = d.PartitionOffsetStatements.Prepare(d.db, d.writer, "federationsender"); err != nil { return nil, err diff --git a/federationsender/storage/shared/storage.go b/federationapi/storage/shared/storage.go similarity index 72% rename from federationsender/storage/shared/storage.go rename to federationapi/storage/shared/storage.go index 45c9febdb..ddd770e2e 100644 --- a/federationsender/storage/shared/storage.go +++ b/federationapi/storage/shared/storage.go @@ -20,26 +20,27 @@ import ( "fmt" "time" - "github.com/matrix-org/dendrite/federationsender/storage/tables" - "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/federationapi/storage/tables" + "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) type Database struct { - DB *sql.DB - Cache caching.FederationSenderCache - Writer sqlutil.Writer - FederationSenderQueuePDUs tables.FederationSenderQueuePDUs - FederationSenderQueueEDUs tables.FederationSenderQueueEDUs - FederationSenderQueueJSON tables.FederationSenderQueueJSON - FederationSenderJoinedHosts tables.FederationSenderJoinedHosts - FederationSenderBlacklist tables.FederationSenderBlacklist - FederationSenderOutboundPeeks tables.FederationSenderOutboundPeeks - FederationSenderInboundPeeks tables.FederationSenderInboundPeeks - NotaryServerKeysJSON tables.FederationSenderNotaryServerKeysJSON - NotaryServerKeysMetadata tables.FederationSenderNotaryServerKeysMetadata + DB *sql.DB + Cache caching.FederationCache + Writer sqlutil.Writer + FederationQueuePDUs tables.FederationQueuePDUs + FederationQueueEDUs tables.FederationQueueEDUs + FederationQueueJSON tables.FederationQueueJSON + FederationJoinedHosts tables.FederationJoinedHosts + FederationBlacklist tables.FederationBlacklist + FederationOutboundPeeks tables.FederationOutboundPeeks + FederationInboundPeeks tables.FederationInboundPeeks + NotaryServerKeysJSON tables.FederationNotaryServerKeysJSON + NotaryServerKeysMetadata tables.FederationNotaryServerKeysMetadata + ServerSigningKeys tables.FederationServerSigningKeys } // An Receipt contains the NIDs of a call to GetNextTransactionPDUs/EDUs. @@ -66,18 +67,18 @@ func (d *Database) UpdateRoom( removeHosts []string, ) (joinedHosts []types.JoinedHost, err error) { err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - joinedHosts, err = d.FederationSenderJoinedHosts.SelectJoinedHostsWithTx(ctx, txn, roomID) + joinedHosts, err = d.FederationJoinedHosts.SelectJoinedHostsWithTx(ctx, txn, roomID) if err != nil { return err } for _, add := range addHosts { - err = d.FederationSenderJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName) + err = d.FederationJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName) if err != nil { return err } } - if err = d.FederationSenderJoinedHosts.DeleteJoinedHosts(ctx, txn, removeHosts); err != nil { + if err = d.FederationJoinedHosts.DeleteJoinedHosts(ctx, txn, removeHosts); err != nil { return err } return nil @@ -91,18 +92,18 @@ func (d *Database) UpdateRoom( func (d *Database) GetJoinedHosts( ctx context.Context, roomID string, ) ([]types.JoinedHost, error) { - return d.FederationSenderJoinedHosts.SelectJoinedHosts(ctx, roomID) + return d.FederationJoinedHosts.SelectJoinedHosts(ctx, roomID) } // GetAllJoinedHosts returns the currently joined hosts for // all rooms known to the federation sender. // Returns an error if something goes wrong. func (d *Database) GetAllJoinedHosts(ctx context.Context) ([]gomatrixserverlib.ServerName, error) { - return d.FederationSenderJoinedHosts.SelectAllJoinedHosts(ctx) + return d.FederationJoinedHosts.SelectAllJoinedHosts(ctx) } func (d *Database) GetJoinedHostsForRooms(ctx context.Context, roomIDs []string) ([]gomatrixserverlib.ServerName, error) { - return d.FederationSenderJoinedHosts.SelectJoinedHostsForRooms(ctx, roomIDs) + return d.FederationJoinedHosts.SelectJoinedHostsForRooms(ctx, roomIDs) } // StoreJSON adds a JSON blob into the queue JSON table and returns @@ -114,7 +115,7 @@ func (d *Database) StoreJSON( var nid int64 var err error _ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - nid, err = d.FederationSenderQueueJSON.InsertQueueJSON(ctx, txn, js) + nid, err = d.FederationQueueJSON.InsertQueueJSON(ctx, txn, js) return err }) if err != nil { @@ -132,8 +133,8 @@ func (d *Database) PurgeRoomState( // If the event is a create event then we'll delete all of the existing // data for the room. The only reason that a create event would be replayed // to us in this way is if we're about to receive the entire room state. - if err := d.FederationSenderJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil { - return fmt.Errorf("d.FederationSenderJoinedHosts.DeleteJoinedHosts: %w", err) + if err := d.FederationJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil { + return fmt.Errorf("d.FederationJoinedHosts.DeleteJoinedHosts: %w", err) } return nil }) @@ -141,64 +142,64 @@ func (d *Database) PurgeRoomState( func (d *Database) AddServerToBlacklist(serverName gomatrixserverlib.ServerName) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.FederationSenderBlacklist.InsertBlacklist(context.TODO(), txn, serverName) + return d.FederationBlacklist.InsertBlacklist(context.TODO(), txn, serverName) }) } func (d *Database) RemoveServerFromBlacklist(serverName gomatrixserverlib.ServerName) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.FederationSenderBlacklist.DeleteBlacklist(context.TODO(), txn, serverName) + return d.FederationBlacklist.DeleteBlacklist(context.TODO(), txn, serverName) }) } func (d *Database) RemoveAllServersFromBlacklist() error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.FederationSenderBlacklist.DeleteAllBlacklist(context.TODO(), txn) + return d.FederationBlacklist.DeleteAllBlacklist(context.TODO(), txn) }) } func (d *Database) IsServerBlacklisted(serverName gomatrixserverlib.ServerName) (bool, error) { - return d.FederationSenderBlacklist.SelectBlacklist(context.TODO(), nil, serverName) + return d.FederationBlacklist.SelectBlacklist(context.TODO(), nil, serverName) } func (d *Database) AddOutboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.FederationSenderOutboundPeeks.InsertOutboundPeek(ctx, txn, serverName, roomID, peekID, renewalInterval) + return d.FederationOutboundPeeks.InsertOutboundPeek(ctx, txn, serverName, roomID, peekID, renewalInterval) }) } func (d *Database) RenewOutboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.FederationSenderOutboundPeeks.RenewOutboundPeek(ctx, txn, serverName, roomID, peekID, renewalInterval) + return d.FederationOutboundPeeks.RenewOutboundPeek(ctx, txn, serverName, roomID, peekID, renewalInterval) }) } func (d *Database) GetOutboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string) (*types.OutboundPeek, error) { - return d.FederationSenderOutboundPeeks.SelectOutboundPeek(ctx, nil, serverName, roomID, peekID) + return d.FederationOutboundPeeks.SelectOutboundPeek(ctx, nil, serverName, roomID, peekID) } func (d *Database) GetOutboundPeeks(ctx context.Context, roomID string) ([]types.OutboundPeek, error) { - return d.FederationSenderOutboundPeeks.SelectOutboundPeeks(ctx, nil, roomID) + return d.FederationOutboundPeeks.SelectOutboundPeeks(ctx, nil, roomID) } func (d *Database) AddInboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.FederationSenderInboundPeeks.InsertInboundPeek(ctx, txn, serverName, roomID, peekID, renewalInterval) + return d.FederationInboundPeeks.InsertInboundPeek(ctx, txn, serverName, roomID, peekID, renewalInterval) }) } func (d *Database) RenewInboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.FederationSenderInboundPeeks.RenewInboundPeek(ctx, txn, serverName, roomID, peekID, renewalInterval) + return d.FederationInboundPeeks.RenewInboundPeek(ctx, txn, serverName, roomID, peekID, renewalInterval) }) } func (d *Database) GetInboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string) (*types.InboundPeek, error) { - return d.FederationSenderInboundPeeks.SelectInboundPeek(ctx, nil, serverName, roomID, peekID) + return d.FederationInboundPeeks.SelectInboundPeek(ctx, nil, serverName, roomID, peekID) } func (d *Database) GetInboundPeeks(ctx context.Context, roomID string) ([]types.InboundPeek, error) { - return d.FederationSenderInboundPeeks.SelectInboundPeeks(ctx, nil, roomID) + return d.FederationInboundPeeks.SelectInboundPeeks(ctx, nil, roomID) } func (d *Database) UpdateNotaryKeys(ctx context.Context, serverName gomatrixserverlib.ServerName, serverKeys gomatrixserverlib.ServerKeys) error { diff --git a/federationsender/storage/shared/storage_edus.go b/federationapi/storage/shared/storage_edus.go similarity index 80% rename from federationsender/storage/shared/storage_edus.go rename to federationapi/storage/shared/storage_edus.go index 86fee1a37..6e3c7e367 100644 --- a/federationsender/storage/shared/storage_edus.go +++ b/federationapi/storage/shared/storage_edus.go @@ -33,12 +33,12 @@ func (d *Database) AssociateEDUWithDestination( receipt *Receipt, ) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - if err := d.FederationSenderQueueEDUs.InsertQueueEDU( + if err := d.FederationQueueEDUs.InsertQueueEDU( ctx, // context txn, // SQL transaction "", // TODO: EDU type for coalescing serverName, // destination server name - receipt.nid, // NID from the federationsender_queue_json table + receipt.nid, // NID from the federationapi_queue_json table ); err != nil { return fmt.Errorf("InsertQueueEDU: %w", err) } @@ -58,21 +58,21 @@ func (d *Database) GetPendingEDUs( ) { edus = make(map[*Receipt]*gomatrixserverlib.EDU) err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - nids, err := d.FederationSenderQueueEDUs.SelectQueueEDUs(ctx, txn, serverName, limit) + nids, err := d.FederationQueueEDUs.SelectQueueEDUs(ctx, txn, serverName, limit) if err != nil { return fmt.Errorf("SelectQueueEDUs: %w", err) } retrieve := make([]int64, 0, len(nids)) for _, nid := range nids { - if edu, ok := d.Cache.GetFederationSenderQueuedEDU(nid); ok { + if edu, ok := d.Cache.GetFederationQueuedEDU(nid); ok { edus[&Receipt{nid}] = edu } else { retrieve = append(retrieve, nid) } } - blobs, err := d.FederationSenderQueueJSON.SelectQueueJSON(ctx, txn, retrieve) + blobs, err := d.FederationQueueJSON.SelectQueueJSON(ctx, txn, retrieve) if err != nil { return fmt.Errorf("SelectQueueJSON: %w", err) } @@ -107,24 +107,24 @@ func (d *Database) CleanEDUs( } return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - if err := d.FederationSenderQueueEDUs.DeleteQueueEDUs(ctx, txn, serverName, nids); err != nil { + if err := d.FederationQueueEDUs.DeleteQueueEDUs(ctx, txn, serverName, nids); err != nil { return err } var deleteNIDs []int64 for _, nid := range nids { - count, err := d.FederationSenderQueueEDUs.SelectQueueEDUReferenceJSONCount(ctx, txn, nid) + count, err := d.FederationQueueEDUs.SelectQueueEDUReferenceJSONCount(ctx, txn, nid) if err != nil { return fmt.Errorf("SelectQueueEDUReferenceJSONCount: %w", err) } if count == 0 { deleteNIDs = append(deleteNIDs, nid) - d.Cache.EvictFederationSenderQueuedEDU(nid) + d.Cache.EvictFederationQueuedEDU(nid) } } if len(deleteNIDs) > 0 { - if err := d.FederationSenderQueueJSON.DeleteQueueJSON(ctx, txn, deleteNIDs); err != nil { + if err := d.FederationQueueJSON.DeleteQueueJSON(ctx, txn, deleteNIDs); err != nil { return fmt.Errorf("DeleteQueueJSON: %w", err) } } @@ -139,7 +139,7 @@ func (d *Database) GetPendingEDUCount( ctx context.Context, serverName gomatrixserverlib.ServerName, ) (int64, error) { - return d.FederationSenderQueueEDUs.SelectQueueEDUCount(ctx, nil, serverName) + return d.FederationQueueEDUs.SelectQueueEDUCount(ctx, nil, serverName) } // GetPendingServerNames returns the server names that have EDUs @@ -147,5 +147,5 @@ func (d *Database) GetPendingEDUCount( func (d *Database) GetPendingEDUServerNames( ctx context.Context, ) ([]gomatrixserverlib.ServerName, error) { - return d.FederationSenderQueueEDUs.SelectQueueEDUServerNames(ctx, nil) + return d.FederationQueueEDUs.SelectQueueEDUServerNames(ctx, nil) } diff --git a/federationapi/storage/shared/storage_keys.go b/federationapi/storage/shared/storage_keys.go new file mode 100644 index 000000000..3222b1224 --- /dev/null +++ b/federationapi/storage/shared/storage_keys.go @@ -0,0 +1,59 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shared + +import ( + "context" + "database/sql" + + "github.com/matrix-org/gomatrixserverlib" +) + +// FetcherName implements KeyFetcher +func (d Database) FetcherName() string { + return "FederationAPIKeyDatabase" +} + +// FetchKeys implements gomatrixserverlib.KeyDatabase +func (d *Database) FetchKeys( + ctx context.Context, + requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, +) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { + return d.ServerSigningKeys.BulkSelectServerKeys(ctx, nil, requests) +} + +// StoreKeys implements gomatrixserverlib.KeyDatabase +func (d *Database) StoreKeys( + ctx context.Context, + keyMap map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, +) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + var lastErr error + for request, keys := range keyMap { + if err := d.ServerSigningKeys.UpsertServerKeys(ctx, txn, request, keys); err != nil { + // Rather than returning immediately on error we try to insert the + // remaining keys. + // Since we are inserting the keys outside of a transaction it is + // possible for some of the inserts to succeed even though some + // of the inserts have failed. + // Ensuring that we always insert all the keys we can means that + // this behaviour won't depend on the iteration order of the map. + lastErr = err + } + } + return lastErr + }) +} diff --git a/federationsender/storage/shared/storage_pdus.go b/federationapi/storage/shared/storage_pdus.go similarity index 81% rename from federationsender/storage/shared/storage_pdus.go rename to federationapi/storage/shared/storage_pdus.go index bc298a905..5a12c388a 100644 --- a/federationsender/storage/shared/storage_pdus.go +++ b/federationapi/storage/shared/storage_pdus.go @@ -34,12 +34,12 @@ func (d *Database) AssociatePDUWithDestination( receipt *Receipt, ) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - if err := d.FederationSenderQueuePDUs.InsertQueuePDU( + if err := d.FederationQueuePDUs.InsertQueuePDU( ctx, // context txn, // SQL transaction transactionID, // transaction ID serverName, // destination server name - receipt.nid, // NID from the federationsender_queue_json table + receipt.nid, // NID from the federationapi_queue_json table ); err != nil { return fmt.Errorf("InsertQueuePDU: %w", err) } @@ -64,21 +64,21 @@ func (d *Database) GetPendingPDUs( // the database. events = make(map[*Receipt]*gomatrixserverlib.HeaderedEvent) err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - nids, err := d.FederationSenderQueuePDUs.SelectQueuePDUs(ctx, txn, serverName, limit) + nids, err := d.FederationQueuePDUs.SelectQueuePDUs(ctx, txn, serverName, limit) if err != nil { return fmt.Errorf("SelectQueuePDUs: %w", err) } retrieve := make([]int64, 0, len(nids)) for _, nid := range nids { - if event, ok := d.Cache.GetFederationSenderQueuedPDU(nid); ok { + if event, ok := d.Cache.GetFederationQueuedPDU(nid); ok { events[&Receipt{nid}] = event } else { retrieve = append(retrieve, nid) } } - blobs, err := d.FederationSenderQueueJSON.SelectQueueJSON(ctx, txn, retrieve) + blobs, err := d.FederationQueueJSON.SelectQueueJSON(ctx, txn, retrieve) if err != nil { return fmt.Errorf("SelectQueueJSON: %w", err) } @@ -89,7 +89,7 @@ func (d *Database) GetPendingPDUs( return fmt.Errorf("json.Unmarshal: %w", err) } events[&Receipt{nid}] = &event - d.Cache.StoreFederationSenderQueuedPDU(nid, &event) + d.Cache.StoreFederationQueuedPDU(nid, &event) } return nil @@ -115,24 +115,24 @@ func (d *Database) CleanPDUs( } return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - if err := d.FederationSenderQueuePDUs.DeleteQueuePDUs(ctx, txn, serverName, nids); err != nil { + if err := d.FederationQueuePDUs.DeleteQueuePDUs(ctx, txn, serverName, nids); err != nil { return err } var deleteNIDs []int64 for _, nid := range nids { - count, err := d.FederationSenderQueuePDUs.SelectQueuePDUReferenceJSONCount(ctx, txn, nid) + count, err := d.FederationQueuePDUs.SelectQueuePDUReferenceJSONCount(ctx, txn, nid) if err != nil { return fmt.Errorf("SelectQueuePDUReferenceJSONCount: %w", err) } if count == 0 { deleteNIDs = append(deleteNIDs, nid) - d.Cache.EvictFederationSenderQueuedPDU(nid) + d.Cache.EvictFederationQueuedPDU(nid) } } if len(deleteNIDs) > 0 { - if err := d.FederationSenderQueueJSON.DeleteQueueJSON(ctx, txn, deleteNIDs); err != nil { + if err := d.FederationQueueJSON.DeleteQueueJSON(ctx, txn, deleteNIDs); err != nil { return fmt.Errorf("DeleteQueueJSON: %w", err) } } @@ -147,7 +147,7 @@ func (d *Database) GetPendingPDUCount( ctx context.Context, serverName gomatrixserverlib.ServerName, ) (int64, error) { - return d.FederationSenderQueuePDUs.SelectQueuePDUCount(ctx, nil, serverName) + return d.FederationQueuePDUs.SelectQueuePDUCount(ctx, nil, serverName) } // GetPendingServerNames returns the server names that have PDUs @@ -155,5 +155,5 @@ func (d *Database) GetPendingPDUCount( func (d *Database) GetPendingPDUServerNames( ctx context.Context, ) ([]gomatrixserverlib.ServerName, error) { - return d.FederationSenderQueuePDUs.SelectQueuePDUServerNames(ctx, nil) + return d.FederationQueuePDUs.SelectQueuePDUServerNames(ctx, nil) } diff --git a/federationsender/storage/sqlite3/blacklist_table.go b/federationapi/storage/sqlite3/blacklist_table.go similarity index 100% rename from federationsender/storage/sqlite3/blacklist_table.go rename to federationapi/storage/sqlite3/blacklist_table.go diff --git a/federationsender/storage/sqlite3/deltas/2021020411080000_rooms.go b/federationapi/storage/sqlite3/deltas/2021020411080000_rooms.go similarity index 100% rename from federationsender/storage/sqlite3/deltas/2021020411080000_rooms.go rename to federationapi/storage/sqlite3/deltas/2021020411080000_rooms.go diff --git a/federationsender/storage/sqlite3/inbound_peeks_table.go b/federationapi/storage/sqlite3/inbound_peeks_table.go similarity index 99% rename from federationsender/storage/sqlite3/inbound_peeks_table.go rename to federationapi/storage/sqlite3/inbound_peeks_table.go index d5eacf9e4..ad3c4a6dd 100644 --- a/federationsender/storage/sqlite3/inbound_peeks_table.go +++ b/federationapi/storage/sqlite3/inbound_peeks_table.go @@ -19,7 +19,7 @@ import ( "database/sql" "time" - "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" diff --git a/federationsender/storage/sqlite3/joined_hosts_table.go b/federationapi/storage/sqlite3/joined_hosts_table.go similarity index 99% rename from federationsender/storage/sqlite3/joined_hosts_table.go rename to federationapi/storage/sqlite3/joined_hosts_table.go index 4c0c1f510..e0e0f2873 100644 --- a/federationsender/storage/sqlite3/joined_hosts_table.go +++ b/federationapi/storage/sqlite3/joined_hosts_table.go @@ -20,7 +20,7 @@ import ( "database/sql" "strings" - "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" diff --git a/federationsender/storage/sqlite3/notary_server_keys_json_table.go b/federationapi/storage/sqlite3/notary_server_keys_json_table.go similarity index 96% rename from federationsender/storage/sqlite3/notary_server_keys_json_table.go rename to federationapi/storage/sqlite3/notary_server_keys_json_table.go index 6990036a2..4a028fa20 100644 --- a/federationsender/storage/sqlite3/notary_server_keys_json_table.go +++ b/federationapi/storage/sqlite3/notary_server_keys_json_table.go @@ -18,7 +18,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/federationsender/storage/tables" + "github.com/matrix-org/dendrite/federationapi/storage/tables" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/federationsender/storage/sqlite3/notary_server_keys_metadata_table.go b/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go similarity index 98% rename from federationsender/storage/sqlite3/notary_server_keys_metadata_table.go rename to federationapi/storage/sqlite3/notary_server_keys_metadata_table.go index a2959407f..55709a962 100644 --- a/federationsender/storage/sqlite3/notary_server_keys_metadata_table.go +++ b/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - "github.com/matrix-org/dendrite/federationsender/storage/tables" + "github.com/matrix-org/dendrite/federationapi/storage/tables" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" diff --git a/federationsender/storage/sqlite3/outbound_peeks_table.go b/federationapi/storage/sqlite3/outbound_peeks_table.go similarity index 99% rename from federationsender/storage/sqlite3/outbound_peeks_table.go rename to federationapi/storage/sqlite3/outbound_peeks_table.go index 02aefce79..e29026fab 100644 --- a/federationsender/storage/sqlite3/outbound_peeks_table.go +++ b/federationapi/storage/sqlite3/outbound_peeks_table.go @@ -19,7 +19,7 @@ import ( "database/sql" "time" - "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" diff --git a/federationsender/storage/sqlite3/queue_edus_table.go b/federationapi/storage/sqlite3/queue_edus_table.go similarity index 100% rename from federationsender/storage/sqlite3/queue_edus_table.go rename to federationapi/storage/sqlite3/queue_edus_table.go diff --git a/federationsender/storage/sqlite3/queue_json_table.go b/federationapi/storage/sqlite3/queue_json_table.go similarity index 100% rename from federationsender/storage/sqlite3/queue_json_table.go rename to federationapi/storage/sqlite3/queue_json_table.go diff --git a/federationsender/storage/sqlite3/queue_pdus_table.go b/federationapi/storage/sqlite3/queue_pdus_table.go similarity index 100% rename from federationsender/storage/sqlite3/queue_pdus_table.go rename to federationapi/storage/sqlite3/queue_pdus_table.go diff --git a/signingkeyserver/storage/sqlite3/server_key_table.go b/federationapi/storage/sqlite3/server_key_table.go similarity index 81% rename from signingkeyserver/storage/sqlite3/server_key_table.go rename to federationapi/storage/sqlite3/server_key_table.go index 2484d6368..9b89649f6 100644 --- a/signingkeyserver/storage/sqlite3/server_key_table.go +++ b/federationapi/storage/sqlite3/server_key_table.go @@ -24,7 +24,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -const serverKeysSchema = ` +const serverSigningKeysSchema = ` -- A cache of signing keys downloaded from remote servers. CREATE TABLE IF NOT EXISTS keydb_server_keys ( -- The name of the matrix server the key is for. @@ -48,43 +48,43 @@ CREATE TABLE IF NOT EXISTS keydb_server_keys ( CREATE INDEX IF NOT EXISTS keydb_server_name_and_key_id ON keydb_server_keys (server_name_and_key_id); ` -const bulkSelectServerKeysSQL = "" + +const bulkSelectServerSigningKeysSQL = "" + "SELECT server_name, server_key_id, valid_until_ts, expired_ts, " + " server_key FROM keydb_server_keys" + " WHERE server_name_and_key_id IN ($1)" -const upsertServerKeysSQL = "" + +const upsertServerSigningKeysSQL = "" + "INSERT INTO keydb_server_keys (server_name, server_key_id," + " server_name_and_key_id, valid_until_ts, expired_ts, server_key)" + " VALUES ($1, $2, $3, $4, $5, $6)" + " ON CONFLICT (server_name, server_key_id)" + " DO UPDATE SET valid_until_ts = $4, expired_ts = $5, server_key = $6" -type serverKeyStatements struct { +type serverSigningKeyStatements struct { db *sql.DB - writer sqlutil.Writer bulkSelectServerKeysStmt *sql.Stmt upsertServerKeysStmt *sql.Stmt } -func (s *serverKeyStatements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) { - s.db = db - s.writer = writer - _, err = db.Exec(serverKeysSchema) +func NewSQLiteServerSigningKeysTable(db *sql.DB) (s *serverSigningKeyStatements, err error) { + s = &serverSigningKeyStatements{ + db: db, + } + _, err = db.Exec(serverSigningKeysSchema) if err != nil { return } - if s.bulkSelectServerKeysStmt, err = db.Prepare(bulkSelectServerKeysSQL); err != nil { + if s.bulkSelectServerKeysStmt, err = db.Prepare(bulkSelectServerSigningKeysSQL); err != nil { return } - if s.upsertServerKeysStmt, err = db.Prepare(upsertServerKeysSQL); err != nil { + if s.upsertServerKeysStmt, err = db.Prepare(upsertServerSigningKeysSQL); err != nil { return } - return + return s, nil } -func (s *serverKeyStatements) bulkSelectServerKeys( - ctx context.Context, +func (s *serverSigningKeyStatements) BulkSelectServerKeys( + ctx context.Context, txn *sql.Tx, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, ) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { nameAndKeyIDs := make([]string, 0, len(requests)) @@ -98,7 +98,7 @@ func (s *serverKeyStatements) bulkSelectServerKeys( } err := sqlutil.RunLimitedVariablesQuery( - ctx, bulkSelectServerKeysSQL, s.db, iKeyIDs, sqlutil.SQLite3MaxVariables, + ctx, bulkSelectServerSigningKeysSQL, s.db, iKeyIDs, sqlutil.SQLite3MaxVariables, func(rows *sql.Rows) error { for rows.Next() { var serverName string @@ -134,24 +134,22 @@ func (s *serverKeyStatements) bulkSelectServerKeys( return results, nil } -func (s *serverKeyStatements) upsertServerKeys( - ctx context.Context, +func (s *serverSigningKeyStatements) UpsertServerKeys( + ctx context.Context, txn *sql.Tx, request gomatrixserverlib.PublicKeyLookupRequest, key gomatrixserverlib.PublicKeyLookupResult, ) error { - return s.writer.Do(s.db, nil, func(txn *sql.Tx) error { - stmt := sqlutil.TxStmt(txn, s.upsertServerKeysStmt) - _, err := stmt.ExecContext( - ctx, - string(request.ServerName), - string(request.KeyID), - nameAndKeyID(request), - key.ValidUntilTS, - key.ExpiredTS, - key.Key.Encode(), - ) - return err - }) + stmt := sqlutil.TxStmt(txn, s.upsertServerKeysStmt) + _, err := stmt.ExecContext( + ctx, + string(request.ServerName), + string(request.KeyID), + nameAndKeyID(request), + key.ValidUntilTS, + key.ExpiredTS, + key.Key.Encode(), + ) + return err } func nameAndKeyID(request gomatrixserverlib.PublicKeyLookupRequest) string { diff --git a/federationsender/storage/sqlite3/storage.go b/federationapi/storage/sqlite3/storage.go similarity index 75% rename from federationsender/storage/sqlite3/storage.go rename to federationapi/storage/sqlite3/storage.go index 18fa418f6..0fe6df5da 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationapi/storage/sqlite3/storage.go @@ -18,8 +18,8 @@ package sqlite3 import ( "database/sql" - "github.com/matrix-org/dendrite/federationsender/storage/shared" - "github.com/matrix-org/dendrite/federationsender/storage/sqlite3/deltas" + "github.com/matrix-org/dendrite/federationapi/storage/shared" + "github.com/matrix-org/dendrite/federationapi/storage/sqlite3/deltas" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" @@ -34,7 +34,7 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationSenderCache) (*Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationCache) (*Database, error) { var d Database var err error if d.db, err = sqlutil.Open(dbProperties); err != nil { @@ -77,24 +77,29 @@ func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationS if err != nil { return nil, err } + serverSigningKeys, err := NewSQLiteServerSigningKeysTable(d.db) + if err != nil { + return nil, err + } m := sqlutil.NewMigrations() deltas.LoadRemoveRoomsTable(m) if err = m.RunDeltas(d.db, dbProperties); err != nil { return nil, err } d.Database = shared.Database{ - DB: d.db, - Cache: cache, - Writer: d.writer, - FederationSenderJoinedHosts: joinedHosts, - FederationSenderQueuePDUs: queuePDUs, - FederationSenderQueueEDUs: queueEDUs, - FederationSenderQueueJSON: queueJSON, - FederationSenderBlacklist: blacklist, - FederationSenderOutboundPeeks: outboundPeeks, - FederationSenderInboundPeeks: inboundPeeks, - NotaryServerKeysJSON: notaryKeys, - NotaryServerKeysMetadata: notaryKeysMetadata, + DB: d.db, + Cache: cache, + Writer: d.writer, + FederationJoinedHosts: joinedHosts, + FederationQueuePDUs: queuePDUs, + FederationQueueEDUs: queueEDUs, + FederationQueueJSON: queueJSON, + FederationBlacklist: blacklist, + FederationOutboundPeeks: outboundPeeks, + FederationInboundPeeks: inboundPeeks, + NotaryServerKeysJSON: notaryKeys, + NotaryServerKeysMetadata: notaryKeysMetadata, + ServerSigningKeys: serverSigningKeys, } if err = d.PartitionOffsetStatements.Prepare(d.db, d.writer, "federationsender"); err != nil { return nil, err diff --git a/federationsender/storage/storage.go b/federationapi/storage/storage.go similarity index 86% rename from federationsender/storage/storage.go rename to federationapi/storage/storage.go index 46e01f256..083f0b302 100644 --- a/federationsender/storage/storage.go +++ b/federationapi/storage/storage.go @@ -20,14 +20,14 @@ package storage import ( "fmt" - "github.com/matrix-org/dendrite/federationsender/storage/postgres" - "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" + "github.com/matrix-org/dendrite/federationapi/storage/postgres" + "github.com/matrix-org/dendrite/federationapi/storage/sqlite3" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/setup/config" ) // NewDatabase opens a new database -func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationSenderCache) (Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationCache) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): return sqlite3.NewDatabase(dbProperties, cache) diff --git a/federationsender/storage/storage_wasm.go b/federationapi/storage/storage_wasm.go similarity index 90% rename from federationsender/storage/storage_wasm.go rename to federationapi/storage/storage_wasm.go index bc52bd9bb..455464e7c 100644 --- a/federationsender/storage/storage_wasm.go +++ b/federationapi/storage/storage_wasm.go @@ -17,13 +17,13 @@ package storage import ( "fmt" - "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" + "github.com/matrix-org/dendrite/federationapi/storage/sqlite3" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/setup/config" ) // NewDatabase opens a new database -func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationSenderCache) (Database, error) { +func NewDatabase(dbProperties *config.DatabaseOptions, cache caching.FederationCache) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): return sqlite3.NewDatabase(dbProperties, cache) diff --git a/federationsender/storage/tables/interface.go b/federationapi/storage/tables/interface.go similarity index 84% rename from federationsender/storage/tables/interface.go rename to federationapi/storage/tables/interface.go index 663a4cb20..19357393d 100644 --- a/federationsender/storage/tables/interface.go +++ b/federationapi/storage/tables/interface.go @@ -18,13 +18,13 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/gomatrixserverlib" ) type NotaryID int64 -type FederationSenderQueuePDUs interface { +type FederationQueuePDUs interface { InsertQueuePDU(ctx context.Context, txn *sql.Tx, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, nid int64) error DeleteQueuePDUs(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, jsonNIDs []int64) error SelectQueuePDUReferenceJSONCount(ctx context.Context, txn *sql.Tx, jsonNID int64) (int64, error) @@ -33,7 +33,7 @@ type FederationSenderQueuePDUs interface { SelectQueuePDUServerNames(ctx context.Context, txn *sql.Tx) ([]gomatrixserverlib.ServerName, error) } -type FederationSenderQueueEDUs interface { +type FederationQueueEDUs interface { InsertQueueEDU(ctx context.Context, txn *sql.Tx, eduType string, serverName gomatrixserverlib.ServerName, nid int64) error DeleteQueueEDUs(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, jsonNIDs []int64) error SelectQueueEDUs(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, limit int) ([]int64, error) @@ -42,13 +42,13 @@ type FederationSenderQueueEDUs interface { SelectQueueEDUServerNames(ctx context.Context, txn *sql.Tx) ([]gomatrixserverlib.ServerName, error) } -type FederationSenderQueueJSON interface { +type FederationQueueJSON interface { InsertQueueJSON(ctx context.Context, txn *sql.Tx, json string) (int64, error) DeleteQueueJSON(ctx context.Context, txn *sql.Tx, nids []int64) error SelectQueueJSON(ctx context.Context, txn *sql.Tx, jsonNIDs []int64) (map[int64][]byte, error) } -type FederationSenderJoinedHosts interface { +type FederationJoinedHosts interface { InsertJoinedHosts(ctx context.Context, txn *sql.Tx, roomID, eventID string, serverName gomatrixserverlib.ServerName) error DeleteJoinedHosts(ctx context.Context, txn *sql.Tx, eventIDs []string) error DeleteJoinedHostsForRoom(ctx context.Context, txn *sql.Tx, roomID string) error @@ -58,14 +58,14 @@ type FederationSenderJoinedHosts interface { SelectJoinedHostsForRooms(ctx context.Context, roomIDs []string) ([]gomatrixserverlib.ServerName, error) } -type FederationSenderBlacklist interface { +type FederationBlacklist interface { InsertBlacklist(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName) error SelectBlacklist(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName) (bool, error) DeleteBlacklist(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName) error DeleteAllBlacklist(ctx context.Context, txn *sql.Tx) error } -type FederationSenderOutboundPeeks interface { +type FederationOutboundPeeks interface { InsertOutboundPeek(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) (err error) RenewOutboundPeek(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) (err error) SelectOutboundPeek(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, roomID, peekID string) (outboundPeek *types.OutboundPeek, err error) @@ -74,7 +74,7 @@ type FederationSenderOutboundPeeks interface { DeleteOutboundPeeks(ctx context.Context, txn *sql.Tx, roomID string) (err error) } -type FederationSenderInboundPeeks interface { +type FederationInboundPeeks interface { InsertInboundPeek(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) (err error) RenewInboundPeek(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) (err error) SelectInboundPeek(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, roomID, peekID string) (inboundPeek *types.InboundPeek, err error) @@ -83,9 +83,9 @@ type FederationSenderInboundPeeks interface { DeleteInboundPeeks(ctx context.Context, txn *sql.Tx, roomID string) (err error) } -// FederationSenderNotaryServerKeysJSON contains the byte-for-byte responses from servers which contain their keys and is signed by them. -type FederationSenderNotaryServerKeysJSON interface { - // InsertJSONResponse inserts a new response JSON. Useless on its own, needs querying via FederationSenderNotaryServerKeysMetadata +// FederationNotaryServerKeysJSON contains the byte-for-byte responses from servers which contain their keys and is signed by them. +type FederationNotaryServerKeysJSON interface { + // InsertJSONResponse inserts a new response JSON. Useless on its own, needs querying via FederationNotaryServerKeysMetadata // `validUntil` should be the value of `valid_until_ts` with the 7-day check applied from: // "Servers MUST use the lesser of this field and 7 days into the future when determining if a key is valid. // This is to avoid a situation where an attacker publishes a key which is valid for a significant amount of time @@ -93,14 +93,19 @@ type FederationSenderNotaryServerKeysJSON interface { InsertJSONResponse(ctx context.Context, txn *sql.Tx, keyQueryResponseJSON gomatrixserverlib.ServerKeys, serverName gomatrixserverlib.ServerName, validUntil gomatrixserverlib.Timestamp) (NotaryID, error) } -// FederationSenderNotaryServerKeysMetadata persists the metadata for FederationSenderNotaryServerKeysJSON -type FederationSenderNotaryServerKeysMetadata interface { +// FederationNotaryServerKeysMetadata persists the metadata for FederationNotaryServerKeysJSON +type FederationNotaryServerKeysMetadata interface { // UpsertKey updates or inserts a (server_name, key_id) tuple, pointing it via NotaryID at the the response which has the longest valid_until_ts // `newNotaryID` and `newValidUntil` should be the notary ID / valid_until which has this (server_name, key_id) tuple already, e.g one you just inserted. UpsertKey(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, keyID gomatrixserverlib.KeyID, newNotaryID NotaryID, newValidUntil gomatrixserverlib.Timestamp) (NotaryID, error) // SelectKeys returns the signed JSON objects which contain the given key IDs. This will be at most the length of `keyIDs` and at least 1 (assuming // the keys exist in the first place). If `keyIDs` is empty, the signed JSON object with the longest valid_until_ts will be returned. SelectKeys(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, keyIDs []gomatrixserverlib.KeyID) ([]gomatrixserverlib.ServerKeys, error) - // DeleteOldJSONResponses removes all responses which are not referenced in FederationSenderNotaryServerKeysMetadata + // DeleteOldJSONResponses removes all responses which are not referenced in FederationNotaryServerKeysMetadata DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error } + +type FederationServerSigningKeys interface { + BulkSelectServerKeys(ctx context.Context, txn *sql.Tx, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) + UpsertServerKeys(ctx context.Context, txn *sql.Tx, request gomatrixserverlib.PublicKeyLookupRequest, key gomatrixserverlib.PublicKeyLookupResult) error +} diff --git a/federationsender/types/types.go b/federationapi/types/types.go similarity index 100% rename from federationsender/types/types.go rename to federationapi/types/types.go diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go deleted file mode 100644 index f34a9ff06..000000000 --- a/federationsender/federationsender.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package federationsender - -import ( - "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/federationsender/consumers" - "github.com/matrix-org/dendrite/federationsender/internal" - "github.com/matrix-org/dendrite/federationsender/inthttp" - "github.com/matrix-org/dendrite/federationsender/queue" - "github.com/matrix-org/dendrite/federationsender/statistics" - "github.com/matrix-org/dendrite/federationsender/storage" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup" - "github.com/matrix-org/dendrite/setup/kafka" - userAPI "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/sirupsen/logrus" -) - -// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions -// on the given input API. -func AddInternalRoutes(router *mux.Router, intAPI api.FederationSenderInternalAPI) { - inthttp.AddRoutes(intAPI, router) -} - -// NewInternalAPI returns a concerete implementation of the internal API. Callers -// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. -func NewInternalAPI( - base *setup.BaseDendrite, - federation *gomatrixserverlib.FederationClient, - rsAPI roomserverAPI.RoomserverInternalAPI, - userAPI userAPI.UserInternalAPI, - keyRing *gomatrixserverlib.KeyRing, - resetBlacklist bool, -) api.FederationSenderInternalAPI { - cfg := &base.Cfg.FederationSender - - federationSenderDB, err := storage.NewDatabase(&cfg.Database, base.Caches) - if err != nil { - logrus.WithError(err).Panic("failed to connect to federation sender db") - } - - if resetBlacklist { - _ = federationSenderDB.RemoveAllServersFromBlacklist() - } - - stats := &statistics.Statistics{ - DB: federationSenderDB, - FailuresUntilBlacklist: cfg.FederationMaxRetries, - } - - consumer, _ := kafka.SetupConsumerProducer(&cfg.Matrix.Kafka) - - queues := queue.NewOutgoingQueues( - federationSenderDB, base.ProcessContext, - cfg.Matrix.DisableFederation, - cfg.Matrix.ServerName, federation, rsAPI, stats, - &queue.SigningInfo{ - KeyID: cfg.Matrix.KeyID, - PrivateKey: cfg.Matrix.PrivateKey, - ServerName: cfg.Matrix.ServerName, - }, - ) - - rsConsumer := consumers.NewOutputRoomEventConsumer( - base.ProcessContext, cfg, consumer, queues, - federationSenderDB, rsAPI, - ) - if err = rsConsumer.Start(); err != nil { - logrus.WithError(err).Panic("failed to start room server consumer") - } - - tsConsumer := consumers.NewOutputEDUConsumer( - base.ProcessContext, cfg, consumer, queues, federationSenderDB, - ) - if err := tsConsumer.Start(); err != nil { - logrus.WithError(err).Panic("failed to start typing server consumer") - } - keyConsumer := consumers.NewKeyChangeConsumer( - base.ProcessContext, &base.Cfg.KeyServer, consumer, queues, federationSenderDB, rsAPI, - ) - if err := keyConsumer.Start(); err != nil { - logrus.WithError(err).Panic("failed to start key server consumer") - } - - return internal.NewFederationSenderInternalAPI(federationSenderDB, cfg, rsAPI, userAPI, federation, keyRing, stats, queues) -} diff --git a/go.mod b/go.mod index 9b970c334..300d22673 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 - github.com/matrix-org/pinecone v0.0.0-20211116111603-febf3501584d + github.com/matrix-org/pinecone v0.0.0-20211125101824-cc7886682cfd github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.8 github.com/morikuni/aec v1.0.0 // indirect diff --git a/go.sum b/go.sum index 2c19e76f0..d56241d35 100644 --- a/go.sum +++ b/go.sum @@ -997,8 +997,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2 h1:RF github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20211116111603-febf3501584d h1:V1b6GZVvL95qTkjYSEWH9Pja6c0WcJKBt2MlAILlw+Q= -github.com/matrix-org/pinecone v0.0.0-20211116111603-febf3501584d/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= +github.com/matrix-org/pinecone v0.0.0-20211125101824-cc7886682cfd h1:/iX6jehN2sO8n4pn63U+7iDoNx18fjC6pQ2RpwyZtKk= +github.com/matrix-org/pinecone v0.0.0-20211125101824-cc7886682cfd/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= diff --git a/internal/caching/cache_federationevents.go b/internal/caching/cache_federationevents.go index a48c11fd2..d10b333a6 100644 --- a/internal/caching/cache_federationevents.go +++ b/internal/caching/cache_federationevents.go @@ -12,19 +12,19 @@ const ( FederationEventCacheMutable = true // to allow use of Unset only ) -// FederationSenderCache contains the subset of functions needed for +// FederationCache contains the subset of functions needed for // a federation event cache. -type FederationSenderCache interface { - GetFederationSenderQueuedPDU(eventNID int64) (event *gomatrixserverlib.HeaderedEvent, ok bool) - StoreFederationSenderQueuedPDU(eventNID int64, event *gomatrixserverlib.HeaderedEvent) - EvictFederationSenderQueuedPDU(eventNID int64) +type FederationCache interface { + GetFederationQueuedPDU(eventNID int64) (event *gomatrixserverlib.HeaderedEvent, ok bool) + StoreFederationQueuedPDU(eventNID int64, event *gomatrixserverlib.HeaderedEvent) + EvictFederationQueuedPDU(eventNID int64) - GetFederationSenderQueuedEDU(eventNID int64) (event *gomatrixserverlib.EDU, ok bool) - StoreFederationSenderQueuedEDU(eventNID int64, event *gomatrixserverlib.EDU) - EvictFederationSenderQueuedEDU(eventNID int64) + GetFederationQueuedEDU(eventNID int64) (event *gomatrixserverlib.EDU, ok bool) + StoreFederationQueuedEDU(eventNID int64, event *gomatrixserverlib.EDU) + EvictFederationQueuedEDU(eventNID int64) } -func (c Caches) GetFederationSenderQueuedPDU(eventNID int64) (*gomatrixserverlib.HeaderedEvent, bool) { +func (c Caches) GetFederationQueuedPDU(eventNID int64) (*gomatrixserverlib.HeaderedEvent, bool) { key := fmt.Sprintf("%d", eventNID) val, found := c.FederationEvents.Get(key) if found && val != nil { @@ -35,17 +35,17 @@ func (c Caches) GetFederationSenderQueuedPDU(eventNID int64) (*gomatrixserverlib return nil, false } -func (c Caches) StoreFederationSenderQueuedPDU(eventNID int64, event *gomatrixserverlib.HeaderedEvent) { +func (c Caches) StoreFederationQueuedPDU(eventNID int64, event *gomatrixserverlib.HeaderedEvent) { key := fmt.Sprintf("%d", eventNID) c.FederationEvents.Set(key, event) } -func (c Caches) EvictFederationSenderQueuedPDU(eventNID int64) { +func (c Caches) EvictFederationQueuedPDU(eventNID int64) { key := fmt.Sprintf("%d", eventNID) c.FederationEvents.Unset(key) } -func (c Caches) GetFederationSenderQueuedEDU(eventNID int64) (*gomatrixserverlib.EDU, bool) { +func (c Caches) GetFederationQueuedEDU(eventNID int64) (*gomatrixserverlib.EDU, bool) { key := fmt.Sprintf("%d", eventNID) val, found := c.FederationEvents.Get(key) if found && val != nil { @@ -56,12 +56,12 @@ func (c Caches) GetFederationSenderQueuedEDU(eventNID int64) (*gomatrixserverlib return nil, false } -func (c Caches) StoreFederationSenderQueuedEDU(eventNID int64, event *gomatrixserverlib.EDU) { +func (c Caches) StoreFederationQueuedEDU(eventNID int64, event *gomatrixserverlib.EDU) { key := fmt.Sprintf("%d", eventNID) c.FederationEvents.Set(key, event) } -func (c Caches) EvictFederationSenderQueuedEDU(eventNID int64) { +func (c Caches) EvictFederationQueuedEDU(eventNID int64) { key := fmt.Sprintf("%d", eventNID) c.FederationEvents.Unset(key) } diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go index 704bdecb6..4defb2335 100644 --- a/internal/httputil/httpapi.go +++ b/internal/httputil/httpapi.go @@ -29,7 +29,7 @@ import ( "github.com/getsentry/sentry-go" "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth" - federationsenderAPI "github.com/matrix-org/dendrite/federationsender/api" + federationAPI "github.com/matrix-org/dendrite/federationapi/api" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -251,7 +251,7 @@ func MakeFedAPI( } type FederationWakeups struct { - FsAPI federationsenderAPI.FederationSenderInternalAPI + FsAPI federationAPI.FederationInternalAPI origins sync.Map } @@ -263,10 +263,10 @@ func (f *FederationWakeups) Wakeup(ctx context.Context, origin gomatrixserverlib return } } - aliveReq := federationsenderAPI.PerformServersAliveRequest{ + aliveReq := federationAPI.PerformServersAliveRequest{ Servers: []gomatrixserverlib.ServerName{origin}, } - aliveRes := federationsenderAPI.PerformServersAliveResponse{} + aliveRes := federationAPI.PerformServersAliveResponse{} if err := f.FsAPI.PerformServersAlive(ctx, &aliveReq, &aliveRes); err != nil { util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{ "origin": origin, diff --git a/clientapi/routing/rate_limiting.go b/internal/httputil/rate_limiting.go similarity index 92% rename from clientapi/routing/rate_limiting.go rename to internal/httputil/rate_limiting.go index 5291cabae..c4f47c7b5 100644 --- a/clientapi/routing/rate_limiting.go +++ b/internal/httputil/rate_limiting.go @@ -1,4 +1,4 @@ -package routing +package httputil import ( "net/http" @@ -10,7 +10,7 @@ import ( "github.com/matrix-org/util" ) -type rateLimits struct { +type RateLimits struct { limits map[string]chan struct{} limitsMutex sync.RWMutex cleanMutex sync.RWMutex @@ -19,8 +19,8 @@ type rateLimits struct { cooloffDuration time.Duration } -func newRateLimits(cfg *config.RateLimiting) *rateLimits { - l := &rateLimits{ +func NewRateLimits(cfg *config.RateLimiting) *RateLimits { + l := &RateLimits{ limits: make(map[string]chan struct{}), enabled: cfg.Enabled, requestThreshold: cfg.Threshold, @@ -32,7 +32,7 @@ func newRateLimits(cfg *config.RateLimiting) *rateLimits { return l } -func (l *rateLimits) clean() { +func (l *RateLimits) clean() { for { // On a 30 second interval, we'll take an exclusive write // lock of the entire map and see if any of the channels are @@ -52,7 +52,7 @@ func (l *rateLimits) clean() { } } -func (l *rateLimits) rateLimit(req *http.Request) *util.JSONResponse { +func (l *RateLimits) Limit(req *http.Request) *util.JSONResponse { // If rate limiting is disabled then do nothing. if !l.enabled { return nil diff --git a/internal/test/config.go b/internal/test/config.go index 7e68d6d2e..826d2ce00 100644 --- a/internal/test/config.go +++ b/internal/test/config.go @@ -50,7 +50,7 @@ const ( // Generates new matrix and TLS keys for the server. func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*config.Dendrite, int, error) { var cfg config.Dendrite - cfg.Defaults() + cfg.Defaults(true) port := startPort assignAddress := func() config.HTTPAddress { @@ -88,11 +88,10 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con // the table names are globally unique. But we might not want to // rely on that in the future. cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(database) - cfg.FederationSender.Database.ConnectionString = config.DataSource(database) + cfg.FederationAPI.Database.ConnectionString = config.DataSource(database) cfg.KeyServer.Database.ConnectionString = config.DataSource(database) cfg.MediaAPI.Database.ConnectionString = config.DataSource(database) cfg.RoomServer.Database.ConnectionString = config.DataSource(database) - cfg.SigningKeyServer.Database.ConnectionString = config.DataSource(database) cfg.SyncAPI.Database.ConnectionString = config.DataSource(database) cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(database) cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(database) @@ -100,22 +99,18 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.AppServiceAPI.InternalAPI.Listen = assignAddress() cfg.EDUServer.InternalAPI.Listen = assignAddress() cfg.FederationAPI.InternalAPI.Listen = assignAddress() - cfg.FederationSender.InternalAPI.Listen = assignAddress() cfg.KeyServer.InternalAPI.Listen = assignAddress() cfg.MediaAPI.InternalAPI.Listen = assignAddress() cfg.RoomServer.InternalAPI.Listen = assignAddress() - cfg.SigningKeyServer.InternalAPI.Listen = assignAddress() cfg.SyncAPI.InternalAPI.Listen = assignAddress() cfg.UserAPI.InternalAPI.Listen = assignAddress() cfg.AppServiceAPI.InternalAPI.Connect = cfg.AppServiceAPI.InternalAPI.Listen cfg.EDUServer.InternalAPI.Connect = cfg.EDUServer.InternalAPI.Listen cfg.FederationAPI.InternalAPI.Connect = cfg.FederationAPI.InternalAPI.Listen - cfg.FederationSender.InternalAPI.Connect = cfg.FederationSender.InternalAPI.Listen cfg.KeyServer.InternalAPI.Connect = cfg.KeyServer.InternalAPI.Listen cfg.MediaAPI.InternalAPI.Connect = cfg.MediaAPI.InternalAPI.Listen cfg.RoomServer.InternalAPI.Connect = cfg.RoomServer.InternalAPI.Listen - cfg.SigningKeyServer.InternalAPI.Connect = cfg.SigningKeyServer.InternalAPI.Listen cfg.SyncAPI.InternalAPI.Connect = cfg.SyncAPI.InternalAPI.Listen cfg.UserAPI.InternalAPI.Connect = cfg.UserAPI.InternalAPI.Listen diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go index 1f7c6e2a0..1b6e2d428 100644 --- a/keyserver/internal/device_list_update.go +++ b/keyserver/internal/device_list_update.go @@ -22,7 +22,7 @@ import ( "sync" "time" - fedsenderapi "github.com/matrix-org/dendrite/federationsender/api" + fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index a546e94b5..3e91962ed 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -22,7 +22,7 @@ import ( "sync" "time" - fedsenderapi "github.com/matrix-org/dendrite/federationsender/api" + fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/keyserver/producers" "github.com/matrix-org/dendrite/keyserver/storage" diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index 603067552..477efafd6 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -16,14 +16,14 @@ package keyserver import ( "github.com/gorilla/mux" - fedsenderapi "github.com/matrix-org/dendrite/federationsender/api" + fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/keyserver/consumers" "github.com/matrix-org/dendrite/keyserver/internal" "github.com/matrix-org/dendrite/keyserver/inthttp" "github.com/matrix-org/dendrite/keyserver/producers" "github.com/matrix-org/dendrite/keyserver/storage" - "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/kafka" "github.com/sirupsen/logrus" @@ -38,7 +38,7 @@ func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI) { // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *setup.BaseDendrite, cfg *config.KeyServer, fedClient fedsenderapi.FederationClient, + base *base.BaseDendrite, cfg *config.KeyServer, fedClient fedsenderapi.FederationClient, ) api.KeyInternalAPI { consumer, producer := kafka.SetupConsumerProducer(&cfg.Matrix.Kafka) diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index 811d8e4a4..c010981c0 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -26,7 +26,9 @@ import ( // AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component. func AddPublicRoutes( - router *mux.Router, cfg *config.MediaAPI, + router *mux.Router, + cfg *config.MediaAPI, + rateLimit *config.RateLimiting, userAPI userapi.UserInternalAPI, client *gomatrixserverlib.Client, ) { @@ -36,6 +38,6 @@ func AddPublicRoutes( } routing.Setup( - router, cfg, mediaDB, userAPI, client, + router, cfg, rateLimit, mediaDB, userAPI, client, ) } diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 917a85964..44f9a9d69 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -15,16 +15,16 @@ package routing import ( + "encoding/json" "net/http" "strings" - userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/gorilla/mux" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/dendrite/setup/config" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" @@ -32,6 +32,12 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" ) +// configResponse is the response to GET /_matrix/media/r0/config +// https://matrix.org/docs/spec/client_server/latest#get-matrix-media-r0-config +type configResponse struct { + UploadSize config.FileSizeBytes `json:"m.upload.size"` +} + // Setup registers the media API HTTP handlers // // Due to Setup being used to call many other functions, a gocyclo nolint is @@ -40,10 +46,13 @@ import ( func Setup( publicAPIMux *mux.Router, cfg *config.MediaAPI, + rateLimit *config.RateLimiting, db storage.Database, userAPI userapi.UserInternalAPI, client *gomatrixserverlib.Client, ) { + rateLimits := httputil.NewRateLimits(rateLimit) + r0mux := publicAPIMux.PathPrefix("/r0").Subrouter() v1mux := publicAPIMux.PathPrefix("/v1").Subrouter() @@ -54,31 +63,46 @@ func Setup( uploadHandler := httputil.MakeAuthAPI( "upload", userAPI, func(req *http.Request, dev *userapi.Device) util.JSONResponse { + if r := rateLimits.Limit(req); r != nil { + return *r + } return Upload(req, cfg, dev, db, activeThumbnailGeneration) }, ) + configHandler := httputil.MakeAuthAPI("config", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + if r := rateLimits.Limit(req); r != nil { + return *r + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: configResponse{UploadSize: *cfg.MaxFileSizeBytes}, + } + }) + r0mux.Handle("/upload", uploadHandler).Methods(http.MethodPost, http.MethodOptions) + r0mux.Handle("/config", configHandler).Methods(http.MethodGet, http.MethodOptions) v1mux.Handle("/upload", uploadHandler).Methods(http.MethodPost, http.MethodOptions) activeRemoteRequests := &types.ActiveRemoteRequests{ MXCToResult: map[string]*types.RemoteRequestResult{}, } - downloadHandler := makeDownloadAPI("download", cfg, db, client, activeRemoteRequests, activeThumbnailGeneration) + downloadHandler := makeDownloadAPI("download", cfg, rateLimits, db, client, activeRemoteRequests, activeThumbnailGeneration) r0mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) v1mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) // TODO: remove when synapse is fixed v1mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) // TODO: remove when synapse is fixed r0mux.Handle("/thumbnail/{serverName}/{mediaId}", - makeDownloadAPI("thumbnail", cfg, db, client, activeRemoteRequests, activeThumbnailGeneration), + makeDownloadAPI("thumbnail", cfg, rateLimits, db, client, activeRemoteRequests, activeThumbnailGeneration), ).Methods(http.MethodGet, http.MethodOptions) } func makeDownloadAPI( name string, cfg *config.MediaAPI, + rateLimits *httputil.RateLimits, db storage.Database, client *gomatrixserverlib.Client, activeRemoteRequests *types.ActiveRemoteRequests, @@ -99,6 +123,16 @@ func makeDownloadAPI( // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors w.Header().Set("Content-Type", "application/json") + // Ratelimit requests + if r := rateLimits.Limit(req); r != nil { + if err := json.NewEncoder(w).Encode(r); err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusTooManyRequests) + return + } + vars, _ := httputil.URLDecodeMapValues(mux.Vars(req)) serverName := gomatrixserverlib.ServerName(vars["serverName"]) diff --git a/roomserver/api/api.go b/roomserver/api/api.go index 72e406ee8..1e882ca10 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -4,15 +4,17 @@ import ( "context" asAPI "github.com/matrix-org/dendrite/appservice/api" - fsAPI "github.com/matrix-org/dendrite/federationsender/api" + fsAPI "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/gomatrixserverlib" ) // RoomserverInputAPI is used to write events to the room server. type RoomserverInternalAPI interface { // needed to avoid chicken and egg scenario when setting up the // interdependencies between the roomserver and other input APIs - SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) + SetFederationAPI(fsAPI fsAPI.FederationInternalAPI) SetAppserviceAPI(asAPI asAPI.AppServiceQueryAPI) + SetKeyring(keyRing *gomatrixserverlib.KeyRing) InputRoomEvents( ctx context.Context, diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go index 1a2b9a490..cb8c471a3 100644 --- a/roomserver/api/api_trace.go +++ b/roomserver/api/api_trace.go @@ -6,7 +6,8 @@ import ( "fmt" asAPI "github.com/matrix-org/dendrite/appservice/api" - fsAPI "github.com/matrix-org/dendrite/federationsender/api" + fsAPI "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -16,8 +17,12 @@ type RoomserverInternalAPITrace struct { Impl RoomserverInternalAPI } -func (t *RoomserverInternalAPITrace) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) { - t.Impl.SetFederationSenderAPI(fsAPI) +func (t *RoomserverInternalAPITrace) SetKeyring(keyRing *gomatrixserverlib.KeyRing) { + t.Impl.SetKeyring(keyRing) +} + +func (t *RoomserverInternalAPITrace) SetFederationAPI(fsAPI fsAPI.FederationInternalAPI) { + t.Impl.SetFederationAPI(fsAPI) } func (t *RoomserverInternalAPITrace) SetAppserviceAPI(asAPI asAPI.AppServiceQueryAPI) { diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index f39b26eaf..bda585cf8 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -6,7 +6,7 @@ import ( "github.com/Shopify/sarama" "github.com/getsentry/sentry-go" asAPI "github.com/matrix-org/dendrite/appservice/api" - fsAPI "github.com/matrix-org/dendrite/federationsender/api" + fsAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/roomserver/acls" "github.com/matrix-org/dendrite/roomserver/api" @@ -37,7 +37,7 @@ type RoomserverInternalAPI struct { Cache caching.RoomServerCaches ServerName gomatrixserverlib.ServerName KeyRing gomatrixserverlib.JSONVerifier - fsAPI fsAPI.FederationSenderInternalAPI + fsAPI fsAPI.FederationInternalAPI asAPI asAPI.AppServiceQueryAPI OutputRoomEventTopic string // Kafka topic for new output room events PerspectiveServerNames []gomatrixserverlib.ServerName @@ -46,7 +46,7 @@ type RoomserverInternalAPI struct { func NewRoomserverAPI( cfg *config.RoomServer, roomserverDB storage.Database, producer sarama.SyncProducer, outputRoomEventTopic string, caches caching.RoomServerCaches, - keyRing gomatrixserverlib.JSONVerifier, perspectiveServerNames []gomatrixserverlib.ServerName, + perspectiveServerNames []gomatrixserverlib.ServerName, ) *RoomserverInternalAPI { serverACLs := acls.NewServerACLs(roomserverDB) a := &RoomserverInternalAPI{ @@ -55,7 +55,6 @@ func NewRoomserverAPI( Cache: caches, ServerName: cfg.Matrix.ServerName, PerspectiveServerNames: perspectiveServerNames, - KeyRing: keyRing, Queryer: &query.Queryer{ DB: roomserverDB, Cache: caches, @@ -74,11 +73,18 @@ func NewRoomserverAPI( return a } -// SetFederationSenderInputAPI passes in a federation sender input API reference -// so that we can avoid the chicken-and-egg problem of both the roomserver input API -// and the federation sender input API being interdependent. -func (r *RoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) { +// SetKeyring sets the keyring to a given keyring. This is only useful for the P2P +// demos and must be called after SetFederationSenderInputAPI. +func (r *RoomserverInternalAPI) SetKeyring(keyRing *gomatrixserverlib.KeyRing) { + r.KeyRing = keyRing +} + +// SetFederationInputAPI passes in a federation input API reference so that we can +// avoid the chicken-and-egg problem of both the roomserver input API and the +// federation input API being interdependent. +func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.FederationInternalAPI) { r.fsAPI = fsAPI + r.SetKeyring(fsAPI.KeyRing()) r.Inviter = &perform.Inviter{ DB: r.DB, diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 2a558c483..d8ce9727f 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -121,20 +121,8 @@ func (r *Inputer) processRoomEvent( } } - // If we don't have a transaction ID then get one. - if input.TransactionID != nil { - tdID := input.TransactionID - eventID, err = r.DB.GetTransactionEventID( - ctx, tdID.TransactionID, tdID.SessionID, event.Sender(), - ) - // On error OR event with the transaction already processed/processesing - if err != nil || eventID != "" { - return - } - } - // Store the event. - _, stateAtEvent, redactionEvent, redactedEventID, err := r.DB.StoreEvent(ctx, event, input.TransactionID, authEventNIDs, isRejected) + _, stateAtEvent, redactionEvent, redactedEventID, err := r.DB.StoreEvent(ctx, event, authEventNIDs, isRejected) if err != nil { return "", fmt.Errorf("r.DB.StoreEvent: %w", err) } diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go index d9d720f26..8c2477dee 100644 --- a/roomserver/internal/perform/perform_backfill.go +++ b/roomserver/internal/perform/perform_backfill.go @@ -18,7 +18,7 @@ import ( "context" "fmt" - federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/auth" @@ -38,7 +38,7 @@ const maxBackfillServers = 5 type Backfiller struct { ServerName gomatrixserverlib.ServerName DB storage.Database - FSAPI federationSenderAPI.FederationSenderInternalAPI + FSAPI federationAPI.FederationInternalAPI KeyRing gomatrixserverlib.JSONVerifier // The servers which should be preferred above other servers when backfilling @@ -224,7 +224,7 @@ func (r *Backfiller) fetchAndStoreMissingEvents(ctx context.Context, roomVer gom // backfillRequester implements gomatrixserverlib.BackfillRequester type backfillRequester struct { db storage.Database - fsAPI federationSenderAPI.FederationSenderInternalAPI + fsAPI federationAPI.FederationInternalAPI thisServer gomatrixserverlib.ServerName preferServer map[gomatrixserverlib.ServerName]bool bwExtrems map[string][]string @@ -236,7 +236,7 @@ type backfillRequester struct { } func newBackfillRequester( - db storage.Database, fsAPI federationSenderAPI.FederationSenderInternalAPI, thisServer gomatrixserverlib.ServerName, + db storage.Database, fsAPI federationAPI.FederationInternalAPI, thisServer gomatrixserverlib.ServerName, bwExtrems map[string][]string, preferServers []gomatrixserverlib.ServerName, ) *backfillRequester { preferServer := make(map[gomatrixserverlib.ServerName]bool) @@ -562,7 +562,7 @@ func persistEvents(ctx context.Context, db storage.Database, events []*gomatrixs var stateAtEvent types.StateAtEvent var redactedEventID string var redactionEvent *gomatrixserverlib.Event - roomNID, stateAtEvent, redactionEvent, redactedEventID, err = db.StoreEvent(ctx, ev.Unwrap(), nil, authNids, false) + roomNID, stateAtEvent, redactionEvent, redactedEventID, err = db.StoreEvent(ctx, ev.Unwrap(), authNids, false) if err != nil { logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to persist event") continue diff --git a/roomserver/internal/perform/perform_inbound_peek.go b/roomserver/internal/perform/perform_inbound_peek.go index eb3c9727d..98f5f6f96 100644 --- a/roomserver/internal/perform/perform_inbound_peek.go +++ b/roomserver/internal/perform/perform_inbound_peek.go @@ -34,7 +34,7 @@ type InboundPeeker struct { } // PerformInboundPeek handles peeking into matrix rooms, including over -// federation by talking to the federationsender. called when a remote server +// federation by talking to the federationapi. called when a remote server // initiates a /peek over federation. // // It should atomically figure out the current state of the room (for the diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go index 024c09306..ca0654685 100644 --- a/roomserver/internal/perform/perform_invite.go +++ b/roomserver/internal/perform/perform_invite.go @@ -18,7 +18,7 @@ import ( "context" "fmt" - federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/internal/helpers" "github.com/matrix-org/dendrite/roomserver/internal/input" @@ -33,7 +33,7 @@ import ( type Inviter struct { DB storage.Database Cfg *config.RoomServer - FSAPI federationSenderAPI.FederationSenderInternalAPI + FSAPI federationAPI.FederationInternalAPI Inputer *input.Inputer } @@ -146,12 +146,12 @@ func (r *Inviter) PerformInvite( // that the remote user doesn't exist, in which case we can give up // processing here. if req.SendAsServer != api.DoNotSendToOtherServers && !isTargetLocal { - fsReq := &federationSenderAPI.PerformInviteRequest{ + fsReq := &federationAPI.PerformInviteRequest{ RoomVersion: req.RoomVersion, Event: event, InviteRoomState: inviteState, } - fsRes := &federationSenderAPI.PerformInviteResponse{} + fsRes := &federationAPI.PerformInviteResponse{} if err = r.FSAPI.PerformInvite(ctx, fsReq, fsRes); err != nil { res.Error = &api.PerformError{ Msg: err.Error(), diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go index 772c9d7dc..75397eb60 100644 --- a/roomserver/internal/perform/perform_join.go +++ b/roomserver/internal/perform/perform_join.go @@ -22,7 +22,7 @@ import ( "time" "github.com/getsentry/sentry-go" - fsAPI "github.com/matrix-org/dendrite/federationsender/api" + fsAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/eventutil" rsAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/internal/helpers" @@ -37,7 +37,7 @@ import ( type Joiner struct { ServerName gomatrixserverlib.ServerName Cfg *config.RoomServer - FSAPI fsAPI.FederationSenderInternalAPI + FSAPI fsAPI.FederationInternalAPI RSAPI rsAPI.RoomserverInternalAPI DB storage.Database @@ -45,7 +45,7 @@ type Joiner struct { Queryer *query.Queryer } -// PerformJoin handles joining matrix rooms, including over federation by talking to the federationsender. +// PerformJoin handles joining matrix rooms, including over federation by talking to the federationapi. func (r *Joiner) PerformJoin( ctx context.Context, req *rsAPI.PerformJoinRequest, diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go index a51de5469..4daeb10af 100644 --- a/roomserver/internal/perform/perform_leave.go +++ b/roomserver/internal/perform/perform_leave.go @@ -19,7 +19,7 @@ import ( "fmt" "strings" - fsAPI "github.com/matrix-org/dendrite/federationsender/api" + fsAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/internal/helpers" "github.com/matrix-org/dendrite/roomserver/internal/input" @@ -32,7 +32,7 @@ import ( type Leaver struct { Cfg *config.RoomServer DB storage.Database - FSAPI fsAPI.FederationSenderInternalAPI + FSAPI fsAPI.FederationInternalAPI Inputer *input.Inputer } diff --git a/roomserver/internal/perform/perform_peek.go b/roomserver/internal/perform/perform_peek.go index bd7996677..6a2c329b9 100644 --- a/roomserver/internal/perform/perform_peek.go +++ b/roomserver/internal/perform/perform_peek.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - fsAPI "github.com/matrix-org/dendrite/federationsender/api" + fsAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/internal/input" "github.com/matrix-org/dendrite/roomserver/storage" @@ -33,13 +33,13 @@ import ( type Peeker struct { ServerName gomatrixserverlib.ServerName Cfg *config.RoomServer - FSAPI fsAPI.FederationSenderInternalAPI + FSAPI fsAPI.FederationInternalAPI DB storage.Database Inputer *input.Inputer } -// PerformPeek handles peeking into matrix rooms, including over federation by talking to the federationsender. +// PerformPeek handles peeking into matrix rooms, including over federation by talking to the federationapi. func (r *Peeker) PerformPeek( ctx context.Context, req *api.PerformPeekRequest, diff --git a/roomserver/internal/perform/perform_unpeek.go b/roomserver/internal/perform/perform_unpeek.go index f71e0007c..16b4eeaed 100644 --- a/roomserver/internal/perform/perform_unpeek.go +++ b/roomserver/internal/perform/perform_unpeek.go @@ -19,7 +19,7 @@ import ( "fmt" "strings" - fsAPI "github.com/matrix-org/dendrite/federationsender/api" + fsAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/internal/input" "github.com/matrix-org/dendrite/roomserver/storage" @@ -30,13 +30,13 @@ import ( type Unpeeker struct { ServerName gomatrixserverlib.ServerName Cfg *config.RoomServer - FSAPI fsAPI.FederationSenderInternalAPI + FSAPI fsAPI.FederationInternalAPI DB storage.Database Inputer *input.Inputer } -// PerformPeek handles peeking into matrix rooms, including over federation by talking to the federationsender. +// PerformPeek handles peeking into matrix rooms, including over federation by talking to the federationapi. func (r *Unpeeker) PerformUnpeek( ctx context.Context, req *api.PerformUnpeekRequest, diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index 6774d102d..2afeb8b13 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -7,10 +7,11 @@ import ( "net/http" asAPI "github.com/matrix-org/dendrite/appservice/api" - fsInputAPI "github.com/matrix-org/dendrite/federationsender/api" + fsInputAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" "github.com/opentracing/opentracing-go" ) @@ -82,8 +83,12 @@ func NewRoomserverClient( }, nil } -// SetFederationSenderInputAPI no-ops in HTTP client mode as there is no chicken/egg scenario -func (h *httpRoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsInputAPI.FederationSenderInternalAPI) { +// SetKeyring no-ops in HTTP client mode as there is no chicken/egg scenario +func (h *httpRoomserverInternalAPI) SetKeyring(keyRing *gomatrixserverlib.KeyRing) { +} + +// SetFederationInputAPI no-ops in HTTP client mode as there is no chicken/egg scenario +func (h *httpRoomserverInternalAPI) SetFederationAPI(fsAPI fsInputAPI.FederationInternalAPI) { } // SetAppserviceAPI no-ops in HTTP client mode as there is no chicken/egg scenario diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 396a1defa..e47421008 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -22,7 +22,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/internal" "github.com/matrix-org/dendrite/roomserver/storage" - "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/kafka" "github.com/sirupsen/logrus" @@ -37,15 +37,14 @@ func AddInternalRoutes(router *mux.Router, intAPI api.RoomserverInternalAPI) { // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *setup.BaseDendrite, - keyRing gomatrixserverlib.JSONVerifier, + base *base.BaseDendrite, ) api.RoomserverInternalAPI { cfg := &base.Cfg.RoomServer _, producer := kafka.SetupConsumerProducer(&cfg.Matrix.Kafka) var perspectiveServerNames []gomatrixserverlib.ServerName - for _, kp := range base.Cfg.SigningKeyServer.KeyPerspectives { + for _, kp := range base.Cfg.FederationAPI.KeyPerspectives { perspectiveServerNames = append(perspectiveServerNames, kp.ServerName) } @@ -56,6 +55,6 @@ func NewInternalAPI( return internal.NewRoomserverAPI( cfg, roomserverDB, producer, string(cfg.Matrix.Kafka.TopicFor(config.TopicOutputRoomEvent)), - base.Caches, keyRing, perspectiveServerNames, + base.Caches, perspectiveServerNames, ) } diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index 5c9540071..40e8e92d1 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -13,11 +13,10 @@ import ( "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/internal/caching" - "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/internal" "github.com/matrix-org/dendrite/roomserver/storage" - "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -159,7 +158,7 @@ func mustLoadRawEvents(t *testing.T, ver gomatrixserverlib.RoomVersion, events [ func mustCreateRoomserverAPI(t *testing.T) (api.RoomserverInternalAPI, *dummyProducer) { t.Helper() cfg := &config.Dendrite{} - cfg.Defaults() + cfg.Defaults(true) cfg.Global.ServerName = testOrigin cfg.Global.Kafka.UseNaffka = true cfg.RoomServer.Database = config.DatabaseOptions{ @@ -172,7 +171,7 @@ func mustCreateRoomserverAPI(t *testing.T) (api.RoomserverInternalAPI, *dummyPro if err != nil { t.Fatalf("failed to make caches: %s", err) } - base := &setup.BaseDendrite{ + base := &base.BaseDendrite{ Caches: cache, Cfg: cfg, } @@ -182,7 +181,7 @@ func mustCreateRoomserverAPI(t *testing.T) (api.RoomserverInternalAPI, *dummyPro } return internal.NewRoomserverAPI( &cfg.RoomServer, roomserverDB, dp, string(cfg.Global.Kafka.TopicFor(config.TopicOutputRoomEvent)), - base.Caches, &test.NopJSONVerifier{}, nil, + base.Caches, nil, ), dp } diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 62aa73ad4..7f6b98557 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -17,7 +17,6 @@ package storage import ( "context" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -69,7 +68,7 @@ type Database interface { SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) // Stores a matrix room event in the database. Returns the room NID, the state snapshot and the redacted event ID if any, or an error. StoreEvent( - ctx context.Context, event *gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID, + ctx context.Context, event *gomatrixserverlib.Event, authEventNIDs []types.EventNID, isRejected bool, ) (types.RoomNID, types.StateAtEvent, *gomatrixserverlib.Event, string, error) // Look up the state entries for a list of string event IDs @@ -92,10 +91,6 @@ type Database interface { // Returns the latest events in the room and the last eventID sent to the log along with an updater. // If this returns an error then no further action is required. GetLatestEventsForUpdate(ctx context.Context, roomInfo types.RoomInfo) (*shared.LatestEventsUpdater, error) - // Look up event ID by transaction's info. - // This is used to determine if the room event is processed/processing already. - // Returns an empty string if no such event exists. - GetTransactionEventID(ctx context.Context, transactionID string, sessionID int64, userID string) (string, error) // Look up event references for the latest events in the room and the current state snapshot. // Returns the latest events, the current state and the maximum depth of the latest events plus 1. // Returns an error if there was a problem talking to the database. diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 863a15939..b5e05c982 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -82,9 +82,6 @@ func (d *Database) create(db *sql.DB) error { if err := createRoomsTable(db); err != nil { return err } - if err := createTransactionsTable(db); err != nil { - return err - } if err := createStateBlockTable(db); err != nil { return err } @@ -134,10 +131,6 @@ func (d *Database) prepare(db *sql.DB, cache caching.RoomServerCaches) error { if err != nil { return err } - transactions, err := prepareTransactionsTable(db) - if err != nil { - return err - } stateBlock, err := prepareStateBlockTable(db) if err != nil { return err @@ -179,7 +172,6 @@ func (d *Database) prepare(db *sql.DB, cache caching.RoomServerCaches) error { EventJSONTable: eventJSON, EventsTable: events, RoomsTable: rooms, - TransactionsTable: transactions, StateBlockTable: stateBlock, StateSnapshotTable: stateSnapshot, PrevEventsTable: prevEvents, diff --git a/roomserver/storage/postgres/transactions_table.go b/roomserver/storage/postgres/transactions_table.go deleted file mode 100644 index af023b740..000000000 --- a/roomserver/storage/postgres/transactions_table.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "context" - "database/sql" - - "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/tables" -) - -const transactionsSchema = ` --- The transactions table holds transaction IDs with sender's info and event ID it belongs to. --- This table is used by roomserver to prevent reprocessing of events. -CREATE TABLE IF NOT EXISTS roomserver_transactions ( - -- The transaction ID of the event. - transaction_id TEXT NOT NULL, - -- The session ID of the originating transaction. - session_id BIGINT NOT NULL, - -- User ID of the sender who authored the event - user_id TEXT NOT NULL, - -- Event ID corresponding to the transaction - -- Required to return event ID to client on a duplicate request. - event_id TEXT NOT NULL, - -- A transaction ID is unique for a user and device - -- This automatically creates an index. - PRIMARY KEY (transaction_id, session_id, user_id) -); -` -const insertTransactionSQL = "" + - "INSERT INTO roomserver_transactions (transaction_id, session_id, user_id, event_id)" + - " VALUES ($1, $2, $3, $4)" - -const selectTransactionEventIDSQL = "" + - "SELECT event_id FROM roomserver_transactions" + - " WHERE transaction_id = $1 AND session_id = $2 AND user_id = $3" - -type transactionStatements struct { - insertTransactionStmt *sql.Stmt - selectTransactionEventIDStmt *sql.Stmt -} - -func createTransactionsTable(db *sql.DB) error { - _, err := db.Exec(transactionsSchema) - return err -} - -func prepareTransactionsTable(db *sql.DB) (tables.Transactions, error) { - s := &transactionStatements{} - - return s, sqlutil.StatementList{ - {&s.insertTransactionStmt, insertTransactionSQL}, - {&s.selectTransactionEventIDStmt, selectTransactionEventIDSQL}, - }.Prepare(db) -} - -func (s *transactionStatements) InsertTransaction( - ctx context.Context, txn *sql.Tx, - transactionID string, - sessionID int64, - userID string, - eventID string, -) (err error) { - _, err = s.insertTransactionStmt.ExecContext( - ctx, transactionID, sessionID, userID, eventID, - ) - return -} - -func (s *transactionStatements) SelectTransactionEventID( - ctx context.Context, - transactionID string, - sessionID int64, - userID string, -) (eventID string, err error) { - err = s.selectTransactionEventIDStmt.QueryRowContext( - ctx, transactionID, sessionID, userID, - ).Scan(&eventID) - return -} diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 3685332fd..dbf706e5d 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -9,7 +9,6 @@ import ( "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -35,7 +34,6 @@ type Database struct { EventTypesTable tables.EventTypes EventStateKeysTable tables.EventStateKeys RoomsTable tables.Rooms - TransactionsTable tables.Transactions StateSnapshotTable tables.StateSnapshot StateBlockTable tables.StateBlock RoomAliasesTable tables.RoomAliases @@ -426,17 +424,6 @@ func (d *Database) Events( return results, nil } -func (d *Database) GetTransactionEventID( - ctx context.Context, transactionID string, - sessionID int64, userID string, -) (string, error) { - eventID, err := d.TransactionsTable.SelectTransactionEventID(ctx, transactionID, sessionID, userID) - if err == sql.ErrNoRows { - return "", nil - } - return eventID, err -} - func (d *Database) MembershipUpdater( ctx context.Context, roomID, targetUserID string, targetLocal bool, roomVersion gomatrixserverlib.RoomVersion, @@ -473,7 +460,7 @@ func (d *Database) GetLatestEventsForUpdate( func (d *Database) StoreEvent( ctx context.Context, event *gomatrixserverlib.Event, - txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID, isRejected bool, + authEventNIDs []types.EventNID, isRejected bool, ) (types.RoomNID, types.StateAtEvent, *gomatrixserverlib.Event, string, error) { var ( roomNID types.RoomNID @@ -487,15 +474,6 @@ func (d *Database) StoreEvent( ) err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - if txnAndSessionID != nil { - if err = d.TransactionsTable.InsertTransaction( - ctx, txn, txnAndSessionID.TransactionID, - txnAndSessionID.SessionID, event.Sender(), event.EventID(), - ); err != nil { - return fmt.Errorf("d.TransactionsTable.InsertTransaction: %w", err) - } - } - // TODO: Here we should aim to have two different code paths for new rooms // vs existing ones. diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index e081acdbd..1fcc7989d 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -90,9 +90,6 @@ func (d *Database) create(db *sql.DB) error { if err := createRoomsTable(db); err != nil { return err } - if err := createTransactionsTable(db); err != nil { - return err - } if err := createStateBlockTable(db); err != nil { return err } @@ -142,10 +139,6 @@ func (d *Database) prepare(db *sql.DB, cache caching.RoomServerCaches) error { if err != nil { return err } - transactions, err := prepareTransactionsTable(db) - if err != nil { - return err - } stateBlock, err := prepareStateBlockTable(db) if err != nil { return err @@ -187,7 +180,6 @@ func (d *Database) prepare(db *sql.DB, cache caching.RoomServerCaches) error { EventStateKeysTable: eventStateKeys, EventJSONTable: eventJSON, RoomsTable: rooms, - TransactionsTable: transactions, StateBlockTable: stateBlock, StateSnapshotTable: stateSnapshot, PrevEventsTable: prevEvents, diff --git a/roomserver/storage/sqlite3/transactions_table.go b/roomserver/storage/sqlite3/transactions_table.go deleted file mode 100644 index 1fb0a831d..000000000 --- a/roomserver/storage/sqlite3/transactions_table.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "context" - "database/sql" - - "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/tables" -) - -const transactionsSchema = ` - CREATE TABLE IF NOT EXISTS roomserver_transactions ( - transaction_id TEXT NOT NULL, - session_id INTEGER NOT NULL, - user_id TEXT NOT NULL, - event_id TEXT NOT NULL, - PRIMARY KEY (transaction_id, session_id, user_id) - ); -` -const insertTransactionSQL = ` - INSERT INTO roomserver_transactions (transaction_id, session_id, user_id, event_id) - VALUES ($1, $2, $3, $4) -` - -const selectTransactionEventIDSQL = ` - SELECT event_id FROM roomserver_transactions - WHERE transaction_id = $1 AND session_id = $2 AND user_id = $3 -` - -type transactionStatements struct { - db *sql.DB - insertTransactionStmt *sql.Stmt - selectTransactionEventIDStmt *sql.Stmt -} - -func createTransactionsTable(db *sql.DB) error { - _, err := db.Exec(transactionsSchema) - return err -} - -func prepareTransactionsTable(db *sql.DB) (tables.Transactions, error) { - s := &transactionStatements{ - db: db, - } - - return s, sqlutil.StatementList{ - {&s.insertTransactionStmt, insertTransactionSQL}, - {&s.selectTransactionEventIDStmt, selectTransactionEventIDSQL}, - }.Prepare(db) -} - -func (s *transactionStatements) InsertTransaction( - ctx context.Context, txn *sql.Tx, - transactionID string, - sessionID int64, - userID string, - eventID string, -) error { - stmt := sqlutil.TxStmt(txn, s.insertTransactionStmt) - _, err := stmt.ExecContext( - ctx, transactionID, sessionID, userID, eventID, - ) - return err -} - -func (s *transactionStatements) SelectTransactionEventID( - ctx context.Context, - transactionID string, - sessionID int64, - userID string, -) (eventID string, err error) { - err = s.selectTransactionEventIDStmt.QueryRowContext( - ctx, transactionID, sessionID, userID, - ).Scan(&eventID) - return -} diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index 8720d4007..6ad7ed2e8 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -76,11 +76,6 @@ type Rooms interface { BulkSelectRoomNIDs(ctx context.Context, roomIDs []string) ([]types.RoomNID, error) } -type Transactions interface { - InsertTransaction(ctx context.Context, txn *sql.Tx, transactionID string, sessionID int64, userID string, eventID string) error - SelectTransactionEventID(ctx context.Context, transactionID string, sessionID int64, userID string) (eventID string, err error) -} - type StateSnapshot interface { InsertState(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, stateBlockNIDs types.StateBlockNIDs) (stateNID types.StateSnapshotNID, err error) BulkSelectStateBlockNIDs(ctx context.Context, stateNIDs []types.StateSnapshotNID) ([]types.StateBlockNIDList, error) diff --git a/setup/base.go b/setup/base/base.go similarity index 91% rename from setup/base.go rename to setup/base/base.go index d4acdbfb9..9ba88bef7 100644 --- a/setup/base.go +++ b/setup/base/base.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package setup +package base import ( "context" @@ -47,15 +47,13 @@ import ( asinthttp "github.com/matrix-org/dendrite/appservice/inthttp" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" eduinthttp "github.com/matrix-org/dendrite/eduserver/inthttp" - federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" - fsinthttp "github.com/matrix-org/dendrite/federationsender/inthttp" + federationAPI "github.com/matrix-org/dendrite/federationapi/api" + federationIntHTTP "github.com/matrix-org/dendrite/federationapi/inthttp" keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" keyinthttp "github.com/matrix-org/dendrite/keyserver/inthttp" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" rsinthttp "github.com/matrix-org/dendrite/roomserver/inthttp" "github.com/matrix-org/dendrite/setup/config" - skapi "github.com/matrix-org/dendrite/signingkeyserver/api" - skinthttp "github.com/matrix-org/dendrite/signingkeyserver/inthttp" userapi "github.com/matrix-org/dendrite/userapi/api" userapiinthttp "github.com/matrix-org/dendrite/userapi/inthttp" "github.com/sirupsen/logrus" @@ -89,15 +87,33 @@ type BaseDendrite struct { // KafkaProducer sarama.SyncProducer } +const NoListener = "" + const HTTPServerTimeout = time.Minute * 5 const HTTPClientTimeout = time.Second * 30 -const NoListener = "" +type BaseDendriteOptions int + +const ( + NoCacheMetrics BaseDendriteOptions = iota + UseHTTPAPIs +) // NewBaseDendrite creates a new instance to be used by a component. // The componentName is used for logging purposes, and should be a friendly name // of the compontent running, e.g. "SyncAPI" -func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs bool) *BaseDendrite { +func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...BaseDendriteOptions) *BaseDendrite { + useHTTPAPIs := false + cacheMetrics := true + for _, opt := range options { + switch opt { + case NoCacheMetrics: + cacheMetrics = false + case UseHTTPAPIs: + useHTTPAPIs = true + } + } + configErrors := &config.ConfigErrors{} cfg.Verify(configErrors, componentName == "Monolith") // TODO: better way? if len(*configErrors) > 0 { @@ -133,7 +149,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo } } - cache, err := caching.NewInMemoryLRUCache(true) + cache, err := caching.NewInMemoryLRUCache(cacheMetrics) if err != nil { logrus.WithError(err).Warnf("Failed to create cache") } @@ -168,10 +184,10 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo }, } client := http.Client{Timeout: HTTPClientTimeout} - if cfg.FederationSender.Proxy.Enabled { + if cfg.FederationAPI.Proxy.Enabled { client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{ - Scheme: cfg.FederationSender.Proxy.Protocol, - Host: fmt.Sprintf("%s:%d", cfg.FederationSender.Proxy.Host, cfg.FederationSender.Proxy.Port), + Scheme: cfg.FederationAPI.Proxy.Protocol, + Host: fmt.Sprintf("%s:%d", cfg.FederationAPI.Proxy.Host, cfg.FederationAPI.Proxy.Port), })} } @@ -248,25 +264,12 @@ func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI { return e } -// FederationSenderHTTPClient returns FederationSenderInternalAPI for hitting -// the federation sender over HTTP -func (b *BaseDendrite) FederationSenderHTTPClient() federationSenderAPI.FederationSenderInternalAPI { - f, err := fsinthttp.NewFederationSenderClient(b.Cfg.FederationSenderURL(), b.apiHttpClient) +// FederationAPIHTTPClient returns FederationInternalAPI for hitting +// the federation API server over HTTP +func (b *BaseDendrite) FederationAPIHTTPClient() federationAPI.FederationInternalAPI { + f, err := federationIntHTTP.NewFederationAPIClient(b.Cfg.FederationAPIURL(), b.apiHttpClient, b.Caches) if err != nil { - logrus.WithError(err).Panic("FederationSenderHTTPClient failed", b.apiHttpClient) - } - return f -} - -// SigningKeyServerHTTPClient returns SigningKeyServer for hitting the signing key server over HTTP -func (b *BaseDendrite) SigningKeyServerHTTPClient() skapi.SigningKeyServerAPI { - f, err := skinthttp.NewSigningKeyServerClient( - b.Cfg.SigningKeyServerURL(), - b.apiHttpClient, - b.Caches, - ) - if err != nil { - logrus.WithError(err).Panic("SigningKeyServerHTTPClient failed", b.httpClient) + logrus.WithError(err).Panic("FederationAPIHTTPClient failed", b.apiHttpClient) } return f } @@ -300,7 +303,7 @@ func (b *BaseDendrite) CreateClient() *gomatrixserverlib.Client { ) } opts := []gomatrixserverlib.ClientOption{ - gomatrixserverlib.WithSkipVerify(b.Cfg.FederationSender.DisableTLSValidation), + gomatrixserverlib.WithSkipVerify(b.Cfg.FederationAPI.DisableTLSValidation), } if b.Cfg.Global.DNSCache.Enabled { opts = append(opts, gomatrixserverlib.WithDNSCache(b.DNSCache)) @@ -321,7 +324,7 @@ func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationCli } opts := []gomatrixserverlib.ClientOption{ gomatrixserverlib.WithTimeout(time.Minute * 5), - gomatrixserverlib.WithSkipVerify(b.Cfg.FederationSender.DisableTLSValidation), + gomatrixserverlib.WithSkipVerify(b.Cfg.FederationAPI.DisableTLSValidation), } if b.Cfg.Global.DNSCache.Enabled { opts = append(opts, gomatrixserverlib.WithDNSCache(b.DNSCache)) diff --git a/setup/federation.go b/setup/base/federation.go similarity index 98% rename from setup/federation.go rename to setup/base/federation.go index 7e9a22b33..804b74e5e 100644 --- a/setup/federation.go +++ b/setup/base/federation.go @@ -1,4 +1,4 @@ -package setup +package base import ( "context" diff --git a/setup/config/config.go b/setup/config/config.go index b91144078..404b7178b 100644 --- a/setup/config/config.go +++ b/setup/config/config.go @@ -53,18 +53,16 @@ type Dendrite struct { // been a breaking change to the config file format. Version int `yaml:"version"` - Global Global `yaml:"global"` - AppServiceAPI AppServiceAPI `yaml:"app_service_api"` - ClientAPI ClientAPI `yaml:"client_api"` - EDUServer EDUServer `yaml:"edu_server"` - FederationAPI FederationAPI `yaml:"federation_api"` - FederationSender FederationSender `yaml:"federation_sender"` - KeyServer KeyServer `yaml:"key_server"` - MediaAPI MediaAPI `yaml:"media_api"` - RoomServer RoomServer `yaml:"room_server"` - SigningKeyServer SigningKeyServer `yaml:"signing_key_server"` - SyncAPI SyncAPI `yaml:"sync_api"` - UserAPI UserAPI `yaml:"user_api"` + Global Global `yaml:"global"` + AppServiceAPI AppServiceAPI `yaml:"app_service_api"` + ClientAPI ClientAPI `yaml:"client_api"` + EDUServer EDUServer `yaml:"edu_server"` + FederationAPI FederationAPI `yaml:"federation_api"` + KeyServer KeyServer `yaml:"key_server"` + MediaAPI MediaAPI `yaml:"media_api"` + RoomServer RoomServer `yaml:"room_server"` + SyncAPI SyncAPI `yaml:"sync_api"` + UserAPI UserAPI `yaml:"user_api"` MSCs MSCs `yaml:"mscs"` @@ -212,7 +210,7 @@ func loadConfig( monolithic bool, ) (*Dendrite, error) { var c Dendrite - c.Defaults() + c.Defaults(false) var err error if err = yaml.Unmarshal(configData, &c); err != nil { @@ -293,22 +291,20 @@ func (config *Dendrite) Derive() error { } // SetDefaults sets default config values if they are not explicitly set. -func (c *Dendrite) Defaults() { +func (c *Dendrite) Defaults(generate bool) { c.Version = 1 - c.Global.Defaults() - c.ClientAPI.Defaults() - c.EDUServer.Defaults() - c.FederationAPI.Defaults() - c.FederationSender.Defaults() - c.KeyServer.Defaults() - c.MediaAPI.Defaults() - c.RoomServer.Defaults() - c.SigningKeyServer.Defaults() - c.SyncAPI.Defaults() - c.UserAPI.Defaults() - c.AppServiceAPI.Defaults() - c.MSCs.Defaults() + c.Global.Defaults(generate) + c.ClientAPI.Defaults(generate) + c.EDUServer.Defaults(generate) + c.FederationAPI.Defaults(generate) + c.KeyServer.Defaults(generate) + c.MediaAPI.Defaults(generate) + c.RoomServer.Defaults(generate) + c.SyncAPI.Defaults(generate) + c.UserAPI.Defaults(generate) + c.AppServiceAPI.Defaults(generate) + c.MSCs.Defaults(generate) c.Wiring() } @@ -319,9 +315,9 @@ func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) { } for _, c := range []verifiable{ &c.Global, &c.ClientAPI, - &c.EDUServer, &c.FederationAPI, &c.FederationSender, + &c.EDUServer, &c.FederationAPI, &c.KeyServer, &c.MediaAPI, &c.RoomServer, - &c.SigningKeyServer, &c.SyncAPI, &c.UserAPI, + &c.SyncAPI, &c.UserAPI, &c.AppServiceAPI, &c.MSCs, } { c.Verify(configErrs, isMonolith) @@ -332,11 +328,9 @@ func (c *Dendrite) Wiring() { c.ClientAPI.Matrix = &c.Global c.EDUServer.Matrix = &c.Global c.FederationAPI.Matrix = &c.Global - c.FederationSender.Matrix = &c.Global c.KeyServer.Matrix = &c.Global c.MediaAPI.Matrix = &c.Global c.RoomServer.Matrix = &c.Global - c.SigningKeyServer.Matrix = &c.Global c.SyncAPI.Matrix = &c.Global c.UserAPI.Matrix = &c.Global c.AppServiceAPI.Matrix = &c.Global @@ -493,6 +487,15 @@ func (config *Dendrite) AppServiceURL() string { return string(config.AppServiceAPI.InternalAPI.Connect) } +// FederationAPIURL returns an HTTP URL for where the federation API is listening. +func (config *Dendrite) FederationAPIURL() string { + // Hard code the federationapi to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return string(config.FederationAPI.InternalAPI.Connect) +} + // RoomServerURL returns an HTTP URL for where the roomserver is listening. func (config *Dendrite) RoomServerURL() string { // Hard code the roomserver to talk HTTP for now. @@ -520,24 +523,6 @@ func (config *Dendrite) EDUServerURL() string { return string(config.EDUServer.InternalAPI.Connect) } -// FederationSenderURL returns an HTTP URL for where the federation sender is listening. -func (config *Dendrite) FederationSenderURL() string { - // Hard code the federation sender server to talk HTTP for now. - // If we support HTTPS we need to think of a practical way to do certificate validation. - // People setting up servers shouldn't need to get a certificate valid for the public - // internet for an internal API. - return string(config.FederationSender.InternalAPI.Connect) -} - -// SigningKeyServerURL returns an HTTP URL for where the signing key server is listening. -func (config *Dendrite) SigningKeyServerURL() string { - // Hard code the signing key server to talk HTTP for now. - // If we support HTTPS we need to think of a practical way to do certificate validation. - // People setting up servers shouldn't need to get a certificate valid for the public - // internet for an internal API. - return string(config.SigningKeyServer.InternalAPI.Connect) -} - // KeyServerURL returns an HTTP URL for where the key server is listening. func (config *Dendrite) KeyServerURL() string { // Hard code the key server to talk HTTP for now. diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go index 9bd8a1b51..4f6553f10 100644 --- a/setup/config/config_appservice.go +++ b/setup/config/config_appservice.go @@ -40,11 +40,13 @@ type AppServiceAPI struct { ConfigFiles []string `yaml:"config_files"` } -func (c *AppServiceAPI) Defaults() { +func (c *AppServiceAPI) Defaults(generate bool) { c.InternalAPI.Listen = "http://localhost:7777" c.InternalAPI.Connect = "http://localhost:7777" c.Database.Defaults(5) - c.Database.ConnectionString = "file:appservice.db" + if generate { + c.Database.ConnectionString = "file:appservice.db" + } } func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go index c7cb9c33e..75f5e3df3 100644 --- a/setup/config/config_clientapi.go +++ b/setup/config/config_clientapi.go @@ -41,7 +41,7 @@ type ClientAPI struct { MSCs *MSCs `yaml:"mscs"` } -func (c *ClientAPI) Defaults() { +func (c *ClientAPI) Defaults(generate bool) { c.InternalAPI.Listen = "http://localhost:7771" c.InternalAPI.Connect = "http://localhost:7771" c.ExternalAPI.Listen = "http://[::]:8071" diff --git a/setup/config/config_eduserver.go b/setup/config/config_eduserver.go index a2ff36973..e7ed36aa0 100644 --- a/setup/config/config_eduserver.go +++ b/setup/config/config_eduserver.go @@ -6,7 +6,7 @@ type EDUServer struct { InternalAPI InternalAPIOptions `yaml:"internal_api"` } -func (c *EDUServer) Defaults() { +func (c *EDUServer) Defaults(generate bool) { c.InternalAPI.Listen = "http://localhost:7778" c.InternalAPI.Connect = "http://localhost:7778" } diff --git a/setup/config/config_federationapi.go b/setup/config/config_federationapi.go index 64803d95e..4f5f49de8 100644 --- a/setup/config/config_federationapi.go +++ b/setup/config/config_federationapi.go @@ -1,23 +1,57 @@ package config +import "github.com/matrix-org/gomatrixserverlib" + type FederationAPI struct { Matrix *Global `yaml:"-"` InternalAPI InternalAPIOptions `yaml:"internal_api"` ExternalAPI ExternalAPIOptions `yaml:"external_api"` + // The database stores information used by the federation destination queues to + // send transactions to remote servers. + Database DatabaseOptions `yaml:"database"` + // List of paths to X509 certificates used by the external federation listeners. // These are used to calculate the TLS fingerprints to publish for this server. // Other matrix servers talking to this server will expect the x509 certificate // to match one of these certificates. // The certificates should be in PEM format. FederationCertificatePaths []Path `yaml:"federation_certificates"` + + // Federation failure threshold. How many consecutive failures that we should + // tolerate when sending federation requests to a specific server. The backoff + // is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. + // The default value is 16 if not specified, which is circa 18 hours. + FederationMaxRetries uint32 `yaml:"send_max_retries"` + + // FederationDisableTLSValidation disables the validation of X.509 TLS certs + // on remote federation endpoints. This is not recommended in production! + DisableTLSValidation bool `yaml:"disable_tls_validation"` + + Proxy Proxy `yaml:"proxy_outbound"` + + // Perspective keyservers, to use as a backup when direct key fetch + // requests don't succeed + KeyPerspectives KeyPerspectives `yaml:"key_perspectives"` + + // Should we prefer direct key fetches over perspective ones? + PreferDirectFetch bool `yaml:"prefer_direct_fetch"` } -func (c *FederationAPI) Defaults() { +func (c *FederationAPI) Defaults(generate bool) { c.InternalAPI.Listen = "http://localhost:7772" c.InternalAPI.Connect = "http://localhost:7772" c.ExternalAPI.Listen = "http://[::]:8072" + c.Database.Defaults(10) + if generate { + c.Database.ConnectionString = "file:federationapi.db" + } + + c.FederationMaxRetries = 16 + c.DisableTLSValidation = false + + c.Proxy.Defaults() } func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { @@ -26,6 +60,48 @@ func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { if !isMonolith { checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen)) } + checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString)) // TODO: not applicable always, e.g. in demos //checkNotZero(configErrs, "federation_api.federation_certificates", int64(len(c.FederationCertificatePaths))) } + +// The config for setting a proxy to use for server->server requests +type Proxy struct { + // Is the proxy enabled? + Enabled bool `yaml:"enabled"` + // The protocol for the proxy (http / https / socks5) + Protocol string `yaml:"protocol"` + // The host where the proxy is listening + Host string `yaml:"host"` + // The port on which the proxy is listening + Port uint16 `yaml:"port"` +} + +func (c *Proxy) Defaults() { + c.Enabled = false + c.Protocol = "http" + c.Host = "localhost" + c.Port = 8080 +} + +func (c *Proxy) Verify(configErrs *ConfigErrors) { +} + +// KeyPerspectives are used to configure perspective key servers for +// retrieving server keys. +type KeyPerspectives []KeyPerspective + +type KeyPerspective struct { + // The server name of the perspective key server + ServerName gomatrixserverlib.ServerName `yaml:"server_name"` + // Server keys for the perspective user, used to verify the + // keys have been signed by the perspective server + Keys []KeyPerspectiveTrustKey `yaml:"keys"` +} + +type KeyPerspectiveTrustKey struct { + // The key ID, e.g. ed25519:auto + KeyID gomatrixserverlib.KeyID `yaml:"key_id"` + // The public key in base64 unpadded format + PublicKey string `yaml:"public_key"` +} diff --git a/setup/config/config_federationsender.go b/setup/config/config_federationsender.go deleted file mode 100644 index 67ee63565..000000000 --- a/setup/config/config_federationsender.go +++ /dev/null @@ -1,63 +0,0 @@ -package config - -type FederationSender struct { - Matrix *Global `yaml:"-"` - - InternalAPI InternalAPIOptions `yaml:"internal_api"` - - // The FederationSender database stores information used by the FederationSender - // It is only accessed by the FederationSender. - Database DatabaseOptions `yaml:"database"` - - // Federation failure threshold. How many consecutive failures that we should - // tolerate when sending federation requests to a specific server. The backoff - // is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. - // The default value is 16 if not specified, which is circa 18 hours. - FederationMaxRetries uint32 `yaml:"send_max_retries"` - - // FederationDisableTLSValidation disables the validation of X.509 TLS certs - // on remote federation endpoints. This is not recommended in production! - DisableTLSValidation bool `yaml:"disable_tls_validation"` - - Proxy Proxy `yaml:"proxy_outbound"` -} - -func (c *FederationSender) Defaults() { - c.InternalAPI.Listen = "http://localhost:7775" - c.InternalAPI.Connect = "http://localhost:7775" - c.Database.Defaults(10) - c.Database.ConnectionString = "file:federationsender.db" - - c.FederationMaxRetries = 16 - c.DisableTLSValidation = false - - c.Proxy.Defaults() -} - -func (c *FederationSender) Verify(configErrs *ConfigErrors, isMonolith bool) { - checkURL(configErrs, "federation_sender.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "federation_sender.internal_api.connect", string(c.InternalAPI.Connect)) - checkNotEmpty(configErrs, "federation_sender.database.connection_string", string(c.Database.ConnectionString)) -} - -// The config for setting a proxy to use for server->server requests -type Proxy struct { - // Is the proxy enabled? - Enabled bool `yaml:"enabled"` - // The protocol for the proxy (http / https / socks5) - Protocol string `yaml:"protocol"` - // The host where the proxy is listening - Host string `yaml:"host"` - // The port on which the proxy is listening - Port uint16 `yaml:"port"` -} - -func (c *Proxy) Defaults() { - c.Enabled = false - c.Protocol = "http" - c.Host = "localhost" - c.Port = 8080 -} - -func (c *Proxy) Verify(configErrs *ConfigErrors) { -} diff --git a/setup/config/config_global.go b/setup/config/config_global.go index 8dfa93fdf..30602abfb 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -62,16 +62,18 @@ type Global struct { DNSCache DNSCacheOptions `yaml:"dns_cache"` } -func (c *Global) Defaults() { - c.ServerName = "localhost" - c.PrivateKeyPath = "matrix_key.pem" - _, c.PrivateKey, _ = ed25519.GenerateKey(rand.New(rand.NewSource(0))) - c.KeyID = "ed25519:auto" +func (c *Global) Defaults(generate bool) { + if generate { + c.ServerName = "localhost" + c.PrivateKeyPath = "matrix_key.pem" + _, c.PrivateKey, _ = ed25519.GenerateKey(rand.New(rand.NewSource(0))) + c.KeyID = "ed25519:auto" + } c.KeyValidityPeriod = time.Hour * 24 * 7 c.PresenceEnabled = true - c.Kafka.Defaults() - c.Metrics.Defaults() + c.Kafka.Defaults(generate) + c.Metrics.Defaults(generate) c.DNSCache.Defaults() c.Sentry.Defaults() } @@ -114,10 +116,12 @@ type Metrics struct { } `yaml:"basic_auth"` } -func (c *Metrics) Defaults() { +func (c *Metrics) Defaults(generate bool) { c.Enabled = false - c.BasicAuth.Username = "metrics" - c.BasicAuth.Password = "metrics" + if generate { + c.BasicAuth.Username = "metrics" + c.BasicAuth.Password = "metrics" + } } func (c *Metrics) Verify(configErrs *ConfigErrors, isMonolith bool) { diff --git a/setup/config/config_kafka.go b/setup/config/config_kafka.go index 637f9df24..b4a392d4c 100644 --- a/setup/config/config_kafka.go +++ b/setup/config/config_kafka.go @@ -36,11 +36,13 @@ func (k *Kafka) TopicFor(name string) string { return fmt.Sprintf("%s%s", k.TopicPrefix, name) } -func (c *Kafka) Defaults() { +func (c *Kafka) Defaults(generate bool) { c.UseNaffka = true c.Database.Defaults(10) - c.Addresses = []string{"localhost:2181"} - c.Database.ConnectionString = DataSource("file:naffka.db") + if generate { + c.Addresses = []string{"localhost:2181"} + c.Database.ConnectionString = DataSource("file:naffka.db") + } c.TopicPrefix = "Dendrite" maxBytes := 1024 * 1024 * 8 // about 8MB diff --git a/setup/config/config_keyserver.go b/setup/config/config_keyserver.go index 62a30dbb9..6180ccbc8 100644 --- a/setup/config/config_keyserver.go +++ b/setup/config/config_keyserver.go @@ -8,11 +8,13 @@ type KeyServer struct { Database DatabaseOptions `yaml:"database"` } -func (c *KeyServer) Defaults() { +func (c *KeyServer) Defaults(generate bool) { c.InternalAPI.Listen = "http://localhost:7779" c.InternalAPI.Connect = "http://localhost:7779" c.Database.Defaults(10) - c.Database.ConnectionString = "file:keyserver.db" + if generate { + c.Database.ConnectionString = "file:keyserver.db" + } } func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) { diff --git a/setup/config/config_mediaapi.go b/setup/config/config_mediaapi.go index c55978e11..9a7d84969 100644 --- a/setup/config/config_mediaapi.go +++ b/setup/config/config_mediaapi.go @@ -38,16 +38,18 @@ type MediaAPI struct { // DefaultMaxFileSizeBytes defines the default file size allowed in transfers var DefaultMaxFileSizeBytes = FileSizeBytes(10485760) -func (c *MediaAPI) Defaults() { +func (c *MediaAPI) Defaults(generate bool) { c.InternalAPI.Listen = "http://localhost:7774" c.InternalAPI.Connect = "http://localhost:7774" c.ExternalAPI.Listen = "http://[::]:8074" c.Database.Defaults(5) - c.Database.ConnectionString = "file:mediaapi.db" + if generate { + c.Database.ConnectionString = "file:mediaapi.db" + c.BasePath = "./media_store" + } c.MaxFileSizeBytes = &DefaultMaxFileSizeBytes c.MaxThumbnailGenerators = 10 - c.BasePath = "./media_store" } func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { diff --git a/setup/config/config_mscs.go b/setup/config/config_mscs.go index a94e5dc56..66a4c80c9 100644 --- a/setup/config/config_mscs.go +++ b/setup/config/config_mscs.go @@ -13,9 +13,11 @@ type MSCs struct { Database DatabaseOptions `yaml:"database"` } -func (c *MSCs) Defaults() { +func (c *MSCs) Defaults(generate bool) { c.Database.Defaults(5) - c.Database.ConnectionString = "file:mscs.db" + if generate { + c.Database.ConnectionString = "file:mscs.db" + } } // Enabled returns true if the given msc is enabled. Should in the form 'msc12345'. diff --git a/setup/config/config_roomserver.go b/setup/config/config_roomserver.go index ffb9b5f9d..73abb4f47 100644 --- a/setup/config/config_roomserver.go +++ b/setup/config/config_roomserver.go @@ -8,11 +8,13 @@ type RoomServer struct { Database DatabaseOptions `yaml:"database"` } -func (c *RoomServer) Defaults() { +func (c *RoomServer) Defaults(generate bool) { c.InternalAPI.Listen = "http://localhost:7770" c.InternalAPI.Connect = "http://localhost:7770" c.Database.Defaults(10) - c.Database.ConnectionString = "file:roomserver.db" + if generate { + c.Database.ConnectionString = "file:roomserver.db" + } } func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) { diff --git a/setup/config/config_signingkeyserver.go b/setup/config/config_signingkeyserver.go deleted file mode 100644 index c192d1fc0..000000000 --- a/setup/config/config_signingkeyserver.go +++ /dev/null @@ -1,52 +0,0 @@ -package config - -import "github.com/matrix-org/gomatrixserverlib" - -type SigningKeyServer struct { - Matrix *Global `yaml:"-"` - - InternalAPI InternalAPIOptions `yaml:"internal_api"` - - // The SigningKeyServer database caches the public keys of remote servers. - // It may be accessed by the FederationAPI, the ClientAPI, and the MediaAPI. - Database DatabaseOptions `yaml:"database"` - - // Perspective keyservers, to use as a backup when direct key fetch - // requests don't succeed - KeyPerspectives KeyPerspectives `yaml:"key_perspectives"` - - // Should we prefer direct key fetches over perspective ones? - PreferDirectFetch bool `yaml:"prefer_direct_fetch"` -} - -func (c *SigningKeyServer) Defaults() { - c.InternalAPI.Listen = "http://localhost:7780" - c.InternalAPI.Connect = "http://localhost:7780" - c.Database.Defaults(10) - c.Database.ConnectionString = "file:signingkeyserver.db" -} - -func (c *SigningKeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) { - checkURL(configErrs, "signing_key_server.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "signing_key_server.internal_api.bind", string(c.InternalAPI.Connect)) - checkNotEmpty(configErrs, "signing_key_server.database.connection_string", string(c.Database.ConnectionString)) -} - -// KeyPerspectives are used to configure perspective key servers for -// retrieving server keys. -type KeyPerspectives []KeyPerspective - -type KeyPerspective struct { - // The server name of the perspective key server - ServerName gomatrixserverlib.ServerName `yaml:"server_name"` - // Server keys for the perspective user, used to verify the - // keys have been signed by the perspective server - Keys []KeyPerspectiveTrustKey `yaml:"keys"` -} - -type KeyPerspectiveTrustKey struct { - // The key ID, e.g. ed25519:auto - KeyID gomatrixserverlib.KeyID `yaml:"key_id"` - // The public key in base64 unpadded format - PublicKey string `yaml:"public_key"` -} diff --git a/setup/config/config_syncapi.go b/setup/config/config_syncapi.go index 4b9bfb161..dc813cb7d 100644 --- a/setup/config/config_syncapi.go +++ b/setup/config/config_syncapi.go @@ -11,12 +11,14 @@ type SyncAPI struct { RealIPHeader string `yaml:"real_ip_header"` } -func (c *SyncAPI) Defaults() { +func (c *SyncAPI) Defaults(generate bool) { c.InternalAPI.Listen = "http://localhost:7773" c.InternalAPI.Connect = "http://localhost:7773" c.ExternalAPI.Listen = "http://localhost:8073" c.Database.Defaults(10) - c.Database.ConnectionString = "file:syncapi.db" + if generate { + c.Database.ConnectionString = "file:syncapi.db" + } } func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { diff --git a/setup/config/config_test.go b/setup/config/config_test.go index 5c51a363f..ffe9edab2 100644 --- a/setup/config/config_test.go +++ b/setup/config/config_test.go @@ -112,7 +112,7 @@ federation_sender: listen: http://localhost:7775 connect: http://localhost:7775 database: - connection_string: file:federationsender.db + connection_string: file:federationapi.db max_open_conns: 100 max_idle_conns: 2 conn_max_lifetime: -1 diff --git a/setup/config/config_userapi.go b/setup/config/config_userapi.go index 70a0d0f57..b830a350f 100644 --- a/setup/config/config_userapi.go +++ b/setup/config/config_userapi.go @@ -25,14 +25,16 @@ type UserAPI struct { const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes -func (c *UserAPI) Defaults() { +func (c *UserAPI) Defaults(generate bool) { c.InternalAPI.Listen = "http://localhost:7781" c.InternalAPI.Connect = "http://localhost:7781" c.AccountDatabase.Defaults(10) c.DeviceDatabase.Defaults(10) - c.AccountDatabase.ConnectionString = "file:userapi_accounts.db" - c.DeviceDatabase.ConnectionString = "file:userapi_devices.db" - c.PresenceDatabase.ConnectionString = "file:userapi_presence.db" + if generate { + c.AccountDatabase.ConnectionString = "file:userapi_accounts.db" + c.DeviceDatabase.ConnectionString = "file:userapi_devices.db" + c.PresenceDatabase.ConnectionString = "file:userapi_presence.db" + } c.BCryptCost = bcrypt.DefaultCost c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS } diff --git a/setup/monolith.go b/setup/monolith.go index a77cdd56a..e6c955222 100644 --- a/setup/monolith.go +++ b/setup/monolith.go @@ -21,14 +21,13 @@ import ( "github.com/matrix-org/dendrite/clientapi/api" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/federationapi" - federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/transactions" keyAPI "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/mediaapi" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" - serverKeyAPI "github.com/matrix-org/dendrite/signingkeyserver/api" "github.com/matrix-org/dendrite/syncapi" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/storage/accounts" @@ -44,13 +43,12 @@ type Monolith struct { Client *gomatrixserverlib.Client FedClient *gomatrixserverlib.FederationClient - AppserviceAPI appserviceAPI.AppServiceQueryAPI - EDUInternalAPI eduServerAPI.EDUServerInputAPI - FederationSenderAPI federationSenderAPI.FederationSenderInternalAPI - RoomserverAPI roomserverAPI.RoomserverInternalAPI - ServerKeyAPI serverKeyAPI.SigningKeyServerAPI - UserAPI userapi.UserInternalAPI - KeyAPI keyAPI.KeyInternalAPI + AppserviceAPI appserviceAPI.AppServiceQueryAPI + EDUInternalAPI eduServerAPI.EDUServerInputAPI + FederationAPI federationAPI.FederationInternalAPI + RoomserverAPI roomserverAPI.RoomserverInternalAPI + UserAPI userapi.UserInternalAPI + KeyAPI keyAPI.KeyInternalAPI // Optional ExtPublicRoomsProvider api.ExtraPublicRoomsProvider @@ -62,15 +60,15 @@ func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ss csMux, synapseMux, &m.Config.ClientAPI, m.AccountDB, m.FedClient, m.RoomserverAPI, m.EDUInternalAPI, m.AppserviceAPI, transactions.New(), - m.FederationSenderAPI, m.UserAPI, m.KeyAPI, m.ExtPublicRoomsProvider, + m.FederationAPI, m.UserAPI, m.KeyAPI, m.ExtPublicRoomsProvider, &m.Config.MSCs, ) federationapi.AddPublicRoutes( ssMux, keyMux, wkMux, &m.Config.FederationAPI, m.UserAPI, m.FedClient, - m.KeyRing, m.RoomserverAPI, m.FederationSenderAPI, + m.KeyRing, m.RoomserverAPI, m.FederationAPI, m.EDUInternalAPI, m.KeyAPI, &m.Config.MSCs, nil, ) - mediaapi.AddPublicRoutes(mediaMux, &m.Config.MediaAPI, m.UserAPI, m.Client) + mediaapi.AddPublicRoutes(mediaMux, &m.Config.MediaAPI, &m.Config.ClientAPI.RateLimiting, m.UserAPI, m.Client) syncapi.AddPublicRoutes( process, csMux, m.UserAPI, m.RoomserverAPI, m.KeyAPI, m.FedClient, &m.Config.SyncAPI, diff --git a/setup/mscs/msc2836/msc2836.go b/setup/mscs/msc2836/msc2836.go index a538299dc..7e2ecfb9d 100644 --- a/setup/mscs/msc2836/msc2836.go +++ b/setup/mscs/msc2836/msc2836.go @@ -28,11 +28,11 @@ import ( "time" "github.com/matrix-org/dendrite/clientapi/jsonerror" - fs "github.com/matrix-org/dendrite/federationsender/api" + fs "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/hooks" "github.com/matrix-org/dendrite/internal/httputil" roomserver "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -93,7 +93,7 @@ func toClientResponse(res *gomatrixserverlib.MSC2836EventRelationshipsResponse) // Enable this MSC func Enable( - base *setup.BaseDendrite, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationSenderInternalAPI, + base *base.BaseDendrite, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationInternalAPI, userAPI userapi.UserInternalAPI, keyRing gomatrixserverlib.JSONVerifier, ) error { db, err := NewDatabase(&base.Cfg.MSCs.Database) @@ -148,10 +148,10 @@ type reqCtx struct { // federated request args isFederatedRequest bool serverName gomatrixserverlib.ServerName - fsAPI fs.FederationSenderInternalAPI + fsAPI fs.FederationInternalAPI } -func eventRelationshipHandler(db Database, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationSenderInternalAPI) func(*http.Request, *userapi.Device) util.JSONResponse { +func eventRelationshipHandler(db Database, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationInternalAPI) func(*http.Request, *userapi.Device) util.JSONResponse { return func(req *http.Request, device *userapi.Device) util.JSONResponse { relation, err := NewEventRelationshipRequest(req.Body) if err != nil { @@ -183,7 +183,7 @@ func eventRelationshipHandler(db Database, rsAPI roomserver.RoomserverInternalAP } func federatedEventRelationship( - ctx context.Context, fedReq *gomatrixserverlib.FederationRequest, db Database, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationSenderInternalAPI, + ctx context.Context, fedReq *gomatrixserverlib.FederationRequest, db Database, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationInternalAPI, ) util.JSONResponse { relation, err := NewEventRelationshipRequest(bytes.NewBuffer(fedReq.Content())) if err != nil { diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go index 51fde6915..18ab08be0 100644 --- a/setup/mscs/msc2836/msc2836_test.go +++ b/setup/mscs/msc2836/msc2836_test.go @@ -19,7 +19,7 @@ import ( "github.com/matrix-org/dendrite/internal/hooks" "github.com/matrix-org/dendrite/internal/httputil" roomserver "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/mscs/msc2836" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -545,11 +545,11 @@ func (r *testRoomserverAPI) QueryMembershipForUser(ctx context.Context, req *roo func injectEvents(t *testing.T, userAPI userapi.UserInternalAPI, rsAPI roomserver.RoomserverInternalAPI, events []*gomatrixserverlib.HeaderedEvent) *mux.Router { t.Helper() cfg := &config.Dendrite{} - cfg.Defaults() + cfg.Defaults(true) cfg.Global.ServerName = "localhost" cfg.MSCs.Database.ConnectionString = "file:msc2836_test.db" cfg.MSCs.MSCs = []string{"msc2836"} - base := &setup.BaseDendrite{ + base := &base.BaseDendrite{ Cfg: cfg, PublicClientAPIMux: mux.NewRouter().PathPrefix(httputil.PublicClientPathPrefix).Subrouter(), PublicFederationAPIMux: mux.NewRouter().PathPrefix(httputil.PublicFederationPathPrefix).Subrouter(), diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go index 121a73fd0..3824c99a2 100644 --- a/setup/mscs/msc2946/msc2946.go +++ b/setup/mscs/msc2946/msc2946.go @@ -27,11 +27,11 @@ import ( "github.com/gorilla/mux" chttputil "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - fs "github.com/matrix-org/dendrite/federationsender/api" + fs "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/hooks" "github.com/matrix-org/dendrite/internal/httputil" roomserver "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -52,8 +52,8 @@ func Defaults(r *gomatrixserverlib.MSC2946SpacesRequest) { // Enable this MSC func Enable( - base *setup.BaseDendrite, rsAPI roomserver.RoomserverInternalAPI, userAPI userapi.UserInternalAPI, - fsAPI fs.FederationSenderInternalAPI, keyRing gomatrixserverlib.JSONVerifier, + base *base.BaseDendrite, rsAPI roomserver.RoomserverInternalAPI, userAPI userapi.UserInternalAPI, + fsAPI fs.FederationInternalAPI, keyRing gomatrixserverlib.JSONVerifier, ) error { db, err := NewDatabase(&base.Cfg.MSCs.Database) if err != nil { @@ -96,7 +96,7 @@ func Enable( func federatedSpacesHandler( ctx context.Context, fedReq *gomatrixserverlib.FederationRequest, roomID string, db Database, - rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationSenderInternalAPI, + rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationInternalAPI, thisServer gomatrixserverlib.ServerName, ) util.JSONResponse { inMemoryBatchCache := make(map[string]set) @@ -128,7 +128,7 @@ func federatedSpacesHandler( } func spacesHandler( - db Database, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationSenderInternalAPI, + db Database, rsAPI roomserver.RoomserverInternalAPI, fsAPI fs.FederationInternalAPI, thisServer gomatrixserverlib.ServerName, ) func(*http.Request, *userapi.Device) util.JSONResponse { return func(req *http.Request, device *userapi.Device) util.JSONResponse { @@ -172,7 +172,7 @@ type walker struct { thisServer gomatrixserverlib.ServerName db Database rsAPI roomserver.RoomserverInternalAPI - fsAPI fs.FederationSenderInternalAPI + fsAPI fs.FederationInternalAPI ctx context.Context // user ID|device ID|batch_num => event/room IDs sent to client diff --git a/setup/mscs/msc2946/msc2946_test.go b/setup/mscs/msc2946/msc2946_test.go index c362d9fbd..441892f3e 100644 --- a/setup/mscs/msc2946/msc2946_test.go +++ b/setup/mscs/msc2946/msc2946_test.go @@ -30,7 +30,7 @@ import ( "github.com/matrix-org/dendrite/internal/hooks" "github.com/matrix-org/dendrite/internal/httputil" roomserver "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/mscs/msc2946" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -411,11 +411,11 @@ func (r *testRoomserverAPI) QueryCurrentState(ctx context.Context, req *roomserv func injectEvents(t *testing.T, userAPI userapi.UserInternalAPI, rsAPI roomserver.RoomserverInternalAPI, events []*gomatrixserverlib.HeaderedEvent) *mux.Router { t.Helper() cfg := &config.Dendrite{} - cfg.Defaults() + cfg.Defaults(true) cfg.Global.ServerName = "localhost" cfg.MSCs.Database.ConnectionString = "file:msc2946_test.db" cfg.MSCs.MSCs = []string{"msc2946"} - base := &setup.BaseDendrite{ + base := &base.BaseDendrite{ Cfg: cfg, PublicClientAPIMux: mux.NewRouter().PathPrefix(httputil.PublicClientPathPrefix).Subrouter(), PublicFederationAPIMux: mux.NewRouter().PathPrefix(httputil.PublicFederationPathPrefix).Subrouter(), diff --git a/setup/mscs/mscs.go b/setup/mscs/mscs.go index da02956b0..10df6a15d 100644 --- a/setup/mscs/mscs.go +++ b/setup/mscs/mscs.go @@ -20,13 +20,14 @@ import ( "fmt" "github.com/matrix-org/dendrite/setup" + "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/mscs/msc2836" "github.com/matrix-org/dendrite/setup/mscs/msc2946" "github.com/matrix-org/util" ) // Enable MSCs - returns an error on unknown MSCs -func Enable(base *setup.BaseDendrite, monolith *setup.Monolith) error { +func Enable(base *base.BaseDendrite, monolith *setup.Monolith) error { for _, msc := range base.Cfg.MSCs.MSCs { util.GetLogger(context.Background()).WithField("msc", msc).Info("Enabling MSC") if err := EnableMSC(base, monolith, msc); err != nil { @@ -36,12 +37,12 @@ func Enable(base *setup.BaseDendrite, monolith *setup.Monolith) error { return nil } -func EnableMSC(base *setup.BaseDendrite, monolith *setup.Monolith, msc string) error { +func EnableMSC(base *base.BaseDendrite, monolith *setup.Monolith, msc string) error { switch msc { case "msc2836": - return msc2836.Enable(base, monolith.RoomserverAPI, monolith.FederationSenderAPI, monolith.UserAPI, monolith.KeyRing) + return msc2836.Enable(base, monolith.RoomserverAPI, monolith.FederationAPI, monolith.UserAPI, monolith.KeyRing) case "msc2946": - return msc2946.Enable(base, monolith.RoomserverAPI, monolith.UserAPI, monolith.FederationSenderAPI, monolith.KeyRing) + return msc2946.Enable(base, monolith.RoomserverAPI, monolith.UserAPI, monolith.FederationAPI, monolith.KeyRing) case "msc2444": // enabled inside federationapi case "msc2753": // enabled inside clientapi default: diff --git a/signingkeyserver/api/api.go b/signingkeyserver/api/api.go deleted file mode 100644 index f053d72e2..000000000 --- a/signingkeyserver/api/api.go +++ /dev/null @@ -1,40 +0,0 @@ -package api - -import ( - "context" - - "github.com/matrix-org/gomatrixserverlib" -) - -type SigningKeyServerAPI interface { - gomatrixserverlib.KeyDatabase - - KeyRing() *gomatrixserverlib.KeyRing - - InputPublicKeys( - ctx context.Context, - request *InputPublicKeysRequest, - response *InputPublicKeysResponse, - ) error - - QueryPublicKeys( - ctx context.Context, - request *QueryPublicKeysRequest, - response *QueryPublicKeysResponse, - ) error -} - -type QueryPublicKeysRequest struct { - Requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp `json:"requests"` -} - -type QueryPublicKeysResponse struct { - Results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult `json:"results"` -} - -type InputPublicKeysRequest struct { - Keys map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult `json:"keys"` -} - -type InputPublicKeysResponse struct { -} diff --git a/signingkeyserver/inthttp/client.go b/signingkeyserver/inthttp/client.go deleted file mode 100644 index 71e40b8f0..000000000 --- a/signingkeyserver/inthttp/client.go +++ /dev/null @@ -1,132 +0,0 @@ -package inthttp - -import ( - "context" - "errors" - "net/http" - - "github.com/matrix-org/dendrite/internal/caching" - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/signingkeyserver/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/opentracing/opentracing-go" -) - -// HTTP paths for the internal HTTP APIs -const ( - ServerKeyInputPublicKeyPath = "/signingkeyserver/inputPublicKey" - ServerKeyQueryPublicKeyPath = "/signingkeyserver/queryPublicKey" -) - -// NewSigningKeyServerClient creates a SigningKeyServerAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewSigningKeyServerClient( - serverKeyAPIURL string, - httpClient *http.Client, - cache caching.ServerKeyCache, -) (api.SigningKeyServerAPI, error) { - if httpClient == nil { - return nil, errors.New("NewSigningKeyServerClient: httpClient is ") - } - return &httpServerKeyInternalAPI{ - serverKeyAPIURL: serverKeyAPIURL, - httpClient: httpClient, - cache: cache, - }, nil -} - -type httpServerKeyInternalAPI struct { - serverKeyAPIURL string - httpClient *http.Client - cache caching.ServerKeyCache -} - -func (s *httpServerKeyInternalAPI) KeyRing() *gomatrixserverlib.KeyRing { - // This is a bit of a cheat - we tell gomatrixserverlib that this API is - // both the key database and the key fetcher. While this does have the - // rather unfortunate effect of preventing gomatrixserverlib from handling - // key fetchers directly, we can at least reimplement this behaviour on - // the other end of the API. - return &gomatrixserverlib.KeyRing{ - KeyDatabase: s, - KeyFetchers: []gomatrixserverlib.KeyFetcher{}, - } -} - -func (s *httpServerKeyInternalAPI) FetcherName() string { - return "httpServerKeyInternalAPI" -} - -func (s *httpServerKeyInternalAPI) StoreKeys( - _ context.Context, - results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, -) error { - // Run in a background context - we don't want to stop this work just - // because the caller gives up waiting. - ctx := context.Background() - request := api.InputPublicKeysRequest{ - Keys: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult), - } - response := api.InputPublicKeysResponse{} - for req, res := range results { - request.Keys[req] = res - s.cache.StoreServerKey(req, res) - } - return s.InputPublicKeys(ctx, &request, &response) -} - -func (s *httpServerKeyInternalAPI) FetchKeys( - _ context.Context, - requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, -) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { - // Run in a background context - we don't want to stop this work just - // because the caller gives up waiting. - ctx := context.Background() - result := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) - request := api.QueryPublicKeysRequest{ - Requests: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp), - } - response := api.QueryPublicKeysResponse{ - Results: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult), - } - for req, ts := range requests { - if res, ok := s.cache.GetServerKey(req, ts); ok { - result[req] = res - continue - } - request.Requests[req] = ts - } - err := s.QueryPublicKeys(ctx, &request, &response) - if err != nil { - return nil, err - } - for req, res := range response.Results { - result[req] = res - s.cache.StoreServerKey(req, res) - } - return result, nil -} - -func (h *httpServerKeyInternalAPI) InputPublicKeys( - ctx context.Context, - request *api.InputPublicKeysRequest, - response *api.InputPublicKeysResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "InputPublicKey") - defer span.Finish() - - apiURL := h.serverKeyAPIURL + ServerKeyInputPublicKeyPath - return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -func (h *httpServerKeyInternalAPI) QueryPublicKeys( - ctx context.Context, - request *api.QueryPublicKeysRequest, - response *api.QueryPublicKeysResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryPublicKey") - defer span.Finish() - - apiURL := h.serverKeyAPIURL + ServerKeyQueryPublicKeyPath - return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} diff --git a/signingkeyserver/inthttp/server.go b/signingkeyserver/inthttp/server.go deleted file mode 100644 index d26f73805..000000000 --- a/signingkeyserver/inthttp/server.go +++ /dev/null @@ -1,43 +0,0 @@ -package inthttp - -import ( - "encoding/json" - "net/http" - - "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/internal/caching" - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/signingkeyserver/api" - "github.com/matrix-org/util" -) - -func AddRoutes(s api.SigningKeyServerAPI, internalAPIMux *mux.Router, cache caching.ServerKeyCache) { - internalAPIMux.Handle(ServerKeyQueryPublicKeyPath, - httputil.MakeInternalAPI("queryPublicKeys", func(req *http.Request) util.JSONResponse { - request := api.QueryPublicKeysRequest{} - response := api.QueryPublicKeysResponse{} - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - keys, err := s.FetchKeys(req.Context(), request.Requests) - if err != nil { - return util.ErrorResponse(err) - } - response.Results = keys - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle(ServerKeyInputPublicKeyPath, - httputil.MakeInternalAPI("inputPublicKeys", func(req *http.Request) util.JSONResponse { - request := api.InputPublicKeysRequest{} - response := api.InputPublicKeysResponse{} - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := s.StoreKeys(req.Context(), request.Keys); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/signingkeyserver/signingkeyserver.go b/signingkeyserver/signingkeyserver.go deleted file mode 100644 index 2b1d6751f..000000000 --- a/signingkeyserver/signingkeyserver.go +++ /dev/null @@ -1,107 +0,0 @@ -package signingkeyserver - -import ( - "crypto/ed25519" - "encoding/base64" - - "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/internal/caching" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/signingkeyserver/api" - "github.com/matrix-org/dendrite/signingkeyserver/internal" - "github.com/matrix-org/dendrite/signingkeyserver/inthttp" - "github.com/matrix-org/dendrite/signingkeyserver/storage" - "github.com/matrix-org/dendrite/signingkeyserver/storage/cache" - "github.com/matrix-org/gomatrixserverlib" - "github.com/sirupsen/logrus" -) - -// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions -// on the given input API. -func AddInternalRoutes(router *mux.Router, intAPI api.SigningKeyServerAPI, caches *caching.Caches) { - inthttp.AddRoutes(intAPI, router, caches) -} - -// NewInternalAPI returns a concerete implementation of the internal API. Callers -// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. -func NewInternalAPI( - cfg *config.SigningKeyServer, - fedClient gomatrixserverlib.KeyClient, - caches *caching.Caches, -) api.SigningKeyServerAPI { - innerDB, err := storage.NewDatabase( - &cfg.Database, - cfg.Matrix.ServerName, - cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), - cfg.Matrix.KeyID, - ) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to server key database") - } - - serverKeyDB, err := cache.NewKeyDatabase(innerDB, caches) - if err != nil { - logrus.WithError(err).Panicf("failed to set up caching wrapper for server key database") - } - - internalAPI := internal.ServerKeyAPI{ - ServerName: cfg.Matrix.ServerName, - ServerPublicKey: cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), - ServerKeyID: cfg.Matrix.KeyID, - ServerKeyValidity: cfg.Matrix.KeyValidityPeriod, - OldServerKeys: cfg.Matrix.OldVerifyKeys, - FedClient: fedClient, - OurKeyRing: gomatrixserverlib.KeyRing{ - KeyFetchers: []gomatrixserverlib.KeyFetcher{}, - KeyDatabase: serverKeyDB, - }, - } - - addDirectFetcher := func() { - internalAPI.OurKeyRing.KeyFetchers = append( - internalAPI.OurKeyRing.KeyFetchers, - &gomatrixserverlib.DirectKeyFetcher{ - Client: fedClient, - }, - ) - } - - if cfg.PreferDirectFetch { - addDirectFetcher() - } else { - defer addDirectFetcher() - } - - var b64e = base64.StdEncoding.WithPadding(base64.NoPadding) - for _, ps := range cfg.KeyPerspectives { - perspective := &gomatrixserverlib.PerspectiveKeyFetcher{ - PerspectiveServerName: ps.ServerName, - PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{}, - Client: fedClient, - } - - for _, key := range ps.Keys { - rawkey, err := b64e.DecodeString(key.PublicKey) - if err != nil { - logrus.WithError(err).WithFields(logrus.Fields{ - "server_name": ps.ServerName, - "public_key": key.PublicKey, - }).Warn("Couldn't parse perspective key") - continue - } - perspective.PerspectiveServerKeys[key.KeyID] = rawkey - } - - internalAPI.OurKeyRing.KeyFetchers = append( - internalAPI.OurKeyRing.KeyFetchers, - perspective, - ) - - logrus.WithFields(logrus.Fields{ - "server_name": ps.ServerName, - "num_public_keys": len(ps.Keys), - }).Info("Enabled perspective key fetcher") - } - - return &internalAPI -} diff --git a/signingkeyserver/storage/interface.go b/signingkeyserver/storage/interface.go deleted file mode 100644 index 3a67ac55a..000000000 --- a/signingkeyserver/storage/interface.go +++ /dev/null @@ -1,13 +0,0 @@ -package storage - -import ( - "context" - - "github.com/matrix-org/gomatrixserverlib" -) - -type Database interface { - FetcherName() string - FetchKeys(ctx context.Context, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) - StoreKeys(ctx context.Context, keyMap map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) error -} diff --git a/signingkeyserver/storage/keydb.go b/signingkeyserver/storage/keydb.go deleted file mode 100644 index 82b6d0ad5..000000000 --- a/signingkeyserver/storage/keydb.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build !wasm -// +build !wasm - -package storage - -import ( - "fmt" - - "golang.org/x/crypto/ed25519" - - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/signingkeyserver/storage/postgres" - "github.com/matrix-org/dendrite/signingkeyserver/storage/sqlite3" - "github.com/matrix-org/gomatrixserverlib" -) - -// NewDatabase opens a database connection. -func NewDatabase( - dbProperties *config.DatabaseOptions, - serverName gomatrixserverlib.ServerName, - serverKey ed25519.PublicKey, - serverKeyID gomatrixserverlib.KeyID, -) (Database, error) { - switch { - case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(dbProperties, serverName, serverKey, serverKeyID) - case dbProperties.ConnectionString.IsPostgres(): - return postgres.NewDatabase(dbProperties, serverName, serverKey, serverKeyID) - default: - return nil, fmt.Errorf("unexpected database type") - } -} diff --git a/signingkeyserver/storage/keydb_wasm.go b/signingkeyserver/storage/keydb_wasm.go deleted file mode 100644 index b112993a1..000000000 --- a/signingkeyserver/storage/keydb_wasm.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build wasm -// +build wasm - -package storage - -import ( - "fmt" - "net/url" - - "golang.org/x/crypto/ed25519" - - "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/signingkeyserver/storage/sqlite3" - "github.com/matrix-org/gomatrixserverlib" -) - -// NewDatabase opens a database connection. -func NewDatabase( - dataSourceName string, - dbProperties sqlutil.DbProperties, // nolint:unparam - serverName gomatrixserverlib.ServerName, - serverKey ed25519.PublicKey, - serverKeyID gomatrixserverlib.KeyID, -) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return nil, err - } - switch uri.Scheme { - case "postgres": - return nil, fmt.Errorf("Cannot use postgres implementation") - case "file": - return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) - default: - return nil, fmt.Errorf("Cannot use postgres implementation") - } -} diff --git a/signingkeyserver/storage/postgres/keydb.go b/signingkeyserver/storage/postgres/keydb.go deleted file mode 100644 index 1b3032de5..000000000 --- a/signingkeyserver/storage/postgres/keydb.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "context" - - "golang.org/x/crypto/ed25519" - - "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/gomatrixserverlib" -) - -// A Database implements gomatrixserverlib.KeyDatabase and is used to store -// the public keys for other matrix servers. -type Database struct { - statements serverKeyStatements -} - -// NewDatabase prepares a new key database. -// It creates the necessary tables if they don't already exist. -// It prepares all the SQL statements that it will use. -// Returns an error if there was a problem talking to the database. -func NewDatabase( - dbProperties *config.DatabaseOptions, - serverName gomatrixserverlib.ServerName, - serverKey ed25519.PublicKey, - serverKeyID gomatrixserverlib.KeyID, -) (*Database, error) { - db, err := sqlutil.Open(dbProperties) - if err != nil { - return nil, err - } - d := &Database{} - err = d.statements.prepare(db) - if err != nil { - return nil, err - } - return d, nil -} - -// FetcherName implements KeyFetcher -func (d Database) FetcherName() string { - return "PostgresKeyDatabase" -} - -// FetchKeys implements gomatrixserverlib.KeyDatabase -func (d *Database) FetchKeys( - ctx context.Context, - requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, -) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { - return d.statements.bulkSelectServerKeys(ctx, requests) -} - -// StoreKeys implements gomatrixserverlib.KeyDatabase -func (d *Database) StoreKeys( - ctx context.Context, - keyMap map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, -) error { - // TODO: Inserting all the keys within a single transaction may - // be more efficient since the transaction overhead can be quite - // high for a single insert statement. - var lastErr error - for request, keys := range keyMap { - if err := d.statements.upsertServerKeys(ctx, request, keys); err != nil { - // Rather than returning immediately on error we try to insert the - // remaining keys. - // Since we are inserting the keys outside of a transaction it is - // possible for some of the inserts to succeed even though some - // of the inserts have failed. - // Ensuring that we always insert all the keys we can means that - // this behaviour won't depend on the iteration order of the map. - lastErr = err - } - } - return lastErr -} diff --git a/signingkeyserver/storage/sqlite3/keydb.go b/signingkeyserver/storage/sqlite3/keydb.go deleted file mode 100644 index 1f85a09c3..000000000 --- a/signingkeyserver/storage/sqlite3/keydb.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "context" - - "golang.org/x/crypto/ed25519" - - "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/gomatrixserverlib" -) - -// A Database implements gomatrixserverlib.KeyDatabase and is used to store -// the public keys for other matrix servers. -type Database struct { - writer sqlutil.Writer - statements serverKeyStatements -} - -// NewDatabase prepares a new key database. -// It creates the necessary tables if they don't already exist. -// It prepares all the SQL statements that it will use. -// Returns an error if there was a problem talking to the database. -func NewDatabase( - dbProperties *config.DatabaseOptions, - serverName gomatrixserverlib.ServerName, - serverKey ed25519.PublicKey, - serverKeyID gomatrixserverlib.KeyID, -) (*Database, error) { - db, err := sqlutil.Open(dbProperties) - if err != nil { - return nil, err - } - d := &Database{ - writer: sqlutil.NewExclusiveWriter(), - } - err = d.statements.prepare(db, d.writer) - if err != nil { - return nil, err - } - if err != nil { - return nil, err - } - return d, nil -} - -// FetcherName implements KeyFetcher -func (d Database) FetcherName() string { - return "SqliteKeyDatabase" -} - -// FetchKeys implements gomatrixserverlib.KeyDatabase -func (d *Database) FetchKeys( - ctx context.Context, - requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, -) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { - return d.statements.bulkSelectServerKeys(ctx, requests) -} - -// StoreKeys implements gomatrixserverlib.KeyDatabase -func (d *Database) StoreKeys( - ctx context.Context, - keyMap map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, -) error { - // TODO: Inserting all the keys within a single transaction may - // be more efficient since the transaction overhead can be quite - // high for a single insert statement. - var lastErr error - for request, keys := range keyMap { - if err := d.statements.upsertServerKeys(ctx, request, keys); err != nil { - // Rather than returning immediately on error we try to insert the - // remaining keys. - // Since we are inserting the keys outside of a transaction it is - // possible for some of the inserts to succeed even though some - // of the inserts have failed. - // Ensuring that we always insert all the keys we can means that - // this behaviour won't depend on the iteration order of the map. - lastErr = err - } - } - return lastErr -} diff --git a/sytest-whitelist b/sytest-whitelist index cfbc80aa0..6c5de16ca 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -564,6 +564,7 @@ Presence changes are reported to local room members Presence changes are also reported to remote room members Presence changes to UNAVAILABLE are reported to local room members Presence changes to UNAVAILABLE are reported to remote room members +Can read configuration endpoint User can create and send/receive messages in a room with version 7 local user can join room with version 7 User can invite local user to room with version 7