From fe82e1f7255c05e0bc7a7872a53cf2a1a78ffaa0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 22 May 2020 11:43:17 +0100 Subject: [PATCH 1/9] Separate muxes for public and internal APIs (#1056) * Separate muxes for public and internal APIs * Update client-api-proxy and federation-api-proxy so they don't add /api to the path * Tidy up * Consistent HTTP setup * Set up prefixes properly --- appservice/appservice.go | 2 +- appservice/routing/routing.go | 2 +- clientapi/clientapi.go | 2 +- clientapi/routing/routing.go | 16 +++++------ cmd/client-api-proxy/main.go | 1 - cmd/dendrite-demo-libp2p/main.go | 14 ++++----- cmd/dendrite-monolith-server/main.go | 16 +++++------ cmd/dendritejs/main.go | 10 +++++-- cmd/federation-api-proxy/main.go | 1 - federationapi/federationapi.go | 2 +- federationapi/routing/routing.go | 14 ++++----- federationsender/federationsender.go | 6 +--- federationsender/internal/api.go | 11 +++---- internal/basecomponent/base.go | 17 ++++++++--- internal/httpapi.go | 13 +++++++-- keyserver/keyserver.go | 2 +- keyserver/routing/routing.go | 6 ++-- mediaapi/mediaapi.go | 2 +- mediaapi/routing/routing.go | 6 ++-- publicroomsapi/publicroomsapi.go | 2 +- publicroomsapi/routing/routing.go | 8 +++--- roomserver/api/alias.go | 10 +++---- roomserver/api/input.go | 2 +- roomserver/api/perform.go | 4 +-- roomserver/api/query.go | 24 ++++++++-------- roomserver/internal/api.go | 43 ++++++++++++++-------------- roomserver/roomserver.go | 6 +--- syncapi/routing/routing.go | 6 ++-- syncapi/syncapi.go | 2 +- 29 files changed, 131 insertions(+), 119 deletions(-) diff --git a/appservice/appservice.go b/appservice/appservice.go index 3c67ce9c1..76181d2a3 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -101,7 +101,7 @@ func SetupAppServiceAPIComponent( // Set up HTTP Endpoints routing.Setup( - base.APIMux, base.Cfg, rsAPI, + base.PublicAPIMux, base.Cfg, rsAPI, accountsDB, federation, transactionsCache, ) diff --git a/appservice/routing/routing.go b/appservice/routing/routing.go index ac1ff32ea..e5ed7ab40 100644 --- a/appservice/routing/routing.go +++ b/appservice/routing/routing.go @@ -27,7 +27,7 @@ import ( "github.com/matrix-org/util" ) -const pathPrefixApp = "/_matrix/app/v1" +const pathPrefixApp = "/app/v1" // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 66317e92e..0ce44c211 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -65,7 +65,7 @@ func SetupClientAPIComponent( } routing.Setup( - base.APIMux, base.Cfg, roomserverProducer, rsAPI, asAPI, + base.PublicAPIMux, base.Cfg, roomserverProducer, rsAPI, asAPI, accountsDB, deviceDB, federation, *keyRing, userUpdateProducer, syncProducer, eduProducer, transactionsCache, fsAPI, ) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 377881cb6..934d9f065 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -36,9 +36,9 @@ import ( "github.com/matrix-org/util" ) -const pathPrefixV1 = "/_matrix/client/api/v1" -const pathPrefixR0 = "/_matrix/client/r0" -const pathPrefixUnstable = "/_matrix/client/unstable" +const pathPrefixV1 = "/client/api/v1" +const pathPrefixR0 = "/client/r0" +const pathPrefixUnstable = "/client/unstable" // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. @@ -47,7 +47,7 @@ const pathPrefixUnstable = "/_matrix/client/unstable" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, cfg *config.Dendrite, + publicAPIMux *mux.Router, cfg *config.Dendrite, producer *producers.RoomserverProducer, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, @@ -62,7 +62,7 @@ func Setup( federationSender federationSenderAPI.FederationSenderInternalAPI, ) { - apiMux.Handle("/_matrix/client/versions", + publicAPIMux.Handle("/client/versions", internal.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse { return util.JSONResponse{ Code: http.StatusOK, @@ -78,9 +78,9 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) - r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() - v1mux := apiMux.PathPrefix(pathPrefixV1).Subrouter() - unstableMux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter() + r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() + v1mux := publicAPIMux.PathPrefix(pathPrefixV1).Subrouter() + unstableMux := publicAPIMux.PathPrefix(pathPrefixUnstable).Subrouter() authData := auth.Data{ AccountDB: accountDB, diff --git a/cmd/client-api-proxy/main.go b/cmd/client-api-proxy/main.go index 27991c109..979b0b042 100644 --- a/cmd/client-api-proxy/main.go +++ b/cmd/client-api-proxy/main.go @@ -75,7 +75,6 @@ func makeProxy(targetURL string) (*httputil.ReverseProxy, error) { // Pratically this means that any distinction between '%2F' and '/' // in the URL will be lost by the time it reaches the target. path := req.URL.Path - path = "api" + path log.WithFields(log.Fields{ "path": path, "url": targetURL, diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index a04a28e06..3f53cb429 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -47,7 +47,6 @@ import ( "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" ) @@ -178,12 +177,13 @@ func main() { publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, rsAPI, federation, nil) // Check this later syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, rsAPI, federation, &cfg) - httpHandler := internal.WrapHandlerInCORS(base.Base.APIMux) - - // Set up the API endpoints we handle. /metrics is for prometheus, and is - // not wrapped by CORS, while everything else is - http.Handle("/metrics", promhttp.Handler()) - http.Handle("/", httpHandler) + internal.SetupHTTPAPI( + http.DefaultServeMux, + base.Base.PublicAPIMux, + base.Base.InternalAPIMux, + &cfg, + base.Base.EnableHTTPAPIs, + ) // Expose the matrix APIs directly rather than putting them under a /api path. go func() { diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 6d3a479ac..0042a9b9c 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -35,7 +35,6 @@ import ( "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/syncapi" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" ) @@ -91,14 +90,13 @@ func main() { publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, nil) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) - httpHandler := internal.WrapHandlerInCORS(base.APIMux) - - // Set up the API endpoints we handle. /metrics is for prometheus, and is - // not wrapped by CORS, while everything else is - if cfg.Metrics.Enabled { - http.Handle("/metrics", internal.WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth)) - } - http.Handle("/", httpHandler) + internal.SetupHTTPAPI( + http.DefaultServeMux, + base.PublicAPIMux, + base.InternalAPIMux, + cfg, + base.EnableHTTPAPIs, + ) // Expose the matrix APIs directly rather than putting them under a /api path. go func() { diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 2609c74cd..1398bc6da 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -227,9 +227,13 @@ func main() { publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, p2pPublicRoomProvider) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) - httpHandler := internal.WrapHandlerInCORS(base.APIMux) - - http.Handle("/", httpHandler) + internal.SetupHTTPAPI( + http.DefaultServeMux, + base.PublicAPIMux, + base.InternalAPIMux, + cfg, + base.EnableHTTPAPIs, + ) // Expose the matrix APIs via libp2p-js - for federation traffic if node != nil { diff --git a/cmd/federation-api-proxy/main.go b/cmd/federation-api-proxy/main.go index fa90482d5..7324de148 100644 --- a/cmd/federation-api-proxy/main.go +++ b/cmd/federation-api-proxy/main.go @@ -73,7 +73,6 @@ func makeProxy(targetURL string) (*httputil.ReverseProxy, error) { // Pratically this means that any distinction between '%2F' and '/' // in the URL will be lost by the time it reaches the target. path := req.URL.Path - path = "api" + path log.WithFields(log.Fields{ "path": path, "url": targetURL, diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index 9e3411403..baeaa36bc 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -44,7 +44,7 @@ func SetupFederationAPIComponent( roomserverProducer := producers.NewRoomserverProducer(rsAPI) routing.Setup( - base.APIMux, base.Cfg, rsAPI, asAPI, roomserverProducer, + base.PublicAPIMux, base.Cfg, rsAPI, asAPI, roomserverProducer, eduProducer, federationSenderAPI, *keyRing, federation, accountsDB, deviceDB, ) diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index c368c16c4..86d3192a2 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -31,9 +31,9 @@ import ( ) const ( - pathPrefixV2Keys = "/_matrix/key/v2" - pathPrefixV1Federation = "/_matrix/federation/v1" - pathPrefixV2Federation = "/_matrix/federation/v2" + pathPrefixV2Keys = "/key/v2" + pathPrefixV1Federation = "/federation/v1" + pathPrefixV2Federation = "/federation/v2" ) // Setup registers HTTP handlers with the given ServeMux. @@ -42,7 +42,7 @@ const ( // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, + publicAPIMux *mux.Router, cfg *config.Dendrite, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, @@ -54,9 +54,9 @@ func Setup( accountDB accounts.Database, deviceDB devices.Database, ) { - v2keysmux := apiMux.PathPrefix(pathPrefixV2Keys).Subrouter() - v1fedmux := apiMux.PathPrefix(pathPrefixV1Federation).Subrouter() - v2fedmux := apiMux.PathPrefix(pathPrefixV2Federation).Subrouter() + v2keysmux := publicAPIMux.PathPrefix(pathPrefixV2Keys).Subrouter() + v1fedmux := publicAPIMux.PathPrefix(pathPrefixV1Federation).Subrouter() + v2fedmux := publicAPIMux.PathPrefix(pathPrefixV2Federation).Subrouter() localKeys := internal.MakeExternalAPI("localkeys", func(req *http.Request) util.JSONResponse { return LocalKeys(cfg) diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 446f920c2..cca847a51 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -15,8 +15,6 @@ package federationsender import ( - "net/http" - "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/consumers" "github.com/matrix-org/dendrite/federationsender/internal" @@ -72,9 +70,7 @@ func SetupFederationSenderComponent( statistics, ) - if base.EnableHTTPAPIs { - queryAPI.SetupHTTP(http.DefaultServeMux) - } + queryAPI.SetupHTTP(base.InternalAPIMux) return queryAPI } diff --git a/federationsender/internal/api.go b/federationsender/internal/api.go index 1f6d857f4..4805e7050 100644 --- a/federationsender/internal/api.go +++ b/federationsender/internal/api.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/dendrite/federationsender/storage" @@ -43,8 +44,8 @@ func NewFederationSenderInternalAPI( } // SetupHTTP adds the FederationSenderInternalAPI handlers to the http.ServeMux. -func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle( +func (f *FederationSenderInternalAPI) SetupHTTP(internalAPIMux *mux.Router) { + internalAPIMux.Handle( api.FederationSenderQueryJoinedHostsInRoomPath, internal.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { var request api.QueryJoinedHostsInRoomRequest @@ -58,7 +59,7 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.FederationSenderQueryJoinedHostServerNamesInRoomPath, internal.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { var request api.QueryJoinedHostServerNamesInRoomRequest @@ -72,7 +73,7 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle(api.FederationSenderPerformJoinRequestPath, + internalAPIMux.Handle(api.FederationSenderPerformJoinRequestPath, internal.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse { var request api.PerformJoinRequest var response api.PerformJoinResponse @@ -85,7 +86,7 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle(api.FederationSenderPerformLeaveRequestPath, + internalAPIMux.Handle(api.FederationSenderPerformLeaveRequestPath, internal.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse { var request api.PerformLeaveRequest var response api.PerformLeaveResponse diff --git a/internal/basecomponent/base.go b/internal/basecomponent/base.go index e9a375a70..0682ae0ea 100644 --- a/internal/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -56,8 +56,9 @@ type BaseDendrite struct { componentName string tracerCloser io.Closer - // APIMux should be used to register new public matrix api endpoints - APIMux *mux.Router + // PublicAPIMux should be used to register new public matrix api endpoints + PublicAPIMux *mux.Router + InternalAPIMux *mux.Router EnableHTTPAPIs bool httpClient *http.Client Cfg *config.Dendrite @@ -95,13 +96,15 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, enableHTTPAPIs logrus.WithError(err).Warnf("Failed to create cache") } + httpmux := mux.NewRouter() return &BaseDendrite{ componentName: componentName, EnableHTTPAPIs: enableHTTPAPIs, tracerCloser: closer, Cfg: cfg, ImmutableCache: cache, - APIMux: mux.NewRouter().UseEncodedPath(), + PublicAPIMux: httpmux.PathPrefix(internal.HTTPPublicPathPrefix).Subrouter().UseEncodedPath(), + InternalAPIMux: httpmux.PathPrefix(internal.HTTPInternalPathPrefix).Subrouter().UseEncodedPath(), httpClient: &http.Client{Timeout: HTTPClientTimeout}, KafkaConsumer: kafkaConsumer, KafkaProducer: kafkaProducer, @@ -221,7 +224,13 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { WriteTimeout: HTTPServerTimeout, } - internal.SetupHTTPAPI(http.DefaultServeMux, internal.WrapHandlerInCORS(b.APIMux), b.Cfg) + internal.SetupHTTPAPI( + http.DefaultServeMux, + b.PublicAPIMux, + b.InternalAPIMux, + b.Cfg, + b.EnableHTTPAPIs, + ) logrus.Infof("Starting %s server on %s", b.componentName, serv.Addr) err := serv.ListenAndServe() diff --git a/internal/httpapi.go b/internal/httpapi.go index b4c53f58d..526ff8a2d 100644 --- a/internal/httpapi.go +++ b/internal/httpapi.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/config" @@ -22,6 +23,11 @@ import ( "github.com/sirupsen/logrus" ) +const ( + HTTPPublicPathPrefix = "/_matrix/" + HTTPInternalPathPrefix = "/api/" +) + // BasicAuth is used for authorization on /metrics handlers type BasicAuth struct { Username string `yaml:"username"` @@ -184,11 +190,14 @@ func MakeFedAPI( // SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics // listener. -func SetupHTTPAPI(servMux *http.ServeMux, apiMux http.Handler, cfg *config.Dendrite) { +func SetupHTTPAPI(servMux *http.ServeMux, publicApiMux *mux.Router, internalApiMux *mux.Router, cfg *config.Dendrite, enableHTTPAPIs bool) { if cfg.Metrics.Enabled { servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth)) } - servMux.Handle("/api/", http.StripPrefix("/api", apiMux)) + if enableHTTPAPIs { + servMux.Handle(HTTPInternalPathPrefix, internalApiMux) + } + servMux.Handle(HTTPPublicPathPrefix, WrapHandlerInCORS(publicApiMux)) } // WrapHandlerInBasicAuth adds basic auth to a handler. Only used for /metrics diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index 0f450165f..4e785cbc7 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -28,5 +28,5 @@ func SetupKeyServerComponent( deviceDB devices.Database, accountsDB accounts.Database, ) { - routing.Setup(base.APIMux, base.Cfg, accountsDB, deviceDB) + routing.Setup(base.PublicAPIMux, base.Cfg, accountsDB, deviceDB) } diff --git a/keyserver/routing/routing.go b/keyserver/routing/routing.go index 3ca68ade4..2d1122c62 100644 --- a/keyserver/routing/routing.go +++ b/keyserver/routing/routing.go @@ -27,7 +27,7 @@ import ( "github.com/matrix-org/util" ) -const pathPrefixR0 = "/_matrix/client/r0" +const pathPrefixR0 = "/client/r0" // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. @@ -36,11 +36,11 @@ const pathPrefixR0 = "/_matrix/client/r0" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, cfg *config.Dendrite, + publicAPIMux *mux.Router, cfg *config.Dendrite, accountDB accounts.Database, deviceDB devices.Database, ) { - r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() + r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() authData := auth.Data{ AccountDB: accountDB, diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index 38572daf8..b5bec3907 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -35,6 +35,6 @@ func SetupMediaAPIComponent( } routing.Setup( - base.APIMux, base.Cfg, mediaDB, deviceDB, gomatrixserverlib.NewClient(), + base.PublicAPIMux, base.Cfg, mediaDB, deviceDB, gomatrixserverlib.NewClient(), ) } diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 4f765d173..11e4ecad5 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -33,7 +33,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" ) -const pathPrefixR0 = "/_matrix/media/r0" +const pathPrefixR0 = "/media/r0" // Setup registers the media API HTTP handlers // @@ -41,13 +41,13 @@ const pathPrefixR0 = "/_matrix/media/r0" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, + publicAPIMux *mux.Router, cfg *config.Dendrite, db storage.Database, deviceDB devices.Database, client *gomatrixserverlib.Client, ) { - r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() + r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() activeThumbnailGeneration := &types.ActiveThumbnailGeneration{ PathToResult: map[string]*types.ThumbnailGenerationResult{}, diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index f767a5303..b53351ffd 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -43,5 +43,5 @@ func SetupPublicRoomsAPIComponent( logrus.WithError(err).Panic("failed to start public rooms server consumer") } - routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsAPI, fedClient, extRoomsProvider) + routing.Setup(base.PublicAPIMux, deviceDB, publicRoomsDB, rsAPI, fedClient, extRoomsProvider) } diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index 39a9dcca1..ee1434392 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -31,7 +31,7 @@ import ( "github.com/matrix-org/util" ) -const pathPrefixR0 = "/_matrix/client/r0" +const pathPrefixR0 = "/client/r0" // Setup configures the given mux with publicroomsapi server listeners // @@ -39,10 +39,10 @@ const pathPrefixR0 = "/_matrix/client/r0" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI, + publicAPIMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI, fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, ) { - r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() + r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() authData := auth.Data{ AccountDB: nil, @@ -79,7 +79,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) // Federation - TODO: should this live here or in federation API? It's sure easier if it's here so here it is. - apiMux.Handle("/_matrix/federation/v1/publicRooms", + publicAPIMux.Handle("/federation/v1/publicRooms", internal.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse { return directory.GetPostPublicRooms(req, publicRoomsDB) }), diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go index 93975d36c..54d2c633b 100644 --- a/roomserver/api/alias.go +++ b/roomserver/api/alias.go @@ -85,19 +85,19 @@ type RemoveRoomAliasRequest struct { type RemoveRoomAliasResponse struct{} // RoomserverSetRoomAliasPath is the HTTP path for the SetRoomAlias API. -const RoomserverSetRoomAliasPath = "/api/roomserver/setRoomAlias" +const RoomserverSetRoomAliasPath = "/roomserver/setRoomAlias" // RoomserverGetRoomIDForAliasPath is the HTTP path for the GetRoomIDForAlias API. -const RoomserverGetRoomIDForAliasPath = "/api/roomserver/GetRoomIDForAlias" +const RoomserverGetRoomIDForAliasPath = "/roomserver/GetRoomIDForAlias" // RoomserverGetAliasesForRoomIDPath is the HTTP path for the GetAliasesForRoomID API. -const RoomserverGetAliasesForRoomIDPath = "/api/roomserver/GetAliasesForRoomID" +const RoomserverGetAliasesForRoomIDPath = "/roomserver/GetAliasesForRoomID" // RoomserverGetCreatorIDForAliasPath is the HTTP path for the GetCreatorIDForAlias API. -const RoomserverGetCreatorIDForAliasPath = "/api/roomserver/GetCreatorIDForAlias" +const RoomserverGetCreatorIDForAliasPath = "/roomserver/GetCreatorIDForAlias" // RoomserverRemoveRoomAliasPath is the HTTP path for the RemoveRoomAlias API. -const RoomserverRemoveRoomAliasPath = "/api/roomserver/removeRoomAlias" +const RoomserverRemoveRoomAliasPath = "/roomserver/removeRoomAlias" // SetRoomAlias implements RoomserverAliasAPI func (h *httpRoomserverInternalAPI) SetRoomAlias( diff --git a/roomserver/api/input.go b/roomserver/api/input.go index f1eec75a8..d35ead764 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -103,7 +103,7 @@ type InputRoomEventsResponse struct { } // RoomserverInputRoomEventsPath is the HTTP path for the InputRoomEvents API. -const RoomserverInputRoomEventsPath = "/api/roomserver/inputRoomEvents" +const RoomserverInputRoomEventsPath = "/roomserver/inputRoomEvents" // InputRoomEvents implements RoomserverInputAPI func (h *httpRoomserverInternalAPI) InputRoomEvents( diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index 93f36f451..c98f91fb4 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -10,10 +10,10 @@ import ( const ( // RoomserverPerformJoinPath is the HTTP path for the PerformJoin API. - RoomserverPerformJoinPath = "/api/roomserver/performJoin" + RoomserverPerformJoinPath = "/roomserver/performJoin" // RoomserverPerformLeavePath is the HTTP path for the PerformLeave API. - RoomserverPerformLeavePath = "/api/roomserver/performLeave" + RoomserverPerformLeavePath = "/roomserver/performLeave" ) type PerformJoinRequest struct { diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 98f46c771..916ecb362 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -273,40 +273,40 @@ type QueryRoomVersionForRoomResponse struct { } // RoomserverQueryLatestEventsAndStatePath is the HTTP path for the QueryLatestEventsAndState API. -const RoomserverQueryLatestEventsAndStatePath = "/api/roomserver/queryLatestEventsAndState" +const RoomserverQueryLatestEventsAndStatePath = "/roomserver/queryLatestEventsAndState" // RoomserverQueryStateAfterEventsPath is the HTTP path for the QueryStateAfterEvents API. -const RoomserverQueryStateAfterEventsPath = "/api/roomserver/queryStateAfterEvents" +const RoomserverQueryStateAfterEventsPath = "/roomserver/queryStateAfterEvents" // RoomserverQueryEventsByIDPath is the HTTP path for the QueryEventsByID API. -const RoomserverQueryEventsByIDPath = "/api/roomserver/queryEventsByID" +const RoomserverQueryEventsByIDPath = "/roomserver/queryEventsByID" // RoomserverQueryMembershipForUserPath is the HTTP path for the QueryMembershipForUser API. -const RoomserverQueryMembershipForUserPath = "/api/roomserver/queryMembershipForUser" +const RoomserverQueryMembershipForUserPath = "/roomserver/queryMembershipForUser" // RoomserverQueryMembershipsForRoomPath is the HTTP path for the QueryMembershipsForRoom API -const RoomserverQueryMembershipsForRoomPath = "/api/roomserver/queryMembershipsForRoom" +const RoomserverQueryMembershipsForRoomPath = "/roomserver/queryMembershipsForRoom" // RoomserverQueryInvitesForUserPath is the HTTP path for the QueryInvitesForUser API -const RoomserverQueryInvitesForUserPath = "/api/roomserver/queryInvitesForUser" +const RoomserverQueryInvitesForUserPath = "/roomserver/queryInvitesForUser" // RoomserverQueryServerAllowedToSeeEventPath is the HTTP path for the QueryServerAllowedToSeeEvent API -const RoomserverQueryServerAllowedToSeeEventPath = "/api/roomserver/queryServerAllowedToSeeEvent" +const RoomserverQueryServerAllowedToSeeEventPath = "/roomserver/queryServerAllowedToSeeEvent" // RoomserverQueryMissingEventsPath is the HTTP path for the QueryMissingEvents API -const RoomserverQueryMissingEventsPath = "/api/roomserver/queryMissingEvents" +const RoomserverQueryMissingEventsPath = "/roomserver/queryMissingEvents" // RoomserverQueryStateAndAuthChainPath is the HTTP path for the QueryStateAndAuthChain API -const RoomserverQueryStateAndAuthChainPath = "/api/roomserver/queryStateAndAuthChain" +const RoomserverQueryStateAndAuthChainPath = "/roomserver/queryStateAndAuthChain" // RoomserverQueryBackfillPath is the HTTP path for the QueryBackfillPath API -const RoomserverQueryBackfillPath = "/api/roomserver/queryBackfill" +const RoomserverQueryBackfillPath = "/roomserver/queryBackfill" // RoomserverQueryRoomVersionCapabilitiesPath is the HTTP path for the QueryRoomVersionCapabilities API -const RoomserverQueryRoomVersionCapabilitiesPath = "/api/roomserver/queryRoomVersionCapabilities" +const RoomserverQueryRoomVersionCapabilitiesPath = "/roomserver/queryRoomVersionCapabilities" // RoomserverQueryRoomVersionForRoomPath is the HTTP path for the QueryRoomVersionForRoom API -const RoomserverQueryRoomVersionForRoomPath = "/api/roomserver/queryRoomVersionForRoom" +const RoomserverQueryRoomVersionForRoomPath = "/roomserver/queryRoomVersionForRoom" // QueryLatestEventsAndState implements RoomserverQueryAPI func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState( diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index 4d223cf36..248e457d1 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/Shopify/sarama" + "github.com/gorilla/mux" fsAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/caching" @@ -32,8 +33,8 @@ type RoomserverInternalAPI struct { // SetupHTTP adds the RoomserverInternalAPI handlers to the http.ServeMux. // nolint: gocyclo -func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle(api.RoomserverInputRoomEventsPath, +func (r *RoomserverInternalAPI) SetupHTTP(internalAPIMux *mux.Router) { + internalAPIMux.Handle(api.RoomserverInputRoomEventsPath, internal.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { var request api.InputRoomEventsRequest var response api.InputRoomEventsResponse @@ -46,7 +47,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle(api.RoomserverPerformJoinPath, + internalAPIMux.Handle(api.RoomserverPerformJoinPath, internal.MakeInternalAPI("performJoin", func(req *http.Request) util.JSONResponse { var request api.PerformJoinRequest var response api.PerformJoinResponse @@ -59,7 +60,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle(api.RoomserverPerformLeavePath, + internalAPIMux.Handle(api.RoomserverPerformLeavePath, internal.MakeInternalAPI("performLeave", func(req *http.Request) util.JSONResponse { var request api.PerformLeaveRequest var response api.PerformLeaveResponse @@ -72,7 +73,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryLatestEventsAndStatePath, internal.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { var request api.QueryLatestEventsAndStateRequest @@ -86,7 +87,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryStateAfterEventsPath, internal.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { var request api.QueryStateAfterEventsRequest @@ -100,7 +101,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryEventsByIDPath, internal.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { var request api.QueryEventsByIDRequest @@ -114,7 +115,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryMembershipForUserPath, internal.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { var request api.QueryMembershipForUserRequest @@ -128,7 +129,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryMembershipsForRoomPath, internal.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { var request api.QueryMembershipsForRoomRequest @@ -142,7 +143,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryInvitesForUserPath, internal.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse { var request api.QueryInvitesForUserRequest @@ -156,7 +157,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryServerAllowedToSeeEventPath, internal.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { var request api.QueryServerAllowedToSeeEventRequest @@ -170,7 +171,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryMissingEventsPath, internal.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { var request api.QueryMissingEventsRequest @@ -184,7 +185,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryStateAndAuthChainPath, internal.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { var request api.QueryStateAndAuthChainRequest @@ -198,7 +199,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryBackfillPath, internal.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse { var request api.QueryBackfillRequest @@ -212,7 +213,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryRoomVersionCapabilitiesPath, internal.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { var request api.QueryRoomVersionCapabilitiesRequest @@ -226,7 +227,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryRoomVersionForRoomPath, internal.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { var request api.QueryRoomVersionForRoomRequest @@ -240,7 +241,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverSetRoomAliasPath, internal.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { var request api.SetRoomAliasRequest @@ -254,7 +255,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverGetRoomIDForAliasPath, internal.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { var request api.GetRoomIDForAliasRequest @@ -268,7 +269,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverGetCreatorIDForAliasPath, internal.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { var request api.GetCreatorIDForAliasRequest @@ -282,7 +283,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverGetAliasesForRoomIDPath, internal.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { var request api.GetAliasesForRoomIDRequest @@ -296,7 +297,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverRemoveRoomAliasPath, internal.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { var request api.RemoveRoomAliasRequest diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 07250e7ce..82934d50d 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -15,8 +15,6 @@ package roomserver import ( - "net/http" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -51,9 +49,7 @@ func SetupRoomServerComponent( KeyRing: keyRing, } - if base.EnableHTTPAPIs { - internalAPI.SetupHTTP(http.DefaultServeMux) - } + internalAPI.SetupHTTP(base.InternalAPIMux) return &internalAPI } diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 6300b17bb..c876164d7 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -30,7 +30,7 @@ import ( "github.com/matrix-org/util" ) -const pathPrefixR0 = "/_matrix/client/r0" +const pathPrefixR0 = "/client/r0" // Setup configures the given mux with sync-server listeners // @@ -38,12 +38,12 @@ const pathPrefixR0 = "/_matrix/client/r0" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, srp *sync.RequestPool, syncDB storage.Database, + publicAPIMux *mux.Router, srp *sync.RequestPool, syncDB storage.Database, deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, rsAPI api.RoomserverInternalAPI, cfg *config.Dendrite, ) { - r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() + r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() authData := auth.Data{ AccountDB: nil, diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 022ed7bab..9251f6182 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -81,5 +81,5 @@ func SetupSyncAPIComponent( logrus.WithError(err).Panicf("failed to start typing server consumer") } - routing.Setup(base.APIMux, requestPool, syncDB, deviceDB, federation, rsAPI, cfg) + routing.Setup(base.PublicAPIMux, requestPool, syncDB, deviceDB, federation, rsAPI, cfg) } From fbdcfdd25674eb73c4804be42031397e415797a9 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 22 May 2020 12:28:36 +0100 Subject: [PATCH 2/9] Use HTTP APIs when -api specified (#1057) --- cmd/dendrite-monolith-server/main.go | 31 +++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 0042a9b9c..ded0934ae 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -27,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/keydb" "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/keyserver" @@ -44,11 +45,23 @@ var ( httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server") certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS") keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS") - enableHTTPAPIs = flag.Bool("api", false, "Expose internal HTTP APIs in monolith mode") + enableHTTPAPIs = flag.Bool("api", false, "Use HTTP APIs instead of short-circuiting (warning: exposes API endpoints!)") ) func main() { cfg := basecomponent.ParseMonolithFlags() + if *enableHTTPAPIs { + // If the HTTP APIs are enabled then we need to update the Listen + // statements in the configuration so that we know where to find + // the API endpoints. They'll listen on the same port as the monolith + // itself. + addr := config.Address(*httpBindAddr) + cfg.Listen.RoomServer = addr + cfg.Listen.EDUServer = addr + cfg.Listen.AppServiceAPI = addr + cfg.Listen.FederationSender = addr + } + base := basecomponent.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs) defer base.Close() // nolint: errcheck @@ -61,15 +74,30 @@ func main() { rsAPI := roomserver.SetupRoomServerComponent( base, keyRing, federation, ) + if base.EnableHTTPAPIs { + rsAPI = base.CreateHTTPRoomserverAPIs() + } + eduInputAPI := eduserver.SetupEDUServerComponent( base, cache.New(), ) + if base.EnableHTTPAPIs { + eduInputAPI = base.CreateHTTPEDUServerAPIs() + } + asAPI := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, rsAPI, transactions.New(), ) + if base.EnableHTTPAPIs { + asAPI = base.CreateHTTPAppServiceAPIs() + } + fsAPI := federationsender.SetupFederationSenderComponent( base, federation, rsAPI, &keyRing, ) + if base.EnableHTTPAPIs { + fsAPI = base.CreateHTTPFederationSenderAPIs() + } rsAPI.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( @@ -77,6 +105,7 @@ func main() { federation, &keyRing, rsAPI, eduInputAPI, asAPI, transactions.New(), fsAPI, ) + keyserver.SetupKeyServerComponent( base, deviceDB, accountDB, ) From 3daa2327edc90e1ff70bcbc578b793ae5ba5b69f Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 22 May 2020 12:28:48 +0100 Subject: [PATCH 3/9] dendritejs tweaks for persisting sqlite DBs (#1058) * Use uri.path so we don't have file: in the filename * New go-sqlite-js version --- appservice/storage/storage_wasm.go | 2 +- .../auth/storage/accounts/storage_wasm.go | 2 +- .../auth/storage/devices/storage_wasm.go | 2 +- cmd/dendritejs/main.go | 20 +++++++++---------- federationsender/storage/storage_wasm.go | 2 +- go.mod | 2 +- go.sum | 2 ++ internal/basecomponent/base.go | 2 +- internal/keydb/keydb_wasm.go | 2 +- mediaapi/storage/storage_wasm.go | 2 +- publicroomsapi/storage/storage_wasm.go | 2 +- 11 files changed, 21 insertions(+), 19 deletions(-) diff --git a/appservice/storage/storage_wasm.go b/appservice/storage/storage_wasm.go index a6144b435..de1acf92f 100644 --- a/appservice/storage/storage_wasm.go +++ b/appservice/storage/storage_wasm.go @@ -34,7 +34,7 @@ func NewDatabase( case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewDatabase(dataSourceName) + return sqlite3.NewDatabase(uri.Path) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/clientapi/auth/storage/accounts/storage_wasm.go b/clientapi/auth/storage/accounts/storage_wasm.go index 81e77cf79..7cf50de79 100644 --- a/clientapi/auth/storage/accounts/storage_wasm.go +++ b/clientapi/auth/storage/accounts/storage_wasm.go @@ -36,7 +36,7 @@ func NewDatabase( case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewDatabase(dataSourceName, serverName) + return sqlite3.NewDatabase(uri.Path, serverName) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/clientapi/auth/storage/devices/storage_wasm.go b/clientapi/auth/storage/devices/storage_wasm.go index 14c19e74b..c89ad887b 100644 --- a/clientapi/auth/storage/devices/storage_wasm.go +++ b/clientapi/auth/storage/devices/storage_wasm.go @@ -36,7 +36,7 @@ func NewDatabase( case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewDatabase(dataSourceName, serverName) + return sqlite3.NewDatabase(uri.Path, serverName) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 1398bc6da..73d381492 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -163,16 +163,16 @@ func main() { cfg := &config.Dendrite{} cfg.SetDefaults() cfg.Kafka.UseNaffka = true - cfg.Database.Account = "file:dendritejs_account.db" - cfg.Database.AppService = "file:dendritejs_appservice.db" - cfg.Database.Device = "file:dendritejs_device.db" - cfg.Database.FederationSender = "file:dendritejs_fedsender.db" - cfg.Database.MediaAPI = "file:dendritejs_mediaapi.db" - cfg.Database.Naffka = "file:dendritejs_naffka.db" - cfg.Database.PublicRoomsAPI = "file:dendritejs_publicrooms.db" - cfg.Database.RoomServer = "file:dendritejs_roomserver.db" - cfg.Database.ServerKey = "file:dendritejs_serverkey.db" - cfg.Database.SyncAPI = "file:dendritejs_syncapi.db" + cfg.Database.Account = "file:/idb/dendritejs_account.db" + cfg.Database.AppService = "file:/idb/dendritejs_appservice.db" + cfg.Database.Device = "file:/idb/dendritejs_device.db" + cfg.Database.FederationSender = "file:/idb/dendritejs_fedsender.db" + cfg.Database.MediaAPI = "file:/idb/dendritejs_mediaapi.db" + cfg.Database.Naffka = "file:/idb/dendritejs_naffka.db" + cfg.Database.PublicRoomsAPI = "file:/idb/dendritejs_publicrooms.db" + cfg.Database.RoomServer = "file:/idb/dendritejs_roomserver.db" + cfg.Database.ServerKey = "file:/idb/dendritejs_serverkey.db" + cfg.Database.SyncAPI = "file:/idb/dendritejs_syncapi.db" cfg.Kafka.Topics.UserUpdates = "user_updates" cfg.Kafka.Topics.OutputTypingEvent = "output_typing_event" cfg.Kafka.Topics.OutputClientData = "output_client_data" diff --git a/federationsender/storage/storage_wasm.go b/federationsender/storage/storage_wasm.go index f593fd448..3d071bfef 100644 --- a/federationsender/storage/storage_wasm.go +++ b/federationsender/storage/storage_wasm.go @@ -33,7 +33,7 @@ func NewDatabase( } switch uri.Scheme { case "file": - return sqlite3.NewDatabase(dataSourceName) + return sqlite3.NewDatabase(uri.Path) case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") default: diff --git a/go.mod b/go.mod index bbac53eca..151fee27a 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/libp2p/go-libp2p-record v0.1.2 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 - github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd + github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f diff --git a/go.sum b/go.sum index d487f42a3..15822b532 100644 --- a/go.sum +++ b/go.sum @@ -354,6 +354,8 @@ github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 h1:eq github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4/go.mod h1:3WluEZ9QXSwU30tWYqktnpC1x9mwZKx1r8uAv8Iq+a4= github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd h1:C1FV4dRKF1uuGK8UH01+IoW6zZpfsTV1MvQimZvt418= github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf/iHhWlLWd+kCgG+Fsg4Dc+xBl7hptfK7lD0zY= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b h1:nAmSc1KvQOumoRTz/LD68KyrB6Q5/6q7CmQ5Bswc2nM= diff --git a/internal/basecomponent/base.go b/internal/basecomponent/base.go index 0682ae0ea..45aa454a1 100644 --- a/internal/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -264,7 +264,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { uri, err := url.Parse(string(cfg.Database.Naffka)) if err != nil || uri.Scheme == "file" { - db, err = sqlutil.Open(internal.SQLiteDriverName(), string(cfg.Database.Naffka), nil) + db, err = sqlutil.Open(internal.SQLiteDriverName(), string(uri.Path), nil) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } diff --git a/internal/keydb/keydb_wasm.go b/internal/keydb/keydb_wasm.go index 349381ee7..c80329027 100644 --- a/internal/keydb/keydb_wasm.go +++ b/internal/keydb/keydb_wasm.go @@ -41,7 +41,7 @@ func NewDatabase( case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) + return sqlite3.NewDatabase(uri.Path, serverName, serverKey, serverKeyID) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go index 78de2cb82..aa188997f 100644 --- a/mediaapi/storage/storage_wasm.go +++ b/mediaapi/storage/storage_wasm.go @@ -35,7 +35,7 @@ func Open( case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.Open(dataSourceName) + return sqlite3.Open(uri.Path) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/publicroomsapi/storage/storage_wasm.go b/publicroomsapi/storage/storage_wasm.go index d00c339d8..2157ce110 100644 --- a/publicroomsapi/storage/storage_wasm.go +++ b/publicroomsapi/storage/storage_wasm.go @@ -31,7 +31,7 @@ func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) { case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewPublicRoomsServerDatabase(dataSourceName) + return sqlite3.NewPublicRoomsServerDatabase(uri.Path) default: return nil, fmt.Errorf("Cannot use postgres implementation") } From 3d06fe91f278413bf883d853575239e322402b2e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 22 May 2020 13:54:04 +0100 Subject: [PATCH 4/9] Fix internal HTTP API calls --- internal/http/http.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/internal/http/http.go b/internal/http/http.go index 3c6475443..d0b4d6c50 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -6,6 +6,8 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" + "strings" opentracing "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" @@ -21,6 +23,14 @@ func PostJSON( return err } + parsedAPIURL, err := url.Parse(apiURL) + if err != nil { + return err + } + + parsedAPIURL.Path = "/api/" + strings.TrimLeft(parsedAPIURL.Path, "/") + apiURL = parsedAPIURL.String() + req, err := http.NewRequest(http.MethodPost, apiURL, bytes.NewReader(jsonBytes)) if err != nil { return err @@ -48,10 +58,10 @@ func PostJSON( var errorBody struct { Message string `json:"message"` } - if err = json.NewDecoder(res.Body).Decode(&errorBody); err != nil { - return err + if msgerr := json.NewDecoder(res.Body).Decode(&errorBody); msgerr == nil { + return fmt.Errorf("api: %d from %s: %s", res.StatusCode, apiURL, errorBody.Message) } - return fmt.Errorf("api: %d: %s", res.StatusCode, errorBody.Message) + return fmt.Errorf("api: %d from %s", res.StatusCode, apiURL) } return json.NewDecoder(res.Body).Decode(response) } From 0978630b552b0f88bf5efc1c59860cbeb706606d Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 22 May 2020 14:11:06 +0100 Subject: [PATCH 5/9] Fix monolith room server-federation sender connection --- cmd/dendrite-monolith-server/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index ded0934ae..f9993774f 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -71,9 +71,10 @@ func main() { federation := base.CreateFederationClient() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - rsAPI := roomserver.SetupRoomServerComponent( + rsComponent := roomserver.SetupRoomServerComponent( base, keyRing, federation, ) + rsAPI := rsComponent if base.EnableHTTPAPIs { rsAPI = base.CreateHTTPRoomserverAPIs() } @@ -98,7 +99,7 @@ func main() { if base.EnableHTTPAPIs { fsAPI = base.CreateHTTPFederationSenderAPIs() } - rsAPI.SetFederationSenderAPI(fsAPI) + rsComponent.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, From 06d5f1e6dcd9f3e1db92880c7c3f4068cc92ab99 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 22 May 2020 14:14:39 +0100 Subject: [PATCH 6/9] Fix API paths --- appservice/api/query.go | 4 ++-- eduserver/api/input.go | 2 +- federationsender/api/perform.go | 6 +++--- federationsender/api/query.go | 4 ++-- internal/http/http.go | 10 ---------- syncapi/api/query.go | 8 ++++---- 6 files changed, 12 insertions(+), 22 deletions(-) diff --git a/appservice/api/query.go b/appservice/api/query.go index 4a4e31a99..d36e138cb 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -84,10 +84,10 @@ type AppServiceQueryAPI interface { } // AppServiceRoomAliasExistsPath is the HTTP path for the RoomAliasExists API -const AppServiceRoomAliasExistsPath = "/api/appservice/RoomAliasExists" +const AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists" // AppServiceUserIDExistsPath is the HTTP path for the UserIDExists API -const AppServiceUserIDExistsPath = "/api/appservice/UserIDExists" +const AppServiceUserIDExistsPath = "/appservice/UserIDExists" // httpAppServiceQueryAPI contains the URL to an appservice query API and a // reference to a httpClient used to reach it diff --git a/eduserver/api/input.go b/eduserver/api/input.go index 999a5b41a..8b5b6d76a 100644 --- a/eduserver/api/input.go +++ b/eduserver/api/input.go @@ -55,7 +55,7 @@ type EDUServerInputAPI interface { } // EDUServerInputTypingEventPath is the HTTP path for the InputTypingEvent API. -const EDUServerInputTypingEventPath = "/api/eduserver/input" +const EDUServerInputTypingEventPath = "/eduserver/input" // NewEDUServerInputAPIHTTP creates a EDUServerInputAPI implemented by talking to a HTTP POST API. func NewEDUServerInputAPIHTTP(eduServerURL string, httpClient *http.Client) (EDUServerInputAPI, error) { diff --git a/federationsender/api/perform.go b/federationsender/api/perform.go index 427e70683..a73ba0475 100644 --- a/federationsender/api/perform.go +++ b/federationsender/api/perform.go @@ -11,13 +11,13 @@ import ( const ( // FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API. - FederationSenderPerformDirectoryLookupRequestPath = "/api/federationsender/performDirectoryLookup" + FederationSenderPerformDirectoryLookupRequestPath = "/federationsender/performDirectoryLookup" // FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API. - FederationSenderPerformJoinRequestPath = "/api/federationsender/performJoinRequest" + FederationSenderPerformJoinRequestPath = "/federationsender/performJoinRequest" // FederationSenderPerformLeaveRequestPath is the HTTP path for the PerformLeaveRequest API. - FederationSenderPerformLeaveRequestPath = "/api/federationsender/performLeaveRequest" + FederationSenderPerformLeaveRequestPath = "/federationsender/performLeaveRequest" ) type PerformDirectoryLookupRequest struct { diff --git a/federationsender/api/query.go b/federationsender/api/query.go index bf58d5cc9..4c0f757b2 100644 --- a/federationsender/api/query.go +++ b/federationsender/api/query.go @@ -11,10 +11,10 @@ import ( ) // FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API. -const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom" +const FederationSenderQueryJoinedHostsInRoomPath = "/federationsender/queryJoinedHostsInRoom" // FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API. -const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom" +const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/federationsender/queryJoinedHostServerNamesInRoom" // QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom type QueryJoinedHostsInRoomRequest struct { diff --git a/internal/http/http.go b/internal/http/http.go index d0b4d6c50..77896a53f 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -6,8 +6,6 @@ import ( "encoding/json" "fmt" "net/http" - "net/url" - "strings" opentracing "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" @@ -23,14 +21,6 @@ func PostJSON( return err } - parsedAPIURL, err := url.Parse(apiURL) - if err != nil { - return err - } - - parsedAPIURL.Path = "/api/" + strings.TrimLeft(parsedAPIURL.Path, "/") - apiURL = parsedAPIURL.String() - req, err := http.NewRequest(http.MethodPost, apiURL, bytes.NewReader(jsonBytes)) if err != nil { return err diff --git a/syncapi/api/query.go b/syncapi/api/query.go index 6585bc093..1da05fcc8 100644 --- a/syncapi/api/query.go +++ b/syncapi/api/query.go @@ -24,10 +24,10 @@ import ( ) const ( - SyncAPIQuerySyncPath = "/api/syncapi/querySync" - SyncAPIQueryStatePath = "/api/syncapi/queryState" - SyncAPIQueryStateTypePath = "/api/syncapi/queryStateType" - SyncAPIQueryMessagesPath = "/api/syncapi/queryMessages" + SyncAPIQuerySyncPath = "/syncapi/querySync" + SyncAPIQueryStatePath = "/syncapi/queryState" + SyncAPIQueryStateTypePath = "/syncapi/queryStateType" + SyncAPIQueryMessagesPath = "/syncapi/queryMessages" ) func NewSyncQueryAPIHTTP(syncapiURL string, httpClient *http.Client) SyncQueryAPI { From 3c3e014901fd03ea46ff4c5dfa125c6aa148d5ca Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 22 May 2020 14:18:41 +0100 Subject: [PATCH 7/9] Define path prefixes in a package that doesn't create import cycles --- internal/basecomponent/base.go | 5 +++-- internal/http/http.go | 11 +++++++++++ internal/httpapi.go | 10 +++------- internal/httpapis/paths.go | 6 ++++++ 4 files changed, 23 insertions(+), 9 deletions(-) create mode 100644 internal/httpapis/paths.go diff --git a/internal/basecomponent/base.go b/internal/basecomponent/base.go index 45aa454a1..3781cea48 100644 --- a/internal/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -24,6 +24,7 @@ import ( "golang.org/x/crypto/ed25519" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httpapis" "github.com/matrix-org/dendrite/internal/keydb" "github.com/matrix-org/dendrite/internal/keydb/cache" "github.com/matrix-org/dendrite/internal/sqlutil" @@ -103,8 +104,8 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, enableHTTPAPIs tracerCloser: closer, Cfg: cfg, ImmutableCache: cache, - PublicAPIMux: httpmux.PathPrefix(internal.HTTPPublicPathPrefix).Subrouter().UseEncodedPath(), - InternalAPIMux: httpmux.PathPrefix(internal.HTTPInternalPathPrefix).Subrouter().UseEncodedPath(), + PublicAPIMux: httpmux.PathPrefix(httpapis.PublicPathPrefix).Subrouter().UseEncodedPath(), + InternalAPIMux: httpmux.PathPrefix(httpapis.InternalPathPrefix).Subrouter().UseEncodedPath(), httpClient: &http.Client{Timeout: HTTPClientTimeout}, KafkaConsumer: kafkaConsumer, KafkaProducer: kafkaProducer, diff --git a/internal/http/http.go b/internal/http/http.go index 77896a53f..da90b3751 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -6,7 +6,10 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" + "strings" + "github.com/matrix-org/dendrite/internal/httpapis" opentracing "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" ) @@ -21,6 +24,14 @@ func PostJSON( return err } + parsedAPIURL, err := url.Parse(apiURL) + if err != nil { + return err + } + + parsedAPIURL.Path = httpapis.InternalPathPrefix + strings.TrimLeft(parsedAPIURL.Path, "/") + apiURL = parsedAPIURL.String() + req, err := http.NewRequest(http.MethodPost, apiURL, bytes.NewReader(jsonBytes)) if err != nil { return err diff --git a/internal/httpapi.go b/internal/httpapi.go index 526ff8a2d..07bbacdd6 100644 --- a/internal/httpapi.go +++ b/internal/httpapi.go @@ -13,6 +13,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httpapis" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" opentracing "github.com/opentracing/opentracing-go" @@ -23,11 +24,6 @@ import ( "github.com/sirupsen/logrus" ) -const ( - HTTPPublicPathPrefix = "/_matrix/" - HTTPInternalPathPrefix = "/api/" -) - // BasicAuth is used for authorization on /metrics handlers type BasicAuth struct { Username string `yaml:"username"` @@ -195,9 +191,9 @@ func SetupHTTPAPI(servMux *http.ServeMux, publicApiMux *mux.Router, internalApiM servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth)) } if enableHTTPAPIs { - servMux.Handle(HTTPInternalPathPrefix, internalApiMux) + servMux.Handle(httpapis.InternalPathPrefix, internalApiMux) } - servMux.Handle(HTTPPublicPathPrefix, WrapHandlerInCORS(publicApiMux)) + servMux.Handle(httpapis.PublicPathPrefix, WrapHandlerInCORS(publicApiMux)) } // WrapHandlerInBasicAuth adds basic auth to a handler. Only used for /metrics diff --git a/internal/httpapis/paths.go b/internal/httpapis/paths.go new file mode 100644 index 000000000..8adec2dff --- /dev/null +++ b/internal/httpapis/paths.go @@ -0,0 +1,6 @@ +package httpapis + +const ( + PublicPathPrefix = "/_matrix/" + InternalPathPrefix = "/api/" +) From 492af0f2ec3850d41dd02c9c512de8361d50d271 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 25 May 2020 15:29:49 +0100 Subject: [PATCH 8/9] Use Opaque in addition to Path to set naffka DB names --- internal/basecomponent/base.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/basecomponent/base.go b/internal/basecomponent/base.go index 3781cea48..e86d91025 100644 --- a/internal/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -265,7 +265,15 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { uri, err := url.Parse(string(cfg.Database.Naffka)) if err != nil || uri.Scheme == "file" { - db, err = sqlutil.Open(internal.SQLiteDriverName(), string(uri.Path), nil) + var cs string + if uri.Opaque != "" { // file:filename.db + cs = uri.Opaque + } else if uri.Path != "" { // file:///path/to/filename.db + cs = uri.Path + } else { + logrus.Panic("file uri has no filename") + } + db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } From 6d50212f29f2ce3231097833de7686e3237359b4 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 26 May 2020 14:41:16 +0100 Subject: [PATCH 9/9] Miscellaneous fixes (#1060) * Add missing routing for PerformDirectoryLookupRequest * Tweak output * Fix some bugs in devices * Don't default to federated room joins in response to invite * Update sytest-whitelist * Update comments * Return correct room ID from PerformJoin * Fix appservice and EDU server API setup, update sytest-whitelist * Update sytest-whitelist --- appservice/appservice.go | 4 +-- appservice/query/query.go | 7 ++-- .../storage/devices/postgres/devices_table.go | 12 +++++-- .../storage/devices/sqlite3/devices_table.go | 12 +++++-- clientapi/routing/joinroom.go | 2 +- eduserver/eduserver.go | 6 +--- eduserver/input/input.go | 5 +-- federationsender/internal/api.go | 13 +++++++ internal/http/http.go | 4 +-- roomserver/api/perform.go | 1 + roomserver/internal/perform_join.go | 36 +++++++++++++------ sytest-whitelist | 1 + 12 files changed, 71 insertions(+), 32 deletions(-) diff --git a/appservice/appservice.go b/appservice/appservice.go index 76181d2a3..be5b30e2b 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -82,9 +82,7 @@ func SetupAppServiceAPIComponent( Cfg: base.Cfg, } - if base.EnableHTTPAPIs { - appserviceQueryAPI.SetupHTTP(http.DefaultServeMux) - } + appserviceQueryAPI.SetupHTTP(base.InternalAPIMux) consumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB, diff --git a/appservice/query/query.go b/appservice/query/query.go index a61997b42..812ca9f49 100644 --- a/appservice/query/query.go +++ b/appservice/query/query.go @@ -23,6 +23,7 @@ import ( "net/url" "time" + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" @@ -182,8 +183,8 @@ func makeHTTPClient() *http.Client { // SetupHTTP adds the AppServiceQueryPAI handlers to the http.ServeMux. This // handles and muxes incoming api requests the to internal AppServiceQueryAPI. -func (a *AppServiceQueryAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle( +func (a *AppServiceQueryAPI) SetupHTTP(internalAPIMux *mux.Router) { + internalAPIMux.Handle( api.AppServiceRoomAliasExistsPath, internal.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse { var request api.RoomAliasExistsRequest @@ -197,7 +198,7 @@ func (a *AppServiceQueryAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.AppServiceUserIDExistsPath, internal.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse { var request api.UserIDExistsRequest diff --git a/clientapi/auth/storage/devices/postgres/devices_table.go b/clientapi/auth/storage/devices/postgres/devices_table.go index 67d573f95..c84e83d3d 100644 --- a/clientapi/auth/storage/devices/postgres/devices_table.go +++ b/clientapi/auth/storage/devices/postgres/devices_table.go @@ -206,9 +206,8 @@ func (s *devicesStatements) selectDeviceByID( ctx context.Context, localpart, deviceID string, ) (*authtypes.Device, error) { var dev authtypes.Device - var created sql.NullInt64 stmt := s.selectDeviceByIDStmt - err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&created) + err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&dev.DisplayName) if err == nil { dev.ID = deviceID dev.UserID = userutil.MakeUserID(localpart, s.serverName) @@ -230,10 +229,17 @@ func (s *devicesStatements) selectDevicesByLocalpart( for rows.Next() { var dev authtypes.Device - err = rows.Scan(&dev.ID, &dev.DisplayName) + var id, displayname sql.NullString + err = rows.Scan(&id, &displayname) if err != nil { return devices, err } + if id.Valid { + dev.ID = id.String + } + if displayname.Valid { + dev.DisplayName = displayname.String + } dev.UserID = userutil.MakeUserID(localpart, s.serverName) devices = append(devices, dev) } diff --git a/clientapi/auth/storage/devices/sqlite3/devices_table.go b/clientapi/auth/storage/devices/sqlite3/devices_table.go index 6ffc1646f..49f3eaed7 100644 --- a/clientapi/auth/storage/devices/sqlite3/devices_table.go +++ b/clientapi/auth/storage/devices/sqlite3/devices_table.go @@ -208,9 +208,8 @@ func (s *devicesStatements) selectDeviceByID( ctx context.Context, localpart, deviceID string, ) (*authtypes.Device, error) { var dev authtypes.Device - var created sql.NullInt64 stmt := s.selectDeviceByIDStmt - err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&created) + err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&dev.DisplayName) if err == nil { dev.ID = deviceID dev.UserID = userutil.MakeUserID(localpart, s.serverName) @@ -231,10 +230,17 @@ func (s *devicesStatements) selectDevicesByLocalpart( for rows.Next() { var dev authtypes.Device - err = rows.Scan(&dev.ID, &dev.DisplayName) + var id, displayname sql.NullString + err = rows.Scan(&id, &displayname) if err != nil { return devices, err } + if id.Valid { + dev.ID = id.String + } + if displayname.Valid { + dev.DisplayName = displayname.String + } dev.UserID = userutil.MakeUserID(localpart, s.serverName) devices = append(devices, dev) } diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 500df337a..a3d676532 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -75,6 +75,6 @@ func JoinRoomByIDOrAlias( // TODO: Put the response struct somewhere internal. JSON: struct { RoomID string `json:"room_id"` - }{joinReq.RoomIDOrAlias}, + }{joinRes.RoomID}, } } diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go index cba60b8ea..14fbd3328 100644 --- a/eduserver/eduserver.go +++ b/eduserver/eduserver.go @@ -13,8 +13,6 @@ package eduserver import ( - "net/http" - "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/eduserver/input" @@ -35,9 +33,7 @@ func SetupEDUServerComponent( OutputTypingEventTopic: string(base.Cfg.Kafka.Topics.OutputTypingEvent), } - if base.EnableHTTPAPIs { - inputAPI.SetupHTTP(http.DefaultServeMux) - } + inputAPI.SetupHTTP(base.InternalAPIMux) return inputAPI } diff --git a/eduserver/input/input.go b/eduserver/input/input.go index 50837154a..73777e323 100644 --- a/eduserver/input/input.go +++ b/eduserver/input/input.go @@ -19,6 +19,7 @@ import ( "time" "github.com/Shopify/sarama" + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/internal" @@ -90,8 +91,8 @@ func (t *EDUServerInputAPI) sendEvent(ite *api.InputTypingEvent) error { } // SetupHTTP adds the EDUServerInputAPI handlers to the http.ServeMux. -func (t *EDUServerInputAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle(api.EDUServerInputTypingEventPath, +func (t *EDUServerInputAPI) SetupHTTP(internalAPIMux *mux.Router) { + internalAPIMux.Handle(api.EDUServerInputTypingEventPath, internal.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse { var request api.InputTypingEventRequest var response api.InputTypingEventResponse diff --git a/federationsender/internal/api.go b/federationsender/internal/api.go index 4805e7050..dd3942583 100644 --- a/federationsender/internal/api.go +++ b/federationsender/internal/api.go @@ -99,4 +99,17 @@ func (f *FederationSenderInternalAPI) SetupHTTP(internalAPIMux *mux.Router) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(api.FederationSenderPerformDirectoryLookupRequestPath, + internal.MakeInternalAPI("PerformDirectoryLookupRequest", func(req *http.Request) util.JSONResponse { + var request api.PerformDirectoryLookupRequest + var response api.PerformDirectoryLookupResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := f.PerformDirectoryLookup(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) } diff --git a/internal/http/http.go b/internal/http/http.go index da90b3751..2b1891403 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -60,9 +60,9 @@ func PostJSON( Message string `json:"message"` } if msgerr := json.NewDecoder(res.Body).Decode(&errorBody); msgerr == nil { - return fmt.Errorf("api: %d from %s: %s", res.StatusCode, apiURL, errorBody.Message) + return fmt.Errorf("Internal API: %d from %s: %s", res.StatusCode, apiURL, errorBody.Message) } - return fmt.Errorf("api: %d from %s", res.StatusCode, apiURL) + return fmt.Errorf("Internal API: %d from %s", res.StatusCode, apiURL) } return json.NewDecoder(res.Body).Decode(response) } diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index c98f91fb4..f5afd67bf 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -24,6 +24,7 @@ type PerformJoinRequest struct { } type PerformJoinResponse struct { + RoomID string `json:"room_id"` } func (h *httpRoomserverInternalAPI) PerformJoin( diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go index dc786b296..75dfdd19c 100644 --- a/roomserver/internal/perform_join.go +++ b/roomserver/internal/perform_join.go @@ -90,6 +90,12 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( req *api.PerformJoinRequest, res *api.PerformJoinResponse, // nolint:unparam ) error { + // By this point, if req.RoomIDOrAlias contained an alias, then + // it will have been overwritten with a room ID by performJoinRoomByAlias. + // We should now include this in the response so that the CS API can + // return the right room ID. + res.RoomID = req.RoomIDOrAlias + // Get the domain part of the room ID. _, domain, err := gomatrixserverlib.SplitID('!', req.RoomIDOrAlias) if err != nil { @@ -121,20 +127,30 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( return fmt.Errorf("eb.SetContent: %w", err) } - // First work out if this is in response to an existing invite. - // If it is then we avoid the situation where we might think we - // know about a room in the following section but don't know the - // latest state as all of our users have left. + // First work out if this is in response to an existing invite + // from a federated server. If it is then we avoid the situation + // where we might think we know about a room in the following + // section but don't know the latest state as all of our users + // have left. isInvitePending, inviteSender, err := r.isInvitePending(ctx, req.RoomIDOrAlias, req.UserID) if err == nil && isInvitePending { - // Add the server of the person who invited us to the server list, - // as they should be a fairly good bet. - if _, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender); ierr == nil { - req.ServerNames = append(req.ServerNames, inviterDomain) + // Check if there's an invite pending. + _, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender) + if ierr != nil { + return fmt.Errorf("gomatrixserverlib.SplitID: %w", err) } - // Perform a federated room join. - return r.performFederatedJoinRoomByID(ctx, req, res) + // Check that the domain isn't ours. If it's local then we don't + // need to do anything as our own copy of the room state will be + // up-to-date. + if inviterDomain != r.Cfg.Matrix.ServerName { + // Add the server of the person who invited us to the server list, + // as they should be a fairly good bet. + req.ServerNames = append(req.ServerNames, inviterDomain) + + // Perform a federated room join. + return r.performFederatedJoinRoomByID(ctx, req, res) + } } // Try to construct an actual join event from the template. diff --git a/sytest-whitelist b/sytest-whitelist index 1b12aba1b..d4e6be9a4 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -288,3 +288,4 @@ New room members see their own join event Existing members see new members' join events Inbound federation can receive events Inbound federation can receive redacted events +Can logout current device