From cd3da772deea254b458df35cb313779100949c3b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 2 Jul 2020 19:31:12 +0100 Subject: [PATCH] Change API and rename to ExtraPublicRoomsProvider --- clientapi/api/api.go | 11 +-- clientapi/clientapi.go | 2 +- clientapi/routing/directory_public.go | 81 +++++++++++++++++----- clientapi/routing/directory_public_test.go | 14 +++- clientapi/routing/routing.go | 8 +-- cmd/dendritejs/publicrooms.go | 4 ++ internal/setup/monolith.go | 2 +- 7 files changed, 91 insertions(+), 31 deletions(-) diff --git a/clientapi/api/api.go b/clientapi/api/api.go index dae462c08..d96b032f0 100644 --- a/clientapi/api/api.go +++ b/clientapi/api/api.go @@ -14,9 +14,10 @@ package api -type ExternalPublicRoomsProvider interface { - // The list of homeserver domains to query. These servers will receive a request - // via this API: https://matrix.org/docs/spec/server_server/latest#public-room-directory - // This will be called -on demand- by clients, so cache appropriately! - Homeservers() []string +import "github.com/matrix-org/gomatrixserverlib" + +// ExtraPublicRoomsProvider provides a way to inject extra published rooms into /publicRooms requests. +type ExtraPublicRoomsProvider interface { + // Rooms returns the extra rooms. This is called on-demand by clients, so cache appropriately. + Rooms() []gomatrixserverlib.PublicRoom } diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index bbce6dccf..029a73daf 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -48,7 +48,7 @@ func AddPublicRoutes( transactionsCache *transactions.Cache, fsAPI federationSenderAPI.FederationSenderInternalAPI, userAPI userapi.UserInternalAPI, - extRoomsProvider api.ExternalPublicRoomsProvider, + extRoomsProvider api.ExtraPublicRoomsProvider, ) { syncProducer := &producers.SyncAPIProducer{ Producer: producer, diff --git a/clientapi/routing/directory_public.go b/clientapi/routing/directory_public.go index 64600cb49..366f89e80 100644 --- a/clientapi/routing/directory_public.go +++ b/clientapi/routing/directory_public.go @@ -22,7 +22,6 @@ import ( "strconv" "strings" "sync" - "time" "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/httputil" @@ -33,6 +32,11 @@ import ( "github.com/matrix-org/util" ) +var ( + cacheMu sync.Mutex + publicRoomsCache []gomatrixserverlib.PublicRoom +) + type PublicRoomReq struct { Since string `json:"since,omitempty"` Limit int16 `json:"limit,omitempty"` @@ -46,13 +50,15 @@ type filter struct { // GetPostPublicRooms implements GET and POST /publicRooms func GetPostPublicRooms( req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI, + extRoomsProvider api.ExtraPublicRoomsProvider, ) util.JSONResponse { var request PublicRoomReq if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil { return *fillErr } - response, err := publicRooms(req.Context(), request, rsAPI, stateAPI) + response, err := publicRooms(req.Context(), request, rsAPI, stateAPI, extRoomsProvider) if err != nil { + util.GetLogger(req.Context()).WithError(err).Errorf("failed to work out public rooms") return jsonerror.InternalServerError() } return util.JSONResponse{ @@ -64,13 +70,13 @@ func GetPostPublicRooms( // GetPostPublicRoomsWithExternal is the same as GetPostPublicRooms but also mixes in public rooms from the provider supplied. func GetPostPublicRoomsWithExternal( req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI, - fedClient *gomatrixserverlib.FederationClient, extRoomsProvider api.ExternalPublicRoomsProvider, + fedClient *gomatrixserverlib.FederationClient, extRoomsProvider api.ExtraPublicRoomsProvider, ) util.JSONResponse { var request PublicRoomReq if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil { return *fillErr } - response, err := publicRooms(req.Context(), request, rsAPI, stateAPI) + response, err := publicRooms(req.Context(), request, rsAPI, stateAPI, extRoomsProvider) if err != nil { return jsonerror.InternalServerError() } @@ -98,8 +104,8 @@ func GetPostPublicRoomsWithExternal( } // downcasting `limit` is safe as we know it isn't bigger than request.Limit which is int16 - fedRooms := bulkFetchPublicRoomsFromServers(req.Context(), fedClient, extRoomsProvider.Homeservers(), int16(limit)) - response.Chunk = append(response.Chunk, fedRooms...) + //fedRooms := bulkFetchPublicRoomsFromServers(req.Context(), fedClient, extRoomsProvider.Homeservers(), int16(limit)) + //response.Chunk = append(response.Chunk, fedRooms...) // de-duplicate rooms with the same room ID. We can join the room via any of these aliases as we know these servers // are alive and well, so we arbitrarily pick one (purposefully shuffling them to spread the load a bit) @@ -128,6 +134,7 @@ func GetPostPublicRoomsWithExternal( } } +/* // bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers. // Returns a list of public rooms up to the limit specified. func bulkFetchPublicRoomsFromServers( @@ -198,9 +205,11 @@ FanIn: return publicRooms } +*/ func publicRooms(ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.RoomserverInternalAPI, - stateAPI currentstateAPI.CurrentStateInternalAPI) (*gomatrixserverlib.RespPublicRooms, error) { + stateAPI currentstateAPI.CurrentStateInternalAPI, extRoomsProvider api.ExtraPublicRoomsProvider, +) (*gomatrixserverlib.RespPublicRooms, error) { var response gomatrixserverlib.RespPublicRooms var limit int16 @@ -216,23 +225,25 @@ func publicRooms(ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI util.GetLogger(ctx).WithError(err).Error("strconv.ParseInt failed") return nil, err } + err = nil - var queryRes roomserverAPI.QueryPublishedRoomsResponse - err = rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed") - return nil, err + var rooms []gomatrixserverlib.PublicRoom + if request.Since == "" { + rooms = refreshPublicRoomCache(ctx, rsAPI, extRoomsProvider, stateAPI) + } else { + rooms = getPublicRoomsFromCache() } - response.TotalRoomCountEstimate = len(queryRes.RoomIDs) - roomIDs, prev, next := sliceInto(queryRes.RoomIDs, offset, limit) + response.TotalRoomCountEstimate = len(rooms) + + chunk, prev, next := sliceInto(rooms, offset, limit) if prev >= 0 { response.PrevBatch = "T" + strconv.Itoa(prev) } if next >= 0 { response.NextBatch = "T" + strconv.Itoa(next) } - response.Chunk, err = fillInRooms(ctx, roomIDs, stateAPI) + response.Chunk = chunk return &response, err } @@ -348,7 +359,7 @@ func fillInRooms(ctx context.Context, roomIDs []string, stateAPI currentstateAPI // limit=3&since=6 => G (prev='3', next='') // // A value of '-1' for prev/next indicates no position. -func sliceInto(slice []string, since int64, limit int16) (subset []string, prev, next int) { +func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int16) (subset []gomatrixserverlib.PublicRoom, prev, next int) { prev = -1 next = -1 @@ -371,3 +382,41 @@ func sliceInto(slice []string, since int64, limit int16) (subset []string, prev, subset = slice[since:nextIndex] return } + +func refreshPublicRoomCache( + ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI, extRoomsProvider api.ExtraPublicRoomsProvider, + stateAPI currentstateAPI.CurrentStateInternalAPI, +) []gomatrixserverlib.PublicRoom { + cacheMu.Lock() + defer cacheMu.Unlock() + var extraRooms []gomatrixserverlib.PublicRoom + if extRoomsProvider != nil { + extraRooms = extRoomsProvider.Rooms() + } + + var queryRes roomserverAPI.QueryPublishedRoomsResponse + err := rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed") + return publicRoomsCache + } + pubRooms, err := fillInRooms(ctx, queryRes.RoomIDs, stateAPI) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("fillInRooms failed") + return publicRoomsCache + } + publicRoomsCache = []gomatrixserverlib.PublicRoom{} + publicRoomsCache = append(publicRoomsCache, pubRooms...) + publicRoomsCache = append(publicRoomsCache, extraRooms...) + // sort by total joined member count (big to small) + sort.SliceStable(publicRoomsCache, func(i, j int) bool { + return publicRoomsCache[i].JoinedMembersCount > publicRoomsCache[j].JoinedMembersCount + }) + return publicRoomsCache +} + +func getPublicRoomsFromCache() []gomatrixserverlib.PublicRoom { + cacheMu.Lock() + defer cacheMu.Unlock() + return publicRoomsCache +} diff --git a/clientapi/routing/directory_public_test.go b/clientapi/routing/directory_public_test.go index f2a1d5515..bb3912b8c 100644 --- a/clientapi/routing/directory_public_test.go +++ b/clientapi/routing/directory_public_test.go @@ -3,16 +3,26 @@ package routing import ( "reflect" "testing" + + "github.com/matrix-org/gomatrixserverlib" ) +func pubRoom(name string) gomatrixserverlib.PublicRoom { + return gomatrixserverlib.PublicRoom{ + Name: name, + } +} + func TestSliceInto(t *testing.T) { - slice := []string{"a", "b", "c", "d", "e", "f", "g"} + slice := []gomatrixserverlib.PublicRoom{ + pubRoom("a"), pubRoom("b"), pubRoom("c"), pubRoom("d"), pubRoom("e"), pubRoom("f"), pubRoom("g"), + } limit := int16(3) testCases := []struct { since int64 wantPrev int wantNext int - wantSubset []string + wantSubset []gomatrixserverlib.PublicRoom }{ { since: 0, diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 85c5c0d9a..754fbca80 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -61,7 +61,7 @@ func Setup( transactionsCache *transactions.Cache, federationSender federationSenderAPI.FederationSenderInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI, - extRoomsProvider api.ExternalPublicRoomsProvider, + extRoomsProvider api.ExtraPublicRoomsProvider, ) { publicAPIMux.Handle("/client/versions", @@ -313,11 +313,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/publicRooms", httputil.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse { - /* TODO: - if extRoomsProvider != nil { - return GetPostPublicRoomsWithExternal(req, stateAPI, fedClient, extRoomsProvider) - } */ - return GetPostPublicRooms(req, rsAPI, stateAPI) + return GetPostPublicRooms(req, rsAPI, stateAPI, extRoomsProvider) }), ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) diff --git a/cmd/dendritejs/publicrooms.go b/cmd/dendritejs/publicrooms.go index 5032bc15f..f339f71ce 100644 --- a/cmd/dendritejs/publicrooms.go +++ b/cmd/dendritejs/publicrooms.go @@ -62,6 +62,10 @@ func (p *libp2pPublicRoomsProvider) foundProviders(peerInfos []go_http_js_libp2p p.providers = peerInfos } +func (p *libp2pPublicRoomsProvider) Rooms() []gomatrixserverlib.PublicRoom { + return nil +} + func (p *libp2pPublicRoomsProvider) Homeservers() []string { result := make([]string, len(p.providers)) for i := range p.providers { diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index ca90986e9..6ace53a99 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -58,7 +58,7 @@ type Monolith struct { StateAPI currentstateAPI.CurrentStateInternalAPI // Optional - ExtPublicRoomsProvider api.ExternalPublicRoomsProvider + ExtPublicRoomsProvider api.ExtraPublicRoomsProvider } // AddAllPublicRoutes attaches all public paths to the given router