mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2024-11-26 00:01:55 -06:00
Code Issues Packages Projects Releases Wiki Activity

Replace publicroomsapi with a combination of clientapi/roomserver/currentstateserver (#1174)

Browse source 
* Use content_value instead of membership

* Fix build

* Replace publicroomsapi with a combination of clientapi/roomserver/currentstateserver

- All public rooms paths are now handled by clientapi
- Requests to (un)publish rooms are sent to the roomserver via `PerformPublish`
  which are stored in a new `published_table.go`
- Requests for public rooms are handled in clientapi by:
    * Fetch all room IDs which are published using `QueryPublishedRooms` on the roomserver.
    * Apply pagination parameters to the slice.
    * Do a `QueryBulkStateContent` request to the currentstateserver to pull out
      required state event *content* (not entire events).
    * Aggregate and return the chunk.

Mostly but not fully implemented (DB queries on currentstateserver are missing)

* Fix pq query

* Make postgres work

* Make sqlite work

* Fix tests

* Unbreak pagination tests

* Linting
This commit is contained in:
Kegsay 2020-07-02 15:41:18 +01:00 committed by GitHub
parent 55bc82c439
commit 4c1e6597c0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 1481 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
clientapi/routing/createroom.go
View file

@ -410,6 +410,19 @@ func createRoom(
}
}
if r.Visibility == "public" {
// expose this room in the published room list
var pubRes roomserverAPI.PerformPublishResponse
rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
RoomID: roomID,
Visibility: "public",
}, &pubRes)
if pubRes.Error != nil {
// treat as non-fatal since the room is already made by this point
util.GetLogger(req.Context()).WithError(pubRes.Error).Error("failed to visibility:public")
}
}
response := createRoomResponse{
RoomID: roomID,
RoomAlias: roomAlias,

90
clientapi/routing/directory.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -20,10 +20,12 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/userapi/api"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
@ -232,3 +234,89 @@ func RemoveLocalAlias(
JSON: struct{}{},
}
}
type roomVisibility struct {
Visibility string `json:"visibility"`
}
// GetVisibility implements GET /directory/list/room/{roomID}
func GetVisibility(
req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI,
roomID string,
) util.JSONResponse {
var res roomserverAPI.QueryPublishedRoomsResponse
err := rsAPI.QueryPublishedRooms(req.Context(), &roomserverAPI.QueryPublishedRoomsRequest{
RoomID: roomID,
}, &res)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("QueryPublishedRooms failed")
return jsonerror.InternalServerError()
}
var v roomVisibility
if len(res.RoomIDs) == 1 {
v.Visibility = gomatrixserverlib.Public
} else {
v.Visibility = "private"
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: v,
}
}
// SetVisibility implements PUT /directory/list/room/{roomID}
// TODO: Allow admin users to edit the room visibility
func SetVisibility(
req *http.Request, stateAPI currentstateAPI.CurrentStateInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, dev *userapi.Device,
roomID string,
) util.JSONResponse {
resErr := checkMemberInRoom(req.Context(), stateAPI, dev.UserID, roomID)
if resErr != nil {
return *resErr
}
queryEventsReq := roomserverAPI.QueryLatestEventsAndStateRequest{
RoomID: roomID,
StateToFetch: []gomatrixserverlib.StateKeyTuple{{
EventType: gomatrixserverlib.MRoomPowerLevels,
StateKey: "",
}},
}
var queryEventsRes roomserverAPI.QueryLatestEventsAndStateResponse
err := rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes)
if err != nil || len(queryEventsRes.StateEvents) == 0 {
util.GetLogger(req.Context()).WithError(err).Error("could not query events from room")
return jsonerror.InternalServerError()
}
// NOTSPEC: Check if the user's power is greater than power required to change m.room.aliases event
power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event)
if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomAliases, true) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("userID doesn't have power level to change visibility"),
}
}
var v roomVisibility
if reqErr := httputil.UnmarshalJSONRequest(req, &v); reqErr != nil {
return *reqErr
}
var publishRes roomserverAPI.PerformPublishResponse
rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
RoomID: roomID,
Visibility: v.Visibility,
}, &publishRes)
if publishRes.Error != nil {
util.GetLogger(req.Context()).WithError(publishRes.Error).Error("PerformPublish failed")
return publishRes.Error.JSONResponse()
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
}
}

373
clientapi/routing/directory_public.go Normal file
View file

@ -0,0 +1,373 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package routing
import (
"context"
"math/rand"
"net/http"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
"github.com/matrix-org/dendrite/publicroomsapi/types"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
type PublicRoomReq struct {
Since string `json:"since,omitempty"`
Limit int16 `json:"limit,omitempty"`
Filter filter `json:"filter,omitempty"`
}
type filter struct {
SearchTerms string `json:"generic_search_term,omitempty"`
}
// GetPostPublicRooms implements GET and POST /publicRooms
func GetPostPublicRooms(
req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI,
) util.JSONResponse {
var request PublicRoomReq
if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil {
return *fillErr
}
response, err := publicRooms(req.Context(), request, rsAPI, stateAPI)
if err != nil {
return jsonerror.InternalServerError()
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: response,
}
}
// GetPostPublicRoomsWithExternal is the same as GetPostPublicRooms but also mixes in public rooms from the provider supplied.
func GetPostPublicRoomsWithExternal(
req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI,
fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider,
) util.JSONResponse {
var request PublicRoomReq
if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil {
return *fillErr
}
response, err := publicRooms(req.Context(), request, rsAPI, stateAPI)
if err != nil {
return jsonerror.InternalServerError()
}
if request.Since != "" {
// TODO: handle pagination tokens sensibly rather than ignoring them.
// ignore paginated requests since we don't handle them yet over federation.
// Only the initial request will contain federated rooms.
return util.JSONResponse{
Code: http.StatusOK,
JSON: response,
}
}
// If we have already hit the limit on the number of rooms, bail.
var limit int
if request.Limit > 0 {
limit = int(request.Limit) - len(response.Chunk)
if limit <= 0 {
return util.JSONResponse{
Code: http.StatusOK,
JSON: response,
}
}
}
// downcasting `limit` is safe as we know it isn't bigger than request.Limit which is int16
fedRooms := bulkFetchPublicRoomsFromServers(req.Context(), fedClient, extRoomsProvider.Homeservers(), int16(limit))
response.Chunk = append(response.Chunk, fedRooms...)
// de-duplicate rooms with the same room ID. We can join the room via any of these aliases as we know these servers
// are alive and well, so we arbitrarily pick one (purposefully shuffling them to spread the load a bit)
var publicRooms []gomatrixserverlib.PublicRoom
haveRoomIDs := make(map[string]bool)
rand.Shuffle(len(response.Chunk), func(i, j int) {
response.Chunk[i], response.Chunk[j] = response.Chunk[j], response.Chunk[i]
})
for _, r := range response.Chunk {
if haveRoomIDs[r.RoomID] {
continue
}
haveRoomIDs[r.RoomID] = true
publicRooms = append(publicRooms, r)
}
// sort by member count
sort.SliceStable(publicRooms, func(i, j int) bool {
return publicRooms[i].JoinedMembersCount > publicRooms[j].JoinedMembersCount
})
response.Chunk = publicRooms
return util.JSONResponse{
Code: http.StatusOK,
JSON: response,
}
}
// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers.
// Returns a list of public rooms up to the limit specified.
func bulkFetchPublicRoomsFromServers(
ctx context.Context, fedClient *gomatrixserverlib.FederationClient, homeservers []string, limit int16,
) (publicRooms []gomatrixserverlib.PublicRoom) {
// follow pipeline semantics, see https://blog.golang.org/pipelines for more info.
// goroutines send rooms to this channel
roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit))
// signalling channel to tell goroutines to stop sending rooms and quit
done := make(chan bool)
// signalling to say when we can close the room channel
var wg sync.WaitGroup
wg.Add(len(homeservers))
// concurrently query for public rooms
for _, hs := range homeservers {
go func(homeserverDomain string) {
defer wg.Done()
util.GetLogger(ctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms")
fres, err := fedClient.GetPublicRooms(ctx, gomatrixserverlib.ServerName(homeserverDomain), int(limit), "", false, "")
if err != nil {
util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Warn(
"bulkFetchPublicRoomsFromServers: failed to query hs",
)
return
}
for _, room := range fres.Chunk {
// atomically send a room or stop
select {
case roomCh <- room:
case <-done:
util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending rooms")
return
}
}
}(hs)
}
// Close the room channel when the goroutines have quit so we don't leak, but don't let it stop the in-flight request.
// This also allows the request to fail fast if all HSes experience errors as it will cause the room channel to be
// closed.
go func() {
wg.Wait()
util.GetLogger(ctx).Info("Cleaning up resources")
close(roomCh)
}()
// fan-in results with timeout. We stop when we reach the limit.
FanIn:
for len(publicRooms) < int(limit) || limit == 0 {
// add a room or timeout
select {
case room, ok := <-roomCh:
if !ok {
util.GetLogger(ctx).Info("All homeservers have been queried, returning results.")
break FanIn
}
publicRooms = append(publicRooms, room)
case <-time.After(15 * time.Second): // we've waited long enough, let's tell the client what we got.
util.GetLogger(ctx).Info("Waited 15s for federated public rooms, returning early")
break FanIn
case <-ctx.Done(): // the client hung up on us, let's stop.
util.GetLogger(ctx).Info("Client hung up, returning early")
break FanIn
}
}
// tell goroutines to stop
close(done)
return publicRooms
}
func publicRooms(ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.RoomserverInternalAPI,
stateAPI currentstateAPI.CurrentStateInternalAPI) (*gomatrixserverlib.RespPublicRooms, error) {
var response gomatrixserverlib.RespPublicRooms
var limit int16
var offset int64
limit = request.Limit
if limit == 0 {
limit = 50
}
offset, err := strconv.ParseInt(request.Since, 10, 64)
// ParseInt returns 0 and an error when trying to parse an empty string
// In that case, we want to assign 0 so we ignore the error
if err != nil && len(request.Since) > 0 {
util.GetLogger(ctx).WithError(err).Error("strconv.ParseInt failed")
return nil, err
}
var queryRes roomserverAPI.QueryPublishedRoomsResponse
err = rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed")
return nil, err
}
response.TotalRoomCountEstimate = len(queryRes.RoomIDs)
roomIDs, prev, next := sliceInto(queryRes.RoomIDs, offset, limit)
if prev >= 0 {
response.PrevBatch = "T" + strconv.Itoa(prev)
}
if next >= 0 {
response.NextBatch = "T" + strconv.Itoa(next)
}
response.Chunk, err = fillInRooms(ctx, roomIDs, stateAPI)
return &response, err
}
// fillPublicRoomsReq fills the Limit, Since and Filter attributes of a GET or POST request
// on /publicRooms by parsing the incoming HTTP request
// Filter is only filled for POST requests
func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSONResponse {
if httpReq.Method != "GET" && httpReq.Method != "POST" {
return &util.JSONResponse{
Code: http.StatusMethodNotAllowed,
JSON: jsonerror.NotFound("Bad method"),
}
}
if httpReq.Method == "GET" {
limit, err := strconv.Atoi(httpReq.FormValue("limit"))
// Atoi returns 0 and an error when trying to parse an empty string
// In that case, we want to assign 0 so we ignore the error
if err != nil && len(httpReq.FormValue("limit")) > 0 {
util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed")
return &util.JSONResponse{
Code: 400,
JSON: jsonerror.BadJSON("limit param is not a number"),
}
}
request.Limit = int16(limit)
request.Since = httpReq.FormValue("since")
} else {
resErr := httputil.UnmarshalJSONRequest(httpReq, request)
if resErr != nil {
return resErr
}
}
// strip the 'T' which is only required because when sytest does pagination tests it stops
// iterating when !prev_batch which then fails if prev_batch==0, so add arbitrary text to
// make it truthy not falsey.
request.Since = strings.TrimPrefix(request.Since, "T")
return nil
}
// due to lots of switches
// nolint:gocyclo
func fillInRooms(ctx context.Context, roomIDs []string, stateAPI currentstateAPI.CurrentStateInternalAPI) ([]gomatrixserverlib.PublicRoom, error) {
avatarTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.avatar", StateKey: ""}
nameTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.name", StateKey: ""}
canonicalTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCanonicalAlias, StateKey: ""}
topicTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.topic", StateKey: ""}
guestTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.guest_access", StateKey: ""}
visibilityTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomHistoryVisibility, StateKey: ""}
joinRuleTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""}
var stateRes currentstateAPI.QueryBulkStateContentResponse
err := stateAPI.QueryBulkStateContent(ctx, &currentstateAPI.QueryBulkStateContentRequest{
RoomIDs: roomIDs,
AllowWildcards: true,
StateTuples: []gomatrixserverlib.StateKeyTuple{
nameTuple, canonicalTuple, topicTuple, guestTuple, visibilityTuple, joinRuleTuple, avatarTuple,
{EventType: gomatrixserverlib.MRoomMember, StateKey: "*"},
},
}, &stateRes)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("QueryBulkStateContent failed")
return nil, err
}
chunk := make([]gomatrixserverlib.PublicRoom, len(roomIDs))
i := 0
for roomID, data := range stateRes.Rooms {
pub := gomatrixserverlib.PublicRoom{
RoomID: roomID,
}
joinCount := 0
var joinRule, guestAccess string
for tuple, contentVal := range data {
if tuple.EventType == gomatrixserverlib.MRoomMember && contentVal == "join" {
joinCount++
continue
}
switch tuple {
case avatarTuple:
pub.AvatarURL = contentVal
case nameTuple:
pub.Name = contentVal
case topicTuple:
pub.Topic = contentVal
case canonicalTuple:
pub.CanonicalAlias = contentVal
case visibilityTuple:
pub.WorldReadable = contentVal == "world_readable"
// need both of these to determine whether guests can join
case joinRuleTuple:
joinRule = contentVal
case guestTuple:
guestAccess = contentVal
}
}
if joinRule == gomatrixserverlib.Public && guestAccess == "can_join" {
pub.GuestCanJoin = true
}
pub.JoinedMembersCount = joinCount
chunk[i] = pub
i++
}
return chunk, nil
}
// sliceInto returns a subslice of `slice` which honours the since/limit values given.
//
// 0 1 2 3 4 5 6 index
// [A, B, C, D, E, F, G] slice
//
// limit=3 => A,B,C (prev='', next='3')
// limit=3&since=3 => D,E,F (prev='0', next='6')
// limit=3&since=6 => G (prev='3', next='')
//
// A value of '-1' for prev/next indicates no position.
func sliceInto(slice []string, since int64, limit int16) (subset []string, prev, next int) {
prev = -1
next = -1
if since > 0 {
prev = int(since) - int(limit)
}
nextIndex := int(since) + int(limit)
if len(slice) > nextIndex { // there are more rooms ahead of us
next = nextIndex
}
// apply sanity caps
if since < 0 {
since = 0
}
if nextIndex > len(slice) {
nextIndex = len(slice)
}
subset = slice[since:nextIndex]
return
}

48
clientapi/routing/directory_public_test.go Normal file
View file

@ -0,0 +1,48 @@
package routing
import (
"reflect"
"testing"
)
func TestSliceInto(t *testing.T) {
slice := []string{"a", "b", "c", "d", "e", "f", "g"}
limit := int16(3)
testCases := []struct {
since int64
wantPrev int
wantNext int
wantSubset []string
}{
{
since: 0,
wantPrev: -1,
wantNext: 3,
wantSubset: slice[0:3],
},
{
since: 3,
wantPrev: 0,
wantNext: 6,
wantSubset: slice[3:6],
},
{
since: 6,
wantPrev: 3,
wantNext: -1,
wantSubset: slice[6:7],
},
}
for _, tc := range testCases {
subset, prev, next := sliceInto(slice, tc.since, limit)
if !reflect.DeepEqual(subset, tc.wantSubset) {
t.Errorf("returned subset is wrong, got %v want %v", subset, tc.wantSubset)
}
if prev != tc.wantPrev {
t.Errorf("returned prev is wrong, got %d want %d", prev, tc.wantPrev)
}
if next != tc.wantNext {
t.Errorf("returned next is wrong, got %d want %d", next, tc.wantNext)
}
}
}

33
clientapi/routing/membership.go
View file

@ -25,6 +25,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/threepid"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/roomserver/api"
@ -358,3 +359,35 @@ func checkAndProcessThreepid(
}
return
}
func checkMemberInRoom(ctx context.Context, stateAPI currentstateAPI.CurrentStateInternalAPI, userID, roomID string) *util.JSONResponse {
tuple := gomatrixserverlib.StateKeyTuple{
EventType: gomatrixserverlib.MRoomMember,
StateKey: userID,
}
var membershipRes currentstateAPI.QueryCurrentStateResponse
err := stateAPI.QueryCurrentState(ctx, &currentstateAPI.QueryCurrentStateRequest{
RoomID: roomID,
StateTuples: []gomatrixserverlib.StateKeyTuple{tuple},
}, &membershipRes)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("QueryCurrentState: could not query membership for user")
e := jsonerror.InternalServerError()
return &e
}
ev, ok := membershipRes.StateEvents[tuple]
if !ok {
return &util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("user does not belong to room"),
}
}
membership, err := ev.Membership()
if err != nil || membership != "join" {
return &util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("user does not belong to room"),
}
}
return nil
}

31
clientapi/routing/routing.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -31,6 +31,7 @@ import (
"github.com/matrix-org/dendrite/internal/transactions"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/userapi/api"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/dendrite/userapi/storage/devices"
"github.com/matrix-org/gomatrixserverlib"
@ -290,6 +291,34 @@ func Setup(
return RemoveLocalAlias(req, device, vars["roomAlias"], rsAPI)
}),
).Methods(http.MethodDelete, http.MethodOptions)
r0mux.Handle("/directory/list/room/{roomID}",
httputil.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return GetVisibility(req, rsAPI, vars["roomID"])
}),
).Methods(http.MethodGet, http.MethodOptions)
// TODO: Add AS support
r0mux.Handle("/directory/list/room/{roomID}",
httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return SetVisibility(req, stateAPI, rsAPI, device, vars["roomID"])
}),
).Methods(http.MethodPut, http.MethodOptions)
r0mux.Handle("/publicRooms",
httputil.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse {
/* TODO:
if extRoomsProvider != nil {
return GetPostPublicRoomsWithExternal(req, stateAPI, fedClient, extRoomsProvider)
} */
return GetPostPublicRooms(req, rsAPI, stateAPI)
}),
).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
r0mux.Handle("/logout",
httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {

2
cmd/dendrite-federation-api-server/main.go
View file

@ -33,7 +33,7 @@ func main() {
federationapi.AddPublicRoutes(
base.PublicAPIMux, base.Cfg, userAPI, federation, keyRing,
rsAPI, fsAPI, base.EDUServerClient(),
rsAPI, fsAPI, base.EDUServerClient(), base.CurrentStateAPIClient(),
)
base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI))

1
cmd/dendritejs/main.go
View file

@ -173,6 +173,7 @@ func main() {
cfg.Database.RoomServer = "file:/idb/dendritejs_roomserver.db"
cfg.Database.ServerKey = "file:/idb/dendritejs_serverkey.db"
cfg.Database.SyncAPI = "file:/idb/dendritejs_syncapi.db"
cfg.Database.CurrentState = "file:/idb/dendritejs_currentstate.db"
cfg.Kafka.Topics.OutputTypingEvent = "output_typing_event"
cfg.Kafka.Topics.OutputSendToDeviceEvent = "output_send_to_device_event"
cfg.Kafka.Topics.OutputClientData = "output_client_data"

26
currentstateserver/api/api.go
View file

@ -29,6 +29,8 @@ type CurrentStateInternalAPI interface {
QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error
// QueryRoomsForUser retrieves a list of room IDs matching the given query.
QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
// QueryBulkStateContent does a bulk query for state event content in the given rooms.
QueryBulkStateContent(ctx context.Context, req *QueryBulkStateContentRequest, res *QueryBulkStateContentResponse) error
}
type QueryRoomsForUserRequest struct {
@ -41,6 +43,30 @@ type QueryRoomsForUserResponse struct {
RoomIDs []string
}
type QueryBulkStateContentRequest struct {
// Returns state events in these rooms
RoomIDs []string
// If true, treats the '*' StateKey as "all state events of this type" rather than a literal value of '*'
AllowWildcards bool
// The state events to return. Only a small subset of tuples are allowed in this request as only certain events
// have their content fields extracted. Specifically, the tuple Type must be one of:
// m.room.avatar
// m.room.create
// m.room.canonical_alias
// m.room.guest_access
// m.room.history_visibility
// m.room.join_rules
// m.room.member
// m.room.name
// m.room.topic
// Any other tuple type will result in the query failing.
StateTuples []gomatrixserverlib.StateKeyTuple
}
type QueryBulkStateContentResponse struct {
// map of room ID -> tuple -> content_value
Rooms map[string]map[gomatrixserverlib.StateKeyTuple]string
}
type QueryCurrentStateRequest struct {
RoomID string
StateTuples []gomatrixserverlib.StateKeyTuple

20
currentstateserver/internal/api.go
View file

@ -48,3 +48,23 @@ func (a *CurrentStateInternalAPI) QueryRoomsForUser(ctx context.Context, req *ap
res.RoomIDs = roomIDs
return nil
}
func (a *CurrentStateInternalAPI) QueryBulkStateContent(ctx context.Context, req *api.QueryBulkStateContentRequest, res *api.QueryBulkStateContentResponse) error {
events, err := a.DB.GetBulkStateContent(ctx, req.RoomIDs, req.StateTuples, req.AllowWildcards)
if err != nil {
return err
}
res.Rooms = make(map[string]map[gomatrixserverlib.StateKeyTuple]string)
for _, ev := range events {
if res.Rooms[ev.RoomID] == nil {
res.Rooms[ev.RoomID] = make(map[gomatrixserverlib.StateKeyTuple]string)
}
room := res.Rooms[ev.RoomID]
room[gomatrixserverlib.StateKeyTuple{
EventType: ev.EventType,
StateKey: ev.StateKey,
}] = ev.ContentValue
res.Rooms[ev.RoomID] = room
}
return nil
}

17
currentstateserver/inthttp/client.go
View file

@ -26,8 +26,9 @@ import (
// HTTP paths for the internal HTTP APIs
const (
QueryCurrentStatePath = "/currentstateserver/queryCurrentState"
QueryRoomsForUserPath = "/currentstateserver/queryRoomsForUser"
QueryCurrentStatePath = "/currentstateserver/queryCurrentState"
QueryRoomsForUserPath = "/currentstateserver/queryRoomsForUser"
QueryBulkStateContentPath = "/currentstateserver/queryBulkStateContent"
)
// NewCurrentStateAPIClient creates a CurrentStateInternalAPI implemented by talking to a HTTP POST API.
@ -73,3 +74,15 @@ func (h *httpCurrentStateInternalAPI) QueryRoomsForUser(
apiURL := h.apiURL + QueryRoomsForUserPath
return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
}
func (h *httpCurrentStateInternalAPI) QueryBulkStateContent(
ctx context.Context,
request *api.QueryBulkStateContentRequest,
response *api.QueryBulkStateContentResponse,
) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "QueryBulkStateContent")
defer span.Finish()
apiURL := h.apiURL + QueryBulkStateContentPath
return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
}

13
currentstateserver/inthttp/server.go
View file

@ -51,4 +51,17 @@ func AddRoutes(internalAPIMux *mux.Router, intAPI api.CurrentStateInternalAPI) {
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
)
internalAPIMux.Handle(QueryBulkStateContentPath,
httputil.MakeInternalAPI("queryBulkStateContent", func(req *http.Request) util.JSONResponse {
request := api.QueryBulkStateContentRequest{}
response := api.QueryBulkStateContentResponse{}
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
return util.MessageResponse(http.StatusBadRequest, err.Error())
}
if err := intAPI.QueryBulkStateContent(req.Context(), &request, &response); err != nil {
return util.ErrorResponse(err)
}
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
)
}

4
currentstateserver/storage/interface.go
View file

@ -17,6 +17,7 @@ package storage
import (
"context"
"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/gomatrixserverlib"
)
@ -31,4 +32,7 @@ type Database interface {
GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error)
// GetRoomsByMembership returns a list of room IDs matching the provided membership and user ID (as state_key).
GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error)
// GetBulkStateContent returns all state events which match a given room ID and a given state key tuple. Both must be satisfied for a match.
// If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned.
GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error)
}

92
currentstateserver/storage/postgres/current_room_state_table.go
View file

@ -18,7 +18,6 @@ import (
"context"
"database/sql"
"encoding/json"
"strconv"
"github.com/lib/pq"
"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
@ -27,8 +26,6 @@ import (
"github.com/matrix-org/gomatrixserverlib"
)
var leaveEnum = strconv.Itoa(tables.MembershipToEnum["leave"])
var currentRoomStateSchema = `
-- Stores the current room state for every room.
CREATE TABLE IF NOT EXISTS currentstate_current_room_state (
@ -44,29 +41,29 @@ CREATE TABLE IF NOT EXISTS currentstate_current_room_state (
state_key TEXT NOT NULL,
-- The JSON for the event. Stored as TEXT because this should be valid UTF-8.
headered_event_json TEXT NOT NULL,
-- The 'content.membership' enum value if this event is an m.room.member event.
membership SMALLINT NOT NULL DEFAULT 0,
-- A piece of extracted content e.g membership for m.room.member events
content_value TEXT NOT NULL DEFAULT '',
-- Clobber based on 3-uple of room_id, type and state_key
CONSTRAINT currentstate_current_room_state_unique UNIQUE (room_id, type, state_key)
);
-- for event deletion
CREATE UNIQUE INDEX IF NOT EXISTS currentstate_event_id_idx ON currentstate_current_room_state(event_id, room_id, type, sender);
-- for querying membership states of users
CREATE INDEX IF NOT EXISTS currentstate_membership_idx ON currentstate_current_room_state(type, state_key, membership)
WHERE membership IS NOT NULL AND membership != ` + leaveEnum + `;
CREATE INDEX IF NOT EXISTS currentstate_membership_idx ON currentstate_current_room_state(type, state_key, content_value)
WHERE type='m.room.member' AND content_value IS NOT NULL AND content_value != 'leave';
`
const upsertRoomStateSQL = "" +
"INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, membership)" +
"INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, content_value)" +
" VALUES ($1, $2, $3, $4, $5, $6, $7)" +
" ON CONFLICT ON CONSTRAINT currentstate_current_room_state_unique" +
" DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, membership = $7"
" DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, content_value = $7"
const deleteRoomStateByEventIDSQL = "" +
"DELETE FROM currentstate_current_room_state WHERE event_id = $1"
const selectRoomIDsWithMembershipSQL = "" +
"SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2"
"SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND content_value = $2"
const selectStateEventSQL = "" +
"SELECT headered_event_json FROM currentstate_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3"
@ -74,12 +71,20 @@ const selectStateEventSQL = "" +
const selectEventsWithEventIDsSQL = "" +
"SELECT headered_event_json FROM currentstate_current_room_state WHERE event_id = ANY($1)"
const selectBulkStateContentSQL = "" +
"SELECT room_id, type, state_key, content_value FROM currentstate_current_room_state WHERE room_id = ANY($1) AND type = ANY($2) AND state_key = ANY($3)"
const selectBulkStateContentWildSQL = "" +
"SELECT room_id, type, state_key, content_value FROM currentstate_current_room_state WHERE room_id = ANY($1) AND type = ANY($2)"
type currentRoomStateStatements struct {
upsertRoomStateStmt *sql.Stmt
deleteRoomStateByEventIDStmt *sql.Stmt
selectRoomIDsWithMembershipStmt *sql.Stmt
selectEventsWithEventIDsStmt *sql.Stmt
selectStateEventStmt *sql.Stmt
selectBulkStateContentStmt *sql.Stmt
selectBulkStateContentWildStmt *sql.Stmt
}
func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) {
@ -103,6 +108,12 @@ func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, erro
if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil {
return nil, err
}
if s.selectBulkStateContentStmt, err = db.Prepare(selectBulkStateContentSQL); err != nil {
return nil, err
}
if s.selectBulkStateContentWildStmt, err = db.Prepare(selectBulkStateContentWildSQL); err != nil {
return nil, err
}
return s, nil
}
@ -111,10 +122,10 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership(
ctx context.Context,
txn *sql.Tx,
userID string,
membershipEnum int,
contentVal string,
) ([]string, error) {
stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt)
rows, err := stmt.QueryContext(ctx, userID, membershipEnum)
rows, err := stmt.QueryContext(ctx, userID, contentVal)
if err != nil {
return nil, err
}
@ -141,7 +152,7 @@ func (s *currentRoomStateStatements) DeleteRoomStateByEventID(
func (s *currentRoomStateStatements) UpsertRoomState(
ctx context.Context, txn *sql.Tx,
event gomatrixserverlib.HeaderedEvent, membershipEnum int,
event gomatrixserverlib.HeaderedEvent, contentVal string,
) error {
headeredJSON, err := json.Marshal(event)
if err != nil {
@ -158,7 +169,7 @@ func (s *currentRoomStateStatements) UpsertRoomState(
event.Sender(),
*event.StateKey(),
headeredJSON,
membershipEnum,
contentVal,
)
return err
}
@ -206,3 +217,56 @@ func (s *currentRoomStateStatements) SelectStateEvent(
}
return &ev, err
}
func (s *currentRoomStateStatements) SelectBulkStateContent(
ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool,
) ([]tables.StrippedEvent, error) {
hasWildcards := false
eventTypeSet := make(map[string]bool)
stateKeySet := make(map[string]bool)
var eventTypes []string
var stateKeys []string
for _, tuple := range tuples {
if !eventTypeSet[tuple.EventType] {
eventTypeSet[tuple.EventType] = true
eventTypes = append(eventTypes, tuple.EventType)
}
if !stateKeySet[tuple.StateKey] {
stateKeySet[tuple.StateKey] = true
stateKeys = append(stateKeys, tuple.StateKey)
}
if tuple.StateKey == "*" {
hasWildcards = true
}
}
var rows *sql.Rows
var err error
if hasWildcards && allowWildcards {
rows, err = s.selectBulkStateContentWildStmt.QueryContext(ctx, pq.StringArray(roomIDs), pq.StringArray(eventTypes))
} else {
rows, err = s.selectBulkStateContentStmt.QueryContext(
ctx, pq.StringArray(roomIDs), pq.StringArray(eventTypes), pq.StringArray(stateKeys),
)
}
if err != nil {
return nil, err
}
strippedEvents := []tables.StrippedEvent{}
defer internal.CloseAndLogIfError(ctx, rows, "SelectBulkStateContent: rows.close() failed")
for rows.Next() {
var roomID string
var eventType string
var stateKey string
var contentVal string
if err = rows.Scan(&roomID, &eventType, &stateKey, &contentVal); err != nil {
return nil, err
}
strippedEvents = append(strippedEvents, tables.StrippedEvent{
RoomID: roomID,
ContentValue: contentVal,
EventType: eventType,
StateKey: stateKey,
})
}
return strippedEvents, rows.Err()
}

26
currentstateserver/storage/shared/storage.go
View file

@ -17,7 +17,6 @@ package shared
import (
"context"
"database/sql"
"fmt"
"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
"github.com/matrix-org/dendrite/internal/sqlutil"
@ -33,6 +32,10 @@ func (d *Database) GetStateEvent(ctx context.Context, roomID, evType, stateKey s
return d.CurrentRoomState.SelectStateEvent(ctx, roomID, evType, stateKey)
}
func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error) {
return d.CurrentRoomState.SelectBulkStateContent(ctx, roomIDs, tuples, allowWildcards)
}
func (d *Database) StoreStateEvents(ctx context.Context, addStateEvents []gomatrixserverlib.HeaderedEvent,
removeStateEventIDs []string) error {
return sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error {
@ -48,20 +51,9 @@ func (d *Database) StoreStateEvents(ctx context.Context, addStateEvents []gomatr
// ignore non state events
continue
}
var membershipEnum int
if event.Type() == "m.room.member" {
membership, err := event.Membership()
if err != nil {
return err
}
enum, ok := tables.MembershipToEnum[membership]
if !ok {
return fmt.Errorf("unknown membership: %s", membership)
}
membershipEnum = enum
}
contentVal := tables.ExtractContentValue(&event)
if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membershipEnum); err != nil {
if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, contentVal); err != nil {
return err
}
}
@ -70,9 +62,5 @@ func (d *Database) StoreStateEvents(ctx context.Context, addStateEvents []gomatr
}
func (d *Database) GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error) {
enum, ok := tables.MembershipToEnum[membership]
if !ok {
return nil, fmt.Errorf("unknown membership: %s", membership)
}
return d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, nil, userID, enum)
return d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, nil, userID, membership)
}

109
currentstateserver/storage/sqlite3/current_room_state_table.go
View file

@ -35,39 +35,39 @@ CREATE TABLE IF NOT EXISTS currentstate_current_room_state (
sender TEXT NOT NULL,
state_key TEXT NOT NULL,
headered_event_json TEXT NOT NULL,
membership INTEGER NOT NULL DEFAULT 0,
content_value TEXT NOT NULL DEFAULT '',
UNIQUE (room_id, type, state_key)
);
-- for event deletion
CREATE UNIQUE INDEX IF NOT EXISTS currentstate_event_id_idx ON currentstate_current_room_state(event_id, room_id, type, sender);
-- for querying membership states of users
-- CREATE INDEX IF NOT EXISTS currentstate_membership_idx ON currentstate_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave';
`
const upsertRoomStateSQL = "" +
"INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, membership)" +
"INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, content_value)" +
" VALUES ($1, $2, $3, $4, $5, $6, $7)" +
" ON CONFLICT (event_id, room_id, type, sender)" +
" DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, membership = $7"
" DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, content_value = $7"
const deleteRoomStateByEventIDSQL = "" +
"DELETE FROM currentstate_current_room_state WHERE event_id = $1"
const selectRoomIDsWithMembershipSQL = "" +
"SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2"
"SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND content_value = $2"
const selectStateEventSQL = "" +
"SELECT headered_event_json FROM currentstate_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3"
const selectEventsWithEventIDsSQL = "" +
// TODO: The session_id and transaction_id blanks are here because otherwise
// the rowsToStreamEvents expects there to be exactly five columns. We need to
// figure out if these really need to be in the DB, and if so, we need a
// better permanent fix for this. - neilalexander, 2 Jan 2020
"SELECT added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" +
" FROM currentstate_current_room_state WHERE event_id IN ($1)"
"SELECT headered_event_json FROM currentstate_current_room_state WHERE event_id IN ($1)"
const selectBulkStateContentSQL = "" +
"SELECT room_id, type, state_key, content_value FROM currentstate_current_room_state WHERE room_id IN ($1) AND type IN ($2) AND state_key IN ($3)"
const selectBulkStateContentWildSQL = "" +
"SELECT room_id, type, state_key, content_value FROM currentstate_current_room_state WHERE room_id IN ($1) AND type IN ($2)"
type currentRoomStateStatements struct {
db *sql.DB
upsertRoomStateStmt *sql.Stmt
deleteRoomStateByEventIDStmt *sql.Stmt
selectRoomIDsWithMembershipStmt *sql.Stmt
@ -75,7 +75,9 @@ type currentRoomStateStatements struct {
}
func NewSqliteCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) {
s := &currentRoomStateStatements{}
s := &currentRoomStateStatements{
db: db,
}
_, err := db.Exec(currentRoomStateSchema)
if err != nil {
return nil, err
@ -100,10 +102,10 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership(
ctx context.Context,
txn *sql.Tx,
userID string,
membershipEnum int,
membership string,
) ([]string, error) {
stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt)
rows, err := stmt.QueryContext(ctx, userID, membershipEnum)
rows, err := stmt.QueryContext(ctx, userID, membership)
if err != nil {
return nil, err
}
@ -130,7 +132,7 @@ func (s *currentRoomStateStatements) DeleteRoomStateByEventID(
func (s *currentRoomStateStatements) UpsertRoomState(
ctx context.Context, txn *sql.Tx,
event gomatrixserverlib.HeaderedEvent, membershipEnum int,
event gomatrixserverlib.HeaderedEvent, contentVal string,
) error {
headeredJSON, err := json.Marshal(event)
if err != nil {
@ -147,7 +149,7 @@ func (s *currentRoomStateStatements) UpsertRoomState(
event.Sender(),
*event.StateKey(),
headeredJSON,
membershipEnum,
contentVal,
)
return err
}
@ -199,3 +201,76 @@ func (s *currentRoomStateStatements) SelectStateEvent(
}
return &ev, err
}
func (s *currentRoomStateStatements) SelectBulkStateContent(
ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool,
) ([]tables.StrippedEvent, error) {
hasWildcards := false
eventTypeSet := make(map[string]bool)
stateKeySet := make(map[string]bool)
var eventTypes []string
var stateKeys []string
for _, tuple := range tuples {
if !eventTypeSet[tuple.EventType] {
eventTypeSet[tuple.EventType] = true
eventTypes = append(eventTypes, tuple.EventType)
}
if !stateKeySet[tuple.StateKey] {
stateKeySet[tuple.StateKey] = true
stateKeys = append(stateKeys, tuple.StateKey)
}
if tuple.StateKey == "*" {
hasWildcards = true
}
}
iRoomIDs := make([]interface{}, len(roomIDs))
for i, v := range roomIDs {
iRoomIDs[i] = v
}
iEventTypes := make([]interface{}, len(eventTypes))
for i, v := range eventTypes {
iEventTypes[i] = v
}
iStateKeys := make([]interface{}, len(stateKeys))
for i, v := range stateKeys {
iStateKeys[i] = v
}
var query string
var args []interface{}
if hasWildcards && allowWildcards {
query = strings.Replace(selectBulkStateContentWildSQL, "($1)", sqlutil.QueryVariadic(len(iRoomIDs)), 1)
query = strings.Replace(query, "($2)", sqlutil.QueryVariadicOffset(len(iEventTypes), len(iRoomIDs)), 1)
args = append(iRoomIDs, iEventTypes...)
} else {
query = strings.Replace(selectBulkStateContentSQL, "($1)", sqlutil.QueryVariadic(len(iRoomIDs)), 1)
query = strings.Replace(query, "($2)", sqlutil.QueryVariadicOffset(len(iEventTypes), len(iRoomIDs)), 1)
query = strings.Replace(query, "($3)", sqlutil.QueryVariadicOffset(len(iStateKeys), len(iEventTypes)+len(iRoomIDs)), 1)
args = append(iRoomIDs, iEventTypes...)
args = append(args, iStateKeys...)
}
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
strippedEvents := []tables.StrippedEvent{}
defer internal.CloseAndLogIfError(ctx, rows, "SelectBulkStateContent: rows.close() failed")
for rows.Next() {
var roomID string
var eventType string
var stateKey string
var contentVal string
if err = rows.Scan(&roomID, &eventType, &stateKey, &contentVal); err != nil {
return nil, err
}
strippedEvents = append(strippedEvents, tables.StrippedEvent{
RoomID: roomID,
ContentValue: contentVal,
EventType: eventType,
StateKey: stateKey,
})
}
return strippedEvents, rows.Err()
}

65
currentstateserver/storage/tables/interface.go
View file

@ -19,26 +19,61 @@ import (
"database/sql"
"github.com/matrix-org/gomatrixserverlib"
"github.com/tidwall/gjson"
)
var MembershipToEnum = map[string]int{
gomatrixserverlib.Invite: 1,
gomatrixserverlib.Join: 2,
gomatrixserverlib.Leave: 3,
gomatrixserverlib.Ban: 4,
}
var EnumToMembership = map[int]string{
1: gomatrixserverlib.Invite,
2: gomatrixserverlib.Join,
3: gomatrixserverlib.Leave,
4: gomatrixserverlib.Ban,
}
type CurrentRoomState interface {
SelectStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error)
SelectEventsWithEventIDs(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error)
UpsertRoomState(ctx context.Context, txn *sql.Tx, event gomatrixserverlib.HeaderedEvent, membershipEnum int) error
// UpsertRoomState stores the given event in the database, along with an extracted piece of content.
// The piece of content will vary depending on the event type, and table implementations may use this information to optimise
// lookups e.g membership lookups. The mapped value of `contentVal` is outlined in ExtractContentValue. An empty `contentVal`
// means there is nothing to store for this field.
UpsertRoomState(ctx context.Context, txn *sql.Tx, event gomatrixserverlib.HeaderedEvent, contentVal string) error
DeleteRoomStateByEventID(ctx context.Context, txn *sql.Tx, eventID string) error
// SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state.
SelectRoomIDsWithMembership(ctx context.Context, txn *sql.Tx, userID string, membershipEnum int) ([]string, error)
SelectRoomIDsWithMembership(ctx context.Context, txn *sql.Tx, userID string, membership string) ([]string, error)
SelectBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]StrippedEvent, error)
}
// StrippedEvent represents a stripped event for returning extracted content values.
type StrippedEvent struct {
RoomID string
EventType string
StateKey string
ContentValue string
}
// ExtractContentValue from the given state event. For example, given an m.room.name event with:
// content: { name: "Foo" }
// this returns "Foo".
func ExtractContentValue(ev *gomatrixserverlib.HeaderedEvent) string {
content := ev.Content()
key := ""
switch ev.Type() {
case gomatrixserverlib.MRoomCreate:
key = "creator"
case gomatrixserverlib.MRoomCanonicalAlias:
key = "alias"
case gomatrixserverlib.MRoomHistoryVisibility:
key = "history_visibility"
case gomatrixserverlib.MRoomJoinRules:
key = "join_rule"
case gomatrixserverlib.MRoomMember:
key = "membership"
case gomatrixserverlib.MRoomName:
key = "name"
case "m.room.avatar":
key = "url"
case "m.room.topic":
key = "topic"
case "m.room.guest_access":
key = "guest_access"
}
result := gjson.GetBytes(content, key)
if !result.Exists() {
return ""
}
// this returns the empty string if this is not a string type
return result.Str
}

4
federationapi/federationapi.go
View file

@ -16,6 +16,7 @@ package federationapi
import (
"github.com/gorilla/mux"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
@ -36,11 +37,12 @@ func AddPublicRoutes(
rsAPI roomserverAPI.RoomserverInternalAPI,
federationSenderAPI federationSenderAPI.FederationSenderInternalAPI,
eduAPI eduserverAPI.EDUServerInputAPI,
stateAPI currentstateAPI.CurrentStateInternalAPI,
) {
routing.Setup(
router, cfg, rsAPI,
eduAPI, federationSenderAPI, keyRing,
federation, userAPI,
federation, userAPI, stateAPI,
)
}

2
federationapi/federationapi_test.go
View file

@ -31,7 +31,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
fsAPI := base.FederationSenderHTTPClient()
// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil)
federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil, nil)
httputil.SetupHTTPAPI(
base.BaseMux,
base.PublicAPIMux,

178
federationapi/routing/publicrooms.go Normal file
View file

@ -0,0 +1,178 @@
package routing
import (
"context"
"net/http"
"strconv"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
type PublicRoomReq struct {
Since string `json:"since,omitempty"`
Limit int16 `json:"limit,omitempty"`
Filter filter `json:"filter,omitempty"`
}
type filter struct {
SearchTerms string `json:"generic_search_term,omitempty"`
}
// GetPostPublicRooms implements GET and POST /publicRooms
func GetPostPublicRooms(req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI) util.JSONResponse {
var request PublicRoomReq
if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil {
return *fillErr
}
if request.Limit == 0 {
request.Limit = 50
}
response, err := publicRooms(req.Context(), request, rsAPI, stateAPI)
if err != nil {
return jsonerror.InternalServerError()
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: response,
}
}
func publicRooms(ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.RoomserverInternalAPI,
stateAPI currentstateAPI.CurrentStateInternalAPI) (*gomatrixserverlib.RespPublicRooms, error) {
var response gomatrixserverlib.RespPublicRooms
var limit int16
var offset int64
limit = request.Limit
offset, err := strconv.ParseInt(request.Since, 10, 64)
// ParseInt returns 0 and an error when trying to parse an empty string
// In that case, we want to assign 0 so we ignore the error
if err != nil && len(request.Since) > 0 {
util.GetLogger(ctx).WithError(err).Error("strconv.ParseInt failed")
return nil, err
}
var queryRes roomserverAPI.QueryPublishedRoomsResponse
err = rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed")
return nil, err
}
response.TotalRoomCountEstimate = len(queryRes.RoomIDs)
if offset > 0 {
response.PrevBatch = strconv.Itoa(int(offset) - 1)
}
nextIndex := int(offset) + int(limit)
if response.TotalRoomCountEstimate > nextIndex {
response.NextBatch = strconv.Itoa(nextIndex)
}
if offset < 0 {
offset = 0
}
if nextIndex > len(queryRes.RoomIDs) {
nextIndex = len(queryRes.RoomIDs)
}
roomIDs := queryRes.RoomIDs[offset:nextIndex]
response.Chunk, err = fillInRooms(ctx, roomIDs, stateAPI)
return &response, err
}
// fillPublicRoomsReq fills the Limit, Since and Filter attributes of a GET or POST request
// on /publicRooms by parsing the incoming HTTP request
// Filter is only filled for POST requests
func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSONResponse {
if httpReq.Method == http.MethodGet {
limit, err := strconv.Atoi(httpReq.FormValue("limit"))
// Atoi returns 0 and an error when trying to parse an empty string
// In that case, we want to assign 0 so we ignore the error
if err != nil && len(httpReq.FormValue("limit")) > 0 {
util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed")
reqErr := jsonerror.InternalServerError()
return &reqErr
}
request.Limit = int16(limit)
request.Since = httpReq.FormValue("since")
return nil
} else if httpReq.Method == http.MethodPost {
return httputil.UnmarshalJSONRequest(httpReq, request)
}
return &util.JSONResponse{
Code: http.StatusMethodNotAllowed,
JSON: jsonerror.NotFound("Bad method"),
}
}
// due to lots of switches
// nolint:gocyclo
func fillInRooms(ctx context.Context, roomIDs []string, stateAPI currentstateAPI.CurrentStateInternalAPI) ([]gomatrixserverlib.PublicRoom, error) {
avatarTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.avatar", StateKey: ""}
nameTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.name", StateKey: ""}
canonicalTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCanonicalAlias, StateKey: ""}
topicTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.topic", StateKey: ""}
guestTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.guest_access", StateKey: ""}
visibilityTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomHistoryVisibility, StateKey: ""}
joinRuleTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""}
var stateRes currentstateAPI.QueryBulkStateContentResponse
err := stateAPI.QueryBulkStateContent(ctx, &currentstateAPI.QueryBulkStateContentRequest{
RoomIDs: roomIDs,
AllowWildcards: true,
StateTuples: []gomatrixserverlib.StateKeyTuple{
nameTuple, canonicalTuple, topicTuple, guestTuple, visibilityTuple, joinRuleTuple, avatarTuple,
{EventType: gomatrixserverlib.MRoomMember, StateKey: "*"},
},
}, &stateRes)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("QueryBulkStateContent failed")
return nil, err
}
util.GetLogger(ctx).Infof("room IDs: %+v", roomIDs)
util.GetLogger(ctx).Infof("State res: %+v", stateRes.Rooms)
chunk := make([]gomatrixserverlib.PublicRoom, len(roomIDs))
i := 0
for roomID, data := range stateRes.Rooms {
pub := gomatrixserverlib.PublicRoom{
RoomID: roomID,
}
joinCount := 0
var joinRule, guestAccess string
for tuple, contentVal := range data {
if tuple.EventType == gomatrixserverlib.MRoomMember && contentVal == "join" {
joinCount++
continue
}
switch tuple {
case avatarTuple:
pub.AvatarURL = contentVal
case nameTuple:
pub.Name = contentVal
case topicTuple:
pub.Topic = contentVal
case canonicalTuple:
pub.CanonicalAlias = contentVal
case visibilityTuple:
pub.WorldReadable = contentVal == "world_readable"
// need both of these to determine whether guests can join
case joinRuleTuple:
joinRule = contentVal
case guestTuple:
guestAccess = contentVal
}
}
if joinRule == gomatrixserverlib.Public && guestAccess == "can_join" {
pub.GuestCanJoin = true
}
pub.JoinedMembersCount = joinCount
chunk[i] = pub
i++
}
return chunk, nil
}

8
federationapi/routing/routing.go
View file

@ -19,6 +19,7 @@ import (
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
@ -52,6 +53,7 @@ func Setup(
keys gomatrixserverlib.JSONVerifier,
federation *gomatrixserverlib.FederationClient,
userAPI userapi.UserInternalAPI,
stateAPI currentstateAPI.CurrentStateInternalAPI,
) {
v2keysmux := publicAPIMux.PathPrefix(pathPrefixV2Keys).Subrouter()
v1fedmux := publicAPIMux.PathPrefix(pathPrefixV1Federation).Subrouter()
@ -291,4 +293,10 @@ func Setup(
return Backfill(httpReq, request, rsAPI, vars["roomID"], cfg)
},
)).Methods(http.MethodGet)
v1fedmux.Handle("/publicRooms",
httputil.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse {
return GetPostPublicRooms(req, rsAPI, stateAPI)
}),
).Methods(http.MethodGet)
}

15
federationapi/routing/send_test.go
View file

@ -111,6 +111,13 @@ func (t *testRoomserverAPI) PerformJoin(
) {
}
func (t *testRoomserverAPI) PerformPublish(
ctx context.Context,
req *api.PerformPublishRequest,
res *api.PerformPublishResponse,
) {
}
func (t *testRoomserverAPI) PerformLeave(
ctx context.Context,
req *api.PerformLeaveRequest,
@ -168,6 +175,14 @@ func (t *testRoomserverAPI) QueryMembershipForUser(
return fmt.Errorf("not implemented")
}
func (t *testRoomserverAPI) QueryPublishedRooms(
ctx context.Context,
request *api.QueryPublishedRoomsRequest,
response *api.QueryPublishedRoomsResponse,
) error {
return fmt.Errorf("not implemented")
}
// Query a list of membership events for a room
func (t *testRoomserverAPI) QueryMembershipsForRoom(
ctx context.Context,

2
go.mod
View file

@ -20,7 +20,7 @@ require (
github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328
github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe
github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f
github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
github.com/mattn/go-sqlite3 v2.0.2+incompatible

2
go.sum
View file

@ -379,6 +379,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200625170349-8ebb44e6775d h1:v1
github.com/matrix-org/gomatrixserverlib v0.0.0-20200625170349-8ebb44e6775d/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328 h1:rz6aiTpUyNPRcWZBWUGDkQjI7lfeLdhzy+x/Pw2jha8=
github.com/matrix-org/gomatrixserverlib v0.0.0-20200626111150-364501214328/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe h1:rCjG+azihYsO+EIdm//Zx5gQ7hzeJVraeSukLsW1Mic=
github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y=
github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go=
github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=

12
internal/setup/monolith.go
View file

@ -27,7 +27,6 @@ import (
"github.com/matrix-org/dendrite/internal/transactions"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/mediaapi"
"github.com/matrix-org/dendrite/publicroomsapi"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/matrix-org/dendrite/publicroomsapi/types"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
@ -81,13 +80,14 @@ func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) {
federationapi.AddPublicRoutes(
publicMux, m.Config, m.UserAPI, m.FedClient,
m.KeyRing, m.RoomserverAPI, m.FederationSenderAPI,
m.EDUInternalAPI,
m.EDUInternalAPI, m.StateAPI,
)
mediaapi.AddPublicRoutes(publicMux, m.Config, m.UserAPI, m.Client)
publicroomsapi.AddPublicRoutes(
publicMux, m.Config, m.KafkaConsumer, m.UserAPI, m.PublicRoomsDB, m.RoomserverAPI, m.FedClient,
m.ExtPublicRoomsProvider,
)
/*
publicroomsapi.AddPublicRoutes(
publicMux, m.Config, m.KafkaConsumer, m.UserAPI, m.PublicRoomsDB, m.RoomserverAPI, m.FedClient,
m.ExtPublicRoomsProvider,
) */
syncapi.AddPublicRoutes(
publicMux, m.KafkaConsumer, m.UserAPI, m.RoomserverAPI, m.FedClient, m.Config,
)

12
roomserver/api/api.go
View file

@ -36,6 +36,18 @@ type RoomserverInternalAPI interface {
res *PerformLeaveResponse,
) error
PerformPublish(
ctx context.Context,
req *PerformPublishRequest,
res *PerformPublishResponse,
)
QueryPublishedRooms(
ctx context.Context,
req *QueryPublishedRoomsRequest,
res *QueryPublishedRoomsResponse,
) error
// Query the latest events and state for a room from the room server.
QueryLatestEventsAndState(
ctx context.Context,

19
roomserver/api/api_trace.go
View file

@ -57,6 +57,25 @@ func (t *RoomserverInternalAPITrace) PerformLeave(
return err
}
func (t *RoomserverInternalAPITrace) PerformPublish(
ctx context.Context,
req *PerformPublishRequest,
res *PerformPublishResponse,
) {
t.Impl.PerformPublish(ctx, req, res)
util.GetLogger(ctx).Infof("PerformPublish req=%+v res=%+v", js(req), js(res))
}
func (t *RoomserverInternalAPITrace) QueryPublishedRooms(
ctx context.Context,
req *QueryPublishedRoomsRequest,
res *QueryPublishedRoomsResponse,
) error {
err := t.Impl.QueryPublishedRooms(ctx, req, res)
util.GetLogger(ctx).WithError(err).Infof("QueryPublishedRooms req=%+v res=%+v", js(req), js(res))
return err
}
func (t *RoomserverInternalAPITrace) QueryLatestEventsAndState(
ctx context.Context,
req *QueryLatestEventsAndStateRequest,

10
roomserver/api/perform.go
View file

@ -136,3 +136,13 @@ type PerformBackfillResponse struct {
// Missing events, arbritrary order.
Events []gomatrixserverlib.HeaderedEvent `json:"events"`
}
type PerformPublishRequest struct {
RoomID string
Visibility string
}
type PerformPublishResponse struct {
// If non-nil, the publish request failed. Contains more information why it failed.
Error *PerformError
}

10
roomserver/api/query.go
View file

@ -215,3 +215,13 @@ type QueryRoomVersionForRoomRequest struct {
type QueryRoomVersionForRoomResponse struct {
RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"`
}
type QueryPublishedRoomsRequest struct {
// Optional. If specified, returns whether this room is published or not.
RoomID string
}
type QueryPublishedRoomsResponse struct {
// The list of published rooms.
RoomIDs []string
}

20
roomserver/internal/perform_publish.go Normal file
View file

@ -0,0 +1,20 @@
package internal
import (
"context"
"github.com/matrix-org/dendrite/roomserver/api"
)
func (r *RoomserverInternalAPI) PerformPublish(
ctx context.Context,
req *api.PerformPublishRequest,
res *api.PerformPublishResponse,
) {
err := r.DB.PublishRoom(ctx, req.RoomID, req.Visibility == "public")
if err != nil {
res.Error = &api.PerformError{
Msg: err.Error(),
}
}
}

13
roomserver/internal/query.go
View file

@ -930,3 +930,16 @@ func (r *RoomserverInternalAPI) QueryRoomVersionForRoom(
r.Cache.StoreRoomVersion(request.RoomID, response.RoomVersion)
return nil
}
func (r *RoomserverInternalAPI) QueryPublishedRooms(
ctx context.Context,
req *api.QueryPublishedRoomsRequest,
res *api.QueryPublishedRoomsResponse,
) error {
rooms, err := r.DB.GetPublishedRooms(ctx)
if err != nil {
return err
}
res.RoomIDs = rooms
return nil
}

31
roomserver/inthttp/client.go
View file

@ -29,6 +29,7 @@ const (
RoomserverPerformJoinPath = "/roomserver/performJoin"
RoomserverPerformLeavePath = "/roomserver/performLeave"
RoomserverPerformBackfillPath = "/roomserver/performBackfill"
RoomserverPerformPublishPath = "/roomserver/performPublish"
// Query operations
RoomserverQueryLatestEventsAndStatePath = "/roomserver/queryLatestEventsAndState"
@ -41,6 +42,7 @@ const (
RoomserverQueryStateAndAuthChainPath = "/roomserver/queryStateAndAuthChain"
RoomserverQueryRoomVersionCapabilitiesPath = "/roomserver/queryRoomVersionCapabilities"
RoomserverQueryRoomVersionForRoomPath = "/roomserver/queryRoomVersionForRoom"
RoomserverQueryPublishedRoomsPath = "/roomserver/queryPublishedRooms"
)
type httpRoomserverInternalAPI struct {
@ -194,6 +196,23 @@ func (h *httpRoomserverInternalAPI) PerformLeave(
return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
}
func (h *httpRoomserverInternalAPI) PerformPublish(
ctx context.Context,
req *api.PerformPublishRequest,
res *api.PerformPublishResponse,
) {
span, ctx := opentracing.StartSpanFromContext(ctx, "PerformPublish")
defer span.Finish()
apiURL := h.roomserverURL + RoomserverPerformPublishPath
err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res)
if err != nil {
res.Error = &api.PerformError{
Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err),
}
}
}
// QueryLatestEventsAndState implements RoomserverQueryAPI
func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState(
ctx context.Context,
@ -233,6 +252,18 @@ func (h *httpRoomserverInternalAPI) QueryEventsByID(
return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
}
func (h *httpRoomserverInternalAPI) QueryPublishedRooms(
ctx context.Context,
request *api.QueryPublishedRoomsRequest,
response *api.QueryPublishedRoomsResponse,
) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "QueryPublishedRooms")
defer span.Finish()
apiURL := h.roomserverURL + RoomserverQueryPublishedRoomsPath
return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
}
// QueryMembershipForUser implements RoomserverQueryAPI
func (h *httpRoomserverInternalAPI) QueryMembershipForUser(
ctx context.Context,

25
roomserver/inthttp/server.go
View file

@ -61,6 +61,31 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) {
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
)
internalAPIMux.Handle(RoomserverPerformPublishPath,
httputil.MakeInternalAPI("performPublish", func(req *http.Request) util.JSONResponse {
var request api.PerformPublishRequest
var response api.PerformPublishResponse
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
return util.MessageResponse(http.StatusBadRequest, err.Error())
}
r.PerformPublish(req.Context(), &request, &response)
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
)
internalAPIMux.Handle(
RoomserverQueryPublishedRoomsPath,
httputil.MakeInternalAPI("queryPublishedRooms", func(req *http.Request) util.JSONResponse {
var request api.QueryPublishedRoomsRequest
var response api.QueryPublishedRoomsResponse
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
return util.ErrorResponse(err)
}
if err := r.QueryPublishedRooms(req.Context(), &request, &response); err != nil {
return util.ErrorResponse(err)
}
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
)
internalAPIMux.Handle(
RoomserverQueryLatestEventsAndStatePath,
httputil.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse {

4
roomserver/storage/interface.go
View file

@ -139,4 +139,8 @@ type Database interface {
EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error)
// Look up the room version for a given room.
GetRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error)
// Publish or unpublish a room from the room directory.
PublishRoom(ctx context.Context, roomID string, publish bool) error
// Returns a list of room IDs for rooms which are published.
GetPublishedRooms(ctx context.Context) ([]string, error)
}

101
roomserver/storage/postgres/published_table.go Normal file
View file

@ -0,0 +1,101 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgres
import (
"context"
"database/sql"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/roomserver/storage/shared"
"github.com/matrix-org/dendrite/roomserver/storage/tables"
)
const publishedSchema = `
-- Stores which rooms are published in the room directory
CREATE TABLE IF NOT EXISTS roomserver_published (
-- The room ID of the room
room_id TEXT NOT NULL PRIMARY KEY,
-- Whether it is published or not
published BOOLEAN NOT NULL DEFAULT false
);
`
const upsertPublishedSQL = "" +
"INSERT INTO roomserver_published (room_id, published) VALUES ($1, $2) " +
"ON CONFLICT (room_id) DO UPDATE SET published=$2"
const selectAllPublishedSQL = "" +
"SELECT room_id FROM roomserver_published WHERE published = $1 ORDER BY room_id ASC"
const selectPublishedSQL = "" +
"SELECT published FROM roomserver_published WHERE room_id = $1"
type publishedStatements struct {
upsertPublishedStmt *sql.Stmt
selectAllPublishedStmt *sql.Stmt
selectPublishedStmt *sql.Stmt
}
func NewPostgresPublishedTable(db *sql.DB) (tables.Published, error) {
s := &publishedStatements{}
_, err := db.Exec(publishedSchema)
if err != nil {
return nil, err
}
return s, shared.StatementList{
{&s.upsertPublishedStmt, upsertPublishedSQL},
{&s.selectAllPublishedStmt, selectAllPublishedSQL},
{&s.selectPublishedStmt, selectPublishedSQL},
}.Prepare(db)
}
func (s *publishedStatements) UpsertRoomPublished(
ctx context.Context, roomID string, published bool,
) (err error) {
_, err = s.upsertPublishedStmt.ExecContext(ctx, roomID, published)
return
}
func (s *publishedStatements) SelectPublishedFromRoomID(
ctx context.Context, roomID string,
) (published bool, err error) {
err = s.selectPublishedStmt.QueryRowContext(ctx, roomID).Scan(&published)
if err == sql.ErrNoRows {
return false, nil
}
return
}
func (s *publishedStatements) SelectAllPublishedRooms(
ctx context.Context, published bool,
) ([]string, error) {
rows, err := s.selectAllPublishedStmt.QueryContext(ctx, published)
if err != nil {
return nil, err
}
defer internal.CloseAndLogIfError(ctx, rows, "selectAllPublishedStmt: rows.close() failed")
var roomIDs []string
for rows.Next() {
var roomID string
if err = rows.Scan(&roomID); err != nil {
return nil, err
}
roomIDs = append(roomIDs, roomID)
}
return roomIDs, rows.Err()
}

5
roomserver/storage/postgres/storage.go
View file

@ -87,6 +87,10 @@ func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database,
if err != nil {
return nil, err
}
published, err := NewPostgresPublishedTable(db)
if err != nil {
return nil, err
}
d.Database = shared.Database{
DB: db,
EventTypesTable: eventTypes,
@ -101,6 +105,7 @@ func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database,
RoomAliasesTable: roomAliases,
InvitesTable: invites,
MembershipTable: membership,
PublishedTable: published,
}
return &d, nil
}

9
roomserver/storage/shared/storage.go
View file

@ -26,6 +26,7 @@ type Database struct {
PrevEventsTable tables.PreviousEvents
InvitesTable tables.Invites
MembershipTable tables.Membership
PublishedTable tables.Published
}
func (d *Database) EventTypeNIDs(
@ -420,6 +421,14 @@ func (d *Database) StoreEvent(
}, nil
}
func (d *Database) PublishRoom(ctx context.Context, roomID string, publish bool) error {
return d.PublishedTable.UpsertRoomPublished(ctx, roomID, publish)
}
func (d *Database) GetPublishedRooms(ctx context.Context) ([]string, error) {
return d.PublishedTable.SelectAllPublishedRooms(ctx, true)
}
func (d *Database) assignRoomNID(
ctx context.Context, txn *sql.Tx,
roomID string, roomVersion gomatrixserverlib.RoomVersion,

100
roomserver/storage/sqlite3/published_table.go Normal file
View file

@ -0,0 +1,100 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sqlite3
import (
"context"
"database/sql"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/roomserver/storage/shared"
"github.com/matrix-org/dendrite/roomserver/storage/tables"
)
const publishedSchema = `
-- Stores which rooms are published in the room directory
CREATE TABLE IF NOT EXISTS roomserver_published (
-- The room ID of the room
room_id TEXT NOT NULL PRIMARY KEY,
-- Whether it is published or not
published BOOLEAN NOT NULL DEFAULT false
);
`
const upsertPublishedSQL = "" +
"INSERT OR REPLACE INTO roomserver_published (room_id, published) VALUES ($1, $2)"
const selectAllPublishedSQL = "" +
"SELECT room_id FROM roomserver_published WHERE published = $1 ORDER BY room_id ASC"
const selectPublishedSQL = "" +
"SELECT published FROM roomserver_published WHERE room_id = $1"
type publishedStatements struct {
upsertPublishedStmt *sql.Stmt
selectAllPublishedStmt *sql.Stmt
selectPublishedStmt *sql.Stmt
}
func NewSqlitePublishedTable(db *sql.DB) (tables.Published, error) {
s := &publishedStatements{}
_, err := db.Exec(publishedSchema)
if err != nil {
return nil, err
}
return s, shared.StatementList{
{&s.upsertPublishedStmt, upsertPublishedSQL},
{&s.selectAllPublishedStmt, selectAllPublishedSQL},
{&s.selectPublishedStmt, selectPublishedSQL},
}.Prepare(db)
}
func (s *publishedStatements) UpsertRoomPublished(
ctx context.Context, roomID string, published bool,
) (err error) {
_, err = s.upsertPublishedStmt.ExecContext(ctx, roomID, published)
return
}
func (s *publishedStatements) SelectPublishedFromRoomID(
ctx context.Context, roomID string,
) (published bool, err error) {
err = s.selectPublishedStmt.QueryRowContext(ctx, roomID).Scan(&published)
if err == sql.ErrNoRows {
return false, nil
}
return
}
func (s *publishedStatements) SelectAllPublishedRooms(
ctx context.Context, published bool,
) ([]string, error) {
rows, err := s.selectAllPublishedStmt.QueryContext(ctx, published)
if err != nil {
return nil, err
}
defer internal.CloseAndLogIfError(ctx, rows, "selectAllPublishedStmt: rows.close() failed")
var roomIDs []string
for rows.Next() {
var roomID string
if err = rows.Scan(&roomID); err != nil {
return nil, err
}
roomIDs = append(roomIDs, roomID)
}
return roomIDs, rows.Err()
}

5
roomserver/storage/sqlite3/storage.go
View file

@ -110,6 +110,10 @@ func Open(dataSourceName string) (*Database, error) {
if err != nil {
return nil, err
}
published, err := NewSqlitePublishedTable(d.db)
if err != nil {
return nil, err
}
d.Database = shared.Database{
DB: d.db,
EventsTable: d.events,
@ -124,6 +128,7 @@ func Open(dataSourceName string) (*Database, error) {
RoomAliasesTable: roomAliases,
InvitesTable: d.invites,
MembershipTable: d.membership,
PublishedTable: published,
}
return &d, nil
}

6
roomserver/storage/tables/interface.go
View file

@ -120,3 +120,9 @@ type Membership interface {
SelectMembershipsFromRoomAndMembership(ctx context.Context, roomNID types.RoomNID, membership MembershipState, localOnly bool) (eventNIDs []types.EventNID, err error)
UpdateMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, senderUserNID types.EventStateKeyNID, membership MembershipState, eventNID types.EventNID) error
}
type Published interface {
UpsertRoomPublished(ctx context.Context, roomID string, published bool) (err error)
SelectPublishedFromRoomID(ctx context.Context, roomID string) (published bool, err error)
SelectAllPublishedRooms(ctx context.Context, published bool) ([]string, error)
}

4
sytest-whitelist
View file

@ -181,7 +181,11 @@ Outbound federation can query profile data
/event/ on joined room works
/event/ does not allow access to events before the user joined
Federation key API allows unsigned requests for keys
GET /publicRooms lists rooms
GET /publicRooms includes avatar URLs
Can paginate public room list
GET /publicRooms lists newly-created room
Name/topic keys are correct
GET /directory/room/:room_alias yields room ID
PUT /directory/room/:room_alias creates alias
Room aliases can contain Unicode