diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index f59f80675..239b0ebf4 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -91,6 +91,7 @@ snd PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id get GET /rooms/:room_id/messages returns a message get GET /rooms/:room_id/messages lazy loads members correctly typ PUT /rooms/:room_id/typing/:user_id sets typing notification +typ Typing notifications don't leak (3 subtests) rst GET /rooms/:room_id/state/m.room.power_levels can fetch levels rst PUT /rooms/:room_id/state/m.room.power_levels can set levels rst PUT power_levels should not explode if the old power levels were empty @@ -853,4 +854,8 @@ fme Outbound federation will ignore a missing event with bad JSON for room versi fbk Outbound federation rejects backfill containing invalid JSON for events in room version 6 jso Invalid JSON integers jso Invalid JSON floats -jso Invalid JSON special values \ No newline at end of file +jso Invalid JSON special values +inv Can invite users to invite-only rooms (2 subtests) +plv setting 'm.room.name' respects room powerlevel (2 subtests) +psh Messages that notify from another user increment notification_count +psh Messages that org.matrix.msc2625.mark_unread from another user increment org.matrix.msc2625.unread_count diff --git a/are-we-synapse-yet.py b/are-we-synapse-yet.py index 30979a129..8cd7ec9fb 100755 --- a/are-we-synapse-yet.py +++ b/are-we-synapse-yet.py @@ -159,6 +159,8 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose): total_tests = 0 for gid, tests in gid_to_tests.items(): group_total = len(tests) + if group_total == 0: + continue group_passing = 0 test_names_and_marks = [] for name, passing in tests.items(): diff --git a/build/docker/config/dendrite-config.yaml b/build/docker/config/dendrite-config.yaml index 53d9f7b02..c8302c0d9 100644 --- a/build/docker/config/dendrite-config.yaml +++ b/build/docker/config/dendrite-config.yaml @@ -98,7 +98,7 @@ database: room_server: "postgres://dendrite:itsasecret@postgres/dendrite_roomserver?sslmode=disable" server_key: "postgres://dendrite:itsasecret@postgres/dendrite_serverkey?sslmode=disable" federation_sender: "postgres://dendrite:itsasecret@postgres/dendrite_federationsender?sslmode=disable" - public_rooms_api: "postgres://dendrite:itsasecret@postgres/dendrite_publicroomsapi?sslmode=disable" + current_state: "postgres://dendrite:itsasecret@postgres/dendrite_currentstate?sslmode=disable" appservice: "postgres://dendrite:itsasecret@postgres/dendrite_appservice?sslmode=disable" # If using naffka you need to specify a naffka database #naffka: "postgres://dendrite:itsasecret@postgres/dendrite_naffka?sslmode=disable" @@ -113,7 +113,7 @@ listen: server_key_api: "server_key_api:7778" sync_api: "sync_api:7773" media_api: "media_api:7774" - public_rooms_api: "public_rooms_api:7775" + current_state_server: "current_state_server:7775" federation_sender: "federation_sender:7776" edu_server: "edu_server:7777" key_server: "key_server:7779" diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index d424d43b1..62ca6763d 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -7,8 +7,7 @@ services: "--bind-address=:8008", "--client-api-server-url=http://client_api:7771", "--sync-api-server-url=http://sync_api:7773", - "--media-api-server-url=http://media_api:7774", - "--public-rooms-api-server-url=http://public_rooms_api:7775" + "--media-api-server-url=http://media_api:7774" ] volumes: - ./config:/etc/dendrite @@ -18,7 +17,6 @@ services: - sync_api - client_api - media_api - - public_rooms_api ports: - "8008:8008" @@ -45,9 +43,9 @@ services: networks: - internal - public_rooms_api: - hostname: public_rooms_api - image: matrixdotorg/dendrite:publicroomsapi + current_state_server: + hostname: current_state_server + image: matrixdotorg/dendrite:currentstateserver command: [ "--config=dendrite.yaml" ] diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index 9ee5a09de..443f30920 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -15,7 +15,7 @@ docker build -t matrixdotorg/dendrite:federationsender --build-arg component=de docker build -t matrixdotorg/dendrite:federationproxy --build-arg component=federation-api-proxy -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:keyserver --build-arg component=dendrite-key-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:publicroomsapi --build-arg component=dendrite-public-rooms-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:currentstateserver --build-arg component=dendrite-current-state-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:serverkeyapi --build-arg component=dendrite-server-key-api-server -f build/docker/Dockerfile.component . diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh index da08a7325..b4a4b2fce 100755 --- a/build/docker/images-pull.sh +++ b/build/docker/images-pull.sh @@ -11,7 +11,7 @@ docker pull matrixdotorg/dendrite:federationsender docker pull matrixdotorg/dendrite:federationproxy docker pull matrixdotorg/dendrite:keyserver docker pull matrixdotorg/dendrite:mediaapi -docker pull matrixdotorg/dendrite:publicroomsapi +docker pull matrixdotorg/dendrite:currentstateserver docker pull matrixdotorg/dendrite:roomserver docker pull matrixdotorg/dendrite:syncapi docker pull matrixdotorg/dendrite:userapi diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index 1ac60b921..ec1e860f0 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -11,7 +11,7 @@ docker push matrixdotorg/dendrite:federationsender docker push matrixdotorg/dendrite:federationproxy docker push matrixdotorg/dendrite:keyserver docker push matrixdotorg/dendrite:mediaapi -docker push matrixdotorg/dendrite:publicroomsapi +docker push matrixdotorg/dendrite:currentstateserver docker push matrixdotorg/dendrite:roomserver docker push matrixdotorg/dendrite:syncapi docker push matrixdotorg/dendrite:serverkeyapi diff --git a/build/docker/postgres/create_db.sh b/build/docker/postgres/create_db.sh index 8ed11db1e..a884b5ccd 100644 --- a/build/docker/postgres/create_db.sh +++ b/build/docker/postgres/create_db.sh @@ -1,5 +1,5 @@ #!/bin/bash -for db in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi appservice naffka; do +for db in account device mediaapi syncapi roomserver serverkey federationsender currentstate appservice naffka; do createdb -U dendrite -O dendrite dendrite_$db done diff --git a/build/gobind/build.sh b/build/gobind/build.sh index 3a80d374a..aa2cdfc5a 100644 --- a/build/gobind/build.sh +++ b/build/gobind/build.sh @@ -1,6 +1,6 @@ #!/bin/sh gomobile bind -v \ - -ldflags "-X $github.com/yggdrasil-network/yggdrasil-go/src/version.buildName=riot-ios-p2p" \ + -ldflags "-X github.com/yggdrasil-network/yggdrasil-go/src/version.buildName=dendrite" \ -target ios \ github.com/matrix-org/dendrite/build/gobind \ No newline at end of file diff --git a/build/gobind/monolith.go b/build/gobind/monolith.go index 750babad8..ea9c62f20 100644 --- a/build/gobind/monolith.go +++ b/build/gobind/monolith.go @@ -11,13 +11,13 @@ import ( "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn" + "github.com/matrix-org/dendrite/currentstateserver" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" - "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" @@ -25,6 +25,7 @@ import ( ) type DendriteMonolith struct { + YggdrasilNode *yggconn.Node StorageDirectory string listener net.Listener } @@ -33,6 +34,10 @@ func (m *DendriteMonolith) BaseURL() string { return fmt.Sprintf("http://%s", m.listener.Addr().String()) } +func (m *DendriteMonolith) PeerCount() int { + return m.YggdrasilNode.PeerCount() +} + func (m *DendriteMonolith) Start() { logger := logrus.Logger{ Out: BindLogger{}, @@ -49,6 +54,7 @@ func (m *DendriteMonolith) Start() { if err != nil { panic(err) } + m.YggdrasilNode = ygg cfg := &config.Dendrite{} cfg.SetDefaults() @@ -68,7 +74,7 @@ func (m *DendriteMonolith) Start() { cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s/dendrite-serverkey.db", m.StorageDirectory)) cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s/dendrite-federationsender.db", m.StorageDirectory)) cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s/dendrite-appservice.db", m.StorageDirectory)) - cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-publicroomsa.db", m.StorageDirectory)) + cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s/dendrite-currentstate.db", m.StorageDirectory)) cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s/dendrite-naffka.db", m.StorageDirectory)) if err = cfg.Derive(); err != nil { panic(err) @@ -103,10 +109,7 @@ func (m *DendriteMonolith) Start() { // This is different to rsAPI which can be the http client which doesn't need this dependency rsAPI.SetFederationSenderAPI(fsAPI) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to public rooms db") - } + stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer) monolith := setup.Monolith{ Config: base.Cfg, @@ -123,9 +126,8 @@ func (m *DendriteMonolith) Start() { FederationSenderAPI: fsAPI, RoomserverAPI: rsAPI, UserAPI: userAPI, + StateAPI: stateAPI, //ServerKeyAPI: serverKeyAPI, - - PublicRoomsDB: publicRoomsDB, } monolith.AddAllPublicRoutes(base.PublicAPIMux) diff --git a/publicroomsapi/types/types.go b/clientapi/api/api.go similarity index 81% rename from publicroomsapi/types/types.go rename to clientapi/api/api.go index 11cb0d204..dae462c08 100644 --- a/publicroomsapi/types/types.go +++ b/clientapi/api/api.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,10 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -package types +package api -// ExternalPublicRoomsProvider provides a list of homeservers who should be queried -// periodically for a list of public rooms on their server. type ExternalPublicRoomsProvider interface { // The list of homeserver domains to query. These servers will receive a request // via this API: https://matrix.org/docs/spec/server_server/latest#public-room-directory diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 174eb1bf1..bbce6dccf 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -18,9 +18,10 @@ import ( "github.com/Shopify/sarama" "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/clientapi/consumers" + "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/clientapi/routing" + currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" @@ -30,14 +31,12 @@ import ( "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/matrix-org/gomatrixserverlib" - "github.com/sirupsen/logrus" ) // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component. func AddPublicRoutes( router *mux.Router, cfg *config.Dendrite, - consumer sarama.Consumer, producer sarama.SyncProducer, deviceDB devices.Database, accountsDB accounts.Database, @@ -45,25 +44,20 @@ func AddPublicRoutes( rsAPI roomserverAPI.RoomserverInternalAPI, eduInputAPI eduServerAPI.EDUServerInputAPI, asAPI appserviceAPI.AppServiceQueryAPI, + stateAPI currentstateAPI.CurrentStateInternalAPI, transactionsCache *transactions.Cache, fsAPI federationSenderAPI.FederationSenderInternalAPI, userAPI userapi.UserInternalAPI, + extRoomsProvider api.ExternalPublicRoomsProvider, ) { syncProducer := &producers.SyncAPIProducer{ Producer: producer, Topic: string(cfg.Kafka.Topics.OutputClientData), } - roomEventConsumer := consumers.NewOutputRoomEventConsumer( - cfg, consumer, accountsDB, rsAPI, - ) - if err := roomEventConsumer.Start(); err != nil { - logrus.WithError(err).Panicf("failed to start room server consumer") - } - routing.Setup( router, cfg, eduInputAPI, rsAPI, asAPI, accountsDB, deviceDB, userAPI, federation, - syncProducer, transactionsCache, fsAPI, + syncProducer, transactionsCache, fsAPI, stateAPI, extRoomsProvider, ) } diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go deleted file mode 100644 index beeda042b..000000000 --- a/clientapi/consumers/roomserver.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package consumers - -import ( - "context" - "encoding/json" - - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/userapi/storage/accounts" - "github.com/matrix-org/gomatrixserverlib" - - "github.com/Shopify/sarama" - log "github.com/sirupsen/logrus" -) - -// OutputRoomEventConsumer consumes events that originated in the room server. -type OutputRoomEventConsumer struct { - rsAPI api.RoomserverInternalAPI - rsConsumer *internal.ContinualConsumer - db accounts.Database - serverName string -} - -// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. -func NewOutputRoomEventConsumer( - cfg *config.Dendrite, - kafkaConsumer sarama.Consumer, - store accounts.Database, - rsAPI api.RoomserverInternalAPI, -) *OutputRoomEventConsumer { - - consumer := internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputRoomEvent), - Consumer: kafkaConsumer, - PartitionStore: store, - } - s := &OutputRoomEventConsumer{ - rsConsumer: &consumer, - db: store, - rsAPI: rsAPI, - serverName: string(cfg.Matrix.ServerName), - } - consumer.ProcessMessage = s.onMessage - - return s -} - -// Start consuming from room servers -func (s *OutputRoomEventConsumer) Start() error { - return s.rsConsumer.Start() -} - -// onMessage is called when the sync server receives a new event from the room server output log. -// It is not safe for this function to be called from multiple goroutines, or else the -// sync stream position may race and be incorrectly calculated. -func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { - // Parse out the event JSON - var output api.OutputEvent - if err := json.Unmarshal(msg.Value, &output); err != nil { - // If the message was invalid, log it and move on to the next message in the stream - log.WithError(err).Errorf("roomserver output log: message parse failure") - return nil - } - - if output.Type != api.OutputTypeNewRoomEvent { - log.WithField("type", output.Type).Debug( - "roomserver output log: ignoring unknown output type", - ) - return nil - } - - return s.db.UpdateMemberships( - context.TODO(), - gomatrixserverlib.UnwrapEventHeaders(output.NewRoomEvent.AddsState()), - output.NewRoomEvent.RemovesStateEventIDs, - ) -} diff --git a/clientapi/jsonerror/jsonerror.go b/clientapi/jsonerror/jsonerror.go index 85e887aec..7f8f264b7 100644 --- a/clientapi/jsonerror/jsonerror.go +++ b/clientapi/jsonerror/jsonerror.go @@ -125,10 +125,20 @@ func GuestAccessForbidden(msg string) *MatrixError { return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg} } +type IncompatibleRoomVersionError struct { + RoomVersion string `json:"room_version"` + Error string `json:"error"` + Code string `json:"errcode"` +} + // IncompatibleRoomVersion is an error which is returned when the client // requests a room with a version that is unsupported. -func IncompatibleRoomVersion(roomVersion gomatrixserverlib.RoomVersion) *MatrixError { - return &MatrixError{"M_INCOMPATIBLE_ROOM_VERSION", string(roomVersion)} +func IncompatibleRoomVersion(roomVersion gomatrixserverlib.RoomVersion) *IncompatibleRoomVersionError { + return &IncompatibleRoomVersionError{ + Code: "M_INCOMPATIBLE_ROOM_VERSION", + RoomVersion: string(roomVersion), + Error: "Your homeserver does not support the features required to join this room", + } } // UnsupportedRoomVersion is an error which is returned when the client diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 8682b03a4..b6a5d1221 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -28,7 +28,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/threepid" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/userapi/storage/accounts" @@ -373,13 +372,9 @@ func createRoom( // If this is a direct message then we should invite the participants. for _, invitee := range r.Invite { - // Build the membership request. - body := threepid.MembershipRequest{ - UserID: invitee, - } // Build the invite event. inviteEvent, err := buildMembershipEvent( - req.Context(), body, accountDB, device, gomatrixserverlib.Invite, + req.Context(), invitee, "", accountDB, device, gomatrixserverlib.Invite, roomID, true, cfg, evTime, rsAPI, asAPI, ) if err != nil { @@ -415,6 +410,19 @@ func createRoom( } } + if r.Visibility == "public" { + // expose this room in the published room list + var pubRes roomserverAPI.PerformPublishResponse + rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{ + RoomID: roomID, + Visibility: "public", + }, &pubRes) + if pubRes.Error != nil { + // treat as non-fatal since the room is already made by this point + util.GetLogger(req.Context()).WithError(pubRes.Error).Error("failed to visibility:public") + } + } + response := createRoomResponse{ RoomID: roomID, RoomAlias: roomAlias, diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 0dc4d5605..0f78f4a24 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,10 +20,12 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" + currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/userapi/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -232,3 +234,89 @@ func RemoveLocalAlias( JSON: struct{}{}, } } + +type roomVisibility struct { + Visibility string `json:"visibility"` +} + +// GetVisibility implements GET /directory/list/room/{roomID} +func GetVisibility( + req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, + roomID string, +) util.JSONResponse { + var res roomserverAPI.QueryPublishedRoomsResponse + err := rsAPI.QueryPublishedRooms(req.Context(), &roomserverAPI.QueryPublishedRoomsRequest{ + RoomID: roomID, + }, &res) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("QueryPublishedRooms failed") + return jsonerror.InternalServerError() + } + + var v roomVisibility + if len(res.RoomIDs) == 1 { + v.Visibility = gomatrixserverlib.Public + } else { + v.Visibility = "private" + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: v, + } +} + +// SetVisibility implements PUT /directory/list/room/{roomID} +// TODO: Allow admin users to edit the room visibility +func SetVisibility( + req *http.Request, stateAPI currentstateAPI.CurrentStateInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, dev *userapi.Device, + roomID string, +) util.JSONResponse { + resErr := checkMemberInRoom(req.Context(), stateAPI, dev.UserID, roomID) + if resErr != nil { + return *resErr + } + + queryEventsReq := roomserverAPI.QueryLatestEventsAndStateRequest{ + RoomID: roomID, + StateToFetch: []gomatrixserverlib.StateKeyTuple{{ + EventType: gomatrixserverlib.MRoomPowerLevels, + StateKey: "", + }}, + } + var queryEventsRes roomserverAPI.QueryLatestEventsAndStateResponse + err := rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes) + if err != nil || len(queryEventsRes.StateEvents) == 0 { + util.GetLogger(req.Context()).WithError(err).Error("could not query events from room") + return jsonerror.InternalServerError() + } + + // NOTSPEC: Check if the user's power is greater than power required to change m.room.aliases event + power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event) + if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomAliases, true) { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("userID doesn't have power level to change visibility"), + } + } + + var v roomVisibility + if reqErr := httputil.UnmarshalJSONRequest(req, &v); reqErr != nil { + return *reqErr + } + + var publishRes roomserverAPI.PerformPublishResponse + rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{ + RoomID: roomID, + Visibility: v.Visibility, + }, &publishRes) + if publishRes.Error != nil { + util.GetLogger(req.Context()).WithError(publishRes.Error).Error("PerformPublish failed") + return publishRes.Error.JSONResponse() + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} diff --git a/publicroomsapi/directory/public_rooms.go b/clientapi/routing/directory_public.go similarity index 57% rename from publicroomsapi/directory/public_rooms.go rename to clientapi/routing/directory_public.go index df9df8ff9..64600cb49 100644 --- a/publicroomsapi/directory/public_rooms.go +++ b/clientapi/routing/directory_public.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package directory +package routing import ( "context" @@ -20,13 +20,15 @@ import ( "net/http" "sort" "strconv" + "strings" "sync" "time" + "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/matrix-org/dendrite/publicroomsapi/types" + currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -43,13 +45,13 @@ type filter struct { // GetPostPublicRooms implements GET and POST /publicRooms func GetPostPublicRooms( - req *http.Request, publicRoomDatabase storage.Database, + req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI, ) util.JSONResponse { var request PublicRoomReq if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil { return *fillErr } - response, err := publicRooms(req.Context(), request, publicRoomDatabase) + response, err := publicRooms(req.Context(), request, rsAPI, stateAPI) if err != nil { return jsonerror.InternalServerError() } @@ -61,14 +63,14 @@ func GetPostPublicRooms( // GetPostPublicRoomsWithExternal is the same as GetPostPublicRooms but also mixes in public rooms from the provider supplied. func GetPostPublicRoomsWithExternal( - req *http.Request, publicRoomDatabase storage.Database, fedClient *gomatrixserverlib.FederationClient, - extRoomsProvider types.ExternalPublicRoomsProvider, + req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI, + fedClient *gomatrixserverlib.FederationClient, extRoomsProvider api.ExternalPublicRoomsProvider, ) util.JSONResponse { var request PublicRoomReq if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil { return *fillErr } - response, err := publicRooms(req.Context(), request, publicRoomDatabase) + response, err := publicRooms(req.Context(), request, rsAPI, stateAPI) if err != nil { return jsonerror.InternalServerError() } @@ -197,11 +199,16 @@ FanIn: return publicRooms } -func publicRooms(ctx context.Context, request PublicRoomReq, publicRoomDatabase storage.Database) (*gomatrixserverlib.RespPublicRooms, error) { +func publicRooms(ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.RoomserverInternalAPI, + stateAPI currentstateAPI.CurrentStateInternalAPI) (*gomatrixserverlib.RespPublicRooms, error) { + var response gomatrixserverlib.RespPublicRooms var limit int16 var offset int64 limit = request.Limit + if limit == 0 { + limit = 50 + } offset, err := strconv.ParseInt(request.Since, 10, 64) // ParseInt returns 0 and an error when trying to parse an empty string // In that case, we want to assign 0 so we ignore the error @@ -210,53 +217,157 @@ func publicRooms(ctx context.Context, request PublicRoomReq, publicRoomDatabase return nil, err } - est, err := publicRoomDatabase.CountPublicRooms(ctx) + var queryRes roomserverAPI.QueryPublishedRoomsResponse + err = rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes) if err != nil { - util.GetLogger(ctx).WithError(err).Error("publicRoomDatabase.CountPublicRooms failed") + util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed") return nil, err } - response.TotalRoomCountEstimate = int(est) + response.TotalRoomCountEstimate = len(queryRes.RoomIDs) - if offset > 0 { - response.PrevBatch = strconv.Itoa(int(offset) - 1) + roomIDs, prev, next := sliceInto(queryRes.RoomIDs, offset, limit) + if prev >= 0 { + response.PrevBatch = "T" + strconv.Itoa(prev) } - nextIndex := int(offset) + int(limit) - if response.TotalRoomCountEstimate > nextIndex { - response.NextBatch = strconv.Itoa(nextIndex) + if next >= 0 { + response.NextBatch = "T" + strconv.Itoa(next) } - - if response.Chunk, err = publicRoomDatabase.GetPublicRooms( - ctx, offset, limit, request.Filter.SearchTerms, - ); err != nil { - util.GetLogger(ctx).WithError(err).Error("publicRoomDatabase.GetPublicRooms failed") - return nil, err - } - - return &response, nil + response.Chunk, err = fillInRooms(ctx, roomIDs, stateAPI) + return &response, err } // fillPublicRoomsReq fills the Limit, Since and Filter attributes of a GET or POST request // on /publicRooms by parsing the incoming HTTP request // Filter is only filled for POST requests func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSONResponse { - if httpReq.Method == http.MethodGet { + if httpReq.Method != "GET" && httpReq.Method != "POST" { + return &util.JSONResponse{ + Code: http.StatusMethodNotAllowed, + JSON: jsonerror.NotFound("Bad method"), + } + } + if httpReq.Method == "GET" { limit, err := strconv.Atoi(httpReq.FormValue("limit")) // Atoi returns 0 and an error when trying to parse an empty string // In that case, we want to assign 0 so we ignore the error if err != nil && len(httpReq.FormValue("limit")) > 0 { util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed") - reqErr := jsonerror.InternalServerError() - return &reqErr + return &util.JSONResponse{ + Code: 400, + JSON: jsonerror.BadJSON("limit param is not a number"), + } } request.Limit = int16(limit) request.Since = httpReq.FormValue("since") - return nil - } else if httpReq.Method == http.MethodPost { - return httputil.UnmarshalJSONRequest(httpReq, request) + } else { + resErr := httputil.UnmarshalJSONRequest(httpReq, request) + if resErr != nil { + return resErr + } } - return &util.JSONResponse{ - Code: http.StatusMethodNotAllowed, - JSON: jsonerror.NotFound("Bad method"), - } + // strip the 'T' which is only required because when sytest does pagination tests it stops + // iterating when !prev_batch which then fails if prev_batch==0, so add arbitrary text to + // make it truthy not falsey. + request.Since = strings.TrimPrefix(request.Since, "T") + return nil +} + +// due to lots of switches +// nolint:gocyclo +func fillInRooms(ctx context.Context, roomIDs []string, stateAPI currentstateAPI.CurrentStateInternalAPI) ([]gomatrixserverlib.PublicRoom, error) { + avatarTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.avatar", StateKey: ""} + nameTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.name", StateKey: ""} + canonicalTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCanonicalAlias, StateKey: ""} + topicTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.topic", StateKey: ""} + guestTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.guest_access", StateKey: ""} + visibilityTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomHistoryVisibility, StateKey: ""} + joinRuleTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""} + + var stateRes currentstateAPI.QueryBulkStateContentResponse + err := stateAPI.QueryBulkStateContent(ctx, ¤tstateAPI.QueryBulkStateContentRequest{ + RoomIDs: roomIDs, + AllowWildcards: true, + StateTuples: []gomatrixserverlib.StateKeyTuple{ + nameTuple, canonicalTuple, topicTuple, guestTuple, visibilityTuple, joinRuleTuple, avatarTuple, + {EventType: gomatrixserverlib.MRoomMember, StateKey: "*"}, + }, + }, &stateRes) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("QueryBulkStateContent failed") + return nil, err + } + chunk := make([]gomatrixserverlib.PublicRoom, len(roomIDs)) + i := 0 + for roomID, data := range stateRes.Rooms { + pub := gomatrixserverlib.PublicRoom{ + RoomID: roomID, + } + joinCount := 0 + var joinRule, guestAccess string + for tuple, contentVal := range data { + if tuple.EventType == gomatrixserverlib.MRoomMember && contentVal == "join" { + joinCount++ + continue + } + switch tuple { + case avatarTuple: + pub.AvatarURL = contentVal + case nameTuple: + pub.Name = contentVal + case topicTuple: + pub.Topic = contentVal + case canonicalTuple: + pub.CanonicalAlias = contentVal + case visibilityTuple: + pub.WorldReadable = contentVal == "world_readable" + // need both of these to determine whether guests can join + case joinRuleTuple: + joinRule = contentVal + case guestTuple: + guestAccess = contentVal + } + } + if joinRule == gomatrixserverlib.Public && guestAccess == "can_join" { + pub.GuestCanJoin = true + } + pub.JoinedMembersCount = joinCount + chunk[i] = pub + i++ + } + return chunk, nil +} + +// sliceInto returns a subslice of `slice` which honours the since/limit values given. +// +// 0 1 2 3 4 5 6 index +// [A, B, C, D, E, F, G] slice +// +// limit=3 => A,B,C (prev='', next='3') +// limit=3&since=3 => D,E,F (prev='0', next='6') +// limit=3&since=6 => G (prev='3', next='') +// +// A value of '-1' for prev/next indicates no position. +func sliceInto(slice []string, since int64, limit int16) (subset []string, prev, next int) { + prev = -1 + next = -1 + + if since > 0 { + prev = int(since) - int(limit) + } + nextIndex := int(since) + int(limit) + if len(slice) > nextIndex { // there are more rooms ahead of us + next = nextIndex + } + + // apply sanity caps + if since < 0 { + since = 0 + } + if nextIndex > len(slice) { + nextIndex = len(slice) + } + + subset = slice[since:nextIndex] + return } diff --git a/clientapi/routing/directory_public_test.go b/clientapi/routing/directory_public_test.go new file mode 100644 index 000000000..f2a1d5515 --- /dev/null +++ b/clientapi/routing/directory_public_test.go @@ -0,0 +1,48 @@ +package routing + +import ( + "reflect" + "testing" +) + +func TestSliceInto(t *testing.T) { + slice := []string{"a", "b", "c", "d", "e", "f", "g"} + limit := int16(3) + testCases := []struct { + since int64 + wantPrev int + wantNext int + wantSubset []string + }{ + { + since: 0, + wantPrev: -1, + wantNext: 3, + wantSubset: slice[0:3], + }, + { + since: 3, + wantPrev: 0, + wantNext: 6, + wantSubset: slice[3:6], + }, + { + since: 6, + wantPrev: 3, + wantNext: -1, + wantSubset: slice[6:7], + }, + } + for _, tc := range testCases { + subset, prev, next := sliceInto(slice, tc.since, limit) + if !reflect.DeepEqual(subset, tc.wantSubset) { + t.Errorf("returned subset is wrong, got %v want %v", subset, tc.wantSubset) + } + if prev != tc.wantPrev { + t.Errorf("returned prev is wrong, got %d want %d", prev, tc.wantPrev) + } + if next != tc.wantNext { + t.Errorf("returned next is wrong, got %d want %d", next, tc.wantNext) + } + } +} diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index aff1730c5..c2145159a 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -25,6 +25,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/threepid" + currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" @@ -38,40 +39,141 @@ import ( var errMissingUserID = errors.New("'user_id' must be supplied") -// SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite) -// by building a m.room.member event then sending it to the room server -// TODO: Can we improve the cyclo count here? Separate code paths for invites? -// nolint:gocyclo -func SendMembership( +func SendBan( req *http.Request, accountDB accounts.Database, device *userapi.Device, - roomID string, membership string, cfg *config.Dendrite, + roomID string, cfg *config.Dendrite, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { - verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} - verRes := api.QueryRoomVersionForRoomResponse{} - if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI) + if reqErr != nil { + return *reqErr + } + return sendMembership(req.Context(), accountDB, device, roomID, "ban", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI) +} + +func sendMembership(ctx context.Context, accountDB accounts.Database, device *userapi.Device, + roomID, membership, reason string, cfg *config.Dendrite, targetUserID string, evTime time.Time, + roomVer gomatrixserverlib.RoomVersion, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI) util.JSONResponse { + + event, err := buildMembershipEvent( + ctx, targetUserID, reason, accountDB, device, membership, + roomID, false, cfg, evTime, rsAPI, asAPI, + ) + if err == errMissingUserID { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.UnsupportedRoomVersion(err.Error()), + JSON: jsonerror.BadJSON(err.Error()), + } + } else if err == eventutil.ErrRoomNoExists { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound(err.Error()), + } + } else if err != nil { + util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed") + return jsonerror.InternalServerError() + } + + _, err = roomserverAPI.SendEvents( + ctx, rsAPI, + []gomatrixserverlib.HeaderedEvent{event.Headered(roomVer)}, + cfg.Matrix.ServerName, + nil, + ) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("SendEvents failed") + return jsonerror.InternalServerError() + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} + +func SendKick( + req *http.Request, accountDB accounts.Database, device *userapi.Device, + roomID string, cfg *config.Dendrite, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, +) util.JSONResponse { + body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI) + if reqErr != nil { + return *reqErr + } + if body.UserID == "" { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.BadJSON("missing user_id"), } } - var body threepid.MembershipRequest - if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { + var queryRes roomserverAPI.QueryMembershipForUserResponse + err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{ + RoomID: roomID, + UserID: body.UserID, + }, &queryRes) + if err != nil { + return util.ErrorResponse(err) + } + // kick is only valid if the user is not currently banned + if queryRes.Membership == "ban" { + return util.JSONResponse{ + Code: 403, + JSON: jsonerror.Unknown("cannot /kick banned users"), + } + } + // TODO: should we be using SendLeave instead? + return sendMembership(req.Context(), accountDB, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI) +} + +func SendUnban( + req *http.Request, accountDB accounts.Database, device *userapi.Device, + roomID string, cfg *config.Dendrite, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, +) util.JSONResponse { + body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI) + if reqErr != nil { + return *reqErr + } + if body.UserID == "" { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.BadJSON("missing user_id"), + } + } + + var queryRes roomserverAPI.QueryMembershipForUserResponse + err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{ + RoomID: roomID, + UserID: body.UserID, + }, &queryRes) + if err != nil { + return util.ErrorResponse(err) + } + // unban is only valid if the user is currently banned + if queryRes.Membership != "ban" { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.Unknown("can only /unban users that are banned"), + } + } + // TODO: should we be using SendLeave instead? + return sendMembership(req.Context(), accountDB, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI) +} + +func SendInvite( + req *http.Request, accountDB accounts.Database, device *userapi.Device, + roomID string, cfg *config.Dendrite, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, +) util.JSONResponse { + body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI) + if reqErr != nil { return *reqErr } - evTime, err := httputil.ParseTSParam(req) - if err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.InvalidArgumentValue(err.Error()), - } - } - inviteStored, jsonErrResp := checkAndProcessThreepid( - req, device, &body, cfg, rsAPI, accountDB, - membership, roomID, evTime, + req, device, body, cfg, rsAPI, accountDB, roomID, evTime, ) if jsonErrResp != nil { return *jsonErrResp @@ -88,7 +190,7 @@ func SendMembership( } event, err := buildMembershipEvent( - req.Context(), body, accountDB, device, membership, + req.Context(), body.UserID, body.Reason, accountDB, device, "invite", roomID, false, cfg, evTime, rsAPI, asAPI, ) if err == errMissingUserID { @@ -106,61 +208,32 @@ func SendMembership( return jsonerror.InternalServerError() } - var returnData interface{} = struct{}{} - - switch membership { - case gomatrixserverlib.Invite: - // Invites need to be handled specially - perr := roomserverAPI.SendInvite( - req.Context(), rsAPI, - event.Headered(verRes.RoomVersion), - nil, // ask the roomserver to draw up invite room state for us - cfg.Matrix.ServerName, - nil, - ) - if perr != nil { - util.GetLogger(req.Context()).WithError(perr).Error("producer.SendInvite failed") - return perr.JSONResponse() - } - case gomatrixserverlib.Join: - // The join membership requires the room id to be sent in the response - returnData = struct { - RoomID string `json:"room_id"` - }{roomID} - fallthrough - default: - _, err = roomserverAPI.SendEvents( - req.Context(), rsAPI, - []gomatrixserverlib.HeaderedEvent{event.Headered(verRes.RoomVersion)}, - cfg.Matrix.ServerName, - nil, - ) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") - return jsonerror.InternalServerError() - } + perr := roomserverAPI.SendInvite( + req.Context(), rsAPI, + event.Headered(roomVer), + nil, // ask the roomserver to draw up invite room state for us + cfg.Matrix.ServerName, + nil, + ) + if perr != nil { + util.GetLogger(req.Context()).WithError(perr).Error("producer.SendInvite failed") + return perr.JSONResponse() } - return util.JSONResponse{ Code: http.StatusOK, - JSON: returnData, + JSON: struct{}{}, } } func buildMembershipEvent( ctx context.Context, - body threepid.MembershipRequest, accountDB accounts.Database, + targetUserID, reason string, accountDB accounts.Database, device *userapi.Device, membership, roomID string, isDirect bool, cfg *config.Dendrite, evTime time.Time, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) (*gomatrixserverlib.Event, error) { - stateKey, reason, err := getMembershipStateKey(body, device, membership) - if err != nil { - return nil, err - } - - profile, err := loadProfile(ctx, stateKey, cfg, accountDB, asAPI) + profile, err := loadProfile(ctx, targetUserID, cfg, accountDB, asAPI) if err != nil { return nil, err } @@ -169,12 +242,7 @@ func buildMembershipEvent( Sender: device.UserID, RoomID: roomID, Type: "m.room.member", - StateKey: &stateKey, - } - - // "unban" or "kick" isn't a valid membership value, change it to "leave" - if membership == "unban" || membership == "kick" { - membership = gomatrixserverlib.Leave + StateKey: &targetUserID, } content := gomatrixserverlib.MemberContent{ @@ -218,29 +286,33 @@ func loadProfile( return profile, err } -// getMembershipStateKey extracts the target user ID of a membership change. -// For "join" and "leave" this will be the ID of the user making the change. -// For "ban", "unban", "kick" and "invite" the target user ID will be in the JSON request body. -// In the latter case, if there was an issue retrieving the user ID from the request body, -// returns a JSONResponse with a corresponding error code and message. -func getMembershipStateKey( - body threepid.MembershipRequest, device *userapi.Device, membership string, -) (stateKey string, reason string, err error) { - if membership == gomatrixserverlib.Ban || membership == "unban" || membership == "kick" || membership == gomatrixserverlib.Invite { - // If we're in this case, the state key is contained in the request body, - // possibly along with a reason (for "kick" and "ban") so we need to parse - // it - if body.UserID == "" { - err = errMissingUserID - return +func extractRequestData(req *http.Request, roomID string, rsAPI api.RoomserverInternalAPI) ( + body *threepid.MembershipRequest, evTime time.Time, roomVer gomatrixserverlib.RoomVersion, resErr *util.JSONResponse, +) { + verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} + verRes := api.QueryRoomVersionForRoomResponse{} + if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + resErr = &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion(err.Error()), } + return + } + roomVer = verRes.RoomVersion - stateKey = body.UserID - reason = body.Reason - } else { - stateKey = device.UserID + if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { + resErr = reqErr + return } + evTime, err := httputil.ParseTSParam(req) + if err != nil { + resErr = &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.InvalidArgumentValue(err.Error()), + } + return + } return } @@ -251,13 +323,13 @@ func checkAndProcessThreepid( cfg *config.Dendrite, rsAPI roomserverAPI.RoomserverInternalAPI, accountDB accounts.Database, - membership, roomID string, + roomID string, evTime time.Time, ) (inviteStored bool, errRes *util.JSONResponse) { inviteStored, err := threepid.CheckAndProcessInvite( req.Context(), device, body, cfg, rsAPI, accountDB, - membership, roomID, evTime, + roomID, evTime, ) if err == threepid.ErrMissingParameter { return inviteStored, &util.JSONResponse{ @@ -287,3 +359,35 @@ func checkAndProcessThreepid( } return } + +func checkMemberInRoom(ctx context.Context, stateAPI currentstateAPI.CurrentStateInternalAPI, userID, roomID string) *util.JSONResponse { + tuple := gomatrixserverlib.StateKeyTuple{ + EventType: gomatrixserverlib.MRoomMember, + StateKey: userID, + } + var membershipRes currentstateAPI.QueryCurrentStateResponse + err := stateAPI.QueryCurrentState(ctx, ¤tstateAPI.QueryCurrentStateRequest{ + RoomID: roomID, + StateTuples: []gomatrixserverlib.StateKeyTuple{tuple}, + }, &membershipRes) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("QueryCurrentState: could not query membership for user") + e := jsonerror.InternalServerError() + return &e + } + ev, ok := membershipRes.StateEvents[tuple] + if !ok { + return &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("user does not belong to room"), + } + } + membership, err := ev.Membership() + if err != nil || membership != "join" { + return &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("user does not belong to room"), + } + } + return nil +} diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index 1c9800b66..9c4cf7497 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -18,9 +18,8 @@ import ( "encoding/json" "net/http" - "github.com/matrix-org/dendrite/userapi/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/jsonerror" + currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -95,20 +94,19 @@ func GetMemberships( func GetJoinedRooms( req *http.Request, device *userapi.Device, - accountsDB accounts.Database, + stateAPI currentstateAPI.CurrentStateInternalAPI, ) util.JSONResponse { - localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) + var res currentstateAPI.QueryRoomsForUserResponse + err := stateAPI.QueryRoomsForUser(req.Context(), ¤tstateAPI.QueryRoomsForUserRequest{ + UserID: device.UserID, + WantMembership: "join", + }, &res) if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") - return jsonerror.InternalServerError() - } - joinedRooms, err := accountsDB.GetRoomIDsByLocalPart(req.Context(), localpart) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountsDB.GetRoomIDsByLocalPart failed") + util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed") return jsonerror.InternalServerError() } return util.JSONResponse{ Code: http.StatusOK, - JSON: getJoinedRoomsResponse{joinedRooms}, + JSON: getJoinedRoomsResponse{res.RoomIDs}, } } diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 7c2cd19bc..1df4c9b33 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" + currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" @@ -93,8 +94,8 @@ func GetAvatarURL( // SetAvatarURL implements PUT /profile/{userID}/avatar_url // nolint:gocyclo func SetAvatarURL( - req *http.Request, accountDB accounts.Database, device *userapi.Device, - userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, + req *http.Request, accountDB accounts.Database, stateAPI currentstateAPI.CurrentStateInternalAPI, + device *userapi.Device, userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -139,9 +140,13 @@ func SetAvatarURL( return jsonerror.InternalServerError() } - memberships, err := accountDB.GetMembershipsByLocalpart(req.Context(), localpart) + var res currentstateAPI.QueryRoomsForUserResponse + err = stateAPI.QueryRoomsForUser(req.Context(), ¤tstateAPI.QueryRoomsForUserRequest{ + UserID: device.UserID, + WantMembership: "join", + }, &res) if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetMembershipsByLocalpart failed") + util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed") return jsonerror.InternalServerError() } @@ -152,7 +157,7 @@ func SetAvatarURL( } events, err := buildMembershipEvents( - req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI, + req.Context(), res.RoomIDs, newProfile, userID, cfg, evTime, rsAPI, ) switch e := err.(type) { case nil: @@ -207,8 +212,8 @@ func GetDisplayName( // SetDisplayName implements PUT /profile/{userID}/displayname // nolint:gocyclo func SetDisplayName( - req *http.Request, accountDB accounts.Database, device *userapi.Device, - userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, + req *http.Request, accountDB accounts.Database, stateAPI currentstateAPI.CurrentStateInternalAPI, + device *userapi.Device, userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -253,9 +258,13 @@ func SetDisplayName( return jsonerror.InternalServerError() } - memberships, err := accountDB.GetMembershipsByLocalpart(req.Context(), localpart) + var res currentstateAPI.QueryRoomsForUserResponse + err = stateAPI.QueryRoomsForUser(req.Context(), ¤tstateAPI.QueryRoomsForUserRequest{ + UserID: device.UserID, + WantMembership: "join", + }, &res) if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetMembershipsByLocalpart failed") + util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed") return jsonerror.InternalServerError() } @@ -266,7 +275,7 @@ func SetDisplayName( } events, err := buildMembershipEvents( - req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI, + req.Context(), res.RoomIDs, newProfile, userID, cfg, evTime, rsAPI, ) switch e := err.(type) { case nil: @@ -335,14 +344,14 @@ func getProfile( func buildMembershipEvents( ctx context.Context, - memberships []authtypes.Membership, + roomIDs []string, newProfile authtypes.Profile, userID string, cfg *config.Dendrite, evTime time.Time, rsAPI api.RoomserverInternalAPI, ) ([]gomatrixserverlib.HeaderedEvent, error) { evs := []gomatrixserverlib.HeaderedEvent{} - for _, membership := range memberships { - verReq := api.QueryRoomVersionForRoomRequest{RoomID: membership.RoomID} + for _, roomID := range roomIDs { + verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { return []gomatrixserverlib.HeaderedEvent{}, err @@ -350,7 +359,7 @@ func buildMembershipEvents( builder := gomatrixserverlib.EventBuilder{ Sender: userID, - RoomID: membership.RoomID, + RoomID: roomID, Type: "m.room.member", StateKey: &userID, } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 825ac50f2..85c5c0d9a 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,15 +21,17 @@ import ( "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" + "github.com/matrix-org/dendrite/clientapi/api" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" + currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/transactions" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/userapi/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/matrix-org/gomatrixserverlib" @@ -53,11 +55,13 @@ func Setup( asAPI appserviceAPI.AppServiceQueryAPI, accountDB accounts.Database, deviceDB devices.Database, - userAPI api.UserInternalAPI, + userAPI userapi.UserInternalAPI, federation *gomatrixserverlib.FederationClient, syncProducer *producers.SyncAPIProducer, transactionsCache *transactions.Cache, federationSender federationSenderAPI.FederationSenderInternalAPI, + stateAPI currentstateAPI.CurrentStateInternalAPI, + extRoomsProvider api.ExternalPublicRoomsProvider, ) { publicAPIMux.Handle("/client/versions", @@ -81,12 +85,12 @@ func Setup( unstableMux := publicAPIMux.PathPrefix(pathPrefixUnstable).Subrouter() r0mux.Handle("/createRoom", - httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return CreateRoom(req, device, cfg, accountDB, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", - httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -97,12 +101,23 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/joined_rooms", - httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { - return GetJoinedRooms(req, device, accountDB) + httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return GetJoinedRooms(req, device, stateAPI) }), ).Methods(http.MethodGet, http.MethodOptions) + r0mux.Handle("/rooms/{roomID}/join", + httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return JoinRoomByIDOrAlias( + req, device, rsAPI, accountDB, vars["roomID"], + ) + }), + ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/leave", - httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -112,17 +127,44 @@ func Setup( ) }), ).Methods(http.MethodPost, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|invite)}", - httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/ban", + httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } - return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, rsAPI, asAPI) + return SendBan(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI) + }), + ).Methods(http.MethodPost, http.MethodOptions) + r0mux.Handle("/rooms/{roomID}/invite", + httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return SendInvite(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI) + }), + ).Methods(http.MethodPost, http.MethodOptions) + r0mux.Handle("/rooms/{roomID}/kick", + httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return SendKick(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI) + }), + ).Methods(http.MethodPost, http.MethodOptions) + r0mux.Handle("/rooms/{roomID}/unban", + httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return SendUnban(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}", - httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -131,7 +173,7 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}", - httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -142,7 +184,7 @@ func Setup( }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/event/{eventID}", - httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -151,7 +193,7 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -159,7 +201,7 @@ func Setup( return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"]) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type:[^/]+/?}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/state/{type:[^/]+/?}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -173,7 +215,7 @@ func Setup( return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], eventType, "", eventFormat) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -183,7 +225,7 @@ func Setup( })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}", - httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -199,7 +241,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}", - httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -232,7 +274,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", - httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -242,7 +284,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", - httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -250,31 +292,59 @@ func Setup( return RemoveLocalAlias(req, device, vars["roomAlias"], rsAPI) }), ).Methods(http.MethodDelete, http.MethodOptions) + r0mux.Handle("/directory/list/room/{roomID}", + httputil.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetVisibility(req, rsAPI, vars["roomID"]) + }), + ).Methods(http.MethodGet, http.MethodOptions) + // TODO: Add AS support + r0mux.Handle("/directory/list/room/{roomID}", + httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return SetVisibility(req, stateAPI, rsAPI, device, vars["roomID"]) + }), + ).Methods(http.MethodPut, http.MethodOptions) + r0mux.Handle("/publicRooms", + httputil.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse { + /* TODO: + if extRoomsProvider != nil { + return GetPostPublicRoomsWithExternal(req, stateAPI, fedClient, extRoomsProvider) + } */ + return GetPostPublicRooms(req, rsAPI, stateAPI) + }), + ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) r0mux.Handle("/logout", - httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return Logout(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/logout/all", - httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return LogoutAll(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/typing/{userID}", - httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } - return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, eduAPI) + return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, eduAPI, stateAPI) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/sendToDevice/{eventType}/{txnID}", - httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -288,7 +358,7 @@ func Setup( // rather than r0. It's an exact duplicate of the above handler. // TODO: Remove this if/when sytest is fixed! unstableMux.Handle("/sendToDevice/{eventType}/{txnID}", - httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -299,7 +369,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/account/whoami", - httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return Whoami(req, device) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -338,26 +408,6 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/user/{userId}/filter", - httputil.MakeAuthAPI("put_filter", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return PutFilter(req, device, accountDB, vars["userId"]) - }), - ).Methods(http.MethodPost, http.MethodOptions) - - r0mux.Handle("/user/{userId}/filter/{filterId}", - httputil.MakeAuthAPI("get_filter", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return GetFilter(req, device, accountDB, vars["userId"], vars["filterId"]) - }), - ).Methods(http.MethodGet, http.MethodOptions) - // Riot user settings r0mux.Handle("/profile/{userID}", @@ -381,12 +431,12 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/avatar_url", - httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } - return SetAvatarURL(req, accountDB, device, vars["userID"], cfg, rsAPI) + return SetAvatarURL(req, accountDB, stateAPI, device, vars["userID"], cfg, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows @@ -403,31 +453,31 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/displayname", - httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } - return SetDisplayName(req, accountDB, device, vars["userID"], cfg, rsAPI) + return SetDisplayName(req, accountDB, stateAPI, device, vars["userID"], cfg, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows // PUT requests, so we need to allow this method r0mux.Handle("/account/3pid", - httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return GetAssociated3PIDs(req, accountDB, device) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/account/3pid", - httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return CheckAndSave3PIDAssociation(req, accountDB, device, cfg) }), ).Methods(http.MethodPost, http.MethodOptions) unstableMux.Handle("/account/3pid/delete", - httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return Forget3PID(req, accountDB) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -450,7 +500,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/voip/turnServer", - httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return RequestTurnServer(req, device, cfg) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -476,7 +526,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userID}/account_data/{type}", - httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -486,7 +536,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}", - httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -496,7 +546,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userID}/account_data/{type}", - httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -506,7 +556,7 @@ func Setup( ).Methods(http.MethodGet) r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}", - httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -516,7 +566,7 @@ func Setup( ).Methods(http.MethodGet) r0mux.Handle("/rooms/{roomID}/members", - httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -526,7 +576,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/joined_members", - httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -543,13 +593,13 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/devices", - httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return GetDevicesByLocalpart(req, deviceDB, device) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - httputil.MakeAuthAPI("get_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("get_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -559,7 +609,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - httputil.MakeAuthAPI("device_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("device_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -569,7 +619,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - httputil.MakeAuthAPI("delete_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("delete_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -579,7 +629,7 @@ func Setup( ).Methods(http.MethodDelete, http.MethodOptions) r0mux.Handle("/delete_devices", - httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return DeleteDevices(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -604,7 +654,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags", - httputil.MakeAuthAPI("get_tags", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("get_tags", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -614,7 +664,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}", - httputil.MakeAuthAPI("put_tag", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("put_tag", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -624,7 +674,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}", - httputil.MakeAuthAPI("delete_tag", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("delete_tag", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -634,7 +684,7 @@ func Setup( ).Methods(http.MethodDelete, http.MethodOptions) r0mux.Handle("/capabilities", - httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return GetCapabilities(req, rsAPI) }), ).Methods(http.MethodGet) diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index d8936f750..aba5f0d51 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -157,6 +157,17 @@ func generateSendEvent( Code: http.StatusBadRequest, JSON: jsonerror.BadJSON(e.Error()), } + } else if e, ok := err.(gomatrixserverlib.EventValidationError); ok { + if e.Code == gomatrixserverlib.EventValidationTooLarge { + return nil, &util.JSONResponse{ + Code: http.StatusRequestEntityTooLarge, + JSON: jsonerror.BadJSON(e.Error()), + } + } + return nil, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON(e.Error()), + } } else if err != nil { util.GetLogger(req.Context()).WithError(err).Error("eventutil.BuildEvent failed") resErr := jsonerror.InternalServerError() diff --git a/clientapi/routing/sendtyping.go b/clientapi/routing/sendtyping.go index 9b6a0b39b..54a822860 100644 --- a/clientapi/routing/sendtyping.go +++ b/clientapi/routing/sendtyping.go @@ -13,15 +13,15 @@ package routing import ( - "database/sql" "net/http" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/userutil" + currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" "github.com/matrix-org/dendrite/eduserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/storage/accounts" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -36,6 +36,7 @@ func SendTyping( req *http.Request, device *userapi.Device, roomID string, userID string, accountDB accounts.Database, eduAPI api.EDUServerInputAPI, + stateAPI currentstateAPI.CurrentStateInternalAPI, ) util.JSONResponse { if device.UserID != userID { return util.JSONResponse{ @@ -44,23 +45,38 @@ func SendTyping( } } - localpart, err := userutil.ParseUsernameParam(userID, nil) + // Verify that the user is a member of this room + tuple := gomatrixserverlib.StateKeyTuple{ + EventType: gomatrixserverlib.MRoomMember, + StateKey: userID, + } + var res currentstateAPI.QueryCurrentStateResponse + err := stateAPI.QueryCurrentState(req.Context(), ¤tstateAPI.QueryCurrentStateRequest{ + RoomID: roomID, + StateTuples: []gomatrixserverlib.StateKeyTuple{tuple}, + }, &res) if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("userutil.ParseUsernameParam failed") + util.GetLogger(req.Context()).WithError(err).Error("QueryCurrentState failed") return jsonerror.InternalServerError() } - - // Verify that the user is a member of this room - _, err = accountDB.GetMembershipInRoomByLocalpart(req.Context(), localpart, roomID) - if err == sql.ErrNoRows { + ev := res.StateEvents[tuple] + if ev == nil { return util.JSONResponse{ Code: http.StatusForbidden, JSON: jsonerror.Forbidden("User not in this room"), } - } else if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetMembershipInRoomByLocalPart failed") + } + membership, err := ev.Membership() + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("Member event isn't valid") return jsonerror.InternalServerError() } + if membership != gomatrixserverlib.Join { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("User not in this room"), + } + } // parse the incoming http request var r typingContentJSON diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index c308cb1f4..89bc86064 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -88,10 +88,10 @@ func CheckAndProcessInvite( ctx context.Context, device *userapi.Device, body *MembershipRequest, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, db accounts.Database, - membership string, roomID string, + roomID string, evTime time.Time, ) (inviteStoredOnIDServer bool, err error) { - if membership != gomatrixserverlib.Invite || (body.Address == "" && body.IDServer == "" && body.Medium == "") { + if body.Address == "" && body.IDServer == "" && body.Medium == "" { // If none of the 3PID-specific fields are supplied, it's a standard invite // so return nil for it to be processed as such return diff --git a/cmd/client-api-proxy/main.go b/cmd/client-api-proxy/main.go index 979b0b042..742ec3e31 100644 --- a/cmd/client-api-proxy/main.go +++ b/cmd/client-api-proxy/main.go @@ -48,19 +48,17 @@ Arguments: ` var ( - syncServerURL = flag.String("sync-api-server-url", "", "The base URL of the listening 'dendrite-sync-api-server' process. E.g. 'http://localhost:4200'") - clientAPIURL = flag.String("client-api-server-url", "", "The base URL of the listening 'dendrite-client-api-server' process. E.g. 'http://localhost:4321'") - mediaAPIURL = flag.String("media-api-server-url", "", "The base URL of the listening 'dendrite-media-api-server' process. E.g. 'http://localhost:7779'") - publicRoomsAPIURL = flag.String("public-rooms-api-server-url", "", "The base URL of the listening 'dendrite-public-rooms-api-server' process. E.g. 'http://localhost:7775'") - bindAddress = flag.String("bind-address", ":8008", "The listening port for the proxy.") - certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS") - keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS") + syncServerURL = flag.String("sync-api-server-url", "", "The base URL of the listening 'dendrite-sync-api-server' process. E.g. 'http://localhost:4200'") + clientAPIURL = flag.String("client-api-server-url", "", "The base URL of the listening 'dendrite-client-api-server' process. E.g. 'http://localhost:4321'") + mediaAPIURL = flag.String("media-api-server-url", "", "The base URL of the listening 'dendrite-media-api-server' process. E.g. 'http://localhost:7779'") + bindAddress = flag.String("bind-address", ":8008", "The listening port for the proxy.") + certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS") + keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS") ) func makeProxy(targetURL string) (*httputil.ReverseProxy, error) { - if !strings.HasSuffix(targetURL, "/") { - targetURL += "/" - } + targetURL = strings.TrimSuffix(targetURL, "/") + // Check that we can parse the URL. _, err := url.Parse(targetURL) if err != nil { @@ -122,13 +120,6 @@ func main() { fmt.Fprintln(os.Stderr, "no --media-api-server-url specified.") os.Exit(1) } - - if *publicRoomsAPIURL == "" { - flag.Usage() - fmt.Fprintln(os.Stderr, "no --public-rooms-api-server-url specified.") - os.Exit(1) - } - syncProxy, err := makeProxy(*syncServerURL) if err != nil { panic(err) @@ -141,14 +132,8 @@ func main() { if err != nil { panic(err) } - publicRoomsProxy, err := makeProxy(*publicRoomsAPIURL) - if err != nil { - panic(err) - } http.Handle("/_matrix/client/r0/sync", syncProxy) - http.Handle("/_matrix/client/r0/directory/list/", publicRoomsProxy) - http.Handle("/_matrix/client/r0/publicRooms", publicRoomsProxy) http.Handle("/_matrix/media/v1/", mediaProxy) http.Handle("/", clientProxy) @@ -160,8 +145,6 @@ func main() { fmt.Println("Proxying requests to:") fmt.Println(" /_matrix/client/r0/sync => ", *syncServerURL+"/api/_matrix/client/r0/sync") - fmt.Println(" /_matrix/client/r0/directory/list => ", *publicRoomsAPIURL+"/_matrix/client/r0/directory/list") - fmt.Println(" /_matrix/client/r0/publicRooms => ", *publicRoomsAPIURL+"/_matrix/media/client/r0/publicRooms") fmt.Println(" /_matrix/media/v1 => ", *mediaAPIURL+"/api/_matrix/media/v1") fmt.Println(" /* => ", *clientAPIURL+"/api/*") fmt.Println("Listening on ", *bindAddress) diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index fe5f30a0e..58c029fed 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -35,10 +35,11 @@ func main() { fsAPI := base.FederationSenderHTTPClient() eduInputAPI := base.EDUServerClient() userAPI := base.UserAPIClient() + stateAPI := base.CurrentStateAPIClient() clientapi.AddPublicRoutes( - base.PublicAPIMux, base.Cfg, base.KafkaConsumer, base.KafkaProducer, deviceDB, accountDB, federation, - rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI, + base.PublicAPIMux, base.Cfg, base.KafkaProducer, deviceDB, accountDB, federation, + rsAPI, eduInputAPI, asQuery, stateAPI, transactions.New(), fsAPI, userAPI, nil, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI)) diff --git a/cmd/dendrite-current-state-server/main.go b/cmd/dendrite-current-state-server/main.go new file mode 100644 index 000000000..0d4eae7b5 --- /dev/null +++ b/cmd/dendrite-current-state-server/main.go @@ -0,0 +1,33 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/matrix-org/dendrite/currentstateserver" + "github.com/matrix-org/dendrite/internal/setup" +) + +func main() { + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "CurrentStateServer", true) + defer base.Close() // nolint: errcheck + + stateAPI := currentstateserver.NewInternalAPI(cfg, base.KafkaConsumer) + + currentstateserver.AddInternalRoutes(base.InternalAPIMux, stateAPI) + + base.SetupAndServeHTTP(string(base.Cfg.Bind.CurrentState), string(base.Cfg.Listen.CurrentState)) + +} diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 356ab5a7f..988f4aa7f 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -29,7 +29,7 @@ import ( p2phttp "github.com/libp2p/go-libp2p-http" p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery" "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage" + "github.com/matrix-org/dendrite/currentstateserver" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/internal/config" @@ -129,8 +129,8 @@ func main() { cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName)) cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) - cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s-publicroomsa.db", *instanceName)) cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) + cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName)) if err = cfg.Derive(); err != nil { panic(err) } @@ -162,10 +162,13 @@ func main() { &base.Base, federation, rsAPI, keyRing, ) rsAPI.SetFederationSenderAPI(fsAPI) + /* TODO: publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub, cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } + */ + stateAPI := currentstateserver.NewInternalAPI(base.Base.Cfg, base.Base.KafkaConsumer) monolith := setup.Monolith{ Config: base.Base.Cfg, @@ -182,9 +185,8 @@ func main() { FederationSenderAPI: fsAPI, RoomserverAPI: rsAPI, ServerKeyAPI: serverKeyAPI, + StateAPI: stateAPI, UserAPI: userAPI, - - PublicRoomsDB: publicRoomsDB, } monolith.AddAllPublicRoutes(base.Base.PublicAPIMux) diff --git a/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go b/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go deleted file mode 100644 index d2cb36a8b..000000000 --- a/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgreswithdht - -import ( - "context" - "encoding/json" - "fmt" - "sync" - "sync/atomic" - "time" - - "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" - "github.com/matrix-org/gomatrixserverlib" - - dht "github.com/libp2p/go-libp2p-kad-dht" -) - -const DHTInterval = time.Second * 10 - -// PublicRoomsServerDatabase represents a public rooms server database. -type PublicRoomsServerDatabase struct { - dht *dht.IpfsDHT - postgres.PublicRoomsServerDatabase - ourRoomsContext context.Context // our current value in the DHT - ourRoomsCancel context.CancelFunc // cancel when we want to expire our value - foundRooms map[string]gomatrixserverlib.PublicRoom // additional rooms we have learned about from the DHT - foundRoomsMutex sync.RWMutex // protects foundRooms - maintenanceTimer *time.Timer // - roomsAdvertised atomic.Value // stores int - roomsDiscovered atomic.Value // stores int -} - -// NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { - pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil, localServerName) - if err != nil { - return nil, err - } - provider := PublicRoomsServerDatabase{ - dht: dht, - PublicRoomsServerDatabase: *pg, - } - go provider.ResetDHTMaintenance() - provider.roomsAdvertised.Store(0) - provider.roomsDiscovered.Store(0) - return &provider, nil -} - -func (d *PublicRoomsServerDatabase) GetRoomVisibility(ctx context.Context, roomID string) (bool, error) { - return d.PublicRoomsServerDatabase.GetRoomVisibility(ctx, roomID) -} - -func (d *PublicRoomsServerDatabase) SetRoomVisibility(ctx context.Context, visible bool, roomID string) error { - d.ResetDHTMaintenance() - return d.PublicRoomsServerDatabase.SetRoomVisibility(ctx, visible, roomID) -} - -func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) { - count, err := d.PublicRoomsServerDatabase.CountPublicRooms(ctx) - if err != nil { - return 0, err - } - d.foundRoomsMutex.RLock() - defer d.foundRoomsMutex.RUnlock() - return count + int64(len(d.foundRooms)), nil -} - -func (d *PublicRoomsServerDatabase) GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) { - realfilter := filter - if realfilter == "__local__" { - realfilter = "" - } - rooms, err := d.PublicRoomsServerDatabase.GetPublicRooms(ctx, offset, limit, realfilter) - if err != nil { - return []gomatrixserverlib.PublicRoom{}, err - } - if filter != "__local__" { - d.foundRoomsMutex.RLock() - defer d.foundRoomsMutex.RUnlock() - for _, room := range d.foundRooms { - rooms = append(rooms, room) - } - } - return rooms, nil -} - -func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error { - return d.PublicRoomsServerDatabase.UpdateRoomFromEvents(ctx, eventsToAdd, eventsToRemove) -} - -func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error { - return d.PublicRoomsServerDatabase.UpdateRoomFromEvent(ctx, event) -} - -func (d *PublicRoomsServerDatabase) ResetDHTMaintenance() { - if d.maintenanceTimer != nil && !d.maintenanceTimer.Stop() { - <-d.maintenanceTimer.C - } - d.Interval() -} - -func (d *PublicRoomsServerDatabase) Interval() { - if err := d.AdvertiseRoomsIntoDHT(); err != nil { - // fmt.Println("Failed to advertise room in DHT:", err) - } - if err := d.FindRoomsInDHT(); err != nil { - // fmt.Println("Failed to find rooms in DHT:", err) - } - fmt.Println("Found", d.roomsDiscovered.Load(), "room(s), advertised", d.roomsAdvertised.Load(), "room(s)") - d.maintenanceTimer = time.AfterFunc(DHTInterval, d.Interval) -} - -func (d *PublicRoomsServerDatabase) AdvertiseRoomsIntoDHT() error { - dbCtx, dbCancel := context.WithTimeout(context.Background(), 3*time.Second) - _ = dbCancel - ourRooms, err := d.GetPublicRooms(dbCtx, 0, 1024, "__local__") - if err != nil { - return err - } - if j, err := json.Marshal(ourRooms); err == nil { - d.roomsAdvertised.Store(len(ourRooms)) - d.ourRoomsContext, d.ourRoomsCancel = context.WithCancel(context.Background()) - if err := d.dht.PutValue(d.ourRoomsContext, "/matrix/publicRooms", j); err != nil { - return err - } - } - return nil -} - -func (d *PublicRoomsServerDatabase) FindRoomsInDHT() error { - d.foundRoomsMutex.Lock() - searchCtx, searchCancel := context.WithTimeout(context.Background(), 10*time.Second) - defer searchCancel() - defer d.foundRoomsMutex.Unlock() - results, err := d.dht.GetValues(searchCtx, "/matrix/publicRooms", 1024) - if err != nil { - return err - } - d.foundRooms = make(map[string]gomatrixserverlib.PublicRoom) - for _, result := range results { - var received []gomatrixserverlib.PublicRoom - if err := json.Unmarshal(result.Val, &received); err != nil { - return err - } - for _, room := range received { - d.foundRooms[room.RoomID] = room - } - } - d.roomsDiscovered.Store(len(d.foundRooms)) - return nil -} diff --git a/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go b/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go deleted file mode 100644 index cf642eb38..000000000 --- a/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgreswithpubsub - -import ( - "context" - "encoding/json" - "fmt" - "sync" - "sync/atomic" - "time" - - "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" - "github.com/matrix-org/gomatrixserverlib" - - pubsub "github.com/libp2p/go-libp2p-pubsub" -) - -const MaintenanceInterval = time.Second * 10 - -type discoveredRoom struct { - time time.Time - room gomatrixserverlib.PublicRoom -} - -// PublicRoomsServerDatabase represents a public rooms server database. -type PublicRoomsServerDatabase struct { - postgres.PublicRoomsServerDatabase // - pubsub *pubsub.PubSub // - subscription *pubsub.Subscription // - foundRooms map[string]discoveredRoom // additional rooms we have learned about from the DHT - foundRoomsMutex sync.RWMutex // protects foundRooms - maintenanceTimer *time.Timer // - roomsAdvertised atomic.Value // stores int -} - -// NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { - pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil, localServerName) - if err != nil { - return nil, err - } - provider := PublicRoomsServerDatabase{ - pubsub: pubsub, - PublicRoomsServerDatabase: *pg, - foundRooms: make(map[string]discoveredRoom), - } - if topic, err := pubsub.Join("/matrix/publicRooms"); err != nil { - return nil, err - } else if sub, err := topic.Subscribe(); err == nil { - provider.subscription = sub - go provider.MaintenanceTimer() - go provider.FindRooms() - provider.roomsAdvertised.Store(0) - return &provider, nil - } else { - return nil, err - } -} - -func (d *PublicRoomsServerDatabase) GetRoomVisibility(ctx context.Context, roomID string) (bool, error) { - return d.PublicRoomsServerDatabase.GetRoomVisibility(ctx, roomID) -} - -func (d *PublicRoomsServerDatabase) SetRoomVisibility(ctx context.Context, visible bool, roomID string) error { - d.MaintenanceTimer() - return d.PublicRoomsServerDatabase.SetRoomVisibility(ctx, visible, roomID) -} - -func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) { - d.foundRoomsMutex.RLock() - defer d.foundRoomsMutex.RUnlock() - return int64(len(d.foundRooms)), nil -} - -func (d *PublicRoomsServerDatabase) GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) { - var rooms []gomatrixserverlib.PublicRoom - if filter == "__local__" { - if r, err := d.PublicRoomsServerDatabase.GetPublicRooms(ctx, offset, limit, ""); err == nil { - rooms = append(rooms, r...) - } else { - return []gomatrixserverlib.PublicRoom{}, err - } - } else { - d.foundRoomsMutex.RLock() - defer d.foundRoomsMutex.RUnlock() - for _, room := range d.foundRooms { - rooms = append(rooms, room.room) - } - } - return rooms, nil -} - -func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error { - return d.PublicRoomsServerDatabase.UpdateRoomFromEvents(ctx, eventsToAdd, eventsToRemove) -} - -func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error { - return d.PublicRoomsServerDatabase.UpdateRoomFromEvent(ctx, event) -} - -func (d *PublicRoomsServerDatabase) MaintenanceTimer() { - if d.maintenanceTimer != nil && !d.maintenanceTimer.Stop() { - <-d.maintenanceTimer.C - } - d.Interval() -} - -func (d *PublicRoomsServerDatabase) Interval() { - d.foundRoomsMutex.Lock() - for k, v := range d.foundRooms { - if time.Since(v.time) > time.Minute { - delete(d.foundRooms, k) - } - } - d.foundRoomsMutex.Unlock() - if err := d.AdvertiseRooms(); err != nil { - fmt.Println("Failed to advertise room in DHT:", err) - } - d.foundRoomsMutex.RLock() - defer d.foundRoomsMutex.RUnlock() - fmt.Println("Found", len(d.foundRooms), "room(s), advertised", d.roomsAdvertised.Load(), "room(s)") - d.maintenanceTimer = time.AfterFunc(MaintenanceInterval, d.Interval) -} - -func (d *PublicRoomsServerDatabase) AdvertiseRooms() error { - dbCtx, dbCancel := context.WithTimeout(context.Background(), 3*time.Second) - _ = dbCancel - ourRooms, err := d.GetPublicRooms(dbCtx, 0, 1024, "__local__") - if err != nil { - return err - } - advertised := 0 - for _, room := range ourRooms { - if j, err := json.Marshal(room); err == nil { - if topic, err := d.pubsub.Join("/matrix/publicRooms"); err != nil { - fmt.Println("Failed to subscribe to topic:", err) - } else if err := topic.Publish(context.TODO(), j); err != nil { - fmt.Println("Failed to publish public room:", err) - } else { - advertised++ - } - } - } - - d.roomsAdvertised.Store(advertised) - return nil -} - -func (d *PublicRoomsServerDatabase) FindRooms() { - for { - msg, err := d.subscription.Next(context.Background()) - if err != nil { - continue - } - received := discoveredRoom{ - time: time.Now(), - } - if err := json.Unmarshal(msg.Data, &received.room); err != nil { - fmt.Println("Unmarshal error:", err) - continue - } - d.foundRoomsMutex.Lock() - d.foundRooms[received.room.RoomID] = received - d.foundRoomsMutex.Unlock() - } -} diff --git a/cmd/dendrite-demo-libp2p/storage/storage.go b/cmd/dendrite-demo-libp2p/storage/storage.go deleted file mode 100644 index 2d8dc1817..000000000 --- a/cmd/dendrite-demo-libp2p/storage/storage.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "net/url" - - dht "github.com/libp2p/go-libp2p-kad-dht" - pubsub "github.com/libp2p/go-libp2p-pubsub" - "github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage/postgreswithdht" - "github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" - "github.com/matrix-org/gomatrixserverlib" -) - -const schemePostgres = "postgres" -const schemeFile = "file" - -// NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabaseWithDHT(dataSourceName string, dht *dht.IpfsDHT, localServerName gomatrixserverlib.ServerName) (storage.Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName) - } - switch uri.Scheme { - case schemePostgres: - return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName) - case schemeFile: - return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName) - default: - return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName) - } -} - -// NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabaseWithPubSub(dataSourceName string, pubsub *pubsub.PubSub, localServerName gomatrixserverlib.ServerName) (storage.Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName) - } - switch uri.Scheme { - case schemePostgres: - return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName) - case schemeFile: - return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName) - default: - return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName) - } -} diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index db05ecb76..7a527d87d 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -27,13 +27,14 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn" + "github.com/matrix-org/dendrite/currentstateserver" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" - "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" @@ -50,6 +51,7 @@ var ( // nolint:gocyclo func main() { flag.Parse() + internal.SetupPprof() ygg, err := yggconn.Setup(*instanceName, *instancePeer, ".") if err != nil { @@ -73,7 +75,7 @@ func main() { cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName)) cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) - cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s-publicroomsa.db", *instanceName)) + cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName)) cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) if err = cfg.Derive(); err != nil { panic(err) @@ -108,13 +110,10 @@ func main() { rsComponent.SetFederationSenderAPI(fsAPI) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to public rooms db") - } - embed.Embed(base.BaseMux, *instancePort, "Yggdrasil Demo") + stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer) + monolith := setup.Monolith{ Config: base.Cfg, AccountDB: accountDB, @@ -130,9 +129,8 @@ func main() { FederationSenderAPI: fsAPI, RoomserverAPI: rsAPI, UserAPI: userAPI, + StateAPI: stateAPI, //ServerKeyAPI: serverKeyAPI, - - PublicRoomsDB: publicRoomsDB, } monolith.AddAllPublicRoutes(base.PublicAPIMux) diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go index c335f2eac..eb176493e 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go @@ -17,6 +17,7 @@ package yggconn import ( "context" "crypto/ed25519" + "crypto/tls" "encoding/hex" "encoding/json" "fmt" @@ -26,10 +27,11 @@ import ( "os" "strings" "sync" + "time" + "github.com/lucas-clemente/quic-go" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert" - "github.com/libp2p/go-yamux" yggdrasiladmin "github.com/yggdrasil-network/yggdrasil-go/src/admin" yggdrasilconfig "github.com/yggdrasil-network/yggdrasil-go/src/config" yggdrasilmulticast "github.com/yggdrasil-network/yggdrasil-go/src/multicast" @@ -39,16 +41,26 @@ import ( ) type Node struct { - core *yggdrasil.Core - config *yggdrasilconfig.NodeConfig - state *yggdrasilconfig.NodeState - admin *yggdrasiladmin.AdminSocket - multicast *yggdrasilmulticast.Multicast - log *gologme.Logger - listener *yggdrasil.Listener - dialer *yggdrasil.Dialer - sessions sync.Map // string -> yamux.Session - incoming chan *yamux.Stream + core *yggdrasil.Core + config *yggdrasilconfig.NodeConfig + state *yggdrasilconfig.NodeState + admin *yggdrasiladmin.AdminSocket + multicast *yggdrasilmulticast.Multicast + log *gologme.Logger + packetConn *yggdrasil.PacketConn + listener quic.Listener + tlsConfig *tls.Config + quicConfig *quic.Config + sessions sync.Map // string -> quic.Session + incoming chan QUICStream +} + +func (n *Node) BuildName() string { + return "dendrite" +} + +func (n *Node) BuildVersion() string { + return "dev" } func (n *Node) Dialer(_, address string) (net.Conn, error) { @@ -74,8 +86,9 @@ func Setup(instanceName, instancePeer, storageDirectory string) (*Node, error) { admin: &yggdrasiladmin.AdminSocket{}, multicast: &yggdrasilmulticast.Multicast{}, log: gologme.New(os.Stdout, "YGG ", log.Flags()), - incoming: make(chan *yamux.Stream), + incoming: make(chan QUICStream), } + n.core.SetBuildInfo(n) yggfile := fmt.Sprintf("%s/%s-yggdrasil.conf", storageDirectory, instanceName) if _, err := os.Stat(yggfile); !os.IsNotExist(err) { @@ -114,29 +127,21 @@ func Setup(instanceName, instancePeer, storageDirectory string) (*Node, error) { panic(err) } } - /* - if err = n.admin.Init(n.core, n.state, n.log, nil); err != nil { - panic(err) - } - if err = n.admin.Start(); err != nil { - panic(err) - } - */ if err = n.multicast.Init(n.core, n.state, n.log, nil); err != nil { panic(err) } if err = n.multicast.Start(); err != nil { panic(err) } - //n.admin.SetupAdminHandlers(n.admin) - //n.multicast.SetupAdminHandlers(n.admin) - n.listener, err = n.core.ConnListen() - if err != nil { - panic(err) - } - n.dialer, err = n.core.ConnDialer() - if err != nil { - panic(err) + + n.packetConn = n.core.PacketConn() + n.tlsConfig = n.generateTLSConfig() + n.quicConfig = &quic.Config{ + MaxIncomingStreams: 0, + MaxIncomingUniStreams: 0, + KeepAlive: true, + MaxIdleTimeout: time.Second * 120, + HandshakeTimeout: time.Second * 30, } n.log.Println("Public curve25519:", n.core.EncryptionPublicKey()) @@ -174,3 +179,7 @@ func (n *Node) SigningPrivateKey() ed25519.PrivateKey { privBytes, _ := hex.DecodeString(n.config.SigningPrivateKey) return ed25519.PrivateKey(privBytes) } + +func (n *Node) PeerCount() int { + return len(n.core.GetPeers()) - 1 +} diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/session.go b/cmd/dendrite-demo-yggdrasil/yggconn/session.go index c50b6b73c..857b2cc9c 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/session.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/session.go @@ -16,60 +16,52 @@ package yggconn import ( "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/pem" + "errors" + "math/big" "net" - "strings" "time" - "github.com/libp2p/go-yamux" + "github.com/lucas-clemente/quic-go" + "github.com/yggdrasil-network/yggdrasil-go/src/crypto" ) -func (n *Node) yamuxConfig() *yamux.Config { - cfg := yamux.DefaultConfig() - cfg.EnableKeepAlive = false - cfg.ConnectionWriteTimeout = time.Second * 15 - cfg.MaxMessageSize = 65535 - cfg.ReadBufSize = 655350 - return cfg -} - func (n *Node) listenFromYgg() { + var err error + n.listener, err = quic.Listen( + n.packetConn, // yggdrasil.PacketConn + n.tlsConfig, // TLS config + n.quicConfig, // QUIC config + ) + if err != nil { + panic(err) + } + for { - conn, err := n.listener.Accept() + session, err := n.listener.Accept(context.TODO()) if err != nil { n.log.Println("n.listener.Accept:", err) return } - var session *yamux.Session - // If the remote address is lower than ours then we'll be the - // server. Otherwse we'll be the client. - if strings.Compare(conn.RemoteAddr().String(), n.DerivedSessionName()) < 0 { - session, err = yamux.Server(conn, n.yamuxConfig()) - } else { - session, err = yamux.Client(conn, n.yamuxConfig()) - } - if err != nil { - return - } - go n.listenFromYggConn(session) + go n.listenFromQUIC(session) } } -func (n *Node) listenFromYggConn(session *yamux.Session) { +func (n *Node) listenFromQUIC(session quic.Session) { n.sessions.Store(session.RemoteAddr().String(), session) defer n.sessions.Delete(session.RemoteAddr()) - defer func() { - if err := session.Close(); err != nil { - n.log.Println("session.Close:", err) - } - }() - for { - st, err := session.AcceptStream() + st, err := session.AcceptStream(context.TODO()) if err != nil { n.log.Println("session.AcceptStream:", err) return } - n.incoming <- st + n.incoming <- QUICStream{st, session} } } @@ -96,29 +88,63 @@ func (n *Node) Dial(network, address string) (net.Conn, error) { // Implements http.Transport.DialContext func (n *Node) DialContext(ctx context.Context, network, address string) (net.Conn, error) { s, ok1 := n.sessions.Load(address) - session, ok2 := s.(*yamux.Session) - if !ok1 || !ok2 || (ok1 && ok2 && session.IsClosed()) { - conn, err := n.dialer.DialContext(ctx, network, address) + session, ok2 := s.(quic.Session) + if !ok1 || !ok2 || (ok1 && ok2 && session.ConnectionState().HandshakeComplete) { + dest, err := hex.DecodeString(address) + if err != nil { + return nil, err + } + if len(dest) != crypto.BoxPubKeyLen { + return nil, errors.New("invalid key length supplied") + } + var pubKey crypto.BoxPubKey + copy(pubKey[:], dest) + + session, err = quic.Dial( + n.packetConn, // yggdrasil.PacketConn + &pubKey, // dial address + address, // dial SNI + n.tlsConfig, // TLS config + n.quicConfig, // QUIC config + ) if err != nil { n.log.Println("n.dialer.DialContext:", err) return nil, err } - // If the remote address is lower than ours then we will be the - // server. Otherwise we'll be the client. - if strings.Compare(conn.RemoteAddr().String(), n.DerivedSessionName()) < 0 { - session, err = yamux.Server(conn, n.yamuxConfig()) - } else { - session, err = yamux.Client(conn, n.yamuxConfig()) - } - if err != nil { - return nil, err - } - go n.listenFromYggConn(session) + go n.listenFromQUIC(session) } st, err := session.OpenStream() if err != nil { n.log.Println("session.OpenStream:", err) return nil, err } - return st, nil + return QUICStream{st, session}, nil +} + +func (n *Node) generateTLSConfig() *tls.Config { + key, err := rsa.GenerateKey(rand.Reader, 1024) + if err != nil { + panic(err) + } + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + NotAfter: time.Now().Add(time.Hour * 24 * 365), + DNSNames: []string{n.DerivedSessionName()}, + } + certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + if err != nil { + panic(err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + panic(err) + } + return &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + NextProtos: []string{"quic-matrix-ygg"}, + InsecureSkipVerify: true, + } } diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/stream.go b/cmd/dendrite-demo-yggdrasil/yggconn/stream.go new file mode 100644 index 000000000..dac7447ee --- /dev/null +++ b/cmd/dendrite-demo-yggdrasil/yggconn/stream.go @@ -0,0 +1,20 @@ +package yggconn + +import ( + "net" + + "github.com/lucas-clemente/quic-go" +) + +type QUICStream struct { + quic.Stream + session quic.Session +} + +func (s QUICStream) LocalAddr() net.Addr { + return s.session.LocalAddr() +} + +func (s QUICStream) RemoteAddr() net.Addr { + return s.session.RemoteAddr() +} diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index e3bf5edc8..1bde56368 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -33,7 +33,7 @@ func main() { federationapi.AddPublicRoutes( base.PublicAPIMux, base.Cfg, userAPI, federation, keyRing, - rsAPI, fsAPI, base.EDUServerClient(), + rsAPI, fsAPI, base.EDUServerClient(), base.CurrentStateAPIClient(), ) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 339bbe699..9ac6941b2 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -20,13 +20,13 @@ import ( "os" "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/currentstateserver" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" - "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/serverkeyapi" @@ -117,10 +117,7 @@ func main() { // This is different to rsAPI which can be the http client which doesn't need this dependency rsImpl.SetFederationSenderAPI(fsAPI) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to public rooms db") - } + stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer) monolith := setup.Monolith{ Config: base.Cfg, @@ -137,9 +134,8 @@ func main() { FederationSenderAPI: fsAPI, RoomserverAPI: rsAPI, ServerKeyAPI: serverKeyAPI, + StateAPI: stateAPI, UserAPI: userAPI, - - PublicRoomsDB: publicRoomsDB, } monolith.AddAllPublicRoutes(base.PublicAPIMux) diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go deleted file mode 100644 index 23866b757..000000000 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "github.com/matrix-org/dendrite/internal/setup" - "github.com/matrix-org/dendrite/publicroomsapi" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/sirupsen/logrus" -) - -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "PublicRoomsAPI", true) - defer base.Close() // nolint: errcheck - - userAPI := base.UserAPIClient() - - rsAPI := base.RoomserverHTTPClient() - - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to public rooms db") - } - publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, base.KafkaConsumer, userAPI, publicRoomsDB, rsAPI, nil, nil) - - base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI)) - -} diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 883b0fad0..6e2bdafec 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -22,13 +22,13 @@ import ( "syscall/js" "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/currentstateserver" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" - "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/userapi" go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p" @@ -168,10 +168,10 @@ func main() { cfg.Database.FederationSender = "file:/idb/dendritejs_fedsender.db" cfg.Database.MediaAPI = "file:/idb/dendritejs_mediaapi.db" cfg.Database.Naffka = "file:/idb/dendritejs_naffka.db" - cfg.Database.PublicRoomsAPI = "file:/idb/dendritejs_publicrooms.db" cfg.Database.RoomServer = "file:/idb/dendritejs_roomserver.db" cfg.Database.ServerKey = "file:/idb/dendritejs_serverkey.db" cfg.Database.SyncAPI = "file:/idb/dendritejs_syncapi.db" + cfg.Database.CurrentState = "file:/idb/dendritejs_currentstate.db" cfg.Kafka.Topics.OutputTypingEvent = "output_typing_event" cfg.Kafka.Topics.OutputSendToDeviceEvent = "output_send_to_device_event" cfg.Kafka.Topics.OutputClientData = "output_client_data" @@ -213,10 +213,7 @@ func main() { rsAPI.SetFederationSenderAPI(fedSenderAPI) p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), cfg.Matrix.ServerName) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to public rooms db") - } + stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer) monolith := setup.Monolith{ Config: base.Cfg, @@ -232,10 +229,9 @@ func main() { EDUInternalAPI: eduInputAPI, FederationSenderAPI: fedSenderAPI, RoomserverAPI: rsAPI, + StateAPI: stateAPI, UserAPI: userAPI, //ServerKeyAPI: serverKeyAPI, - - PublicRoomsDB: publicRoomsDB, ExtPublicRoomsProvider: p2pPublicRoomProvider, } monolith.AddAllPublicRoutes(base.PublicAPIMux) diff --git a/currentstateserver/api/api.go b/currentstateserver/api/api.go new file mode 100644 index 000000000..729a66baf --- /dev/null +++ b/currentstateserver/api/api.go @@ -0,0 +1,104 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/matrix-org/gomatrixserverlib" +) + +type CurrentStateInternalAPI interface { + // QueryCurrentState retrieves the requested state events. If state events are not found, they will be missing from + // the response. + QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error + // QueryRoomsForUser retrieves a list of room IDs matching the given query. + QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error + // QueryBulkStateContent does a bulk query for state event content in the given rooms. + QueryBulkStateContent(ctx context.Context, req *QueryBulkStateContentRequest, res *QueryBulkStateContentResponse) error +} + +type QueryRoomsForUserRequest struct { + UserID string + // The desired membership of the user. If this is the empty string then no rooms are returned. + WantMembership string +} + +type QueryRoomsForUserResponse struct { + RoomIDs []string +} + +type QueryBulkStateContentRequest struct { + // Returns state events in these rooms + RoomIDs []string + // If true, treats the '*' StateKey as "all state events of this type" rather than a literal value of '*' + AllowWildcards bool + // The state events to return. Only a small subset of tuples are allowed in this request as only certain events + // have their content fields extracted. Specifically, the tuple Type must be one of: + // m.room.avatar + // m.room.create + // m.room.canonical_alias + // m.room.guest_access + // m.room.history_visibility + // m.room.join_rules + // m.room.member + // m.room.name + // m.room.topic + // Any other tuple type will result in the query failing. + StateTuples []gomatrixserverlib.StateKeyTuple +} +type QueryBulkStateContentResponse struct { + // map of room ID -> tuple -> content_value + Rooms map[string]map[gomatrixserverlib.StateKeyTuple]string +} + +type QueryCurrentStateRequest struct { + RoomID string + StateTuples []gomatrixserverlib.StateKeyTuple +} + +type QueryCurrentStateResponse struct { + StateEvents map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent +} + +// MarshalJSON stringifies the StateKeyTuple keys so they can be sent over the wire in HTTP API mode. +func (r *QueryCurrentStateResponse) MarshalJSON() ([]byte, error) { + se := make(map[string]*gomatrixserverlib.HeaderedEvent, len(r.StateEvents)) + for k, v := range r.StateEvents { + // use 0x1F (unit separator) as the delimiter between type/state key, + se[fmt.Sprintf("%s\x1F%s", k.EventType, k.StateKey)] = v + } + return json.Marshal(se) +} + +func (r *QueryCurrentStateResponse) UnmarshalJSON(data []byte) error { + res := make(map[string]*gomatrixserverlib.HeaderedEvent) + err := json.Unmarshal(data, &res) + if err != nil { + return err + } + r.StateEvents = make(map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent, len(res)) + for k, v := range res { + fields := strings.Split(k, "\x1F") + r.StateEvents[gomatrixserverlib.StateKeyTuple{ + EventType: fields[0], + StateKey: fields[1], + }] = v + } + return nil +} diff --git a/currentstateserver/consumers/roomserver.go b/currentstateserver/consumers/roomserver.go new file mode 100644 index 000000000..9e2694b0c --- /dev/null +++ b/currentstateserver/consumers/roomserver.go @@ -0,0 +1,140 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "context" + "encoding/json" + + "github.com/Shopify/sarama" + "github.com/matrix-org/dendrite/currentstateserver/storage" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" + log "github.com/sirupsen/logrus" +) + +type OutputRoomEventConsumer struct { + rsConsumer *internal.ContinualConsumer + db storage.Database +} + +func NewOutputRoomEventConsumer(topicName string, kafkaConsumer sarama.Consumer, store storage.Database) *OutputRoomEventConsumer { + consumer := &internal.ContinualConsumer{ + Topic: topicName, + Consumer: kafkaConsumer, + PartitionStore: store, + } + s := &OutputRoomEventConsumer{ + rsConsumer: consumer, + db: store, + } + consumer.ProcessMessage = s.onMessage + + return s +} + +func (c *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { + // Parse out the event JSON + var output api.OutputEvent + if err := json.Unmarshal(msg.Value, &output); err != nil { + // If the message was invalid, log it and move on to the next message in the stream + log.WithError(err).Errorf("roomserver output log: message parse failure") + return nil + } + + switch output.Type { + case api.OutputTypeNewRoomEvent: + return c.onNewRoomEvent(context.TODO(), *output.NewRoomEvent) + case api.OutputTypeNewInviteEvent: + case api.OutputTypeRetireInviteEvent: + default: + log.WithField("type", output.Type).Debug( + "roomserver output log: ignoring unknown output type", + ) + } + return nil +} + +func (c *OutputRoomEventConsumer) onNewRoomEvent( + ctx context.Context, msg api.OutputNewRoomEvent, +) error { + ev := msg.Event + + addsStateEvents := msg.AddsState() + + ev, err := c.updateStateEvent(ev) + if err != nil { + return err + } + + for i := range addsStateEvents { + addsStateEvents[i], err = c.updateStateEvent(addsStateEvents[i]) + if err != nil { + return err + } + } + + err = c.db.StoreStateEvents( + ctx, + addsStateEvents, + msg.RemovesStateEventIDs, + ) + if err != nil { + // panic rather than continue with an inconsistent database + log.WithFields(log.Fields{ + "event": string(ev.JSON()), + log.ErrorKey: err, + "add": msg.AddsStateEventIDs, + "del": msg.RemovesStateEventIDs, + }).Panicf("roomserver output log: write event failure") + } + return nil +} + +// Start consuming from room servers +func (c *OutputRoomEventConsumer) Start() error { + return c.rsConsumer.Start() +} + +func (c *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.HeaderedEvent) (gomatrixserverlib.HeaderedEvent, error) { + var stateKey string + if event.StateKey() == nil { + stateKey = "" + } else { + stateKey = *event.StateKey() + } + + prevEvent, err := c.db.GetStateEvent( + context.TODO(), event.RoomID(), event.Type(), stateKey, + ) + if err != nil { + return event, err + } + + if prevEvent == nil { + return event, nil + } + + prev := types.PrevEventRef{ + PrevContent: prevEvent.Content(), + ReplacesState: prevEvent.EventID(), + PrevSender: prevEvent.Sender(), + } + + event.Event, err = event.SetUnsigned(prev) + return event, err +} diff --git a/currentstateserver/currentstateserver.go b/currentstateserver/currentstateserver.go new file mode 100644 index 000000000..07d5e54ad --- /dev/null +++ b/currentstateserver/currentstateserver.go @@ -0,0 +1,51 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package currentstateserver + +import ( + "github.com/Shopify/sarama" + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/currentstateserver/api" + "github.com/matrix-org/dendrite/currentstateserver/consumers" + "github.com/matrix-org/dendrite/currentstateserver/internal" + "github.com/matrix-org/dendrite/currentstateserver/inthttp" + "github.com/matrix-org/dendrite/currentstateserver/storage" + "github.com/matrix-org/dendrite/internal/config" + "github.com/sirupsen/logrus" +) + +// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions +// on the given input API. +func AddInternalRoutes(router *mux.Router, intAPI api.CurrentStateInternalAPI) { + inthttp.AddRoutes(router, intAPI) +} + +// NewInternalAPI returns a concrete implementation of the internal API. Callers +// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. +func NewInternalAPI(cfg *config.Dendrite, consumer sarama.Consumer) api.CurrentStateInternalAPI { + csDB, err := storage.NewDatabase(string(cfg.Database.CurrentState), cfg.DbProperties()) + if err != nil { + logrus.WithError(err).Panicf("failed to open database") + } + roomConsumer := consumers.NewOutputRoomEventConsumer( + string(cfg.Kafka.Topics.OutputRoomEvent), consumer, csDB, + ) + if err = roomConsumer.Start(); err != nil { + logrus.WithError(err).Panicf("failed to start room server consumer") + } + return &internal.CurrentStateInternalAPI{ + DB: csDB, + } +} diff --git a/currentstateserver/currentstateserver_test.go b/currentstateserver/currentstateserver_test.go new file mode 100644 index 000000000..a0627fea7 --- /dev/null +++ b/currentstateserver/currentstateserver_test.go @@ -0,0 +1,180 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package currentstateserver + +import ( + "context" + "encoding/json" + "net/http" + "reflect" + "testing" + "time" + + "github.com/Shopify/sarama" + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/currentstateserver/api" + "github.com/matrix-org/dendrite/currentstateserver/inthttp" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/internal/test" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/naffka" +) + +var ( + testRoomVersion = gomatrixserverlib.RoomVersionV1 + testData = []json.RawMessage{ + []byte(`{"auth_events":[],"content":{"creator":"@userid:kaer.morhen"},"depth":0,"event_id":"$0ok8ynDp7kjc95e3:kaer.morhen","hashes":{"sha256":"17kPoH+h0Dk4Omn7Sus0qMb6+oGcf+CZFEgDhv7UKWs"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"jP4a04f5/F10Pw95FPpdCyKAO44JOwUQ/MZOOeA/RTU1Dn+AHPMzGSaZnuGjRr/xQuADt+I3ctb5ZQfLKNzHDw"}},"state_key":"","type":"m.room.create"}`), + []byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}]],"content":{"membership":"join"},"depth":1,"event_id":"$LEwEu0kxrtu5fOiS:kaer.morhen","hashes":{"sha256":"B7M88PhXf3vd1LaFtjQutFu4x/w7fHD28XKZ4sAsJTo"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"p2vqmuJn7ZBRImctSaKbXCAxCcBlIjPH9JHte1ouIUGy84gpu4eLipOvSBCLL26hXfC0Zrm4WUto6Hr+ohdrCg"}},"state_key":"@userid:kaer.morhen","type":"m.room.member"}`), + []byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"join_rule":"public"},"depth":2,"event_id":"$SMHlqUrNhhBBRLeN:kaer.morhen","hashes":{"sha256":"vIuJQvmMjrGxshAkj1SXe0C4RqvMbv4ZADDw9pFCWqQ"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"hBMsb3Qppo3RaqqAl4JyTgaiWEbW5hlckATky6PrHun+F3YM203TzG7w9clwuQU5F5pZoB1a6nw+to0hN90FAw"}},"state_key":"","type":"m.room.join_rules"}`), + []byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"history_visibility":"shared"},"depth":3,"event_id":"$6F1yGIbO0J7TM93h:kaer.morhen","hashes":{"sha256":"Mr23GKSlZW7UCCYLgOWawI2Sg6KIoMjUWO2TDenuOgw"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$SMHlqUrNhhBBRLeN:kaer.morhen",{"sha256":"SylzE8U02I+6eyEHgL+FlU0L5YdqrVp8OOlxKS9VQW0"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"sHLKrFI3hKGrEJfpMVZSDS3LvLasQsy50CTsOwru9XTVxgRsPo6wozNtRVjxo1J3Rk18RC9JppovmQ5VR5EcDw"}},"state_key":"","type":"m.room.history_visibility"}`), + []byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"ban":50,"events":null,"events_default":0,"invite":0,"kick":50,"redact":50,"state_default":50,"users":null,"users_default":0},"depth":4,"event_id":"$UKNe10XzYzG0TeA9:kaer.morhen","hashes":{"sha256":"ngbP3yja9U5dlckKerUs/fSOhtKxZMCVvsfhPURSS28"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$6F1yGIbO0J7TM93h:kaer.morhen",{"sha256":"A4CucrKSoWX4IaJXhq02mBg1sxIyZEftbC+5p3fZAvk"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"zOmwlP01QL3yFchzuR9WHvogOoBZA3oVtNIF3lM0ZfDnqlSYZB9sns27G/4HVq0k7alaK7ZE3oGoCrVnMkPNCw"}},"state_key":"","type":"m.room.power_levels"}`), + // messages + []byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":5,"event_id":"$gl2T9l3qm0kUbiIJ:kaer.morhen","hashes":{"sha256":"Qx3nRMHLDPSL5hBAzuX84FiSSP0K0Kju2iFoBWH4Za8"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$UKNe10XzYzG0TeA9:kaer.morhen",{"sha256":"KtSRyMjt0ZSjsv2koixTRCxIRCGoOp6QrKscsW97XRo"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"sqDgv3EG7ml5VREzmT9aZeBpS4gAPNIaIeJOwqjDhY0GPU/BcpX5wY4R7hYLrNe5cChgV+eFy/GWm1Zfg5FfDg"}},"type":"m.room.message"}`), + []byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":6,"event_id":"$MYSbs8m4rEbsCWXD:kaer.morhen","hashes":{"sha256":"kgbYM7v4Ud2YaBsjBTolM4ySg6rHcJNYI6nWhMSdFUA"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$gl2T9l3qm0kUbiIJ:kaer.morhen",{"sha256":"C/rD04h9wGxRdN2G/IBfrgoE1UovzLZ+uskwaKZ37/Q"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"x0UoKh968jj/F5l1/R7Ew0T6CTKuew3PLNHASNxqck/bkNe8yYQiDHXRr+kZxObeqPZZTpaF1+EI+bLU9W8GDQ"}},"type":"m.room.message"}`), + []byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":7,"event_id":"$N5x9WJkl9ClPrAEg:kaer.morhen","hashes":{"sha256":"FWM8oz4yquTunRZ67qlW2gzPDzdWfBP6RPHXhK1I/x8"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$MYSbs8m4rEbsCWXD:kaer.morhen",{"sha256":"fatqgW+SE8mb2wFn3UN+drmluoD4UJ/EcSrL6Ur9q1M"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"Y+LX/xcyufoXMOIoqQBNOzy6lZfUGB1ffgXIrSugk6obMiyAsiRejHQN/pciZXsHKxMJLYRFAz4zSJoS/LGPAA"}},"type":"m.room.message"}`), + } + testEvents = []gomatrixserverlib.HeaderedEvent{} + testStateEvents = make(map[gomatrixserverlib.StateKeyTuple]gomatrixserverlib.HeaderedEvent) + + kafkaTopic = "room_events" +) + +func init() { + for _, j := range testData { + e, err := gomatrixserverlib.NewEventFromTrustedJSON(j, false, testRoomVersion) + if err != nil { + panic("cannot load test data: " + err.Error()) + } + h := e.Headered(testRoomVersion) + testEvents = append(testEvents, h) + if e.StateKey() != nil { + testStateEvents[gomatrixserverlib.StateKeyTuple{ + EventType: e.Type(), + StateKey: *e.StateKey(), + }] = h + } + } +} + +func MustWriteOutputEvent(t *testing.T, producer sarama.SyncProducer, out *roomserverAPI.OutputNewRoomEvent) error { + value, err := json.Marshal(roomserverAPI.OutputEvent{ + Type: roomserverAPI.OutputTypeNewRoomEvent, + NewRoomEvent: out, + }) + if err != nil { + t.Fatalf("failed to marshal output event: %s", err) + } + _, _, err = producer.SendMessage(&sarama.ProducerMessage{ + Topic: kafkaTopic, + Key: sarama.StringEncoder(out.Event.RoomID()), + Value: sarama.ByteEncoder(value), + }) + if err != nil { + t.Fatalf("failed to send message: %s", err) + } + return nil +} + +func MustMakeInternalAPI(t *testing.T) (api.CurrentStateInternalAPI, sarama.SyncProducer) { + cfg := &config.Dendrite{} + cfg.Kafka.Topics.OutputRoomEvent = config.Topic(kafkaTopic) + cfg.Database.CurrentState = config.DataSource("file::memory:") + db, err := sqlutil.Open(sqlutil.SQLiteDriverName(), "file::memory:", nil) + if err != nil { + t.Fatalf("Failed to open naffka database: %s", err) + } + naffkaDB, err := naffka.NewSqliteDatabase(db) + if err != nil { + t.Fatalf("Failed to setup naffka database: %s", err) + } + naff, err := naffka.New(naffkaDB) + if err != nil { + t.Fatalf("Failed to create naffka consumer: %s", err) + } + return NewInternalAPI(cfg, naff), naff +} + +func TestQueryCurrentState(t *testing.T) { + currStateAPI, producer := MustMakeInternalAPI(t) + plTuple := gomatrixserverlib.StateKeyTuple{ + EventType: "m.room.power_levels", + StateKey: "", + } + plEvent := testEvents[4] + MustWriteOutputEvent(t, producer, &roomserverAPI.OutputNewRoomEvent{ + Event: plEvent, + AddsStateEventIDs: []string{plEvent.EventID()}, + }) + // we have no good way to know /when/ the server has consumed the event + time.Sleep(100 * time.Millisecond) + + testCases := []struct { + req api.QueryCurrentStateRequest + wantRes api.QueryCurrentStateResponse + wantErr error + }{ + { + req: api.QueryCurrentStateRequest{ + RoomID: plEvent.RoomID(), + StateTuples: []gomatrixserverlib.StateKeyTuple{ + plTuple, + }, + }, + wantRes: api.QueryCurrentStateResponse{ + StateEvents: map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent{ + plTuple: &plEvent, + }, + }, + }, + } + + runCases := func(testAPI api.CurrentStateInternalAPI) { + for _, tc := range testCases { + var gotRes api.QueryCurrentStateResponse + gotErr := testAPI.QueryCurrentState(context.TODO(), &tc.req, &gotRes) + if tc.wantErr == nil && gotErr != nil || tc.wantErr != nil && gotErr == nil { + t.Errorf("QueryCurrentState error, got %s want %s", gotErr, tc.wantErr) + continue + } + for tuple, wantEvent := range tc.wantRes.StateEvents { + gotEvent, ok := gotRes.StateEvents[tuple] + if !ok { + t.Errorf("QueryCurrentState want tuple %+v but it is missing from the response", tuple) + continue + } + if !reflect.DeepEqual(gotEvent.JSON(), wantEvent.JSON()) { + t.Errorf("QueryCurrentState tuple %+v got event JSON %s want %s", tuple, string(gotEvent.JSON()), string(wantEvent.JSON())) + } + } + } + } + t.Run("HTTP API", func(t *testing.T) { + router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter() + AddInternalRoutes(router, currStateAPI) + apiURL, cancel := test.ListenAndServe(t, router, false) + defer cancel() + httpAPI, err := inthttp.NewCurrentStateAPIClient(apiURL, &http.Client{}) + if err != nil { + t.Fatalf("failed to create HTTP client") + } + runCases(httpAPI) + }) + t.Run("Monolith", func(t *testing.T) { + runCases(currStateAPI) + }) +} diff --git a/currentstateserver/internal/api.go b/currentstateserver/internal/api.go new file mode 100644 index 000000000..c28760477 --- /dev/null +++ b/currentstateserver/internal/api.go @@ -0,0 +1,70 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "context" + + "github.com/matrix-org/dendrite/currentstateserver/api" + "github.com/matrix-org/dendrite/currentstateserver/storage" + "github.com/matrix-org/gomatrixserverlib" +) + +type CurrentStateInternalAPI struct { + DB storage.Database +} + +func (a *CurrentStateInternalAPI) QueryCurrentState(ctx context.Context, req *api.QueryCurrentStateRequest, res *api.QueryCurrentStateResponse) error { + res.StateEvents = make(map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent) + for _, tuple := range req.StateTuples { + ev, err := a.DB.GetStateEvent(ctx, req.RoomID, tuple.EventType, tuple.StateKey) + if err != nil { + return err + } + if ev != nil { + res.StateEvents[tuple] = ev + } + } + return nil +} + +func (a *CurrentStateInternalAPI) QueryRoomsForUser(ctx context.Context, req *api.QueryRoomsForUserRequest, res *api.QueryRoomsForUserResponse) error { + roomIDs, err := a.DB.GetRoomsByMembership(ctx, req.UserID, req.WantMembership) + if err != nil { + return err + } + res.RoomIDs = roomIDs + return nil +} + +func (a *CurrentStateInternalAPI) QueryBulkStateContent(ctx context.Context, req *api.QueryBulkStateContentRequest, res *api.QueryBulkStateContentResponse) error { + events, err := a.DB.GetBulkStateContent(ctx, req.RoomIDs, req.StateTuples, req.AllowWildcards) + if err != nil { + return err + } + res.Rooms = make(map[string]map[gomatrixserverlib.StateKeyTuple]string) + for _, ev := range events { + if res.Rooms[ev.RoomID] == nil { + res.Rooms[ev.RoomID] = make(map[gomatrixserverlib.StateKeyTuple]string) + } + room := res.Rooms[ev.RoomID] + room[gomatrixserverlib.StateKeyTuple{ + EventType: ev.EventType, + StateKey: ev.StateKey, + }] = ev.ContentValue + res.Rooms[ev.RoomID] = room + } + return nil +} diff --git a/currentstateserver/inthttp/client.go b/currentstateserver/inthttp/client.go new file mode 100644 index 000000000..b8c6a1198 --- /dev/null +++ b/currentstateserver/inthttp/client.go @@ -0,0 +1,88 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inthttp + +import ( + "context" + "errors" + "net/http" + + "github.com/matrix-org/dendrite/currentstateserver/api" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/opentracing/opentracing-go" +) + +// HTTP paths for the internal HTTP APIs +const ( + QueryCurrentStatePath = "/currentstateserver/queryCurrentState" + QueryRoomsForUserPath = "/currentstateserver/queryRoomsForUser" + QueryBulkStateContentPath = "/currentstateserver/queryBulkStateContent" +) + +// NewCurrentStateAPIClient creates a CurrentStateInternalAPI implemented by talking to a HTTP POST API. +// If httpClient is nil an error is returned +func NewCurrentStateAPIClient( + apiURL string, + httpClient *http.Client, +) (api.CurrentStateInternalAPI, error) { + if httpClient == nil { + return nil, errors.New("NewCurrentStateAPIClient: httpClient is ") + } + return &httpCurrentStateInternalAPI{ + apiURL: apiURL, + httpClient: httpClient, + }, nil +} + +type httpCurrentStateInternalAPI struct { + apiURL string + httpClient *http.Client +} + +func (h *httpCurrentStateInternalAPI) QueryCurrentState( + ctx context.Context, + request *api.QueryCurrentStateRequest, + response *api.QueryCurrentStateResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryCurrentState") + defer span.Finish() + + apiURL := h.apiURL + QueryCurrentStatePath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +func (h *httpCurrentStateInternalAPI) QueryRoomsForUser( + ctx context.Context, + request *api.QueryRoomsForUserRequest, + response *api.QueryRoomsForUserResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRoomsForUser") + defer span.Finish() + + apiURL := h.apiURL + QueryRoomsForUserPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +func (h *httpCurrentStateInternalAPI) QueryBulkStateContent( + ctx context.Context, + request *api.QueryBulkStateContentRequest, + response *api.QueryBulkStateContentResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryBulkStateContent") + defer span.Finish() + + apiURL := h.apiURL + QueryBulkStateContentPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/currentstateserver/inthttp/server.go b/currentstateserver/inthttp/server.go new file mode 100644 index 000000000..dafb9f643 --- /dev/null +++ b/currentstateserver/inthttp/server.go @@ -0,0 +1,67 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inthttp + +import ( + "encoding/json" + "net/http" + + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/currentstateserver/api" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/util" +) + +func AddRoutes(internalAPIMux *mux.Router, intAPI api.CurrentStateInternalAPI) { + internalAPIMux.Handle(QueryCurrentStatePath, + httputil.MakeInternalAPI("queryCurrentState", func(req *http.Request) util.JSONResponse { + request := api.QueryCurrentStateRequest{} + response := api.QueryCurrentStateResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := intAPI.QueryCurrentState(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(QueryRoomsForUserPath, + httputil.MakeInternalAPI("queryRoomsForUser", func(req *http.Request) util.JSONResponse { + request := api.QueryRoomsForUserRequest{} + response := api.QueryRoomsForUserResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := intAPI.QueryRoomsForUser(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(QueryBulkStateContentPath, + httputil.MakeInternalAPI("queryBulkStateContent", func(req *http.Request) util.JSONResponse { + request := api.QueryBulkStateContentRequest{} + response := api.QueryBulkStateContentResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := intAPI.QueryBulkStateContent(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/currentstateserver/storage/interface.go b/currentstateserver/storage/interface.go new file mode 100644 index 000000000..04636bafb --- /dev/null +++ b/currentstateserver/storage/interface.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + "github.com/matrix-org/dendrite/currentstateserver/storage/tables" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + internal.PartitionStorer + // StoreStateEvents updates the database with new events from the roomserver. + StoreStateEvents(ctx context.Context, addStateEvents []gomatrixserverlib.HeaderedEvent, removeStateEventIDs []string) error + // GetStateEvent returns the state event of a given type for a given room with a given state key + // If no event could be found, returns nil + // If there was an issue during the retrieval, returns an error + GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) + // GetRoomsByMembership returns a list of room IDs matching the provided membership and user ID (as state_key). + GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error) + // GetBulkStateContent returns all state events which match a given room ID and a given state key tuple. Both must be satisfied for a match. + // If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned. + GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error) +} diff --git a/currentstateserver/storage/postgres/current_room_state_table.go b/currentstateserver/storage/postgres/current_room_state_table.go new file mode 100644 index 000000000..bd2e075f0 --- /dev/null +++ b/currentstateserver/storage/postgres/current_room_state_table.go @@ -0,0 +1,272 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + "encoding/json" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/currentstateserver/storage/tables" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/gomatrixserverlib" +) + +var currentRoomStateSchema = ` +-- Stores the current room state for every room. +CREATE TABLE IF NOT EXISTS currentstate_current_room_state ( + -- The 'room_id' key for the state event. + room_id TEXT NOT NULL, + -- The state event ID + event_id TEXT NOT NULL, + -- The state event type e.g 'm.room.member' + type TEXT NOT NULL, + -- The 'sender' property of the event. + sender TEXT NOT NULL, + -- The state_key value for this state event e.g '' + state_key TEXT NOT NULL, + -- The JSON for the event. Stored as TEXT because this should be valid UTF-8. + headered_event_json TEXT NOT NULL, + -- A piece of extracted content e.g membership for m.room.member events + content_value TEXT NOT NULL DEFAULT '', + -- Clobber based on 3-uple of room_id, type and state_key + CONSTRAINT currentstate_current_room_state_unique UNIQUE (room_id, type, state_key) +); +-- for event deletion +CREATE UNIQUE INDEX IF NOT EXISTS currentstate_event_id_idx ON currentstate_current_room_state(event_id, room_id, type, sender); +-- for querying membership states of users +CREATE INDEX IF NOT EXISTS currentstate_membership_idx ON currentstate_current_room_state(type, state_key, content_value) +WHERE type='m.room.member' AND content_value IS NOT NULL AND content_value != 'leave'; +` + +const upsertRoomStateSQL = "" + + "INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, content_value)" + + " VALUES ($1, $2, $3, $4, $5, $6, $7)" + + " ON CONFLICT ON CONSTRAINT currentstate_current_room_state_unique" + + " DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, content_value = $7" + +const deleteRoomStateByEventIDSQL = "" + + "DELETE FROM currentstate_current_room_state WHERE event_id = $1" + +const selectRoomIDsWithMembershipSQL = "" + + "SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND content_value = $2" + +const selectStateEventSQL = "" + + "SELECT headered_event_json FROM currentstate_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" + +const selectEventsWithEventIDsSQL = "" + + "SELECT headered_event_json FROM currentstate_current_room_state WHERE event_id = ANY($1)" + +const selectBulkStateContentSQL = "" + + "SELECT room_id, type, state_key, content_value FROM currentstate_current_room_state WHERE room_id = ANY($1) AND type = ANY($2) AND state_key = ANY($3)" + +const selectBulkStateContentWildSQL = "" + + "SELECT room_id, type, state_key, content_value FROM currentstate_current_room_state WHERE room_id = ANY($1) AND type = ANY($2)" + +type currentRoomStateStatements struct { + upsertRoomStateStmt *sql.Stmt + deleteRoomStateByEventIDStmt *sql.Stmt + selectRoomIDsWithMembershipStmt *sql.Stmt + selectEventsWithEventIDsStmt *sql.Stmt + selectStateEventStmt *sql.Stmt + selectBulkStateContentStmt *sql.Stmt + selectBulkStateContentWildStmt *sql.Stmt +} + +func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) { + s := ¤tRoomStateStatements{} + _, err := db.Exec(currentRoomStateSchema) + if err != nil { + return nil, err + } + if s.upsertRoomStateStmt, err = db.Prepare(upsertRoomStateSQL); err != nil { + return nil, err + } + if s.deleteRoomStateByEventIDStmt, err = db.Prepare(deleteRoomStateByEventIDSQL); err != nil { + return nil, err + } + if s.selectRoomIDsWithMembershipStmt, err = db.Prepare(selectRoomIDsWithMembershipSQL); err != nil { + return nil, err + } + if s.selectEventsWithEventIDsStmt, err = db.Prepare(selectEventsWithEventIDsSQL); err != nil { + return nil, err + } + if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil { + return nil, err + } + if s.selectBulkStateContentStmt, err = db.Prepare(selectBulkStateContentSQL); err != nil { + return nil, err + } + if s.selectBulkStateContentWildStmt, err = db.Prepare(selectBulkStateContentWildSQL); err != nil { + return nil, err + } + return s, nil +} + +// SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state. +func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( + ctx context.Context, + txn *sql.Tx, + userID string, + contentVal string, +) ([]string, error) { + stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) + rows, err := stmt.QueryContext(ctx, userID, contentVal) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithMembership: rows.close() failed") + + var result []string + for rows.Next() { + var roomID string + if err := rows.Scan(&roomID); err != nil { + return nil, err + } + result = append(result, roomID) + } + return result, rows.Err() +} + +func (s *currentRoomStateStatements) DeleteRoomStateByEventID( + ctx context.Context, txn *sql.Tx, eventID string, +) error { + stmt := sqlutil.TxStmt(txn, s.deleteRoomStateByEventIDStmt) + _, err := stmt.ExecContext(ctx, eventID) + return err +} + +func (s *currentRoomStateStatements) UpsertRoomState( + ctx context.Context, txn *sql.Tx, + event gomatrixserverlib.HeaderedEvent, contentVal string, +) error { + headeredJSON, err := json.Marshal(event) + if err != nil { + return err + } + + // upsert state event + stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt) + _, err = stmt.ExecContext( + ctx, + event.RoomID(), + event.EventID(), + event.Type(), + event.Sender(), + *event.StateKey(), + headeredJSON, + contentVal, + ) + return err +} + +func (s *currentRoomStateStatements) SelectEventsWithEventIDs( + ctx context.Context, txn *sql.Tx, eventIDs []string, +) ([]gomatrixserverlib.HeaderedEvent, error) { + stmt := sqlutil.TxStmt(txn, s.selectEventsWithEventIDsStmt) + rows, err := stmt.QueryContext(ctx, pq.StringArray(eventIDs)) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed") + result := []gomatrixserverlib.HeaderedEvent{} + for rows.Next() { + var eventBytes []byte + if err := rows.Scan(&eventBytes); err != nil { + return nil, err + } + // TODO: Handle redacted events + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { + return nil, err + } + result = append(result, ev) + } + return result, rows.Err() +} + +func (s *currentRoomStateStatements) SelectStateEvent( + ctx context.Context, roomID, evType, stateKey string, +) (*gomatrixserverlib.HeaderedEvent, error) { + stmt := s.selectStateEventStmt + var res []byte + err := stmt.QueryRowContext(ctx, roomID, evType, stateKey).Scan(&res) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, err + } + var ev gomatrixserverlib.HeaderedEvent + if err = json.Unmarshal(res, &ev); err != nil { + return nil, err + } + return &ev, err +} + +func (s *currentRoomStateStatements) SelectBulkStateContent( + ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool, +) ([]tables.StrippedEvent, error) { + hasWildcards := false + eventTypeSet := make(map[string]bool) + stateKeySet := make(map[string]bool) + var eventTypes []string + var stateKeys []string + for _, tuple := range tuples { + if !eventTypeSet[tuple.EventType] { + eventTypeSet[tuple.EventType] = true + eventTypes = append(eventTypes, tuple.EventType) + } + if !stateKeySet[tuple.StateKey] { + stateKeySet[tuple.StateKey] = true + stateKeys = append(stateKeys, tuple.StateKey) + } + if tuple.StateKey == "*" { + hasWildcards = true + } + } + var rows *sql.Rows + var err error + if hasWildcards && allowWildcards { + rows, err = s.selectBulkStateContentWildStmt.QueryContext(ctx, pq.StringArray(roomIDs), pq.StringArray(eventTypes)) + } else { + rows, err = s.selectBulkStateContentStmt.QueryContext( + ctx, pq.StringArray(roomIDs), pq.StringArray(eventTypes), pq.StringArray(stateKeys), + ) + } + if err != nil { + return nil, err + } + strippedEvents := []tables.StrippedEvent{} + defer internal.CloseAndLogIfError(ctx, rows, "SelectBulkStateContent: rows.close() failed") + for rows.Next() { + var roomID string + var eventType string + var stateKey string + var contentVal string + if err = rows.Scan(&roomID, &eventType, &stateKey, &contentVal); err != nil { + return nil, err + } + strippedEvents = append(strippedEvents, tables.StrippedEvent{ + RoomID: roomID, + ContentValue: contentVal, + EventType: eventType, + StateKey: stateKey, + }) + } + return strippedEvents, rows.Err() +} diff --git a/currentstateserver/storage/postgres/storage.go b/currentstateserver/storage/postgres/storage.go new file mode 100644 index 000000000..f8edb94e6 --- /dev/null +++ b/currentstateserver/storage/postgres/storage.go @@ -0,0 +1,35 @@ +package postgres + +import ( + "database/sql" + + "github.com/matrix-org/dendrite/currentstateserver/storage/shared" + "github.com/matrix-org/dendrite/internal/sqlutil" +) + +type Database struct { + shared.Database + db *sql.DB + sqlutil.PartitionOffsetStatements +} + +// NewDatabase creates a new sync server database +func NewDatabase(dbDataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) { + var d Database + var err error + if d.db, err = sqlutil.Open("postgres", dbDataSourceName, dbProperties); err != nil { + return nil, err + } + if err = d.PartitionOffsetStatements.Prepare(d.db, "currentstate"); err != nil { + return nil, err + } + currRoomState, err := NewPostgresCurrentRoomStateTable(d.db) + if err != nil { + return nil, err + } + d.Database = shared.Database{ + DB: d.db, + CurrentRoomState: currRoomState, + } + return &d, nil +} diff --git a/currentstateserver/storage/shared/storage.go b/currentstateserver/storage/shared/storage.go new file mode 100644 index 000000000..cd59ac129 --- /dev/null +++ b/currentstateserver/storage/shared/storage.go @@ -0,0 +1,66 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shared + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/currentstateserver/storage/tables" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database struct { + DB *sql.DB + CurrentRoomState tables.CurrentRoomState +} + +func (d *Database) GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) { + return d.CurrentRoomState.SelectStateEvent(ctx, roomID, evType, stateKey) +} + +func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error) { + return d.CurrentRoomState.SelectBulkStateContent(ctx, roomIDs, tuples, allowWildcards) +} + +func (d *Database) StoreStateEvents(ctx context.Context, addStateEvents []gomatrixserverlib.HeaderedEvent, + removeStateEventIDs []string) error { + return sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { + // remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add. + for _, eventID := range removeStateEventIDs { + if err := d.CurrentRoomState.DeleteRoomStateByEventID(ctx, txn, eventID); err != nil { + return err + } + } + + for _, event := range addStateEvents { + if event.StateKey() == nil { + // ignore non state events + continue + } + contentVal := tables.ExtractContentValue(&event) + + if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, contentVal); err != nil { + return err + } + } + return nil + }) +} + +func (d *Database) GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error) { + return d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, nil, userID, membership) +} diff --git a/currentstateserver/storage/sqlite3/current_room_state_table.go b/currentstateserver/storage/sqlite3/current_room_state_table.go new file mode 100644 index 000000000..95185d9a8 --- /dev/null +++ b/currentstateserver/storage/sqlite3/current_room_state_table.go @@ -0,0 +1,276 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "encoding/json" + "strings" + + "github.com/matrix-org/dendrite/currentstateserver/storage/tables" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/gomatrixserverlib" +) + +const currentRoomStateSchema = ` +-- Stores the current room state for every room. +CREATE TABLE IF NOT EXISTS currentstate_current_room_state ( + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + type TEXT NOT NULL, + sender TEXT NOT NULL, + state_key TEXT NOT NULL, + headered_event_json TEXT NOT NULL, + content_value TEXT NOT NULL DEFAULT '', + UNIQUE (room_id, type, state_key) +); +-- for event deletion +CREATE UNIQUE INDEX IF NOT EXISTS currentstate_event_id_idx ON currentstate_current_room_state(event_id, room_id, type, sender); +` + +const upsertRoomStateSQL = "" + + "INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, content_value)" + + " VALUES ($1, $2, $3, $4, $5, $6, $7)" + + " ON CONFLICT (event_id, room_id, type, sender)" + + " DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, content_value = $7" + +const deleteRoomStateByEventIDSQL = "" + + "DELETE FROM currentstate_current_room_state WHERE event_id = $1" + +const selectRoomIDsWithMembershipSQL = "" + + "SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND content_value = $2" + +const selectStateEventSQL = "" + + "SELECT headered_event_json FROM currentstate_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" + +const selectEventsWithEventIDsSQL = "" + + "SELECT headered_event_json FROM currentstate_current_room_state WHERE event_id IN ($1)" + +const selectBulkStateContentSQL = "" + + "SELECT room_id, type, state_key, content_value FROM currentstate_current_room_state WHERE room_id IN ($1) AND type IN ($2) AND state_key IN ($3)" + +const selectBulkStateContentWildSQL = "" + + "SELECT room_id, type, state_key, content_value FROM currentstate_current_room_state WHERE room_id IN ($1) AND type IN ($2)" + +type currentRoomStateStatements struct { + db *sql.DB + upsertRoomStateStmt *sql.Stmt + deleteRoomStateByEventIDStmt *sql.Stmt + selectRoomIDsWithMembershipStmt *sql.Stmt + selectStateEventStmt *sql.Stmt +} + +func NewSqliteCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) { + s := ¤tRoomStateStatements{ + db: db, + } + _, err := db.Exec(currentRoomStateSchema) + if err != nil { + return nil, err + } + if s.upsertRoomStateStmt, err = db.Prepare(upsertRoomStateSQL); err != nil { + return nil, err + } + if s.deleteRoomStateByEventIDStmt, err = db.Prepare(deleteRoomStateByEventIDSQL); err != nil { + return nil, err + } + if s.selectRoomIDsWithMembershipStmt, err = db.Prepare(selectRoomIDsWithMembershipSQL); err != nil { + return nil, err + } + if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil { + return nil, err + } + return s, nil +} + +// SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state. +func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( + ctx context.Context, + txn *sql.Tx, + userID string, + membership string, +) ([]string, error) { + stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) + rows, err := stmt.QueryContext(ctx, userID, membership) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithMembership: rows.close() failed") + + var result []string + for rows.Next() { + var roomID string + if err := rows.Scan(&roomID); err != nil { + return nil, err + } + result = append(result, roomID) + } + return result, nil +} + +func (s *currentRoomStateStatements) DeleteRoomStateByEventID( + ctx context.Context, txn *sql.Tx, eventID string, +) error { + stmt := sqlutil.TxStmt(txn, s.deleteRoomStateByEventIDStmt) + _, err := stmt.ExecContext(ctx, eventID) + return err +} + +func (s *currentRoomStateStatements) UpsertRoomState( + ctx context.Context, txn *sql.Tx, + event gomatrixserverlib.HeaderedEvent, contentVal string, +) error { + headeredJSON, err := json.Marshal(event) + if err != nil { + return err + } + + // upsert state event + stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt) + _, err = stmt.ExecContext( + ctx, + event.RoomID(), + event.EventID(), + event.Type(), + event.Sender(), + *event.StateKey(), + headeredJSON, + contentVal, + ) + return err +} + +func (s *currentRoomStateStatements) SelectEventsWithEventIDs( + ctx context.Context, txn *sql.Tx, eventIDs []string, +) ([]gomatrixserverlib.HeaderedEvent, error) { + iEventIDs := make([]interface{}, len(eventIDs)) + for k, v := range eventIDs { + iEventIDs[k] = v + } + query := strings.Replace(selectEventsWithEventIDsSQL, "($1)", sqlutil.QueryVariadic(len(iEventIDs)), 1) + rows, err := txn.QueryContext(ctx, query, iEventIDs...) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed") + result := []gomatrixserverlib.HeaderedEvent{} + for rows.Next() { + var eventBytes []byte + if err := rows.Scan(&eventBytes); err != nil { + return nil, err + } + // TODO: Handle redacted events + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { + return nil, err + } + result = append(result, ev) + } + return result, nil +} + +func (s *currentRoomStateStatements) SelectStateEvent( + ctx context.Context, roomID, evType, stateKey string, +) (*gomatrixserverlib.HeaderedEvent, error) { + stmt := s.selectStateEventStmt + var res []byte + err := stmt.QueryRowContext(ctx, roomID, evType, stateKey).Scan(&res) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, err + } + var ev gomatrixserverlib.HeaderedEvent + if err = json.Unmarshal(res, &ev); err != nil { + return nil, err + } + return &ev, err +} + +func (s *currentRoomStateStatements) SelectBulkStateContent( + ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool, +) ([]tables.StrippedEvent, error) { + hasWildcards := false + eventTypeSet := make(map[string]bool) + stateKeySet := make(map[string]bool) + var eventTypes []string + var stateKeys []string + for _, tuple := range tuples { + if !eventTypeSet[tuple.EventType] { + eventTypeSet[tuple.EventType] = true + eventTypes = append(eventTypes, tuple.EventType) + } + if !stateKeySet[tuple.StateKey] { + stateKeySet[tuple.StateKey] = true + stateKeys = append(stateKeys, tuple.StateKey) + } + if tuple.StateKey == "*" { + hasWildcards = true + } + } + + iRoomIDs := make([]interface{}, len(roomIDs)) + for i, v := range roomIDs { + iRoomIDs[i] = v + } + iEventTypes := make([]interface{}, len(eventTypes)) + for i, v := range eventTypes { + iEventTypes[i] = v + } + iStateKeys := make([]interface{}, len(stateKeys)) + for i, v := range stateKeys { + iStateKeys[i] = v + } + + var query string + var args []interface{} + if hasWildcards && allowWildcards { + query = strings.Replace(selectBulkStateContentWildSQL, "($1)", sqlutil.QueryVariadic(len(iRoomIDs)), 1) + query = strings.Replace(query, "($2)", sqlutil.QueryVariadicOffset(len(iEventTypes), len(iRoomIDs)), 1) + args = append(iRoomIDs, iEventTypes...) + } else { + query = strings.Replace(selectBulkStateContentSQL, "($1)", sqlutil.QueryVariadic(len(iRoomIDs)), 1) + query = strings.Replace(query, "($2)", sqlutil.QueryVariadicOffset(len(iEventTypes), len(iRoomIDs)), 1) + query = strings.Replace(query, "($3)", sqlutil.QueryVariadicOffset(len(iStateKeys), len(iEventTypes)+len(iRoomIDs)), 1) + args = append(iRoomIDs, iEventTypes...) + args = append(args, iStateKeys...) + } + rows, err := s.db.QueryContext(ctx, query, args...) + + if err != nil { + return nil, err + } + strippedEvents := []tables.StrippedEvent{} + defer internal.CloseAndLogIfError(ctx, rows, "SelectBulkStateContent: rows.close() failed") + for rows.Next() { + var roomID string + var eventType string + var stateKey string + var contentVal string + if err = rows.Scan(&roomID, &eventType, &stateKey, &contentVal); err != nil { + return nil, err + } + strippedEvents = append(strippedEvents, tables.StrippedEvent{ + RoomID: roomID, + ContentValue: contentVal, + EventType: eventType, + StateKey: stateKey, + }) + } + return strippedEvents, rows.Err() +} diff --git a/currentstateserver/storage/sqlite3/storage.go b/currentstateserver/storage/sqlite3/storage.go new file mode 100644 index 000000000..6975e40ba --- /dev/null +++ b/currentstateserver/storage/sqlite3/storage.go @@ -0,0 +1,39 @@ +package sqlite3 + +import ( + "database/sql" + + "github.com/matrix-org/dendrite/currentstateserver/storage/shared" + "github.com/matrix-org/dendrite/internal/sqlutil" +) + +type Database struct { + shared.Database + db *sql.DB + sqlutil.PartitionOffsetStatements +} + +// NewDatabase creates a new sync server database +// nolint: gocyclo +func NewDatabase(dataSourceName string) (*Database, error) { + var d Database + cs, err := sqlutil.ParseFileURI(dataSourceName) + if err != nil { + return nil, err + } + if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { + return nil, err + } + if err = d.PartitionOffsetStatements.Prepare(d.db, "currentstate"); err != nil { + return nil, err + } + currRoomState, err := NewSqliteCurrentRoomStateTable(d.db) + if err != nil { + return nil, err + } + d.Database = shared.Database{ + DB: d.db, + CurrentRoomState: currRoomState, + } + return &d, nil +} diff --git a/currentstateserver/storage/storage.go b/currentstateserver/storage/storage.go new file mode 100644 index 000000000..ad04cf414 --- /dev/null +++ b/currentstateserver/storage/storage.go @@ -0,0 +1,41 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !wasm + +package storage + +import ( + "net/url" + + "github.com/matrix-org/dendrite/currentstateserver/storage/postgres" + "github.com/matrix-org/dendrite/currentstateserver/storage/sqlite3" + "github.com/matrix-org/dendrite/internal/sqlutil" +) + +// NewDatabase opens a database connection. +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return postgres.NewDatabase(dataSourceName, dbProperties) + } + switch uri.Scheme { + case "postgres": + return postgres.NewDatabase(dataSourceName, dbProperties) + case "file": + return sqlite3.NewDatabase(dataSourceName) + default: + return postgres.NewDatabase(dataSourceName, dbProperties) + } +} diff --git a/publicroomsapi/storage/storage_wasm.go b/currentstateserver/storage/storage_wasm.go similarity index 70% rename from publicroomsapi/storage/storage_wasm.go rename to currentstateserver/storage/storage_wasm.go index 70ceeaf85..aa46c44df 100644 --- a/publicroomsapi/storage/storage_wasm.go +++ b/currentstateserver/storage/storage_wasm.go @@ -18,21 +18,24 @@ import ( "fmt" "net/url" - "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/currentstateserver/storage/sqlite3" + "github.com/matrix-org/dendrite/internal/sqlutil" ) -// NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatrixserverlib.ServerName) (Database, error) { +// NewDatabase opens a database connection. +func NewDatabase( + dataSourceName string, + dbProperties sqlutil.DbProperties, // nolint:unparam +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return nil, err + return nil, fmt.Errorf("Cannot use postgres implementation") } switch uri.Scheme { case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName) + return sqlite3.NewDatabase(dataSourceName) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/currentstateserver/storage/tables/interface.go b/currentstateserver/storage/tables/interface.go new file mode 100644 index 000000000..8ba4e4eb9 --- /dev/null +++ b/currentstateserver/storage/tables/interface.go @@ -0,0 +1,79 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tables + +import ( + "context" + "database/sql" + + "github.com/matrix-org/gomatrixserverlib" + "github.com/tidwall/gjson" +) + +type CurrentRoomState interface { + SelectStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) + SelectEventsWithEventIDs(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) + // UpsertRoomState stores the given event in the database, along with an extracted piece of content. + // The piece of content will vary depending on the event type, and table implementations may use this information to optimise + // lookups e.g membership lookups. The mapped value of `contentVal` is outlined in ExtractContentValue. An empty `contentVal` + // means there is nothing to store for this field. + UpsertRoomState(ctx context.Context, txn *sql.Tx, event gomatrixserverlib.HeaderedEvent, contentVal string) error + DeleteRoomStateByEventID(ctx context.Context, txn *sql.Tx, eventID string) error + // SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state. + SelectRoomIDsWithMembership(ctx context.Context, txn *sql.Tx, userID string, membership string) ([]string, error) + SelectBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]StrippedEvent, error) +} + +// StrippedEvent represents a stripped event for returning extracted content values. +type StrippedEvent struct { + RoomID string + EventType string + StateKey string + ContentValue string +} + +// ExtractContentValue from the given state event. For example, given an m.room.name event with: +// content: { name: "Foo" } +// this returns "Foo". +func ExtractContentValue(ev *gomatrixserverlib.HeaderedEvent) string { + content := ev.Content() + key := "" + switch ev.Type() { + case gomatrixserverlib.MRoomCreate: + key = "creator" + case gomatrixserverlib.MRoomCanonicalAlias: + key = "alias" + case gomatrixserverlib.MRoomHistoryVisibility: + key = "history_visibility" + case gomatrixserverlib.MRoomJoinRules: + key = "join_rule" + case gomatrixserverlib.MRoomMember: + key = "membership" + case gomatrixserverlib.MRoomName: + key = "name" + case "m.room.avatar": + key = "url" + case "m.room.topic": + key = "topic" + case "m.room.guest_access": + key = "guest_access" + } + result := gjson.GetBytes(content, key) + if !result.Exists() { + return "" + } + // this returns the empty string if this is not a string type + return result.Str +} diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 73bfec247..2b95c102b 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -120,7 +120,7 @@ database: server_key: "postgres://dendrite:itsasecret@localhost/dendrite_serverkey?sslmode=disable" federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable" appservice: "postgres://dendrite:itsasecret@localhost/dendrite_appservice?sslmode=disable" - public_rooms_api: "postgres://dendrite:itsasecret@localhost/dendrite_publicroomsapi?sslmode=disable" + current_state: "postgres://dendrite:itsasecret@localhost/dendrite_currentstate?sslmode=disable" max_open_conns: 100 max_idle_conns: 2 conn_max_lifetime: -1 @@ -136,13 +136,13 @@ listen: federation_api: "localhost:7772" sync_api: "localhost:7773" media_api: "localhost:7774" - public_rooms_api: "localhost:7775" federation_sender: "localhost:7776" appservice_api: "localhost:7777" edu_server: "localhost:7778" key_server: "localhost:7779" server_key_api: "localhost:7780" user_api: "localhost:7781" + current_state_server: "localhost:7782" # The configuration for tracing the dendrite components. tracing: diff --git a/docs/INSTALL.md b/docs/INSTALL.md index b4c81a42b..c97351809 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -108,7 +108,7 @@ Assuming that Postgres 9.5 (or later) is installed: * Create the component databases: ```bash - for i in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi appservice naffka; do + for i in account device mediaapi syncapi roomserver serverkey federationsender currentstate appservice naffka; do sudo -u postgres createdb -O dendrite dendrite_$i done ``` @@ -176,17 +176,17 @@ The following contains scripts which will run all the required processes in orde | | :7774 | | | | - | | /directory +----------------------------------+ - | | +--------->| dendrite-public-rooms-api-server |<========++ - | | | +----------------------------------+ || - | | | :7775 | || - | | | +<-----------+ || - | | | | || - | | | /sync +--------------------------+ || + | | + | | + | | + | | + | | + | | + | | /sync +--------------------------+ | | +--------->| dendrite-sync-api-server |<================++ - | | | | +--------------------------+ || - | | | | :7773 | ^^ || -Matrix +------------------+ | | | | || client_data || + | | | +--------------------------+ || + | | | :7773 | ^^ || +Matrix +------------------+ | | | || client_data || Clients --->| client-api-proxy |-------+ +<-----------+ ++=============++ || +------------------+ | | | || || :8008 | | CS API +----------------------------+ || || @@ -232,7 +232,6 @@ your client at `http://localhost:8008`. --client-api-server-url "http://localhost:7771" \ --sync-api-server-url "http://localhost:7773" \ --media-api-server-url "http://localhost:7774" \ ---public-rooms-api-server-url "http://localhost:7775" \ ``` ### Federation proxy @@ -282,15 +281,6 @@ order to upload and retrieve media. ./bin/dendrite-media-api-server --config dendrite.yaml ``` -### Public room server - -This implements `/directory` requests. Clients talk to this via the proxy -in order to retrieve room directory listings. - -```bash -./bin/dendrite-public-rooms-api-server --config dendrite.yaml -``` - ### Federation API server This implements federation requests. Servers talk to this via the proxy in diff --git a/docs/WIRING-Current.md b/docs/WIRING-Current.md index ec539d4e9..b74f341e5 100644 --- a/docs/WIRING-Current.md +++ b/docs/WIRING-Current.md @@ -9,23 +9,22 @@ a request/response model like HTTP or RPC. Therefore, components can expose "int Note in Monolith mode these are actually direct function calls and are not serialised HTTP requests. ``` - Tier 1 Sync PublicRooms FederationAPI ClientAPI MediaAPI -Public Facing | .-----1------` | | | | | | | | | - 2 | .-------3-----------------` | | | `--------|-|-|-|--11--------------------. - | | | .--------4----------------------------------` | | | | - | | | | .---5-----------` | | | | | | - | | | | | .---6----------------------------` | | | - | | | | | | | .-----7----------` | | - | | | | | | 8 | | 10 | - | | | | | | | | `---9----. | | - V V V V V V V V V V V + Tier 1 Sync FederationAPI ClientAPI MediaAPI +Public Facing | | | | | | | | | | + 2 .-------3-----------------` | | | `--------|-|-|-|--11--------------------. + | | .--------4----------------------------------` | | | | + | | | .---5-----------` | | | | | | + | | | | .---6----------------------------` | | | + | | | | | | .-----7----------` | | + | | | | | 8 | | 10 | + | | | | | | | `---9----. | | + V V V V V V V V V V Tier 2 Roomserver EDUServer FedSender AppService KeyServer ServerKeyAPI Internal only | `------------------------12----------^ ^ `------------------------------------------------------------13----------` Client ---> Server ``` -- 1 (PublicRooms -> Roomserver): Calculating current auth for changing visibility - 2 (Sync -> Roomserver): When making backfill requests - 3 (FedAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing backfill/state/state_ids requests - 4 (ClientAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing /state requests @@ -46,20 +45,20 @@ In addition to this, all public facing components (Tier 1) talk to the `UserAPI` ``` .----1--------------------------------------------. V | - Tier 1 Sync PublicRooms FederationAPI ClientAPI MediaAPI -Public Facing ^ ^ ^ ^ - | | | | - 2 | | | + Tier 1 Sync FederationAPI ClientAPI MediaAPI +Public Facing ^ ^ ^ + | | | + 2 | | | `-3------------. | - | | | | - | | | | - | .------4------` | | - | | .--------5-----|------------------------------` - | | | | + | | | + | | | + | | | + | .--------4-----|------------------------------` + | | | Tier 2 Roomserver EDUServer FedSender AppService KeyServer ServerKeyAPI Internal only | | ^ ^ - | `-----6----------` | - `--------------------7--------` + | `-----5----------` | + `--------------------6--------` Producer ----> Consumer @@ -67,7 +66,6 @@ Producer ----> Consumer - 1 (ClientAPI -> Sync): For tracking account data - 2 (Roomserver -> Sync): For all data to send to clients - 3 (EDUServer -> Sync): For typing/send-to-device data to send to clients -- 4 (Roomserver -> PublicRooms): For tracking the current room name/topic/joined count/etc. -- 5 (Roomserver -> ClientAPI): For tracking memberships for profile updates. -- 6 (EDUServer -> FedSender): For sending EDUs over federation -- 7 (Roomserver -> FedSender): For sending PDUs over federation, for tracking joined hosts. +- 4 (Roomserver -> ClientAPI): For tracking memberships for profile updates. +- 5 (EDUServer -> FedSender): For sending EDUs over federation +- 6 (Roomserver -> FedSender): For sending PDUs over federation, for tracking joined hosts. diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index c0c000434..7d1994b25 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -16,6 +16,7 @@ package federationapi import ( "github.com/gorilla/mux" + currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" @@ -36,11 +37,12 @@ func AddPublicRoutes( rsAPI roomserverAPI.RoomserverInternalAPI, federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, eduAPI eduserverAPI.EDUServerInputAPI, + stateAPI currentstateAPI.CurrentStateInternalAPI, ) { routing.Setup( router, cfg, rsAPI, eduAPI, federationSenderAPI, keyRing, - federation, userAPI, + federation, userAPI, stateAPI, ) } diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index cc85c61bf..6bbe9d80e 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -31,7 +31,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) { fsAPI := base.FederationSenderHTTPClient() // TODO: This is pretty fragile, as if anything calls anything on these nils this test will break. // Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing. - federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil) + federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil, nil) httputil.SetupHTTPAPI( base.BaseMux, base.PublicAPIMux, diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index b1d84f254..4a49463a2 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -15,6 +15,7 @@ package routing import ( + "context" "encoding/json" "fmt" "net/http" @@ -27,8 +28,8 @@ import ( "github.com/matrix-org/util" ) -// Invite implements /_matrix/federation/v2/invite/{roomID}/{eventID} -func Invite( +// InviteV2 implements /_matrix/federation/v2/invite/{roomID}/{eventID} +func InviteV2( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, roomID string, @@ -44,14 +45,58 @@ func Invite( JSON: jsonerror.NotJSON("The request body could not be decoded into an invite request. " + err.Error()), } } - event := inviteReq.Event() + return processInvite( + httpReq.Context(), inviteReq.Event(), inviteReq.RoomVersion(), inviteReq.InviteRoomState(), roomID, eventID, cfg, rsAPI, keys, + ) +} + +// InviteV1 implements /_matrix/federation/v1/invite/{roomID}/{eventID} +func InviteV1( + httpReq *http.Request, + request *gomatrixserverlib.FederationRequest, + roomID string, + eventID string, + cfg *config.Dendrite, + rsAPI api.RoomserverInternalAPI, + keys gomatrixserverlib.JSONVerifier, +) util.JSONResponse { + roomVer := gomatrixserverlib.RoomVersionV1 + body := request.Content() + event, err := gomatrixserverlib.NewEventFromTrustedJSON(body, false, roomVer) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.NotJSON("The request body could not be decoded into an invite v1 request: " + err.Error()), + } + } + var strippedState []gomatrixserverlib.InviteV2StrippedState + if err := json.Unmarshal(event.Unsigned(), &strippedState); err != nil { + // just warn, they may not have added any. + util.GetLogger(httpReq.Context()).Warnf("failed to extract stripped state from invite event") + } + return processInvite( + httpReq.Context(), event, roomVer, strippedState, roomID, eventID, cfg, rsAPI, keys, + ) +} + +func processInvite( + ctx context.Context, + event gomatrixserverlib.Event, + roomVer gomatrixserverlib.RoomVersion, + strippedState []gomatrixserverlib.InviteV2StrippedState, + roomID string, + eventID string, + cfg *config.Dendrite, + rsAPI api.RoomserverInternalAPI, + keys gomatrixserverlib.JSONVerifier, +) util.JSONResponse { // Check that we can accept invites for this room version. - if _, err := roomserverVersion.SupportedRoomVersion(inviteReq.RoomVersion()); err != nil { + if _, err := roomserverVersion.SupportedRoomVersion(roomVer); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion( - fmt.Sprintf("Room version %q is not supported by this server.", inviteReq.RoomVersion()), + fmt.Sprintf("Room version %q is not supported by this server.", roomVer), ), } } @@ -80,9 +125,9 @@ func Invite( AtTS: event.OriginServerTS(), StrictValidityChecking: true, }} - verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests) + verifyResults, err := keys.VerifyJSONs(ctx, verifyRequests) if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("keys.VerifyJSONs failed") + util.GetLogger(ctx).WithError(err).Error("keys.VerifyJSONs failed") return jsonerror.InternalServerError() } if verifyResults[0].Error != nil { @@ -99,13 +144,9 @@ func Invite( // Add the invite event to the roomserver. if perr := api.SendInvite( - httpReq.Context(), rsAPI, - signedEvent.Headered(inviteReq.RoomVersion()), - inviteReq.InviteRoomState(), - event.Origin(), - nil, + ctx, rsAPI, signedEvent.Headered(roomVer), strippedState, event.Origin(), nil, ); perr != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendInvite failed") + util.GetLogger(ctx).WithError(err).Error("producer.SendInvite failed") return perr.JSONResponse() } diff --git a/federationapi/routing/publicrooms.go b/federationapi/routing/publicrooms.go new file mode 100644 index 000000000..3807a5183 --- /dev/null +++ b/federationapi/routing/publicrooms.go @@ -0,0 +1,178 @@ +package routing + +import ( + "context" + "net/http" + "strconv" + + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +type PublicRoomReq struct { + Since string `json:"since,omitempty"` + Limit int16 `json:"limit,omitempty"` + Filter filter `json:"filter,omitempty"` +} + +type filter struct { + SearchTerms string `json:"generic_search_term,omitempty"` +} + +// GetPostPublicRooms implements GET and POST /publicRooms +func GetPostPublicRooms(req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI) util.JSONResponse { + var request PublicRoomReq + if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil { + return *fillErr + } + if request.Limit == 0 { + request.Limit = 50 + } + response, err := publicRooms(req.Context(), request, rsAPI, stateAPI) + if err != nil { + return jsonerror.InternalServerError() + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: response, + } +} + +func publicRooms(ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.RoomserverInternalAPI, + stateAPI currentstateAPI.CurrentStateInternalAPI) (*gomatrixserverlib.RespPublicRooms, error) { + + var response gomatrixserverlib.RespPublicRooms + var limit int16 + var offset int64 + limit = request.Limit + offset, err := strconv.ParseInt(request.Since, 10, 64) + // ParseInt returns 0 and an error when trying to parse an empty string + // In that case, we want to assign 0 so we ignore the error + if err != nil && len(request.Since) > 0 { + util.GetLogger(ctx).WithError(err).Error("strconv.ParseInt failed") + return nil, err + } + + var queryRes roomserverAPI.QueryPublishedRoomsResponse + err = rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed") + return nil, err + } + response.TotalRoomCountEstimate = len(queryRes.RoomIDs) + + if offset > 0 { + response.PrevBatch = strconv.Itoa(int(offset) - 1) + } + nextIndex := int(offset) + int(limit) + if response.TotalRoomCountEstimate > nextIndex { + response.NextBatch = strconv.Itoa(nextIndex) + } + + if offset < 0 { + offset = 0 + } + if nextIndex > len(queryRes.RoomIDs) { + nextIndex = len(queryRes.RoomIDs) + } + roomIDs := queryRes.RoomIDs[offset:nextIndex] + response.Chunk, err = fillInRooms(ctx, roomIDs, stateAPI) + return &response, err +} + +// fillPublicRoomsReq fills the Limit, Since and Filter attributes of a GET or POST request +// on /publicRooms by parsing the incoming HTTP request +// Filter is only filled for POST requests +func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSONResponse { + if httpReq.Method == http.MethodGet { + limit, err := strconv.Atoi(httpReq.FormValue("limit")) + // Atoi returns 0 and an error when trying to parse an empty string + // In that case, we want to assign 0 so we ignore the error + if err != nil && len(httpReq.FormValue("limit")) > 0 { + util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed") + reqErr := jsonerror.InternalServerError() + return &reqErr + } + request.Limit = int16(limit) + request.Since = httpReq.FormValue("since") + return nil + } else if httpReq.Method == http.MethodPost { + return httputil.UnmarshalJSONRequest(httpReq, request) + } + + return &util.JSONResponse{ + Code: http.StatusMethodNotAllowed, + JSON: jsonerror.NotFound("Bad method"), + } +} + +// due to lots of switches +// nolint:gocyclo +func fillInRooms(ctx context.Context, roomIDs []string, stateAPI currentstateAPI.CurrentStateInternalAPI) ([]gomatrixserverlib.PublicRoom, error) { + avatarTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.avatar", StateKey: ""} + nameTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.name", StateKey: ""} + canonicalTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCanonicalAlias, StateKey: ""} + topicTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.topic", StateKey: ""} + guestTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.guest_access", StateKey: ""} + visibilityTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomHistoryVisibility, StateKey: ""} + joinRuleTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""} + + var stateRes currentstateAPI.QueryBulkStateContentResponse + err := stateAPI.QueryBulkStateContent(ctx, ¤tstateAPI.QueryBulkStateContentRequest{ + RoomIDs: roomIDs, + AllowWildcards: true, + StateTuples: []gomatrixserverlib.StateKeyTuple{ + nameTuple, canonicalTuple, topicTuple, guestTuple, visibilityTuple, joinRuleTuple, avatarTuple, + {EventType: gomatrixserverlib.MRoomMember, StateKey: "*"}, + }, + }, &stateRes) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("QueryBulkStateContent failed") + return nil, err + } + util.GetLogger(ctx).Infof("room IDs: %+v", roomIDs) + util.GetLogger(ctx).Infof("State res: %+v", stateRes.Rooms) + chunk := make([]gomatrixserverlib.PublicRoom, len(roomIDs)) + i := 0 + for roomID, data := range stateRes.Rooms { + pub := gomatrixserverlib.PublicRoom{ + RoomID: roomID, + } + joinCount := 0 + var joinRule, guestAccess string + for tuple, contentVal := range data { + if tuple.EventType == gomatrixserverlib.MRoomMember && contentVal == "join" { + joinCount++ + continue + } + switch tuple { + case avatarTuple: + pub.AvatarURL = contentVal + case nameTuple: + pub.Name = contentVal + case topicTuple: + pub.Topic = contentVal + case canonicalTuple: + pub.CanonicalAlias = contentVal + case visibilityTuple: + pub.WorldReadable = contentVal == "world_readable" + // need both of these to determine whether guests can join + case joinRuleTuple: + joinRule = contentVal + case guestTuple: + guestAccess = contentVal + } + } + if joinRule == gomatrixserverlib.Public && guestAccess == "can_join" { + pub.GuestCanJoin = true + } + pub.JoinedMembersCount = joinCount + chunk[i] = pub + i++ + } + return chunk, nil +} diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 645f397de..cd97f2978 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -19,6 +19,7 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/jsonerror" + currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" @@ -52,6 +53,7 @@ func Setup( keys gomatrixserverlib.JSONVerifier, federation *gomatrixserverlib.FederationClient, userAPI userapi.UserInternalAPI, + stateAPI currentstateAPI.CurrentStateInternalAPI, ) { v2keysmux := publicAPIMux.PathPrefix(pathPrefixV2Keys).Subrouter() v1fedmux := publicAPIMux.PathPrefix(pathPrefixV1Federation).Subrouter() @@ -83,10 +85,26 @@ func Setup( }, )).Methods(http.MethodPut, http.MethodOptions) + v1fedmux.Handle("/invite/{roomID}/{eventID}", httputil.MakeFedAPI( + "federation_invite", cfg.Matrix.ServerName, keys, wakeup, + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { + res := InviteV1( + httpReq, request, vars["roomID"], vars["eventID"], + cfg, rsAPI, keys, + ) + return util.JSONResponse{ + Code: res.Code, + JSON: []interface{}{ + res.Code, res.JSON, + }, + } + }, + )).Methods(http.MethodPut, http.MethodOptions) + v2fedmux.Handle("/invite/{roomID}/{eventID}", httputil.MakeFedAPI( "federation_invite", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { - return Invite( + return InviteV2( httpReq, request, vars["roomID"], vars["eventID"], cfg, rsAPI, keys, ) @@ -275,4 +293,10 @@ func Setup( return Backfill(httpReq, request, rsAPI, vars["roomID"], cfg) }, )).Methods(http.MethodGet) + + v1fedmux.Handle("/publicRooms", + httputil.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse { + return GetPostPublicRooms(req, rsAPI, stateAPI) + }), + ).Methods(http.MethodGet) } diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 53f951951..680eaccd3 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -404,8 +404,7 @@ func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event, roomVer // at this point we know we're going to have a gap: we need to work out the room state at the new backwards extremity. // security: we have to do state resolution on the new backwards extremity (TODO: WHY) // Therefore, we cannot just query /state_ids with this event to get the state before. Instead, we need to query - // the state AFTER all the prev_events for this event, then mix in our current room state and apply state resolution - // to that to get the state before the event. + // the state AFTER all the prev_events for this event, then apply state resolution to that to get the state before the event. var states []*gomatrixserverlib.RespState needed := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{*backwardsExtremity}).Tuples() for _, prevEventID := range backwardsExtremity.PrevEventIDs() { @@ -417,13 +416,6 @@ func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event, roomVer } states = append(states, prevState) } - // mix in the current room state - currState, err := t.lookupCurrentState(backwardsExtremity) - if err != nil { - util.GetLogger(t.context).WithError(err).Errorf("Failed to lookup current room state") - return err - } - states = append(states, currState) resolvedState, err := t.resolveStatesAndCheck(roomVersion, states, backwardsExtremity) if err != nil { util.GetLogger(t.context).WithError(err).Errorf("Failed to resolve state conflicts for event %s", backwardsExtremity.EventID()) @@ -526,23 +518,6 @@ func (t *txnReq) lookupStateAfterEventLocally(roomID, eventID string, needed []g } } -func (t *txnReq) lookupCurrentState(newEvent *gomatrixserverlib.Event) (*gomatrixserverlib.RespState, error) { - // Ask the roomserver for information about this room - queryReq := api.QueryLatestEventsAndStateRequest{ - RoomID: newEvent.RoomID(), - StateToFetch: gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{*newEvent}).Tuples(), - } - var queryRes api.QueryLatestEventsAndStateResponse - if err := t.rsAPI.QueryLatestEventsAndState(t.context, &queryReq, &queryRes); err != nil { - return nil, fmt.Errorf("lookupCurrentState rsAPI.QueryLatestEventsAndState: %w", err) - } - evs := gomatrixserverlib.UnwrapEventHeaders(queryRes.StateEvents) - return &gomatrixserverlib.RespState{ - StateEvents: evs, - AuthEvents: evs, - }, nil -} - // lookuptStateBeforeEvent returns the room state before the event e, which is just /state_ids and/or /state depending on what // the server supports. func (t *txnReq) lookupStateBeforeEvent(roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) ( diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index 3f5d5f4e0..bfbdaa5ff 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -111,6 +111,13 @@ func (t *testRoomserverAPI) PerformJoin( ) { } +func (t *testRoomserverAPI) PerformPublish( + ctx context.Context, + req *api.PerformPublishRequest, + res *api.PerformPublishResponse, +) { +} + func (t *testRoomserverAPI) PerformLeave( ctx context.Context, req *api.PerformLeaveRequest, @@ -168,6 +175,14 @@ func (t *testRoomserverAPI) QueryMembershipForUser( return fmt.Errorf("not implemented") } +func (t *testRoomserverAPI) QueryPublishedRooms( + ctx context.Context, + request *api.QueryPublishedRoomsRequest, + response *api.QueryPublishedRoomsResponse, +) error { + return fmt.Errorf("not implemented") +} + // Query a list of membership events for a room func (t *testRoomserverAPI) QueryMembershipsForRoom( ctx context.Context, diff --git a/federationsender/api/api.go b/federationsender/api/api.go index 02c762582..d90ffd290 100644 --- a/federationsender/api/api.go +++ b/federationsender/api/api.go @@ -4,6 +4,7 @@ import ( "context" "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" ) @@ -28,7 +29,7 @@ type FederationSenderInternalAPI interface { ctx context.Context, request *PerformJoinRequest, response *PerformJoinResponse, - ) error + ) // Handle an instruction to make_leave & send_leave with a remote server. PerformLeave( ctx context.Context, @@ -62,6 +63,7 @@ type PerformJoinRequest struct { } type PerformJoinResponse struct { + LastError *gomatrix.HTTPError } type PerformLeaveRequest struct { diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 10ac51c8a..acf524146 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -50,7 +50,8 @@ func NewInternalAPI( statistics := &types.Statistics{} queues := queue.NewOutgoingQueues( - base.Cfg.Matrix.ServerName, federation, rsAPI, statistics, &queue.SigningInfo{ + federationSenderDB, base.Cfg.Matrix.ServerName, federation, rsAPI, statistics, + &queue.SigningInfo{ KeyID: base.Cfg.Matrix.KeyID, PrivateKey: base.Cfg.Matrix.PrivateKey, ServerName: base.Cfg.Matrix.ServerName, diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index 7ced4af86..96b1149d9 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -2,6 +2,7 @@ package internal import ( "context" + "errors" "fmt" "time" @@ -9,6 +10,7 @@ import ( "github.com/matrix-org/dendrite/federationsender/internal/perform" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/version" + "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/sirupsen/logrus" @@ -40,7 +42,7 @@ func (r *FederationSenderInternalAPI) PerformJoin( ctx context.Context, request *api.PerformJoinRequest, response *api.PerformJoinResponse, -) (err error) { +) { // Look up the supported room versions. var supportedVersions []gomatrixserverlib.RoomVersion for version := range version.SupportedRoomVersions() { @@ -63,6 +65,7 @@ func (r *FederationSenderInternalAPI) PerformJoin( // Try each server that we were provided until we land on one that // successfully completes the make-join send-join dance. + var lastErr error for _, serverName := range request.ServerNames { if err := r.performJoinUsingServer( ctx, @@ -76,17 +79,32 @@ func (r *FederationSenderInternalAPI) PerformJoin( "server_name": serverName, "room_id": request.RoomID, }).Warnf("Failed to join room through server") + lastErr = err continue } // We're all good. - return nil + return } // If we reach here then we didn't complete a join for some reason. - return fmt.Errorf( - "failed to join user %q to room %q through %d server(s)", - request.UserID, request.RoomID, len(request.ServerNames), + var httpErr gomatrix.HTTPError + if ok := errors.As(lastErr, &httpErr); ok { + httpErr.Message = string(httpErr.Contents) + // Clear the wrapped error, else serialising to JSON (in polylith mode) will fail + httpErr.WrappedError = nil + response.LastError = &httpErr + } else { + response.LastError = &gomatrix.HTTPError{ + Code: 0, + WrappedError: nil, + Message: lastErr.Error(), + } + } + + logrus.Errorf( + "failed to join user %q to room %q through %d server(s): last error %s", + request.UserID, request.RoomID, len(request.ServerNames), lastErr, ) } diff --git a/federationsender/inthttp/client.go b/federationsender/inthttp/client.go index 5da4b35f9..25de99cce 100644 --- a/federationsender/inthttp/client.go +++ b/federationsender/inthttp/client.go @@ -7,6 +7,7 @@ import ( "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/gomatrix" "github.com/opentracing/opentracing-go" ) @@ -77,12 +78,19 @@ func (h *httpFederationSenderInternalAPI) PerformJoin( ctx context.Context, request *api.PerformJoinRequest, response *api.PerformJoinResponse, -) error { +) { span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoinRequest") defer span.Finish() apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath - return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + if err != nil { + response.LastError = &gomatrix.HTTPError{ + Message: err.Error(), + Code: 0, + WrappedError: err, + } + } } // Handle an instruction to make_join & send_join with a remote server. diff --git a/federationsender/inthttp/server.go b/federationsender/inthttp/server.go index babd3ae13..a4f3d63d0 100644 --- a/federationsender/inthttp/server.go +++ b/federationsender/inthttp/server.go @@ -33,9 +33,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route if err := json.NewDecoder(req.Body).Decode(&request); err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } - if err := intAPI.PerformJoin(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } + intAPI.PerformJoin(req.Context(), &request, &response) return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index 4449f9e63..ce706768e 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -18,8 +18,10 @@ import ( "context" "encoding/json" "fmt" + "sync" "time" + "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrix" @@ -29,11 +31,14 @@ import ( "go.uber.org/atomic" ) +const maxPDUsPerTransaction = 50 + // destinationQueue is a queue of events for a single destination. // It is responsible for sending the events to the destination and // ensures that only one request is in flight to a given destination // at a time. type destinationQueue struct { + db storage.Database signing *SigningInfo rsAPI api.RoomserverInternalAPI client *gomatrixserverlib.FederationClient // federation client @@ -42,13 +47,15 @@ type destinationQueue struct { running atomic.Bool // is the queue worker running? backingOff atomic.Bool // true if we're backing off statistics *types.ServerStatistics // statistics about this remote server - incomingPDUs chan *gomatrixserverlib.HeaderedEvent // PDUs to send - incomingEDUs chan *gomatrixserverlib.EDU // EDUs to send incomingInvites chan *gomatrixserverlib.InviteV2Request // invites to send - lastTransactionIDs []gomatrixserverlib.TransactionID // last transaction ID - pendingPDUs []*gomatrixserverlib.HeaderedEvent // owned by backgroundSend + incomingEDUs chan *gomatrixserverlib.EDU // EDUs to send + transactionIDMutex sync.Mutex // protects transactionID + transactionID gomatrixserverlib.TransactionID // last transaction ID + transactionCount atomic.Int32 // how many events in this transaction so far + pendingPDUs atomic.Int64 // how many PDUs are waiting to be sent pendingEDUs []*gomatrixserverlib.EDU // owned by backgroundSend pendingInvites []*gomatrixserverlib.InviteV2Request // owned by backgroundSend + wakeServerCh chan bool // interrupts idle wait retryServerCh chan bool // interrupts backoff } @@ -79,15 +86,44 @@ func (oq *destinationQueue) retry() { // Send event adds the event to the pending queue for the destination. // If the queue is empty then it starts a background goroutine to // start sending events to that destination. -func (oq *destinationQueue) sendEvent(ev *gomatrixserverlib.HeaderedEvent) { +func (oq *destinationQueue) sendEvent(nid int64) { if oq.statistics.Blacklisted() { // If the destination is blacklisted then drop the event. return } - if !oq.running.Load() { - go oq.backgroundSend() + oq.wakeQueueIfNeeded() + // Create a transaction ID. We'll either do this if we don't have + // one made up yet, or if we've exceeded the number of maximum + // events allowed in a single tranaction. We'll reset the counter + // when we do. + oq.transactionIDMutex.Lock() + if oq.transactionID == "" || oq.transactionCount.Load() >= maxPDUsPerTransaction { + now := gomatrixserverlib.AsTimestamp(time.Now()) + oq.transactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, oq.statistics.SuccessCount())) + oq.transactionCount.Store(0) + } + oq.transactionIDMutex.Unlock() + // Create a database entry that associates the given PDU NID with + // this destination queue. We'll then be able to retrieve the PDU + // later. + if err := oq.db.AssociatePDUWithDestination( + context.TODO(), + oq.transactionID, // the current transaction ID + oq.destination, // the destination server name + []int64{nid}, // NID from federationsender_queue_json table + ); err != nil { + log.WithError(err).Errorf("failed to associate PDU NID %d with destination %q", nid, oq.destination) + return + } + // We've successfully added a PDU to the transaction so increase + // the counter. + oq.transactionCount.Add(1) + // Signal that we've sent a new PDU. This will cause the queue to + // wake up if it's asleep. The return to the Add function will only + // be 1 if the previous value was 0, e.g. nothing was waiting before. + if oq.pendingPDUs.Add(1) == 1 { + oq.wakeServerCh <- true } - oq.incomingPDUs <- ev } // sendEDU adds the EDU event to the pending queue for the destination. @@ -98,9 +134,7 @@ func (oq *destinationQueue) sendEDU(ev *gomatrixserverlib.EDU) { // If the destination is blacklisted then drop the event. return } - if !oq.running.Load() { - go oq.backgroundSend() - } + oq.wakeQueueIfNeeded() oq.incomingEDUs <- ev } @@ -112,10 +146,30 @@ func (oq *destinationQueue) sendInvite(ev *gomatrixserverlib.InviteV2Request) { // If the destination is blacklisted then drop the event. return } + oq.wakeQueueIfNeeded() + oq.incomingInvites <- ev +} + +func (oq *destinationQueue) wakeQueueIfNeeded() { if !oq.running.Load() { + // Look up how many events are pending in this queue. We need + // to do this so that the queue thinks it has work to do. + count, err := oq.db.GetPendingPDUCount( + context.TODO(), + oq.destination, + ) + if err == nil { + oq.pendingPDUs.Store(count) + log.Printf("Destination queue %q has %d pending PDUs", oq.destination, count) + } else { + log.WithError(err).Errorf("Can't get pending PDU count for %q destination queue", oq.destination) + } + if count > 0 { + oq.wakeServerCh <- true + } + // Then start the queue. go oq.backgroundSend() } - oq.incomingInvites <- ev } // backgroundSend is the worker goroutine for sending events. @@ -129,26 +183,16 @@ func (oq *destinationQueue) backgroundSend() { defer oq.running.Store(false) for { - // Wait either for incoming events, or until we hit an - // idle timeout. + // If we have nothing to do then wait either for incoming events, or + // until we hit an idle timeout. select { - case pdu := <-oq.incomingPDUs: - // Ordering of PDUs is important so we add them to the end - // of the queue and they will all be added to transactions - // in order. - oq.pendingPDUs = append(oq.pendingPDUs, pdu) - // If there are any more things waiting in the channel queue - // then read them. This is safe because we guarantee only - // having one goroutine per destination queue, so the channel - // isn't being consumed anywhere else. - for len(oq.incomingPDUs) > 0 { - oq.pendingPDUs = append(oq.pendingPDUs, <-oq.incomingPDUs) - } + case <-oq.wakeServerCh: + // We were woken up because there are new PDUs waiting in the + // database. case edu := <-oq.incomingEDUs: - // Likewise for EDUs, although we should probably not try - // too hard with some EDUs (like typing notifications) after - // a certain amount of time has passed. - // TODO: think about EDU expiry some more + // EDUs are handled in-memory for now. We will try to keep + // the ordering intact. + // TODO: Certain EDU types need persistence, e.g. send-to-device oq.pendingEDUs = append(oq.pendingEDUs, edu) // If there are any more things waiting in the channel queue // then read them. This is safe because we guarantee only @@ -175,9 +219,9 @@ func (oq *destinationQueue) backgroundSend() { oq.pendingInvites = append(oq.pendingInvites, <-oq.incomingInvites) } case <-time.After(time.Second * 30): - // The worker is idle so stop the goroutine. It'll - // get restarted automatically the next time we - // get an event. + // The worker is idle so stop the goroutine. It'll get + // restarted automatically the next time we have an event to + // send. return } @@ -193,47 +237,31 @@ func (oq *destinationQueue) backgroundSend() { oq.backingOff.Store(false) } - // How many things do we have waiting? - numPDUs := len(oq.pendingPDUs) - numEDUs := len(oq.pendingEDUs) - numInvites := len(oq.pendingInvites) - // If we have pending PDUs or EDUs then construct a transaction. - if numPDUs > 0 || numEDUs > 0 { + if oq.pendingPDUs.Load() > 0 || len(oq.pendingEDUs) > 0 { // Try sending the next transaction and see what happens. - transaction, terr := oq.nextTransaction(oq.pendingPDUs, oq.pendingEDUs, oq.statistics.SuccessCount()) + transaction, terr := oq.nextTransaction(oq.pendingEDUs) if terr != nil { // We failed to send the transaction. if giveUp := oq.statistics.Failure(); giveUp { - // It's been suggested that we should give up because - // the backoff has exceeded a maximum allowable value. + // It's been suggested that we should give up because the backoff + // has exceeded a maximum allowable value. Clean up the in-memory + // buffers at this point. The PDU clean-up is already on a defer. + oq.cleanPendingEDUs() + oq.cleanPendingInvites() return } } else if transaction { // If we successfully sent the transaction then clear out - // the pending events and EDUs. + // the pending events and EDUs, and wipe our transaction ID. oq.statistics.Success() - // Reallocate so that the underlying arrays can be GC'd, as - // opposed to growing forever. - for i := 0; i < numPDUs; i++ { - oq.pendingPDUs[i] = nil - } - for i := 0; i < numEDUs; i++ { - oq.pendingEDUs[i] = nil - } - oq.pendingPDUs = append( - []*gomatrixserverlib.HeaderedEvent{}, - oq.pendingPDUs[numPDUs:]..., - ) - oq.pendingEDUs = append( - []*gomatrixserverlib.EDU{}, - oq.pendingEDUs[numEDUs:]..., - ) + // Clean up the in-memory buffers. + oq.cleanPendingEDUs() } } // Try sending the next invite and see what happens. - if numInvites > 0 { + if len(oq.pendingInvites) > 0 { sent, ierr := oq.nextInvites(oq.pendingInvites) if ierr != nil { // We failed to send the transaction so increase the @@ -249,59 +277,119 @@ func (oq *destinationQueue) backgroundSend() { oq.statistics.Success() // Reallocate so that the underlying array can be GC'd, as // opposed to growing forever. - oq.pendingInvites = append( - []*gomatrixserverlib.InviteV2Request{}, - oq.pendingInvites[sent:]..., - ) + oq.cleanPendingInvites() } } } } +// cleanPendingEDUs cleans out the pending EDU buffer, removing +// all references so that the underlying objects can be GC'd. +func (oq *destinationQueue) cleanPendingEDUs() { + for i := 0; i < len(oq.pendingEDUs); i++ { + oq.pendingEDUs[i] = nil + } + oq.pendingEDUs = []*gomatrixserverlib.EDU{} +} + +// cleanPendingInvites cleans out the pending invite buffer, +// removing all references so that the underlying objects can +// be GC'd. +func (oq *destinationQueue) cleanPendingInvites() { + for i := 0; i < len(oq.pendingInvites); i++ { + oq.pendingInvites[i] = nil + } + oq.pendingInvites = []*gomatrixserverlib.InviteV2Request{} +} + // nextTransaction creates a new transaction from the pending event // queue and sends it. Returns true if a transaction was sent or // false otherwise. func (oq *destinationQueue) nextTransaction( - pendingPDUs []*gomatrixserverlib.HeaderedEvent, pendingEDUs []*gomatrixserverlib.EDU, - sentCounter uint32, ) (bool, error) { + // Before we do anything, we need to roll over the transaction + // ID that is being used to coalesce events into the next TX. + // Otherwise it's possible that we'll pick up an incomplete + // transaction and end up nuking the rest of the events at the + // cleanup stage. + oq.transactionIDMutex.Lock() + oq.transactionID = "" + oq.transactionIDMutex.Unlock() + oq.transactionCount.Store(0) + + // Create the transaction. t := gomatrixserverlib.Transaction{ PDUs: []json.RawMessage{}, EDUs: []gomatrixserverlib.EDU{}, } - now := gomatrixserverlib.AsTimestamp(time.Now()) - t.TransactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, sentCounter)) t.Origin = oq.origin t.Destination = oq.destination - t.OriginServerTS = now - t.PreviousIDs = oq.lastTransactionIDs - if t.PreviousIDs == nil { - t.PreviousIDs = []gomatrixserverlib.TransactionID{} + t.OriginServerTS = gomatrixserverlib.AsTimestamp(time.Now()) + + // Ask the database for any pending PDUs from the next transaction. + // maxPDUsPerTransaction is an upper limit but we probably won't + // actually retrieve that many events. + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + txid, pdus, err := oq.db.GetNextTransactionPDUs( + ctx, // context + oq.destination, // server name + maxPDUsPerTransaction, // max events to retrieve + ) + if err != nil { + log.WithError(err).Errorf("failed to get next transaction PDUs for server %q", oq.destination) + return false, fmt.Errorf("oq.db.GetNextTransactionPDUs: %w", err) } - oq.lastTransactionIDs = []gomatrixserverlib.TransactionID{t.TransactionID} + // If we didn't get anything from the database and there are no + // pending EDUs then there's nothing to do - stop here. + if len(pdus) == 0 && len(pendingEDUs) == 0 { + return false, nil + } - for _, pdu := range pendingPDUs { + // Pick out the transaction ID from the database. If we didn't + // get a transaction ID (i.e. because there are no PDUs but only + // EDUs) then generate a transaction ID. + t.TransactionID = txid + if t.TransactionID == "" { + now := gomatrixserverlib.AsTimestamp(time.Now()) + t.TransactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, oq.statistics.SuccessCount())) + } + + // Go through PDUs that we retrieved from the database, if any, + // and add them into the transaction. + for _, pdu := range pdus { // Append the JSON of the event, since this is a json.RawMessage type in the // gomatrixserverlib.Transaction struct t.PDUs = append(t.PDUs, (*pdu).JSON()) } + // Do the same for pending EDUS in the queue. for _, edu := range pendingEDUs { t.EDUs = append(t.EDUs, *edu) } logrus.WithField("server_name", oq.destination).Infof("Sending transaction %q containing %d PDUs, %d EDUs", t.TransactionID, len(t.PDUs), len(t.EDUs)) + // Try to send the transaction to the destination server. // TODO: we should check for 500-ish fails vs 400-ish here, // since we shouldn't queue things indefinitely in response // to a 400-ish error - _, err := oq.client.SendTransaction(context.TODO(), t) + _, err = oq.client.SendTransaction(context.TODO(), t) switch e := err.(type) { case nil: // No error was returned so the transaction looks to have // been successfully sent. + oq.pendingPDUs.Sub(int64(len(t.PDUs))) + // Clean up the transaction in the database. + if err = oq.db.CleanTransactionPDUs( + context.TODO(), + t.Destination, + t.TransactionID, + ); err != nil { + log.WithError(err).Errorf("failed to clean transaction %q for server %q", t.TransactionID, t.Destination) + } return true, nil case gomatrix.HTTPError: // We received a HTTP error back. In this instance we only diff --git a/federationsender/queue/queue.go b/federationsender/queue/queue.go index 240343559..492d5f553 100644 --- a/federationsender/queue/queue.go +++ b/federationsender/queue/queue.go @@ -15,10 +15,13 @@ package queue import ( + "context" "crypto/ed25519" + "encoding/json" "fmt" "sync" + "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -29,6 +32,7 @@ import ( // OutgoingQueues is a collection of queues for sending transactions to other // matrix servers type OutgoingQueues struct { + db storage.Database rsAPI api.RoomserverInternalAPI origin gomatrixserverlib.ServerName client *gomatrixserverlib.FederationClient @@ -40,6 +44,7 @@ type OutgoingQueues struct { // NewOutgoingQueues makes a new OutgoingQueues func NewOutgoingQueues( + db storage.Database, origin gomatrixserverlib.ServerName, client *gomatrixserverlib.FederationClient, rsAPI api.RoomserverInternalAPI, @@ -47,6 +52,7 @@ func NewOutgoingQueues( signing *SigningInfo, ) *OutgoingQueues { return &OutgoingQueues{ + db: db, rsAPI: rsAPI, origin: origin, client: client, @@ -76,14 +82,15 @@ func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *d oq := oqs.queues[destination] if oq == nil { oq = &destinationQueue{ + db: oqs.db, rsAPI: oqs.rsAPI, origin: oqs.origin, destination: destination, client: oqs.client, statistics: oqs.statistics.ForServer(destination), - incomingPDUs: make(chan *gomatrixserverlib.HeaderedEvent, 128), incomingEDUs: make(chan *gomatrixserverlib.EDU, 128), incomingInvites: make(chan *gomatrixserverlib.InviteV2Request, 128), + wakeServerCh: make(chan bool, 128), retryServerCh: make(chan bool), signing: oqs.signing, } @@ -115,8 +122,18 @@ func (oqs *OutgoingQueues) SendEvent( "destinations": destinations, "event": ev.EventID(), }).Info("Sending event") + headeredJSON, err := json.Marshal(ev) + if err != nil { + return fmt.Errorf("json.Marshal: %w", err) + } + + nid, err := oqs.db.StoreJSON(context.TODO(), string(headeredJSON)) + if err != nil { + return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err) + } + for _, destination := range destinations { - oqs.getQueue(destination).sendEvent(ev) + oqs.getQueue(destination).sendEvent(nid) } return nil diff --git a/federationsender/storage/interface.go b/federationsender/storage/interface.go index be195382b..09d74ed7e 100644 --- a/federationsender/storage/interface.go +++ b/federationsender/storage/interface.go @@ -19,10 +19,16 @@ import ( "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/gomatrixserverlib" ) type Database interface { internal.PartitionStorer UpdateRoom(ctx context.Context, roomID, oldEventID, newEventID string, addHosts []types.JoinedHost, removeHosts []string) (joinedHosts []types.JoinedHost, err error) GetJoinedHosts(ctx context.Context, roomID string) ([]types.JoinedHost, error) + StoreJSON(ctx context.Context, js string) (int64, error) + AssociatePDUWithDestination(ctx context.Context, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, nids []int64) error + GetNextTransactionPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (gomatrixserverlib.TransactionID, []*gomatrixserverlib.HeaderedEvent, error) + CleanTransactionPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, transactionID gomatrixserverlib.TransactionID) error + GetPendingPDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error) } diff --git a/federationsender/storage/postgres/queue_json_table.go b/federationsender/storage/postgres/queue_json_table.go new file mode 100644 index 000000000..eac2ea988 --- /dev/null +++ b/federationsender/storage/postgres/queue_json_table.go @@ -0,0 +1,111 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" +) + +const queueJSONSchema = ` +-- The federationsender_queue_json table contains event contents that +-- we failed to send. +CREATE TABLE IF NOT EXISTS federationsender_queue_json ( + -- The JSON NID. This allows the federationsender_queue_retry table to + -- cross-reference to find the JSON blob. + json_nid BIGSERIAL, + -- The JSON body. Text so that we preserve UTF-8. + json_body TEXT NOT NULL +); +` + +const insertJSONSQL = "" + + "INSERT INTO federationsender_queue_json (json_body)" + + " VALUES ($1)" + + " RETURNING json_nid" + +const deleteJSONSQL = "" + + "DELETE FROM federationsender_queue_json WHERE json_nid = ANY($1)" + +const selectJSONSQL = "" + + "SELECT json_nid, json_body FROM federationsender_queue_json" + + " WHERE json_nid = ANY($1)" + +type queueJSONStatements struct { + insertJSONStmt *sql.Stmt + deleteJSONStmt *sql.Stmt + selectJSONStmt *sql.Stmt +} + +func (s *queueJSONStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(queueJSONSchema) + if err != nil { + return + } + if s.insertJSONStmt, err = db.Prepare(insertJSONSQL); err != nil { + return + } + if s.deleteJSONStmt, err = db.Prepare(deleteJSONSQL); err != nil { + return + } + if s.selectJSONStmt, err = db.Prepare(selectJSONSQL); err != nil { + return + } + return +} + +func (s *queueJSONStatements) insertQueueJSON( + ctx context.Context, txn *sql.Tx, json string, +) (int64, error) { + stmt := sqlutil.TxStmt(txn, s.insertJSONStmt) + var lastid int64 + if err := stmt.QueryRowContext(ctx, json).Scan(&lastid); err != nil { + return 0, err + } + return lastid, nil +} + +func (s *queueJSONStatements) deleteQueueJSON( + ctx context.Context, txn *sql.Tx, nids []int64, +) error { + stmt := sqlutil.TxStmt(txn, s.deleteJSONStmt) + _, err := stmt.ExecContext(ctx, pq.Int64Array(nids)) + return err +} + +func (s *queueJSONStatements) selectQueueJSON( + ctx context.Context, txn *sql.Tx, jsonNIDs []int64, +) (map[int64][]byte, error) { + blobs := map[int64][]byte{} + stmt := sqlutil.TxStmt(txn, s.selectJSONStmt) + rows, err := stmt.QueryContext(ctx, pq.Int64Array(jsonNIDs)) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectJSON: rows.close() failed") + for rows.Next() { + var nid int64 + var blob []byte + if err = rows.Scan(&nid, &blob); err != nil { + return nil, err + } + blobs[nid] = blob + } + return blobs, err +} diff --git a/federationsender/storage/postgres/queue_pdus_table.go b/federationsender/storage/postgres/queue_pdus_table.go new file mode 100644 index 000000000..bc22825d8 --- /dev/null +++ b/federationsender/storage/postgres/queue_pdus_table.go @@ -0,0 +1,192 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/gomatrixserverlib" +) + +const queuePDUsSchema = ` +CREATE TABLE IF NOT EXISTS federationsender_queue_pdus ( + -- The transaction ID that was generated before persisting the event. + transaction_id TEXT NOT NULL, + -- The destination server that we will send the event to. + server_name TEXT NOT NULL, + -- The JSON NID from the federationsender_queue_pdus_json table. + json_nid BIGINT NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_pdus_pdus_json_nid_idx + ON federationsender_queue_pdus (json_nid, server_name); +` + +const insertQueuePDUSQL = "" + + "INSERT INTO federationsender_queue_pdus (transaction_id, server_name, json_nid)" + + " VALUES ($1, $2, $3)" + +const deleteQueueTransactionPDUsSQL = "" + + "DELETE FROM federationsender_queue_pdus WHERE server_name = $1 AND transaction_id = $2" + +const selectQueueNextTransactionIDSQL = "" + + "SELECT transaction_id FROM federationsender_queue_pdus" + + " WHERE server_name = $1" + + " ORDER BY transaction_id ASC" + + " LIMIT 1" + +const selectQueuePDUsByTransactionSQL = "" + + "SELECT json_nid FROM federationsender_queue_pdus" + + " WHERE server_name = $1 AND transaction_id = $2" + + " LIMIT $3" + +const selectQueueReferenceJSONCountSQL = "" + + "SELECT COUNT(*) FROM federationsender_queue_pdus" + + " WHERE json_nid = $1" + +const selectQueuePDUsCountSQL = "" + + "SELECT COUNT(*) FROM federationsender_queue_pdus" + + " WHERE server_name = $1" + +type queuePDUsStatements struct { + insertQueuePDUStmt *sql.Stmt + deleteQueueTransactionPDUsStmt *sql.Stmt + selectQueueNextTransactionIDStmt *sql.Stmt + selectQueuePDUsByTransactionStmt *sql.Stmt + selectQueueReferenceJSONCountStmt *sql.Stmt + selectQueuePDUsCountStmt *sql.Stmt +} + +func (s *queuePDUsStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(queuePDUsSchema) + if err != nil { + return + } + if s.insertQueuePDUStmt, err = db.Prepare(insertQueuePDUSQL); err != nil { + return + } + if s.deleteQueueTransactionPDUsStmt, err = db.Prepare(deleteQueueTransactionPDUsSQL); err != nil { + return + } + if s.selectQueueNextTransactionIDStmt, err = db.Prepare(selectQueueNextTransactionIDSQL); err != nil { + return + } + if s.selectQueuePDUsByTransactionStmt, err = db.Prepare(selectQueuePDUsByTransactionSQL); err != nil { + return + } + if s.selectQueueReferenceJSONCountStmt, err = db.Prepare(selectQueueReferenceJSONCountSQL); err != nil { + return + } + if s.selectQueuePDUsCountStmt, err = db.Prepare(selectQueuePDUsCountSQL); err != nil { + return + } + return +} + +func (s *queuePDUsStatements) insertQueuePDU( + ctx context.Context, + txn *sql.Tx, + transactionID gomatrixserverlib.TransactionID, + serverName gomatrixserverlib.ServerName, + nid int64, +) error { + stmt := sqlutil.TxStmt(txn, s.insertQueuePDUStmt) + _, err := stmt.ExecContext( + ctx, + transactionID, // the transaction ID that we initially attempted + serverName, // destination server name + nid, // JSON blob NID + ) + return err +} + +func (s *queuePDUsStatements) deleteQueueTransaction( + ctx context.Context, txn *sql.Tx, + serverName gomatrixserverlib.ServerName, + transactionID gomatrixserverlib.TransactionID, +) error { + stmt := sqlutil.TxStmt(txn, s.deleteQueueTransactionPDUsStmt) + _, err := stmt.ExecContext(ctx, serverName, transactionID) + return err +} + +func (s *queuePDUsStatements) selectQueueNextTransactionID( + ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, +) (gomatrixserverlib.TransactionID, error) { + var transactionID gomatrixserverlib.TransactionID + stmt := sqlutil.TxStmt(txn, s.selectQueueNextTransactionIDStmt) + err := stmt.QueryRowContext(ctx, serverName).Scan(&transactionID) + if err == sql.ErrNoRows { + return "", nil + } + return transactionID, err +} + +func (s *queuePDUsStatements) selectQueueReferenceJSONCount( + ctx context.Context, txn *sql.Tx, jsonNID int64, +) (int64, error) { + var count int64 + stmt := sqlutil.TxStmt(txn, s.selectQueueReferenceJSONCountStmt) + err := stmt.QueryRowContext(ctx, jsonNID).Scan(&count) + if err == sql.ErrNoRows { + // It's acceptable for there to be no rows referencing a given + // JSON NID but it's not an error condition. Just return as if + // there's a zero count. + return 0, nil + } + return count, err +} + +func (s *queuePDUsStatements) selectQueuePDUCount( + ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, +) (int64, error) { + var count int64 + stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsCountStmt) + err := stmt.QueryRowContext(ctx, serverName).Scan(&count) + if err == sql.ErrNoRows { + // It's acceptable for there to be no rows referencing a given + // JSON NID but it's not an error condition. Just return as if + // there's a zero count. + return 0, nil + } + return count, err +} + +func (s *queuePDUsStatements) selectQueuePDUs( + ctx context.Context, txn *sql.Tx, + serverName gomatrixserverlib.ServerName, + transactionID gomatrixserverlib.TransactionID, + limit int, +) ([]int64, error) { + stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsByTransactionStmt) + rows, err := stmt.QueryContext(ctx, serverName, transactionID, limit) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "queueFromStmt: rows.close() failed") + var result []int64 + for rows.Next() { + var nid int64 + if err = rows.Scan(&nid); err != nil { + return nil, err + } + result = append(result, nid) + } + + return result, rows.Err() +} diff --git a/federationsender/storage/postgres/storage.go b/federationsender/storage/postgres/storage.go index 8fd4c11a3..be28c15dc 100644 --- a/federationsender/storage/postgres/storage.go +++ b/federationsender/storage/postgres/storage.go @@ -18,15 +18,20 @@ package postgres import ( "context" "database/sql" + "encoding/json" + "fmt" "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/gomatrixserverlib" ) // Database stores information needed by the federation sender type Database struct { joinedHostsStatements roomStatements + queuePDUsStatements + queueJSONStatements sqlutil.PartitionOffsetStatements db *sql.DB } @@ -55,6 +60,14 @@ func (d *Database) prepare() error { return err } + if err = d.queuePDUsStatements.prepare(d.db); err != nil { + return err + } + + if err = d.queueJSONStatements.prepare(d.db); err != nil { + return err + } + return d.PartitionOffsetStatements.Prepare(d.db, "federationsender") } @@ -120,3 +133,134 @@ func (d *Database) GetJoinedHosts( ) ([]types.JoinedHost, error) { return d.selectJoinedHosts(ctx, roomID) } + +// StoreJSON adds a JSON blob into the queue JSON table and returns +// a NID. The NID will then be used when inserting the per-destination +// metadata entries. +func (d *Database) StoreJSON( + ctx context.Context, js string, +) (int64, error) { + nid, err := d.insertQueueJSON(ctx, nil, js) + if err != nil { + return 0, fmt.Errorf("d.insertQueueJSON: %w", err) + } + return nid, nil +} + +// AssociatePDUWithDestination creates an association that the +// destination queues will use to determine which JSON blobs to send +// to which servers. +func (d *Database) AssociatePDUWithDestination( + ctx context.Context, + transactionID gomatrixserverlib.TransactionID, + serverName gomatrixserverlib.ServerName, + nids []int64, +) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { + for _, nid := range nids { + if err := d.insertQueuePDU( + ctx, // context + txn, // SQL transaction + transactionID, // transaction ID + serverName, // destination server name + nid, // NID from the federationsender_queue_json table + ); err != nil { + return fmt.Errorf("d.insertQueueRetryStmt.ExecContext: %w", err) + } + } + return nil + }) +} + +// GetNextTransactionPDUs retrieves events from the database for +// the next pending transaction, up to the limit specified. +func (d *Database) GetNextTransactionPDUs( + ctx context.Context, + serverName gomatrixserverlib.ServerName, + limit int, +) ( + transactionID gomatrixserverlib.TransactionID, + events []*gomatrixserverlib.HeaderedEvent, + err error, +) { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { + transactionID, err = d.selectQueueNextTransactionID(ctx, txn, serverName) + if err != nil { + return fmt.Errorf("d.selectQueueNextTransactionID: %w", err) + } + + if transactionID == "" { + return nil + } + + nids, err := d.selectQueuePDUs(ctx, txn, serverName, transactionID, limit) + if err != nil { + return fmt.Errorf("d.selectQueuePDUs: %w", err) + } + + blobs, err := d.selectQueueJSON(ctx, txn, nids) + if err != nil { + return fmt.Errorf("d.selectJSON: %w", err) + } + + for _, blob := range blobs { + var event gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(blob, &event); err != nil { + return fmt.Errorf("json.Unmarshal: %w", err) + } + events = append(events, &event) + } + + return nil + }) + return +} + +// CleanTransactionPDUs cleans up all associated events for a +// given transaction. This is done when the transaction was sent +// successfully. +func (d *Database) CleanTransactionPDUs( + ctx context.Context, + serverName gomatrixserverlib.ServerName, + transactionID gomatrixserverlib.TransactionID, +) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { + nids, err := d.selectQueuePDUs(ctx, txn, serverName, transactionID, 50) + if err != nil { + return fmt.Errorf("d.selectQueuePDUs: %w", err) + } + + if err = d.deleteQueueTransaction(ctx, txn, serverName, transactionID); err != nil { + return fmt.Errorf("d.deleteQueueTransaction: %w", err) + } + + var count int64 + var deleteNIDs []int64 + for _, nid := range nids { + count, err = d.selectQueueReferenceJSONCount(ctx, txn, nid) + if err != nil { + return fmt.Errorf("d.selectQueueReferenceJSONCount: %w", err) + } + if count == 0 { + deleteNIDs = append(deleteNIDs, nid) + } + } + + if len(deleteNIDs) > 0 { + if err = d.deleteQueueJSON(ctx, txn, deleteNIDs); err != nil { + return fmt.Errorf("d.deleteQueueJSON: %w", err) + } + } + + return nil + }) +} + +// GetPendingPDUCount returns the number of PDUs waiting to be +// sent for a given servername. +func (d *Database) GetPendingPDUCount( + ctx context.Context, + serverName gomatrixserverlib.ServerName, +) (int64, error) { + return d.selectQueuePDUCount(ctx, nil, serverName) +} diff --git a/federationsender/storage/sqlite3/queue_json_table.go b/federationsender/storage/sqlite3/queue_json_table.go new file mode 100644 index 000000000..01b7160db --- /dev/null +++ b/federationsender/storage/sqlite3/queue_json_table.go @@ -0,0 +1,132 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" +) + +const queueJSONSchema = ` +-- The queue_retry_json table contains event contents that +-- we failed to send. +CREATE TABLE IF NOT EXISTS federationsender_queue_json ( + -- The JSON NID. This allows the federationsender_queue_retry table to + -- cross-reference to find the JSON blob. + json_nid INTEGER PRIMARY KEY AUTOINCREMENT, + -- The JSON body. Text so that we preserve UTF-8. + json_body TEXT NOT NULL +); +` + +const insertJSONSQL = "" + + "INSERT INTO federationsender_queue_json (json_body)" + + " VALUES ($1)" + +const deleteJSONSQL = "" + + "DELETE FROM federationsender_queue_json WHERE json_nid IN ($1)" + +const selectJSONSQL = "" + + "SELECT json_nid, json_body FROM federationsender_queue_json" + + " WHERE json_nid IN ($1)" + +type queueJSONStatements struct { + insertJSONStmt *sql.Stmt + //deleteJSONStmt *sql.Stmt - prepared at runtime due to variadic + //selectJSONStmt *sql.Stmt - prepared at runtime due to variadic +} + +func (s *queueJSONStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(queueJSONSchema) + if err != nil { + return + } + if s.insertJSONStmt, err = db.Prepare(insertJSONSQL); err != nil { + return + } + return +} + +func (s *queueJSONStatements) insertQueueJSON( + ctx context.Context, txn *sql.Tx, json string, +) (int64, error) { + stmt := sqlutil.TxStmt(txn, s.insertJSONStmt) + res, err := stmt.ExecContext(ctx, json) + if err != nil { + return 0, fmt.Errorf("stmt.QueryContext: %w", err) + } + lastid, err := res.LastInsertId() + if err != nil { + return 0, fmt.Errorf("res.LastInsertId: %w", err) + } + return lastid, nil +} + +func (s *queueJSONStatements) deleteQueueJSON( + ctx context.Context, txn *sql.Tx, nids []int64, +) error { + deleteSQL := strings.Replace(deleteJSONSQL, "($1)", sqlutil.QueryVariadic(len(nids)), 1) + deleteStmt, err := txn.Prepare(deleteSQL) + if err != nil { + return fmt.Errorf("s.deleteQueueJSON s.db.Prepare: %w", err) + } + + iNIDs := make([]interface{}, len(nids)) + for k, v := range nids { + iNIDs[k] = v + } + + stmt := sqlutil.TxStmt(txn, deleteStmt) + _, err = stmt.ExecContext(ctx, iNIDs...) + return err +} + +func (s *queueJSONStatements) selectQueueJSON( + ctx context.Context, txn *sql.Tx, jsonNIDs []int64, +) (map[int64][]byte, error) { + selectSQL := strings.Replace(selectJSONSQL, "($1)", sqlutil.QueryVariadic(len(jsonNIDs)), 1) + selectStmt, err := txn.Prepare(selectSQL) + if err != nil { + return nil, fmt.Errorf("s.selectQueueJSON s.db.Prepare: %w", err) + } + + iNIDs := make([]interface{}, len(jsonNIDs)) + for k, v := range jsonNIDs { + iNIDs[k] = v + } + + blobs := map[int64][]byte{} + stmt := sqlutil.TxStmt(txn, selectStmt) + rows, err := stmt.QueryContext(ctx, iNIDs...) + if err != nil { + return nil, fmt.Errorf("s.selectQueueJSON stmt.QueryContext: %w", err) + } + defer internal.CloseAndLogIfError(ctx, rows, "selectJSON: rows.close() failed") + for rows.Next() { + var nid int64 + var blob []byte + if err = rows.Scan(&nid, &blob); err != nil { + return nil, fmt.Errorf("s.selectQueueJSON rows.Scan: %w", err) + } + blobs[nid] = blob + } + return blobs, err +} diff --git a/federationsender/storage/sqlite3/queue_pdus_table.go b/federationsender/storage/sqlite3/queue_pdus_table.go new file mode 100644 index 000000000..955ff507d --- /dev/null +++ b/federationsender/storage/sqlite3/queue_pdus_table.go @@ -0,0 +1,190 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/gomatrixserverlib" +) + +const queuePDUsSchema = ` +CREATE TABLE IF NOT EXISTS federationsender_queue_pdus ( + -- The transaction ID that was generated before persisting the event. + transaction_id TEXT NOT NULL, + -- The domain part of the user ID the m.room.member event is for. + server_name TEXT NOT NULL, + -- The JSON NID from the federationsender_queue_pdus_json table. + json_nid BIGINT NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_pdus_pdus_json_nid_idx + ON federationsender_queue_pdus (json_nid, server_name); +` + +const insertQueuePDUSQL = "" + + "INSERT INTO federationsender_queue_pdus (transaction_id, server_name, json_nid)" + + " VALUES ($1, $2, $3)" + +const deleteQueueTransactionPDUsSQL = "" + + "DELETE FROM federationsender_queue_pdus WHERE server_name = $1 AND transaction_id = $2" + +const selectQueueNextTransactionIDSQL = "" + + "SELECT transaction_id FROM federationsender_queue_pdus" + + " WHERE server_name = $1" + + " ORDER BY transaction_id ASC" + + " LIMIT 1" + +const selectQueuePDUsByTransactionSQL = "" + + "SELECT json_nid FROM federationsender_queue_pdus" + + " WHERE server_name = $1 AND transaction_id = $2" + + " LIMIT $3" + +const selectQueueReferenceJSONCountSQL = "" + + "SELECT COUNT(*) FROM federationsender_queue_pdus" + + " WHERE json_nid = $1" + +const selectQueuePDUsCountSQL = "" + + "SELECT COUNT(*) FROM federationsender_queue_pdus" + + " WHERE server_name = $1" + +type queuePDUsStatements struct { + insertQueuePDUStmt *sql.Stmt + deleteQueueTransactionPDUsStmt *sql.Stmt + selectQueueNextTransactionIDStmt *sql.Stmt + selectQueuePDUsByTransactionStmt *sql.Stmt + selectQueueReferenceJSONCountStmt *sql.Stmt + selectQueuePDUsCountStmt *sql.Stmt +} + +func (s *queuePDUsStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(queuePDUsSchema) + if err != nil { + return + } + if s.insertQueuePDUStmt, err = db.Prepare(insertQueuePDUSQL); err != nil { + return + } + if s.deleteQueueTransactionPDUsStmt, err = db.Prepare(deleteQueueTransactionPDUsSQL); err != nil { + return + } + if s.selectQueueNextTransactionIDStmt, err = db.Prepare(selectQueueNextTransactionIDSQL); err != nil { + return + } + if s.selectQueuePDUsByTransactionStmt, err = db.Prepare(selectQueuePDUsByTransactionSQL); err != nil { + return + } + if s.selectQueueReferenceJSONCountStmt, err = db.Prepare(selectQueueReferenceJSONCountSQL); err != nil { + return + } + if s.selectQueuePDUsCountStmt, err = db.Prepare(selectQueuePDUsCountSQL); err != nil { + return + } + return +} + +func (s *queuePDUsStatements) insertQueuePDU( + ctx context.Context, + txn *sql.Tx, + transactionID gomatrixserverlib.TransactionID, + serverName gomatrixserverlib.ServerName, + nid int64, +) error { + stmt := sqlutil.TxStmt(txn, s.insertQueuePDUStmt) + _, err := stmt.ExecContext( + ctx, + transactionID, // the transaction ID that we initially attempted + serverName, // destination server name + nid, // JSON blob NID + ) + return err +} + +func (s *queuePDUsStatements) deleteQueueTransaction( + ctx context.Context, txn *sql.Tx, + serverName gomatrixserverlib.ServerName, + transactionID gomatrixserverlib.TransactionID, +) error { + stmt := sqlutil.TxStmt(txn, s.deleteQueueTransactionPDUsStmt) + _, err := stmt.ExecContext(ctx, serverName, transactionID) + return err +} + +func (s *queuePDUsStatements) selectQueueNextTransactionID( + ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, +) (gomatrixserverlib.TransactionID, error) { + var transactionID gomatrixserverlib.TransactionID + stmt := sqlutil.TxStmt(txn, s.selectQueueNextTransactionIDStmt) + err := stmt.QueryRowContext(ctx, serverName).Scan(&transactionID) + if err == sql.ErrNoRows { + return "", nil + } + return transactionID, err +} + +func (s *queuePDUsStatements) selectQueueReferenceJSONCount( + ctx context.Context, txn *sql.Tx, jsonNID int64, +) (int64, error) { + var count int64 + stmt := sqlutil.TxStmt(txn, s.selectQueueReferenceJSONCountStmt) + err := stmt.QueryRowContext(ctx, jsonNID).Scan(&count) + if err == sql.ErrNoRows { + return -1, nil + } + return count, err +} + +func (s *queuePDUsStatements) selectQueuePDUCount( + ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, +) (int64, error) { + var count int64 + stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsCountStmt) + err := stmt.QueryRowContext(ctx, serverName).Scan(&count) + if err == sql.ErrNoRows { + // It's acceptable for there to be no rows referencing a given + // JSON NID but it's not an error condition. Just return as if + // there's a zero count. + return 0, nil + } + return count, err +} + +func (s *queuePDUsStatements) selectQueuePDUs( + ctx context.Context, txn *sql.Tx, + serverName gomatrixserverlib.ServerName, + transactionID gomatrixserverlib.TransactionID, + limit int, +) ([]int64, error) { + stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsByTransactionStmt) + rows, err := stmt.QueryContext(ctx, serverName, transactionID, limit) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "queueFromStmt: rows.close() failed") + var result []int64 + for rows.Next() { + var nid int64 + if err = rows.Scan(&nid); err != nil { + return nil, err + } + result = append(result, nid) + } + + return result, rows.Err() +} diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go index ac303f646..30ac81bfd 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationsender/storage/sqlite3/storage.go @@ -18,17 +18,22 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" + "fmt" _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/gomatrixserverlib" ) // Database stores information needed by the federation sender type Database struct { joinedHostsStatements roomStatements + queuePDUsStatements + queueJSONStatements sqlutil.PartitionOffsetStatements db *sql.DB } @@ -61,6 +66,14 @@ func (d *Database) prepare() error { return err } + if err = d.queuePDUsStatements.prepare(d.db); err != nil { + return err + } + + if err = d.queueJSONStatements.prepare(d.db); err != nil { + return err + } + return d.PartitionOffsetStatements.Prepare(d.db, "federationsender") } @@ -126,3 +139,134 @@ func (d *Database) GetJoinedHosts( ) ([]types.JoinedHost, error) { return d.selectJoinedHosts(ctx, roomID) } + +// StoreJSON adds a JSON blob into the queue JSON table and returns +// a NID. The NID will then be used when inserting the per-destination +// metadata entries. +func (d *Database) StoreJSON( + ctx context.Context, js string, +) (int64, error) { + nid, err := d.insertQueueJSON(ctx, nil, js) + if err != nil { + return 0, fmt.Errorf("d.insertQueueJSON: %w", err) + } + return nid, nil +} + +// AssociatePDUWithDestination creates an association that the +// destination queues will use to determine which JSON blobs to send +// to which servers. +func (d *Database) AssociatePDUWithDestination( + ctx context.Context, + transactionID gomatrixserverlib.TransactionID, + serverName gomatrixserverlib.ServerName, + nids []int64, +) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { + for _, nid := range nids { + if err := d.insertQueuePDU( + ctx, // context + txn, // SQL transaction + transactionID, // transaction ID + serverName, // destination server name + nid, // NID from the federationsender_queue_json table + ); err != nil { + return fmt.Errorf("d.insertQueueRetryStmt.ExecContext: %w", err) + } + } + return nil + }) +} + +// GetNextTransactionPDUs retrieves events from the database for +// the next pending transaction, up to the limit specified. +func (d *Database) GetNextTransactionPDUs( + ctx context.Context, + serverName gomatrixserverlib.ServerName, + limit int, +) ( + transactionID gomatrixserverlib.TransactionID, + events []*gomatrixserverlib.HeaderedEvent, + err error, +) { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { + transactionID, err = d.selectQueueNextTransactionID(ctx, txn, serverName) + if err != nil { + return fmt.Errorf("d.selectQueueNextTransactionID: %w", err) + } + + if transactionID == "" { + return nil + } + + nids, err := d.selectQueuePDUs(ctx, txn, serverName, transactionID, limit) + if err != nil { + return fmt.Errorf("d.selectQueuePDUs: %w", err) + } + + blobs, err := d.selectQueueJSON(ctx, txn, nids) + if err != nil { + return fmt.Errorf("d.selectJSON: %w", err) + } + + for _, blob := range blobs { + var event gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(blob, &event); err != nil { + return fmt.Errorf("json.Unmarshal: %w", err) + } + events = append(events, &event) + } + + return nil + }) + return +} + +// CleanTransactionPDUs cleans up all associated events for a +// given transaction. This is done when the transaction was sent +// successfully. +func (d *Database) CleanTransactionPDUs( + ctx context.Context, + serverName gomatrixserverlib.ServerName, + transactionID gomatrixserverlib.TransactionID, +) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { + nids, err := d.selectQueuePDUs(ctx, txn, serverName, transactionID, 50) + if err != nil { + return fmt.Errorf("d.selectQueuePDUs: %w", err) + } + + if err = d.deleteQueueTransaction(ctx, txn, serverName, transactionID); err != nil { + return fmt.Errorf("d.deleteQueueTransaction: %w", err) + } + + var count int64 + var deleteNIDs []int64 + for _, nid := range nids { + count, err = d.selectQueueReferenceJSONCount(ctx, txn, nid) + if err != nil { + return fmt.Errorf("d.selectQueueReferenceJSONCount: %w", err) + } + if count == 0 { + deleteNIDs = append(deleteNIDs, nid) + } + } + + if len(deleteNIDs) > 0 { + if err = d.deleteQueueJSON(ctx, txn, deleteNIDs); err != nil { + return fmt.Errorf("d.deleteQueueJSON: %w", err) + } + } + + return nil + }) +} + +// GetPendingPDUCount returns the number of PDUs waiting to be +// sent for a given servername. +func (d *Database) GetPendingPDUCount( + ctx context.Context, + serverName gomatrixserverlib.ServerName, +) (int64, error) { + return d.selectQueuePDUCount(ctx, nil, serverName) +} diff --git a/go.mod b/go.mod index 6bfce8441..5a31df464 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/matrix-org/dendrite require ( github.com/Shopify/sarama v1.26.1 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect + github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/go-metrics v0.0.1 // indirect github.com/gologme/log v1.2.0 github.com/gorilla/mux v1.7.3 github.com/hashicorp/golang-lru v0.5.4 @@ -15,12 +17,13 @@ require ( github.com/libp2p/go-libp2p-kad-dht v0.5.0 github.com/libp2p/go-libp2p-pubsub v0.2.5 github.com/libp2p/go-libp2p-record v0.1.2 - github.com/libp2p/go-yamux v1.3.7 + github.com/libp2p/go-yamux v1.3.7 // indirect + github.com/lucas-clemente/quic-go v0.17.2 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200623103809-13ff8109e137 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible @@ -35,9 +38,9 @@ require ( github.com/uber-go/atomic v1.3.0 // indirect github.com/uber/jaeger-client-go v2.15.0+incompatible github.com/uber/jaeger-lib v1.5.0 - github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b + github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200702163833-11ecfa688d93 go.uber.org/atomic v1.4.0 - golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d + golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 gopkg.in/h2non/bimg.v1 v1.0.18 gopkg.in/yaml.v2 v2.2.8 ) diff --git a/go.sum b/go.sum index 6178f152b..48773c6ec 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,12 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= +dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= +dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= +dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= +dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= +git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0 h1:p3puK8Sl2xK+2FnnIvY/C0N1aqJo2kbEsdAzU+Tnv48= @@ -17,11 +25,13 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= @@ -35,8 +45,11 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= +github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/cheggaaa/pb/v3 v3.0.4/go.mod h1:7rgWxLrAUcFMkvJuv09+DYi7mMUYi8nO9iOWcvGJPfw= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= @@ -46,6 +59,7 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= @@ -59,6 +73,10 @@ github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhY github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -67,14 +85,20 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= +github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -88,18 +112,29 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U= github.com/gologme/log v1.2.0 h1:Ya5Ip/KD6FX7uH0S31QO87nCCSucKtF44TLbTtO7V4c= github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= @@ -107,13 +142,22 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= @@ -185,11 +229,14 @@ github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr1 github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0/go.mod h1:rUi0/YffDo1oXBOGn1KRq7Fr07LX48XEBecQnmwjsAo= @@ -215,6 +262,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= @@ -356,11 +404,18 @@ github.com/libp2p/go-yamux v1.3.0 h1:FsYzT16Wq2XqUGJsBbOxoz9g+dFklvNi7jN6YFPfl7U github.com/libp2p/go-yamux v1.3.0/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.3.7 h1:v40A1eSPJDIZwz2AvrV3cxpTZEGDP11QJbukmEhYyQI= github.com/libp2p/go-yamux v1.3.7/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= +github.com/lucas-clemente/quic-go v0.17.2 h1:4iQInIuNQkPNZmsy9rCnwuOzpH0qGnDo4jn0QfI/qE4= +github.com/lucas-clemente/quic-go v0.17.2/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE= +github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lxn/walk v0.0.0-20191128110447-55ccb3a9f5c1/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ= github.com/lxn/win v0.0.0-20191128105842-2da648fda5b4/go.mod h1:ouWl4wViUNh8tPSIwxTVMuS014WakR1hqvBc2I0bMoA= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI= +github.com/marten-seemann/qtls v0.9.1 h1:O0YKQxNVPaiFgMng0suWEOY2Sb4LT2sRn9Qimq3Z1IQ= +github.com/marten-seemann/qtls v0.9.1/go.mod h1:T1MmAdDPyISzxlK6kjRr0pcZFBVd1OZbBb/j3cvzHhk= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= @@ -371,8 +426,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200623103809-13ff8109e137 h1:+eBh4L04+08IslvFM071TNrQTggU317GsQKzZ1SGEVo= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200623103809-13ff8109e137/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe h1:rCjG+azihYsO+EIdm//Zx5gQ7hzeJVraeSukLsW1Mic= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= @@ -392,6 +447,7 @@ github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.1.12 h1:WMhc1ik4LNkTg8U9l3hI1LvxKmIL+f1+WV/SZtCbDDA= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= @@ -457,6 +513,8 @@ github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 h1:evlcQnJY+v8XRRchV3hXzpHDl6GcEZeLXAhlH9Csdww= @@ -465,18 +523,21 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -487,30 +548,61 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= +github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= +github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= +github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= +github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= +github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= +github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= +github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= +github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= +github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= +github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= +github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= +github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= +github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= +github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= +github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= +github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= +github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= @@ -530,6 +622,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= @@ -546,6 +639,8 @@ github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= +github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= @@ -565,8 +660,9 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhe github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yggdrasil-network/yggdrasil-extras v0.0.0-20200525205615-6c8a4a2e8855/go.mod h1:xQdsh08Io6nV4WRnOVTe6gI8/2iTvfLDQ0CYa5aMt+I= -github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b h1:ELOisSxFXCcptRs4LFub+Hz5fYUvV12wZrTps99Eb3E= -github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b/go.mod h1:d+Nz6SPeG6kmeSPFL0cvfWfgwEql75fUnZiAONgvyBE= +github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200702163833-11ecfa688d93 h1:DX2HXQHoejo9GqkvFuRS9iHrjhfv/9WgL3TjmUz/AaY= +github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200702163833-11ecfa688d93/go.mod h1:d+Nz6SPeG6kmeSPFL0cvfWfgwEql75fUnZiAONgvyBE= +go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -578,14 +674,18 @@ go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -596,7 +696,10 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 h1:Q7tZBpemrlsc2I7IyODzhtallWRSm4Q0d09pL6XbQtU= +golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -604,10 +707,15 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= @@ -619,6 +727,10 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -631,17 +743,20 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191003212358-c178f38b412c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -656,19 +771,26 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200301040627-c5d0d7b4ec88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3-0.20191230102452-929e72ca90de h1:aYKJLPSrddB2N7/6OKyFqJ337SXpo61bBuvO5p1+7iY= golang.org/x/text v0.3.3-0.20191230102452-929e72ca90de/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -676,14 +798,32 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.zx2c4.com/wireguard v0.0.20200122-0.20200214175355-9cbcff10dd3e/go.mod h1:P2HsVp8SKwZEufsnezXZA4GRX/T49/HlU7DGuelXsU4= golang.zx2c4.com/wireguard v0.0.20200320/go.mod h1:lDian4Sw4poJ04SgHh35nzMVwGSYlPumkdnHcucAQoY= golang.zx2c4.com/wireguard/windows v0.1.0/go.mod h1:EK7CxrFnicmYJ0ZCF6crBh2/EMMeSxMlqgLlwN0Kv9s= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= @@ -699,6 +839,7 @@ gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= @@ -723,5 +864,12 @@ gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099 h1:XJP7lxbSxWLOMNdBE4B/STaqVy6L73o0knwj2vIlxnw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/internal/config/config.go b/internal/config/config.go index baa82be23..777bd6a39 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -160,10 +160,13 @@ type Dendrite struct { // Postgres Config Database struct { // The Account database stores the login details and account information - // for local users. It is accessed by the ClientAPI. + // for local users. It is accessed by the UserAPI. Account DataSource `yaml:"account"` + // The CurrentState database stores the current state of all rooms. + // It is accessed by the CurrentStateServer. + CurrentState DataSource `yaml:"current_state"` // The Device database stores session information for the devices of logged - // in local users. It is accessed by the ClientAPI, the MediaAPI and the SyncAPI. + // in local users. It is accessed by the UserAPI. Device DataSource `yaml:"device"` // The MediaAPI database stores information about files uploaded and downloaded // by local users. It is only accessed by the MediaAPI. @@ -183,9 +186,6 @@ type Dendrite struct { // The AppServices database stores information used by the AppService component. // It is only accessed by the AppService component. AppService DataSource `yaml:"appservice"` - // The PublicRoomsAPI database stores information used to compute the public - // room directory. It is only accessed by the PublicRoomsAPI server. - PublicRoomsAPI DataSource `yaml:"public_rooms_api"` // The Naffka database is used internally by the naffka library, if used. Naffka DataSource `yaml:"naffka,omitempty"` // Maximum open connections to the DB (0 = use default, negative means unlimited) @@ -222,6 +222,7 @@ type Dendrite struct { Bind struct { MediaAPI Address `yaml:"media_api"` ClientAPI Address `yaml:"client_api"` + CurrentState Address `yaml:"current_state_server"` FederationAPI Address `yaml:"federation_api"` ServerKeyAPI Address `yaml:"server_key_api"` AppServiceAPI Address `yaml:"appservice_api"` @@ -229,7 +230,6 @@ type Dendrite struct { UserAPI Address `yaml:"user_api"` RoomServer Address `yaml:"room_server"` FederationSender Address `yaml:"federation_sender"` - PublicRoomsAPI Address `yaml:"public_rooms_api"` EDUServer Address `yaml:"edu_server"` KeyServer Address `yaml:"key_server"` } `yaml:"bind"` @@ -238,6 +238,7 @@ type Dendrite struct { Listen struct { MediaAPI Address `yaml:"media_api"` ClientAPI Address `yaml:"client_api"` + CurrentState Address `yaml:"current_state_server"` FederationAPI Address `yaml:"federation_api"` ServerKeyAPI Address `yaml:"server_key_api"` AppServiceAPI Address `yaml:"appservice_api"` @@ -245,7 +246,6 @@ type Dendrite struct { UserAPI Address `yaml:"user_api"` RoomServer Address `yaml:"room_server"` FederationSender Address `yaml:"federation_sender"` - PublicRoomsAPI Address `yaml:"public_rooms_api"` EDUServer Address `yaml:"edu_server"` KeyServer Address `yaml:"key_server"` } `yaml:"listen"` @@ -601,6 +601,7 @@ func (config *Dendrite) checkDatabase(configErrs *configErrors) { checkNotEmpty(configErrs, "database.media_api", string(config.Database.MediaAPI)) checkNotEmpty(configErrs, "database.sync_api", string(config.Database.SyncAPI)) checkNotEmpty(configErrs, "database.room_server", string(config.Database.RoomServer)) + checkNotEmpty(configErrs, "database.current_state", string(config.Database.CurrentState)) } // checkListen verifies the parameters listen.* are valid. @@ -613,6 +614,7 @@ func (config *Dendrite) checkListen(configErrs *configErrors) { checkNotEmpty(configErrs, "listen.edu_server", string(config.Listen.EDUServer)) checkNotEmpty(configErrs, "listen.server_key_api", string(config.Listen.EDUServer)) checkNotEmpty(configErrs, "listen.user_api", string(config.Listen.UserAPI)) + checkNotEmpty(configErrs, "listen.current_state_server", string(config.Listen.CurrentState)) } // checkLogging verifies the parameters logging.* are valid. @@ -735,6 +737,15 @@ func (config *Dendrite) UserAPIURL() string { return "http://" + string(config.Listen.UserAPI) } +// CurrentStateAPIURL returns an HTTP URL for where the currentstateserver is listening. +func (config *Dendrite) CurrentStateAPIURL() string { + // Hard code the currentstateserver to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return "http://" + string(config.Listen.CurrentState) +} + // EDUServerURL returns an HTTP URL for where the EDU server is listening. func (config *Dendrite) EDUServerURL() string { // Hard code the EDU server to talk HTTP for now. @@ -753,7 +764,7 @@ func (config *Dendrite) FederationSenderURL() string { return "http://" + string(config.Listen.FederationSender) } -// FederationSenderURL returns an HTTP URL for where the federation sender is listening. +// ServerKeyAPIURL returns an HTTP URL for where the federation sender is listening. func (config *Dendrite) ServerKeyAPIURL() string { // Hard code the server key API server to talk HTTP for now. // If we support HTTPS we need to think of a practical way to do certificate validation. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 9a543e763..9b776a50f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -55,6 +55,7 @@ database: sync_api: "postgresql:///syn_api" room_server: "postgresql:///room_server" appservice: "postgresql:///appservice" + current_state: "postgresql:///current_state" listen: room_server: "localhost:7770" client_api: "localhost:7771" @@ -64,6 +65,7 @@ listen: appservice_api: "localhost:7777" edu_server: "localhost:7778" user_api: "localhost:7779" + current_state_server: "localhost:7775" logging: - type: "file" level: "info" diff --git a/internal/setup/base.go b/internal/setup/base.go index 66424a609..ddf8e0fad 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -22,6 +22,7 @@ import ( "net/url" "time" + currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/sqlutil" @@ -37,6 +38,7 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" asinthttp "github.com/matrix-org/dendrite/appservice/inthttp" + currentstateinthttp "github.com/matrix-org/dendrite/currentstateserver/inthttp" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" eduinthttp "github.com/matrix-org/dendrite/eduserver/inthttp" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" @@ -171,6 +173,15 @@ func (b *BaseDendrite) UserAPIClient() userapi.UserInternalAPI { return userAPI } +// CurrentStateAPIClient returns CurrentStateInternalAPI for hitting the currentstateserver over HTTP. +func (b *BaseDendrite) CurrentStateAPIClient() currentstateAPI.CurrentStateInternalAPI { + stateAPI, err := currentstateinthttp.NewCurrentStateAPIClient(b.Cfg.CurrentStateAPIURL(), b.httpClient) + if err != nil { + logrus.WithError(err).Panic("UserAPIClient failed", b.httpClient) + } + return stateAPI +} + // EDUServerClient returns EDUServerInputAPI for hitting the EDU server over HTTP func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI { e, err := eduinthttp.NewEDUServerClient(b.Cfg.EDUServerURL(), b.httpClient) diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index 24bee9502..ca90986e9 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -19,6 +19,8 @@ import ( "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi" + "github.com/matrix-org/dendrite/clientapi/api" + currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/federationapi" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" @@ -26,9 +28,6 @@ import ( "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/mediaapi" - "github.com/matrix-org/dendrite/publicroomsapi" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/matrix-org/dendrite/publicroomsapi/types" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/dendrite/syncapi" @@ -56,36 +55,28 @@ type Monolith struct { RoomserverAPI roomserverAPI.RoomserverInternalAPI ServerKeyAPI serverKeyAPI.ServerKeyInternalAPI UserAPI userapi.UserInternalAPI - - // TODO: can we remove this? It's weird that we are required the database - // yet every other component can do that on its own. libp2p-demo uses a custom - // database though annoyingly. - PublicRoomsDB storage.Database + StateAPI currentstateAPI.CurrentStateInternalAPI // Optional - ExtPublicRoomsProvider types.ExternalPublicRoomsProvider + ExtPublicRoomsProvider api.ExternalPublicRoomsProvider } // AddAllPublicRoutes attaches all public paths to the given router func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) { clientapi.AddPublicRoutes( - publicMux, m.Config, m.KafkaConsumer, m.KafkaProducer, m.DeviceDB, m.AccountDB, + publicMux, m.Config, m.KafkaProducer, m.DeviceDB, m.AccountDB, m.FedClient, m.RoomserverAPI, - m.EDUInternalAPI, m.AppserviceAPI, transactions.New(), - m.FederationSenderAPI, m.UserAPI, + m.EDUInternalAPI, m.AppserviceAPI, m.StateAPI, transactions.New(), + m.FederationSenderAPI, m.UserAPI, m.ExtPublicRoomsProvider, ) keyserver.AddPublicRoutes(publicMux, m.Config, m.UserAPI) federationapi.AddPublicRoutes( publicMux, m.Config, m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationSenderAPI, - m.EDUInternalAPI, + m.EDUInternalAPI, m.StateAPI, ) mediaapi.AddPublicRoutes(publicMux, m.Config, m.UserAPI, m.Client) - publicroomsapi.AddPublicRoutes( - publicMux, m.Config, m.KafkaConsumer, m.UserAPI, m.PublicRoomsDB, m.RoomserverAPI, m.FedClient, - m.ExtPublicRoomsProvider, - ) syncapi.AddPublicRoutes( publicMux, m.KafkaConsumer, m.UserAPI, m.RoomserverAPI, m.FedClient, m.Config, ) diff --git a/internal/test/config.go b/internal/test/config.go index 951f65a12..bbcc9bed2 100644 --- a/internal/test/config.go +++ b/internal/test/config.go @@ -96,7 +96,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Database.RoomServer = config.DataSource(database) cfg.Database.ServerKey = config.DataSource(database) cfg.Database.SyncAPI = config.DataSource(database) - cfg.Database.PublicRoomsAPI = config.DataSource(database) + cfg.Database.CurrentState = config.DataSource(database) cfg.Listen.ClientAPI = assignAddress() cfg.Listen.AppServiceAPI = assignAddress() @@ -104,7 +104,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Listen.MediaAPI = assignAddress() cfg.Listen.RoomServer = assignAddress() cfg.Listen.SyncAPI = assignAddress() - cfg.Listen.PublicRoomsAPI = assignAddress() + cfg.Listen.CurrentState = assignAddress() cfg.Listen.EDUServer = assignAddress() // Bind to the same address as the listen address @@ -115,7 +115,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Bind.MediaAPI = cfg.Listen.MediaAPI cfg.Bind.RoomServer = cfg.Listen.RoomServer cfg.Bind.SyncAPI = cfg.Listen.SyncAPI - cfg.Bind.PublicRoomsAPI = cfg.Listen.PublicRoomsAPI + cfg.Bind.CurrentState = cfg.Listen.CurrentState cfg.Bind.EDUServer = cfg.Listen.EDUServer return &cfg, port, nil diff --git a/internal/test/server.go b/internal/test/server.go index c3348d533..2d4117f4c 100644 --- a/internal/test/server.go +++ b/internal/test/server.go @@ -99,7 +99,6 @@ func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) { "--sync-api-server-url", "http://" + string(cfg.Listen.SyncAPI), "--client-api-server-url", "http://" + string(cfg.Listen.ClientAPI), "--media-api-server-url", "http://" + string(cfg.Listen.MediaAPI), - "--public-rooms-api-server-url", "http://" + string(cfg.Listen.PublicRoomsAPI), "--tls-cert", "server.crt", "--tls-key", "server.key", } diff --git a/publicroomsapi/README.md b/publicroomsapi/README.md deleted file mode 100644 index 594fe29c5..000000000 --- a/publicroomsapi/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Public rooms API - -This server is responsible for serving requests hitting `/publicRooms` and `/directory/list/room/{roomID}` as per: - -https://matrix.org/docs/spec/client_server/r0.2.0.html#listing-rooms diff --git a/publicroomsapi/consumers/roomserver.go b/publicroomsapi/consumers/roomserver.go deleted file mode 100644 index b9686d56d..000000000 --- a/publicroomsapi/consumers/roomserver.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package consumers - -import ( - "context" - "encoding/json" - - "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrixserverlib" - log "github.com/sirupsen/logrus" -) - -// OutputRoomEventConsumer consumes events that originated in the room server. -type OutputRoomEventConsumer struct { - rsAPI api.RoomserverInternalAPI - rsConsumer *internal.ContinualConsumer - db storage.Database -} - -// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. -func NewOutputRoomEventConsumer( - cfg *config.Dendrite, - kafkaConsumer sarama.Consumer, - store storage.Database, - rsAPI api.RoomserverInternalAPI, -) *OutputRoomEventConsumer { - consumer := internal.ContinualConsumer{ - Topic: string(cfg.Kafka.Topics.OutputRoomEvent), - Consumer: kafkaConsumer, - PartitionStore: store, - } - s := &OutputRoomEventConsumer{ - rsConsumer: &consumer, - db: store, - rsAPI: rsAPI, - } - consumer.ProcessMessage = s.onMessage - - return s -} - -// Start consuming from room servers -func (s *OutputRoomEventConsumer) Start() error { - return s.rsConsumer.Start() -} - -// onMessage is called when the sync server receives a new event from the room server output log. -func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { - // Parse out the event JSON - var output api.OutputEvent - if err := json.Unmarshal(msg.Value, &output); err != nil { - // If the message was invalid, log it and move on to the next message in the stream - log.WithError(err).Errorf("roomserver output log: message parse failure") - return nil - } - - if output.Type != api.OutputTypeNewRoomEvent { - log.WithField("type", output.Type).Debug( - "roomserver output log: ignoring unknown output type", - ) - return nil - } - - var remQueryRes api.QueryEventsByIDResponse - if len(output.NewRoomEvent.RemovesStateEventIDs) > 0 { - remQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.RemovesStateEventIDs} - if err := s.rsAPI.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil { - log.Warn(err) - return err - } - } - - var addQueryEvents, remQueryEvents []gomatrixserverlib.Event - for _, headeredEvent := range output.NewRoomEvent.AddsState() { - addQueryEvents = append(addQueryEvents, headeredEvent.Event) - } - for _, headeredEvent := range remQueryRes.Events { - remQueryEvents = append(remQueryEvents, headeredEvent.Event) - } - - return s.db.UpdateRoomFromEvents(context.TODO(), addQueryEvents, remQueryEvents) -} diff --git a/publicroomsapi/directory/directory.go b/publicroomsapi/directory/directory.go deleted file mode 100644 index 8b68279aa..000000000 --- a/publicroomsapi/directory/directory.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package directory - -import ( - "net/http" - - "github.com/matrix-org/dendrite/roomserver/api" - userapi "github.com/matrix-org/dendrite/userapi/api" - - "github.com/matrix-org/dendrite/clientapi/httputil" - "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/matrix-org/gomatrixserverlib" - - "github.com/matrix-org/util" -) - -type roomVisibility struct { - Visibility string `json:"visibility"` -} - -// GetVisibility implements GET /directory/list/room/{roomID} -func GetVisibility( - req *http.Request, publicRoomsDatabase storage.Database, - roomID string, -) util.JSONResponse { - isPublic, err := publicRoomsDatabase.GetRoomVisibility(req.Context(), roomID) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("publicRoomsDatabase.GetRoomVisibility failed") - return jsonerror.InternalServerError() - } - - var v roomVisibility - if isPublic { - v.Visibility = gomatrixserverlib.Public - } else { - v.Visibility = "private" - } - - return util.JSONResponse{ - Code: http.StatusOK, - JSON: v, - } -} - -// SetVisibility implements PUT /directory/list/room/{roomID} -// TODO: Allow admin users to edit the room visibility -func SetVisibility( - req *http.Request, publicRoomsDatabase storage.Database, rsAPI api.RoomserverInternalAPI, dev *userapi.Device, - roomID string, -) util.JSONResponse { - queryMembershipReq := api.QueryMembershipForUserRequest{ - RoomID: roomID, - UserID: dev.UserID, - } - var queryMembershipRes api.QueryMembershipForUserResponse - err := rsAPI.QueryMembershipForUser(req.Context(), &queryMembershipReq, &queryMembershipRes) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("could not query membership for user") - return jsonerror.InternalServerError() - } - // Check if user id is in room - if !queryMembershipRes.IsInRoom { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("user does not belong to room"), - } - } - queryEventsReq := api.QueryLatestEventsAndStateRequest{ - RoomID: roomID, - StateToFetch: []gomatrixserverlib.StateKeyTuple{{ - EventType: gomatrixserverlib.MRoomPowerLevels, - StateKey: "", - }}, - } - var queryEventsRes api.QueryLatestEventsAndStateResponse - err = rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes) - if err != nil || len(queryEventsRes.StateEvents) == 0 { - util.GetLogger(req.Context()).WithError(err).Error("could not query events from room") - return jsonerror.InternalServerError() - } - - // NOTSPEC: Check if the user's power is greater than power required to change m.room.aliases event - power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event) - if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomAliases, true) { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("userID doesn't have power level to change visibility"), - } - } - - var v roomVisibility - if reqErr := httputil.UnmarshalJSONRequest(req, &v); reqErr != nil { - return *reqErr - } - - isPublic := v.Visibility == gomatrixserverlib.Public - if err := publicRoomsDatabase.SetRoomVisibility(req.Context(), isPublic, roomID); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("publicRoomsDatabase.SetRoomVisibility failed") - return jsonerror.InternalServerError() - } - - return util.JSONResponse{ - Code: http.StatusOK, - JSON: struct{}{}, - } -} diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go deleted file mode 100644 index b9baa1056..000000000 --- a/publicroomsapi/publicroomsapi.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package publicroomsapi - -import ( - "github.com/Shopify/sarama" - "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/publicroomsapi/consumers" - "github.com/matrix-org/dendrite/publicroomsapi/routing" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/matrix-org/dendrite/publicroomsapi/types" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/sirupsen/logrus" -) - -// AddPublicRoutes sets up and registers HTTP handlers for the PublicRoomsAPI -// component. -func AddPublicRoutes( - router *mux.Router, - cfg *config.Dendrite, - consumer sarama.Consumer, - userAPI userapi.UserInternalAPI, - publicRoomsDB storage.Database, - rsAPI roomserverAPI.RoomserverInternalAPI, - fedClient *gomatrixserverlib.FederationClient, - extRoomsProvider types.ExternalPublicRoomsProvider, -) { - rsConsumer := consumers.NewOutputRoomEventConsumer( - cfg, consumer, publicRoomsDB, rsAPI, - ) - if err := rsConsumer.Start(); err != nil { - logrus.WithError(err).Panic("failed to start public rooms server consumer") - } - - routing.Setup(router, userAPI, publicRoomsDB, rsAPI, fedClient, extRoomsProvider) -} diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go deleted file mode 100644 index 9c82d3508..000000000 --- a/publicroomsapi/routing/routing.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package routing - -import ( - "net/http" - - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/roomserver/api" - userapi "github.com/matrix-org/dendrite/userapi/api" - - "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/publicroomsapi/directory" - "github.com/matrix-org/dendrite/publicroomsapi/storage" - "github.com/matrix-org/dendrite/publicroomsapi/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" -) - -const pathPrefixR0 = "/client/r0" - -// Setup configures the given mux with publicroomsapi server listeners -// -// Due to Setup being used to call many other functions, a gocyclo nolint is -// applied: -// nolint: gocyclo -func Setup( - publicAPIMux *mux.Router, userAPI userapi.UserInternalAPI, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI, - fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, -) { - r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() - - r0mux.Handle("/directory/list/room/{roomID}", - httputil.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return directory.GetVisibility(req, publicRoomsDB, vars["roomID"]) - }), - ).Methods(http.MethodGet, http.MethodOptions) - // TODO: Add AS support - r0mux.Handle("/directory/list/room/{roomID}", - httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return directory.SetVisibility(req, publicRoomsDB, rsAPI, device, vars["roomID"]) - }), - ).Methods(http.MethodPut, http.MethodOptions) - r0mux.Handle("/publicRooms", - httputil.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse { - if extRoomsProvider != nil { - return directory.GetPostPublicRoomsWithExternal(req, publicRoomsDB, fedClient, extRoomsProvider) - } - return directory.GetPostPublicRooms(req, publicRoomsDB) - }), - ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) - - // Federation - TODO: should this live here or in federation API? It's sure easier if it's here so here it is. - publicAPIMux.Handle("/federation/v1/publicRooms", - httputil.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse { - return directory.GetPostPublicRooms(req, publicRoomsDB) - }), - ).Methods(http.MethodGet) -} diff --git a/publicroomsapi/storage/interface.go b/publicroomsapi/storage/interface.go deleted file mode 100644 index 0ca6f455c..000000000 --- a/publicroomsapi/storage/interface.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "context" - - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/gomatrixserverlib" -) - -type Database interface { - internal.PartitionStorer - GetRoomVisibility(ctx context.Context, roomID string) (bool, error) - SetRoomVisibility(ctx context.Context, visible bool, roomID string) error - CountPublicRooms(ctx context.Context) (int64, error) - GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) - UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error - UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error -} diff --git a/publicroomsapi/storage/postgres/prepare.go b/publicroomsapi/storage/postgres/prepare.go deleted file mode 100644 index 70b6e5161..000000000 --- a/publicroomsapi/storage/postgres/prepare.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "database/sql" -) - -// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement. -type statementList []struct { - statement **sql.Stmt - sql string -} - -// prepare the SQL for each statement in the list and assign the result to the prepared statement. -func (s statementList) prepare(db *sql.DB) (err error) { - for _, statement := range s { - if *statement.statement, err = db.Prepare(statement.sql); err != nil { - return - } - } - return -} diff --git a/publicroomsapi/storage/postgres/public_rooms_table.go b/publicroomsapi/storage/postgres/public_rooms_table.go deleted file mode 100644 index 39e355368..000000000 --- a/publicroomsapi/storage/postgres/public_rooms_table.go +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "context" - "database/sql" - "errors" - "fmt" - - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/gomatrixserverlib" - - "github.com/lib/pq" -) - -var editableAttributes = []string{ - "aliases", - "canonical_alias", - "name", - "topic", - "world_readable", - "guest_can_join", - "avatar_url", - "visibility", -} - -const publicRoomsSchema = ` --- Stores all of the rooms with data needed to create the server's room directory -CREATE TABLE IF NOT EXISTS publicroomsapi_public_rooms( - -- The room's ID - room_id TEXT NOT NULL PRIMARY KEY, - -- Number of joined members in the room - joined_members INTEGER NOT NULL DEFAULT 0, - -- Aliases of the room (empty array if none) - aliases TEXT[] NOT NULL DEFAULT '{}'::TEXT[], - -- Canonical alias of the room (empty string if none) - canonical_alias TEXT NOT NULL DEFAULT '', - -- Name of the room (empty string if none) - name TEXT NOT NULL DEFAULT '', - -- Topic of the room (empty string if none) - topic TEXT NOT NULL DEFAULT '', - -- Is the room world readable? - world_readable BOOLEAN NOT NULL DEFAULT false, - -- Can guest join the room? - guest_can_join BOOLEAN NOT NULL DEFAULT false, - -- URL of the room avatar (empty string if none) - avatar_url TEXT NOT NULL DEFAULT '', - -- Visibility of the room: true means the room is publicly visible, false - -- means the room is private - visibility BOOLEAN NOT NULL DEFAULT false -); -` - -const countPublicRoomsSQL = "" + - "SELECT COUNT(*) FROM publicroomsapi_public_rooms" + - " WHERE visibility = true" - -const selectPublicRoomsSQL = "" + - "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + - " FROM publicroomsapi_public_rooms WHERE visibility = true" + - " ORDER BY joined_members DESC" + - " OFFSET $1" - -const selectPublicRoomsWithLimitSQL = "" + - "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + - " FROM publicroomsapi_public_rooms WHERE visibility = true" + - " ORDER BY joined_members DESC" + - " OFFSET $1 LIMIT $2" - -const selectPublicRoomsWithFilterSQL = "" + - "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + - " FROM publicroomsapi_public_rooms" + - " WHERE visibility = true" + - " AND (LOWER(name) LIKE LOWER($1)" + - " OR LOWER(topic) LIKE LOWER($1)" + - " OR LOWER(ARRAY_TO_STRING(aliases, ',')) LIKE LOWER($1))" + - " ORDER BY joined_members DESC" + - " OFFSET $2" - -const selectPublicRoomsWithLimitAndFilterSQL = "" + - "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + - " FROM publicroomsapi_public_rooms" + - " WHERE visibility = true" + - " AND (LOWER(name) LIKE LOWER($1)" + - " OR LOWER(topic) LIKE LOWER($1)" + - " OR LOWER(ARRAY_TO_STRING(aliases, ',')) LIKE LOWER($1))" + - " ORDER BY joined_members DESC" + - " OFFSET $2 LIMIT $3" - -const selectRoomVisibilitySQL = "" + - "SELECT visibility FROM publicroomsapi_public_rooms" + - " WHERE room_id = $1" - -const insertNewRoomSQL = "" + - "INSERT INTO publicroomsapi_public_rooms(room_id)" + - " VALUES ($1)" - -const incrementJoinedMembersInRoomSQL = "" + - "UPDATE publicroomsapi_public_rooms" + - " SET joined_members = joined_members + 1" + - " WHERE room_id = $1" - -const decrementJoinedMembersInRoomSQL = "" + - "UPDATE publicroomsapi_public_rooms" + - " SET joined_members = joined_members - 1" + - " WHERE room_id = $1" - -const updateRoomAttributeSQL = "" + - "UPDATE publicroomsapi_public_rooms" + - " SET %s = $1" + - " WHERE room_id = $2" - -type publicRoomsStatements struct { - countPublicRoomsStmt *sql.Stmt - selectPublicRoomsStmt *sql.Stmt - selectPublicRoomsWithLimitStmt *sql.Stmt - selectPublicRoomsWithFilterStmt *sql.Stmt - selectPublicRoomsWithLimitAndFilterStmt *sql.Stmt - selectRoomVisibilityStmt *sql.Stmt - insertNewRoomStmt *sql.Stmt - incrementJoinedMembersInRoomStmt *sql.Stmt - decrementJoinedMembersInRoomStmt *sql.Stmt - updateRoomAttributeStmts map[string]*sql.Stmt -} - -func (s *publicRoomsStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(publicRoomsSchema) - if err != nil { - return - } - - stmts := statementList{ - {&s.countPublicRoomsStmt, countPublicRoomsSQL}, - {&s.selectPublicRoomsStmt, selectPublicRoomsSQL}, - {&s.selectPublicRoomsWithLimitStmt, selectPublicRoomsWithLimitSQL}, - {&s.selectPublicRoomsWithFilterStmt, selectPublicRoomsWithFilterSQL}, - {&s.selectPublicRoomsWithLimitAndFilterStmt, selectPublicRoomsWithLimitAndFilterSQL}, - {&s.selectRoomVisibilityStmt, selectRoomVisibilitySQL}, - {&s.insertNewRoomStmt, insertNewRoomSQL}, - {&s.incrementJoinedMembersInRoomStmt, incrementJoinedMembersInRoomSQL}, - {&s.decrementJoinedMembersInRoomStmt, decrementJoinedMembersInRoomSQL}, - } - - if err = stmts.prepare(db); err != nil { - return - } - - s.updateRoomAttributeStmts = make(map[string]*sql.Stmt) - for _, editable := range editableAttributes { - stmt := fmt.Sprintf(updateRoomAttributeSQL, editable) - if s.updateRoomAttributeStmts[editable], err = db.Prepare(stmt); err != nil { - return - } - } - - return -} - -func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64, err error) { - err = s.countPublicRoomsStmt.QueryRowContext(ctx).Scan(&nb) - return -} - -func (s *publicRoomsStatements) selectPublicRooms( - ctx context.Context, offset int64, limit int16, filter string, -) ([]gomatrixserverlib.PublicRoom, error) { - var rows *sql.Rows - var err error - - if len(filter) > 0 { - pattern := "%" + filter + "%" - if limit == 0 { - rows, err = s.selectPublicRoomsWithFilterStmt.QueryContext( - ctx, pattern, offset, - ) - } else { - rows, err = s.selectPublicRoomsWithLimitAndFilterStmt.QueryContext( - ctx, pattern, offset, limit, - ) - } - } else { - if limit == 0 { - rows, err = s.selectPublicRoomsStmt.QueryContext(ctx, offset) - } else { - rows, err = s.selectPublicRoomsWithLimitStmt.QueryContext( - ctx, offset, limit, - ) - } - } - - if err != nil { - return []gomatrixserverlib.PublicRoom{}, nil - } - defer internal.CloseAndLogIfError(ctx, rows, "selectPublicRooms: rows.close() failed") - - rooms := []gomatrixserverlib.PublicRoom{} - for rows.Next() { - var r gomatrixserverlib.PublicRoom - var aliases pq.StringArray - - err = rows.Scan( - &r.RoomID, &r.JoinedMembersCount, &aliases, &r.CanonicalAlias, - &r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL, - ) - if err != nil { - return rooms, err - } - - r.Aliases = aliases - - rooms = append(rooms, r) - } - - return rooms, rows.Err() -} - -func (s *publicRoomsStatements) selectRoomVisibility( - ctx context.Context, roomID string, -) (v bool, err error) { - err = s.selectRoomVisibilityStmt.QueryRowContext(ctx, roomID).Scan(&v) - return -} - -func (s *publicRoomsStatements) insertNewRoom( - ctx context.Context, roomID string, -) error { - _, err := s.insertNewRoomStmt.ExecContext(ctx, roomID) - return err -} - -func (s *publicRoomsStatements) incrementJoinedMembersInRoom( - ctx context.Context, roomID string, -) error { - _, err := s.incrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID) - return err -} - -func (s *publicRoomsStatements) decrementJoinedMembersInRoom( - ctx context.Context, roomID string, -) error { - _, err := s.decrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID) - return err -} - -func (s *publicRoomsStatements) updateRoomAttribute( - ctx context.Context, attrName string, attrValue attributeValue, roomID string, -) error { - stmt, isEditable := s.updateRoomAttributeStmts[attrName] - - if !isEditable { - return errors.New("Cannot edit " + attrName) - } - - var value interface{} - switch v := attrValue.(type) { - case []string: - value = pq.StringArray(v) - case bool, string: - value = attrValue - default: - return errors.New("Unsupported attribute type, must be bool, string or []string") - } - - _, err := stmt.ExecContext(ctx, value, roomID) - return err -} diff --git a/publicroomsapi/storage/postgres/storage.go b/publicroomsapi/storage/postgres/storage.go deleted file mode 100644 index 36c6aec64..000000000 --- a/publicroomsapi/storage/postgres/storage.go +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "context" - "database/sql" - "encoding/json" - - "github.com/matrix-org/dendrite/internal/eventutil" - "github.com/matrix-org/dendrite/internal/sqlutil" - - "github.com/matrix-org/gomatrixserverlib" -) - -// PublicRoomsServerDatabase represents a public rooms server database. -type PublicRoomsServerDatabase struct { - db *sql.DB - sqlutil.PartitionOffsetStatements - statements publicRoomsStatements - localServerName gomatrixserverlib.ServerName -} - -type attributeValue interface{} - -// NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { - var db *sql.DB - var err error - if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { - return nil, err - } - storage := PublicRoomsServerDatabase{ - db: db, - localServerName: localServerName, - } - if err = storage.PartitionOffsetStatements.Prepare(db, "publicroomsapi"); err != nil { - return nil, err - } - if err = storage.statements.prepare(db); err != nil { - return nil, err - } - return &storage, nil -} - -// GetRoomVisibility returns the room visibility as a boolean: true if the room -// is publicly visible, false if not. -// Returns an error if the retrieval failed. -func (d *PublicRoomsServerDatabase) GetRoomVisibility( - ctx context.Context, roomID string, -) (bool, error) { - return d.statements.selectRoomVisibility(ctx, roomID) -} - -// SetRoomVisibility updates the visibility attribute of a room. This attribute -// must be set to true if the room is publicly visible, false if not. -// Returns an error if the update failed. -func (d *PublicRoomsServerDatabase) SetRoomVisibility( - ctx context.Context, visible bool, roomID string, -) error { - return d.statements.updateRoomAttribute(ctx, "visibility", visible, roomID) -} - -// CountPublicRooms returns the number of room set as publicly visible on the server. -// Returns an error if the retrieval failed. -func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) { - return d.statements.countPublicRooms(ctx) -} - -// GetPublicRooms returns an array containing the local rooms set as publicly visible, ordered by their number -// of joined members. This array can be limited by a given number of elements, and offset by a given value. -// If the limit is 0, doesn't limit the number of results. If the offset is 0 too, the array contains all -// the rooms set as publicly visible on the server. -// Returns an error if the retrieval failed. -func (d *PublicRoomsServerDatabase) GetPublicRooms( - ctx context.Context, offset int64, limit int16, filter string, -) ([]gomatrixserverlib.PublicRoom, error) { - return d.statements.selectPublicRooms(ctx, offset, limit, filter) -} - -// UpdateRoomFromEvents iterate over a slice of state events and call -// UpdateRoomFromEvent on each of them to update the database representation of -// the rooms updated by each event. -// The slice of events to remove is used to update the number of joined members -// for the room in the database. -// If the update triggered by one of the events failed, aborts the process and -// returns an error. -func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents( - ctx context.Context, - eventsToAdd []gomatrixserverlib.Event, - eventsToRemove []gomatrixserverlib.Event, -) error { - for _, event := range eventsToAdd { - if err := d.UpdateRoomFromEvent(ctx, event); err != nil { - return err - } - } - - for _, event := range eventsToRemove { - if event.Type() == "m.room.member" { - if err := d.updateNumJoinedUsers(ctx, event, true); err != nil { - return err - } - } - } - - return nil -} - -// UpdateRoomFromEvent updates the database representation of a room from a Matrix event, by -// checking the event's type to know which attribute to change and using the event's content -// to define the new value of the attribute. -// If the event doesn't match with any property used to compute the public room directory, -// does nothing. -// If something went wrong during the process, returns an error. -func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent( - ctx context.Context, event gomatrixserverlib.Event, -) error { - // Process the event according to its type - switch event.Type() { - case "m.room.create": - return d.statements.insertNewRoom(ctx, event.RoomID()) - case "m.room.member": - return d.updateNumJoinedUsers(ctx, event, false) - case "m.room.aliases": - return d.updateRoomAliases(ctx, event) - case "m.room.canonical_alias": - var content eventutil.CanonicalAliasContent - field := &(content.Alias) - attrName := "canonical_alias" - return d.updateStringAttribute(ctx, attrName, event, &content, field) - case "m.room.name": - var content eventutil.NameContent - field := &(content.Name) - attrName := "name" - return d.updateStringAttribute(ctx, attrName, event, &content, field) - case "m.room.topic": - var content eventutil.TopicContent - field := &(content.Topic) - attrName := "topic" - return d.updateStringAttribute(ctx, attrName, event, &content, field) - case "m.room.avatar": - var content eventutil.AvatarContent - field := &(content.URL) - attrName := "avatar_url" - return d.updateStringAttribute(ctx, attrName, event, &content, field) - case "m.room.history_visibility": - var content eventutil.HistoryVisibilityContent - field := &(content.HistoryVisibility) - attrName := "world_readable" - strForTrue := "world_readable" - return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue) - case "m.room.guest_access": - var content eventutil.GuestAccessContent - field := &(content.GuestAccess) - attrName := "guest_can_join" - strForTrue := "can_join" - return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue) - } - - // If the event type didn't match, return with no error - return nil -} - -// updateNumJoinedUsers updates the number of joined user in the database representation -// of a room using a given "m.room.member" Matrix event. -// If the membership property of the event isn't "join", ignores it and returs nil. -// If the remove parameter is set to false, increments the joined members counter in the -// database, if set to truem decrements it. -// Returns an error if the update failed. -func (d *PublicRoomsServerDatabase) updateNumJoinedUsers( - ctx context.Context, membershipEvent gomatrixserverlib.Event, remove bool, -) error { - membership, err := membershipEvent.Membership() - if err != nil { - return err - } - - if membership != gomatrixserverlib.Join { - return nil - } - - if remove { - return d.statements.decrementJoinedMembersInRoom(ctx, membershipEvent.RoomID()) - } - return d.statements.incrementJoinedMembersInRoom(ctx, membershipEvent.RoomID()) -} - -// updateStringAttribute updates a given string attribute in the database -// representation of a room using a given string data field from content of the -// Matrix event triggering the update. -// Returns an error if decoding the Matrix event's content or updating the attribute -// failed. -func (d *PublicRoomsServerDatabase) updateStringAttribute( - ctx context.Context, attrName string, event gomatrixserverlib.Event, - content interface{}, field *string, -) error { - if err := json.Unmarshal(event.Content(), content); err != nil { - return err - } - - return d.statements.updateRoomAttribute(ctx, attrName, *field, event.RoomID()) -} - -// updateBooleanAttribute updates a given boolean attribute in the database -// representation of a room using a given string data field from content of the -// Matrix event triggering the update. -// The attribute is set to true if the field matches a given string, false if not. -// Returns an error if decoding the Matrix event's content or updating the attribute -// failed. -func (d *PublicRoomsServerDatabase) updateBooleanAttribute( - ctx context.Context, attrName string, event gomatrixserverlib.Event, - content interface{}, field *string, strForTrue string, -) error { - if err := json.Unmarshal(event.Content(), content); err != nil { - return err - } - - var attrValue bool - if *field == strForTrue { - attrValue = true - } else { - attrValue = false - } - - return d.statements.updateRoomAttribute(ctx, attrName, attrValue, event.RoomID()) -} - -// updateRoomAliases decodes the content of a "m.room.aliases" Matrix event and update the list of aliases of -// a given room with it. -// Returns an error if decoding the Matrix event or updating the list failed. -func (d *PublicRoomsServerDatabase) updateRoomAliases( - ctx context.Context, aliasesEvent gomatrixserverlib.Event, -) error { - if aliasesEvent.StateKey() == nil || *aliasesEvent.StateKey() != string(d.localServerName) { - return nil // only store our own aliases - } - var content eventutil.AliasesContent - if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil { - return err - } - - return d.statements.updateRoomAttribute( - ctx, "aliases", content.Aliases, aliasesEvent.RoomID(), - ) -} diff --git a/publicroomsapi/storage/sqlite3/prepare.go b/publicroomsapi/storage/sqlite3/prepare.go deleted file mode 100644 index 482dfa2b9..000000000 --- a/publicroomsapi/storage/sqlite3/prepare.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "database/sql" -) - -// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement. -type statementList []struct { - statement **sql.Stmt - sql string -} - -// prepare the SQL for each statement in the list and assign the result to the prepared statement. -func (s statementList) prepare(db *sql.DB) (err error) { - for _, statement := range s { - if *statement.statement, err = db.Prepare(statement.sql); err != nil { - return - } - } - return -} diff --git a/publicroomsapi/storage/sqlite3/public_rooms_table.go b/publicroomsapi/storage/sqlite3/public_rooms_table.go deleted file mode 100644 index 7b332e175..000000000 --- a/publicroomsapi/storage/sqlite3/public_rooms_table.go +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" - - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/gomatrixserverlib" -) - -var editableAttributes = []string{ - "aliases", - "canonical_alias", - "name", - "topic", - "world_readable", - "guest_can_join", - "avatar_url", - "visibility", -} - -const publicRoomsSchema = ` --- Stores all of the rooms with data needed to create the server's room directory -CREATE TABLE IF NOT EXISTS publicroomsapi_public_rooms( - room_id TEXT NOT NULL PRIMARY KEY, - joined_members INTEGER NOT NULL DEFAULT 0, - aliases TEXT NOT NULL DEFAULT '', - canonical_alias TEXT NOT NULL DEFAULT '', - name TEXT NOT NULL DEFAULT '', - topic TEXT NOT NULL DEFAULT '', - world_readable BOOLEAN NOT NULL DEFAULT false, - guest_can_join BOOLEAN NOT NULL DEFAULT false, - avatar_url TEXT NOT NULL DEFAULT '', - visibility BOOLEAN NOT NULL DEFAULT false -); -` - -const countPublicRoomsSQL = "" + - "SELECT COUNT(*) FROM publicroomsapi_public_rooms" + - " WHERE visibility = true" - -const selectPublicRoomsSQL = "" + - "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + - " FROM publicroomsapi_public_rooms WHERE visibility = true" + - " ORDER BY joined_members DESC" + - " LIMIT 30 OFFSET $1" - -const selectPublicRoomsWithLimitSQL = "" + - "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + - " FROM publicroomsapi_public_rooms WHERE visibility = true" + - " ORDER BY joined_members DESC" + - " LIMIT $1 OFFSET $2" - -const selectPublicRoomsWithFilterSQL = "" + - "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + - " FROM publicroomsapi_public_rooms" + - " WHERE visibility = true" + - " AND (LOWER(name) LIKE LOWER($1)" + - " OR LOWER(topic) LIKE LOWER($1)" + - " OR LOWER(aliases) LIKE LOWER($1))" + // TODO: Is there a better way to search aliases? - " ORDER BY joined_members DESC" + - " LIMIT 30 OFFSET $2" - -const selectPublicRoomsWithLimitAndFilterSQL = "" + - "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + - " FROM publicroomsapi_public_rooms" + - " WHERE visibility = true" + - " AND (LOWER(name) LIKE LOWER($1)" + - " OR LOWER(topic) LIKE LOWER($1)" + - " OR LOWER(aliases) LIKE LOWER($1))" + // TODO: Is there a better way to search aliases? - " ORDER BY joined_members DESC" + - " LIMIT $3 OFFSET $2" - -const selectRoomVisibilitySQL = "" + - "SELECT visibility FROM publicroomsapi_public_rooms" + - " WHERE room_id = $1" - -const insertNewRoomSQL = "" + - "INSERT INTO publicroomsapi_public_rooms(room_id)" + - " VALUES ($1)" - -const incrementJoinedMembersInRoomSQL = "" + - "UPDATE publicroomsapi_public_rooms" + - " SET joined_members = joined_members + 1" + - " WHERE room_id = $1" - -const decrementJoinedMembersInRoomSQL = "" + - "UPDATE publicroomsapi_public_rooms" + - " SET joined_members = joined_members - 1" + - " WHERE room_id = $1" - -const updateRoomAttributeSQL = "" + - "UPDATE publicroomsapi_public_rooms" + - " SET %s = $1" + - " WHERE room_id = $2" - -type publicRoomsStatements struct { - countPublicRoomsStmt *sql.Stmt - selectPublicRoomsStmt *sql.Stmt - selectPublicRoomsWithLimitStmt *sql.Stmt - selectPublicRoomsWithFilterStmt *sql.Stmt - selectPublicRoomsWithLimitAndFilterStmt *sql.Stmt - selectRoomVisibilityStmt *sql.Stmt - insertNewRoomStmt *sql.Stmt - incrementJoinedMembersInRoomStmt *sql.Stmt - decrementJoinedMembersInRoomStmt *sql.Stmt - updateRoomAttributeStmts map[string]*sql.Stmt -} - -func (s *publicRoomsStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(publicRoomsSchema) - if err != nil { - return - } - - stmts := statementList{ - {&s.countPublicRoomsStmt, countPublicRoomsSQL}, - {&s.selectPublicRoomsStmt, selectPublicRoomsSQL}, - {&s.selectPublicRoomsWithLimitStmt, selectPublicRoomsWithLimitSQL}, - {&s.selectPublicRoomsWithFilterStmt, selectPublicRoomsWithFilterSQL}, - {&s.selectPublicRoomsWithLimitAndFilterStmt, selectPublicRoomsWithLimitAndFilterSQL}, - {&s.selectRoomVisibilityStmt, selectRoomVisibilitySQL}, - {&s.insertNewRoomStmt, insertNewRoomSQL}, - {&s.incrementJoinedMembersInRoomStmt, incrementJoinedMembersInRoomSQL}, - {&s.decrementJoinedMembersInRoomStmt, decrementJoinedMembersInRoomSQL}, - } - - if err = stmts.prepare(db); err != nil { - return - } - - s.updateRoomAttributeStmts = make(map[string]*sql.Stmt) - for _, editable := range editableAttributes { - stmt := fmt.Sprintf(updateRoomAttributeSQL, editable) - if s.updateRoomAttributeStmts[editable], err = db.Prepare(stmt); err != nil { - return - } - } - - return -} - -func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64, err error) { - err = s.countPublicRoomsStmt.QueryRowContext(ctx).Scan(&nb) - return -} - -func (s *publicRoomsStatements) selectPublicRooms( - ctx context.Context, offset int64, limit int16, filter string, -) ([]gomatrixserverlib.PublicRoom, error) { - var rows *sql.Rows - var err error - - if len(filter) > 0 { - pattern := "%" + filter + "%" - if limit == 0 { - rows, err = s.selectPublicRoomsWithFilterStmt.QueryContext( - ctx, pattern, offset, - ) - } else { - rows, err = s.selectPublicRoomsWithLimitAndFilterStmt.QueryContext( - ctx, pattern, limit, offset, - ) - } - } else { - if limit == 0 { - rows, err = s.selectPublicRoomsStmt.QueryContext(ctx, offset) - } else { - rows, err = s.selectPublicRoomsWithLimitStmt.QueryContext( - ctx, limit, offset, - ) - } - } - - if err != nil { - return []gomatrixserverlib.PublicRoom{}, nil - } - defer internal.CloseAndLogIfError(ctx, rows, "selectPublicRooms failed to close rows") - - rooms := []gomatrixserverlib.PublicRoom{} - for rows.Next() { - var r gomatrixserverlib.PublicRoom - var aliasesJSON string - - err = rows.Scan( - &r.RoomID, &r.JoinedMembersCount, &aliasesJSON, &r.CanonicalAlias, - &r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL, - ) - if err != nil { - return rooms, err - } - - if len(aliasesJSON) > 0 { - if err := json.Unmarshal([]byte(aliasesJSON), &r.Aliases); err != nil { - return rooms, err - } - } - - rooms = append(rooms, r) - } - - return rooms, nil -} - -func (s *publicRoomsStatements) selectRoomVisibility( - ctx context.Context, roomID string, -) (v bool, err error) { - err = s.selectRoomVisibilityStmt.QueryRowContext(ctx, roomID).Scan(&v) - return -} - -func (s *publicRoomsStatements) insertNewRoom( - ctx context.Context, roomID string, -) error { - _, err := s.insertNewRoomStmt.ExecContext(ctx, roomID) - return err -} - -func (s *publicRoomsStatements) incrementJoinedMembersInRoom( - ctx context.Context, roomID string, -) error { - _, err := s.incrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID) - return err -} - -func (s *publicRoomsStatements) decrementJoinedMembersInRoom( - ctx context.Context, roomID string, -) error { - _, err := s.decrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID) - return err -} - -func (s *publicRoomsStatements) updateRoomAttribute( - ctx context.Context, attrName string, attrValue attributeValue, roomID string, -) error { - stmt, isEditable := s.updateRoomAttributeStmts[attrName] - - if !isEditable { - return errors.New("Cannot edit " + attrName) - } - - var value interface{} - switch v := attrValue.(type) { - case []string: - b, _ := json.Marshal(v) - value = string(b) - case bool, string: - value = attrValue - default: - return errors.New("Unsupported attribute type, must be bool, string or []string") - } - - _, err := stmt.ExecContext(ctx, value, roomID) - return err -} diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go deleted file mode 100644 index 5c685d131..000000000 --- a/publicroomsapi/storage/sqlite3/storage.go +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "context" - "database/sql" - "encoding/json" - - _ "github.com/mattn/go-sqlite3" - - "github.com/matrix-org/dendrite/internal/eventutil" - "github.com/matrix-org/dendrite/internal/sqlutil" - - "github.com/matrix-org/gomatrixserverlib" -) - -// PublicRoomsServerDatabase represents a public rooms server database. -type PublicRoomsServerDatabase struct { - db *sql.DB - sqlutil.PartitionOffsetStatements - statements publicRoomsStatements - localServerName gomatrixserverlib.ServerName -} - -type attributeValue interface{} - -// NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { - var db *sql.DB - var err error - cs, err := sqlutil.ParseFileURI(dataSourceName) - if err != nil { - return nil, err - } - if db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { - return nil, err - } - storage := PublicRoomsServerDatabase{ - db: db, - localServerName: localServerName, - } - if err = storage.PartitionOffsetStatements.Prepare(db, "publicroomsapi"); err != nil { - return nil, err - } - if err = storage.statements.prepare(db); err != nil { - return nil, err - } - return &storage, nil -} - -// GetRoomVisibility returns the room visibility as a boolean: true if the room -// is publicly visible, false if not. -// Returns an error if the retrieval failed. -func (d *PublicRoomsServerDatabase) GetRoomVisibility( - ctx context.Context, roomID string, -) (bool, error) { - return d.statements.selectRoomVisibility(ctx, roomID) -} - -// SetRoomVisibility updates the visibility attribute of a room. This attribute -// must be set to true if the room is publicly visible, false if not. -// Returns an error if the update failed. -func (d *PublicRoomsServerDatabase) SetRoomVisibility( - ctx context.Context, visible bool, roomID string, -) error { - return d.statements.updateRoomAttribute(ctx, "visibility", visible, roomID) -} - -// CountPublicRooms returns the number of room set as publicly visible on the server. -// Returns an error if the retrieval failed. -func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) { - return d.statements.countPublicRooms(ctx) -} - -// GetPublicRooms returns an array containing the local rooms set as publicly visible, ordered by their number -// of joined members. This array can be limited by a given number of elements, and offset by a given value. -// If the limit is 0, doesn't limit the number of results. If the offset is 0 too, the array contains all -// the rooms set as publicly visible on the server. -// Returns an error if the retrieval failed. -func (d *PublicRoomsServerDatabase) GetPublicRooms( - ctx context.Context, offset int64, limit int16, filter string, -) ([]gomatrixserverlib.PublicRoom, error) { - return d.statements.selectPublicRooms(ctx, offset, limit, filter) -} - -// UpdateRoomFromEvents iterate over a slice of state events and call -// UpdateRoomFromEvent on each of them to update the database representation of -// the rooms updated by each event. -// The slice of events to remove is used to update the number of joined members -// for the room in the database. -// If the update triggered by one of the events failed, aborts the process and -// returns an error. -func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents( - ctx context.Context, - eventsToAdd []gomatrixserverlib.Event, - eventsToRemove []gomatrixserverlib.Event, -) error { - for _, event := range eventsToAdd { - if err := d.UpdateRoomFromEvent(ctx, event); err != nil { - return err - } - } - - for _, event := range eventsToRemove { - if event.Type() == "m.room.member" { - if err := d.updateNumJoinedUsers(ctx, event, true); err != nil { - return err - } - } - } - - return nil -} - -// UpdateRoomFromEvent updates the database representation of a room from a Matrix event, by -// checking the event's type to know which attribute to change and using the event's content -// to define the new value of the attribute. -// If the event doesn't match with any property used to compute the public room directory, -// does nothing. -// If something went wrong during the process, returns an error. -func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent( - ctx context.Context, event gomatrixserverlib.Event, -) error { - // Process the event according to its type - switch event.Type() { - case "m.room.create": - return d.statements.insertNewRoom(ctx, event.RoomID()) - case "m.room.member": - return d.updateNumJoinedUsers(ctx, event, false) - case "m.room.aliases": - return d.updateRoomAliases(ctx, event) - case "m.room.canonical_alias": - var content eventutil.CanonicalAliasContent - field := &(content.Alias) - attrName := "canonical_alias" - return d.updateStringAttribute(ctx, attrName, event, &content, field) - case "m.room.name": - var content eventutil.NameContent - field := &(content.Name) - attrName := "name" - return d.updateStringAttribute(ctx, attrName, event, &content, field) - case "m.room.topic": - var content eventutil.TopicContent - field := &(content.Topic) - attrName := "topic" - return d.updateStringAttribute(ctx, attrName, event, &content, field) - case "m.room.avatar": - var content eventutil.AvatarContent - field := &(content.URL) - attrName := "avatar_url" - return d.updateStringAttribute(ctx, attrName, event, &content, field) - case "m.room.history_visibility": - var content eventutil.HistoryVisibilityContent - field := &(content.HistoryVisibility) - attrName := "world_readable" - strForTrue := "world_readable" - return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue) - case "m.room.guest_access": - var content eventutil.GuestAccessContent - field := &(content.GuestAccess) - attrName := "guest_can_join" - strForTrue := "can_join" - return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue) - } - - // If the event type didn't match, return with no error - return nil -} - -// updateNumJoinedUsers updates the number of joined user in the database representation -// of a room using a given "m.room.member" Matrix event. -// If the membership property of the event isn't "join", ignores it and returs nil. -// If the remove parameter is set to false, increments the joined members counter in the -// database, if set to truem decrements it. -// Returns an error if the update failed. -func (d *PublicRoomsServerDatabase) updateNumJoinedUsers( - ctx context.Context, membershipEvent gomatrixserverlib.Event, remove bool, -) error { - membership, err := membershipEvent.Membership() - if err != nil { - return err - } - - if membership != gomatrixserverlib.Join { - return nil - } - - if remove { - return d.statements.decrementJoinedMembersInRoom(ctx, membershipEvent.RoomID()) - } - return d.statements.incrementJoinedMembersInRoom(ctx, membershipEvent.RoomID()) -} - -// updateStringAttribute updates a given string attribute in the database -// representation of a room using a given string data field from content of the -// Matrix event triggering the update. -// Returns an error if decoding the Matrix event's content or updating the attribute -// failed. -func (d *PublicRoomsServerDatabase) updateStringAttribute( - ctx context.Context, attrName string, event gomatrixserverlib.Event, - content interface{}, field *string, -) error { - if err := json.Unmarshal(event.Content(), content); err != nil { - return err - } - - return d.statements.updateRoomAttribute(ctx, attrName, *field, event.RoomID()) -} - -// updateBooleanAttribute updates a given boolean attribute in the database -// representation of a room using a given string data field from content of the -// Matrix event triggering the update. -// The attribute is set to true if the field matches a given string, false if not. -// Returns an error if decoding the Matrix event's content or updating the attribute -// failed. -func (d *PublicRoomsServerDatabase) updateBooleanAttribute( - ctx context.Context, attrName string, event gomatrixserverlib.Event, - content interface{}, field *string, strForTrue string, -) error { - if err := json.Unmarshal(event.Content(), content); err != nil { - return err - } - - var attrValue bool - if *field == strForTrue { - attrValue = true - } else { - attrValue = false - } - - return d.statements.updateRoomAttribute(ctx, attrName, attrValue, event.RoomID()) -} - -// updateRoomAliases decodes the content of a "m.room.aliases" Matrix event and update the list of aliases of -// a given room with it. -// Returns an error if decoding the Matrix event or updating the list failed. -func (d *PublicRoomsServerDatabase) updateRoomAliases( - ctx context.Context, aliasesEvent gomatrixserverlib.Event, -) error { - if aliasesEvent.StateKey() == nil || *aliasesEvent.StateKey() != string(d.localServerName) { - return nil // only store our own aliases - } - var content eventutil.AliasesContent - if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil { - return err - } - - return d.statements.updateRoomAttribute( - ctx, "aliases", content.Aliases, aliasesEvent.RoomID(), - ) -} diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go deleted file mode 100644 index f66188040..000000000 --- a/publicroomsapi/storage/storage.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !wasm - -package storage - -import ( - "net/url" - - "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" - "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" - "github.com/matrix-org/gomatrixserverlib" -) - -const schemePostgres = "postgres" -const schemeFile = "file" - -// NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, localServerName gomatrixserverlib.ServerName) (Database, error) { - uri, err := url.Parse(dataSourceName) - if err != nil { - return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName) - } - switch uri.Scheme { - case schemePostgres: - return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName) - case schemeFile: - return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName) - default: - return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName) - } -} diff --git a/roomserver/api/api.go b/roomserver/api/api.go index 26ec8ca1d..0a5845dd6 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -36,6 +36,18 @@ type RoomserverInternalAPI interface { res *PerformLeaveResponse, ) error + PerformPublish( + ctx context.Context, + req *PerformPublishRequest, + res *PerformPublishResponse, + ) + + QueryPublishedRooms( + ctx context.Context, + req *QueryPublishedRoomsRequest, + res *QueryPublishedRoomsResponse, + ) error + // Query the latest events and state for a room from the room server. QueryLatestEventsAndState( ctx context.Context, diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go index 8645b6f28..bdebc57b0 100644 --- a/roomserver/api/api_trace.go +++ b/roomserver/api/api_trace.go @@ -57,6 +57,25 @@ func (t *RoomserverInternalAPITrace) PerformLeave( return err } +func (t *RoomserverInternalAPITrace) PerformPublish( + ctx context.Context, + req *PerformPublishRequest, + res *PerformPublishResponse, +) { + t.Impl.PerformPublish(ctx, req, res) + util.GetLogger(ctx).Infof("PerformPublish req=%+v res=%+v", js(req), js(res)) +} + +func (t *RoomserverInternalAPITrace) QueryPublishedRooms( + ctx context.Context, + req *QueryPublishedRoomsRequest, + res *QueryPublishedRoomsResponse, +) error { + err := t.Impl.QueryPublishedRooms(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryPublishedRooms req=%+v res=%+v", js(req), js(res)) + return err +} + func (t *RoomserverInternalAPITrace) QueryLatestEventsAndState( ctx context.Context, req *QueryLatestEventsAndStateRequest, diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index 0b8e6df25..9e8447339 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -1,6 +1,7 @@ package api import ( + "encoding/json" "fmt" "net/http" @@ -12,8 +13,9 @@ import ( type PerformErrorCode int type PerformError struct { - Msg string - Code PerformErrorCode + Msg string + RemoteCode int // remote HTTP status code, for PerformErrRemote + Code PerformErrorCode } func (p *PerformError) Error() string { @@ -38,6 +40,22 @@ func (p *PerformError) JSONResponse() util.JSONResponse { Code: http.StatusForbidden, JSON: jsonerror.Forbidden(p.Msg), } + case PerformErrorNoOperation: + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden(p.Msg), + } + case PerformErrRemote: + // if the code is 0 then something bad happened and it isn't + // a remote HTTP error being encapsulated, e.g network error to remote. + if p.RemoteCode == 0 { + return util.ErrorResponse(fmt.Errorf("%s", p.Msg)) + } + return util.JSONResponse{ + Code: p.RemoteCode, + // TODO: Should we assert this is in fact JSON? E.g gjson parse? + JSON: json.RawMessage(p.Msg), + } default: return util.ErrorResponse(p) } @@ -52,6 +70,8 @@ const ( PerformErrorNoRoom PerformErrorCode = 3 // PerformErrorNoOperation means that the request resulted in nothing happening e.g invite->invite or leave->leave. PerformErrorNoOperation PerformErrorCode = 4 + // PerformErrRemote means that the request failed and the PerformError.Msg is the raw remote JSON error response + PerformErrRemote PerformErrorCode = 5 ) type PerformJoinRequest struct { @@ -116,3 +136,13 @@ type PerformBackfillResponse struct { // Missing events, arbritrary order. Events []gomatrixserverlib.HeaderedEvent `json:"events"` } + +type PerformPublishRequest struct { + RoomID string + Visibility string +} + +type PerformPublishResponse struct { + // If non-nil, the publish request failed. Contains more information why it failed. + Error *PerformError +} diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 6586b1af3..4e1d09c30 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -112,6 +112,8 @@ type QueryMembershipForUserResponse struct { HasBeenInRoom bool `json:"has_been_in_room"` // True if the user is in room. IsInRoom bool `json:"is_in_room"` + // The current membership + Membership string } // QueryMembershipsForRoomRequest is a request to QueryMembershipsForRoom @@ -213,3 +215,13 @@ type QueryRoomVersionForRoomRequest struct { type QueryRoomVersionForRoomResponse struct { RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` } + +type QueryPublishedRoomsRequest struct { + // Optional. If specified, returns whether this room is published or not. + RoomID string +} + +type QueryPublishedRoomsResponse struct { + // The list of published rooms. + RoomIDs []string +} diff --git a/roomserver/internal/perform_invite.go b/roomserver/internal/perform_invite.go index c65c87f91..4600bec0b 100644 --- a/roomserver/internal/perform_invite.go +++ b/roomserver/internal/perform_invite.go @@ -55,6 +55,7 @@ func (r *RoomserverInternalAPI) performInvite(ctx context.Context, return nil } +// nolint:gocyclo func (r *RoomserverInternalAPI) processInviteEvent( ctx context.Context, ow *RoomserverInternalAPI, @@ -135,6 +136,25 @@ func (r *RoomserverInternalAPI) processInviteEvent( } event := input.Event.Unwrap() + + // check that the user is allowed to do this. We can only do this check if it is + // a local invite as we have the auth events, else we have to take it on trust. + if loopback != nil { + _, err = checkAuthEvents(ctx, r.DB, input.Event, input.Event.AuthEventIDs()) + if err != nil { + log.WithError(err).WithField("event_id", event.EventID()).WithField("auth_event_ids", event.AuthEventIDs()).Error( + "processInviteEvent.checkAuthEvents failed for event", + ) + if _, ok := err.(*gomatrixserverlib.NotAllowed); ok { + return nil, &api.PerformError{ + Msg: err.Error(), + Code: api.PerformErrorNotAllowed, + } + } + return nil, err + } + } + if len(input.InviteRoomState) > 0 { // If we were supplied with some invite room state already (which is // most likely to be if the event came in over federation) then use diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go index d409b6849..c5480a5bd 100644 --- a/roomserver/internal/perform_join.go +++ b/roomserver/internal/perform_join.go @@ -124,7 +124,13 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( Msg: fmt.Sprintf("Room ID %q is invalid: %s", req.RoomIDOrAlias, err), } } - req.ServerNames = append(req.ServerNames, domain) + + // If the server name in the room ID isn't ours then it's a + // possible candidate for finding the room via federation. Add + // it to the list of servers to try. + if domain != r.Cfg.Matrix.ServerName { + req.ServerNames = append(req.ServerNames, domain) + } // Prepare the template for the join event. userID := req.UserID @@ -155,7 +161,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( // where we might think we know about a room in the following // section but don't know the latest state as all of our users // have left. - isInvitePending, inviteSender, err := r.isInvitePending(ctx, req.RoomIDOrAlias, req.UserID) + isInvitePending, inviteSender, _, err := r.isInvitePending(ctx, req.RoomIDOrAlias, req.UserID) if err == nil && isInvitePending { // Check if there's an invite pending. _, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender) @@ -233,13 +239,18 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( } case eventutil.ErrRoomNoExists: - // The room doesn't exist. First of all check if the room is a local - // room. If it is then there's nothing more to do - the room just - // hasn't been created yet. + // The room doesn't exist locally. If the room ID looks like it should + // be ours then this probably means that we've nuked our database at + // some point. if domain == r.Cfg.Matrix.ServerName { - return "", &api.PerformError{ - Code: api.PerformErrorNoRoom, - Msg: fmt.Sprintf("Room ID %q does not exist", req.RoomIDOrAlias), + // If there are no more server names to try then give up here. + // Otherwise we'll try a federated join as normal, since it's quite + // possible that the room still exists on other servers. + if len(req.ServerNames) == 0 { + return "", &api.PerformError{ + Code: api.PerformErrorNoRoom, + Msg: fmt.Sprintf("Room ID %q does not exist", req.RoomIDOrAlias), + } } } @@ -270,9 +281,13 @@ func (r *RoomserverInternalAPI) performFederatedJoinRoomByID( Content: req.Content, // the membership event content } fedRes := fsAPI.PerformJoinResponse{} - if err := r.fsAPI.PerformJoin(ctx, &fedReq, &fedRes); err != nil { - return fmt.Errorf("Error joining federated room: %q", err) + r.fsAPI.PerformJoin(ctx, &fedReq, &fedRes) + if fedRes.LastError != nil { + return &api.PerformError{ + Code: api.PerformErrRemote, + Msg: fedRes.LastError.Message, + RemoteCode: fedRes.LastError.Code, + } } - return nil } diff --git a/roomserver/internal/perform_leave.go b/roomserver/internal/perform_leave.go index 880c8b203..a19d0da9f 100644 --- a/roomserver/internal/perform_leave.go +++ b/roomserver/internal/perform_leave.go @@ -9,6 +9,7 @@ import ( fsAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -38,9 +39,9 @@ func (r *RoomserverInternalAPI) performLeaveRoomByID( ) error { // If there's an invite outstanding for the room then respond to // that. - isInvitePending, senderUser, err := r.isInvitePending(ctx, req.RoomID, req.UserID) + isInvitePending, senderUser, eventID, err := r.isInvitePending(ctx, req.RoomID, req.UserID) if err == nil && isInvitePending { - return r.performRejectInvite(ctx, req, res, senderUser) + return r.performRejectInvite(ctx, req, res, senderUser, eventID) } // There's no invite pending, so first of all we want to find out @@ -134,7 +135,7 @@ func (r *RoomserverInternalAPI) performRejectInvite( ctx context.Context, req *api.PerformLeaveRequest, res *api.PerformLeaveResponse, // nolint:unparam - senderUser string, + senderUser, eventID string, ) error { _, domain, err := gomatrixserverlib.SplitID('@', senderUser) if err != nil { @@ -152,56 +153,68 @@ func (r *RoomserverInternalAPI) performRejectInvite( return err } - // TODO: Withdraw the invite, so that the sync API etc are + // Withdraw the invite, so that the sync API etc are // notified that we rejected it. - - return nil + return r.WriteOutputEvents(req.RoomID, []api.OutputEvent{ + { + Type: api.OutputTypeRetireInviteEvent, + RetireInviteEvent: &api.OutputRetireInviteEvent{ + EventID: eventID, + Membership: "leave", + TargetUserID: req.UserID, + }, + }, + }) } func (r *RoomserverInternalAPI) isInvitePending( ctx context.Context, roomID, userID string, -) (bool, string, error) { +) (bool, string, string, error) { // Look up the room NID for the supplied room ID. roomNID, err := r.DB.RoomNID(ctx, roomID) if err != nil { - return false, "", fmt.Errorf("r.DB.RoomNID: %w", err) + return false, "", "", fmt.Errorf("r.DB.RoomNID: %w", err) } // Look up the state key NID for the supplied user ID. targetUserNIDs, err := r.DB.EventStateKeyNIDs(ctx, []string{userID}) if err != nil { - return false, "", fmt.Errorf("r.DB.EventStateKeyNIDs: %w", err) + return false, "", "", fmt.Errorf("r.DB.EventStateKeyNIDs: %w", err) } targetUserNID, targetUserFound := targetUserNIDs[userID] if !targetUserFound { - return false, "", fmt.Errorf("missing NID for user %q (%+v)", userID, targetUserNIDs) + return false, "", "", fmt.Errorf("missing NID for user %q (%+v)", userID, targetUserNIDs) } // Let's see if we have an event active for the user in the room. If // we do then it will contain a server name that we can direct the // send_leave to. - senderUserNIDs, err := r.DB.GetInvitesForUser(ctx, roomNID, targetUserNID) + senderUserNIDs, eventIDs, err := r.DB.GetInvitesForUser(ctx, roomNID, targetUserNID) if err != nil { - return false, "", fmt.Errorf("r.DB.GetInvitesForUser: %w", err) + return false, "", "", fmt.Errorf("r.DB.GetInvitesForUser: %w", err) } if len(senderUserNIDs) == 0 { - return false, "", nil + return false, "", "", nil + } + userNIDToEventID := make(map[types.EventStateKeyNID]string) + for i, nid := range senderUserNIDs { + userNIDToEventID[nid] = eventIDs[i] } // Look up the user ID from the NID. senderUsers, err := r.DB.EventStateKeys(ctx, senderUserNIDs) if err != nil { - return false, "", fmt.Errorf("r.DB.EventStateKeys: %w", err) + return false, "", "", fmt.Errorf("r.DB.EventStateKeys: %w", err) } if len(senderUsers) == 0 { - return false, "", fmt.Errorf("no senderUsers") + return false, "", "", fmt.Errorf("no senderUsers") } senderUser, senderUserFound := senderUsers[senderUserNIDs[0]] if !senderUserFound { - return false, "", fmt.Errorf("missing user for NID %d (%+v)", senderUserNIDs[0], senderUsers) + return false, "", "", fmt.Errorf("missing user for NID %d (%+v)", senderUserNIDs[0], senderUsers) } - return true, senderUser, nil + return true, senderUser, userNIDToEventID[senderUserNIDs[0]], nil } diff --git a/roomserver/internal/perform_publish.go b/roomserver/internal/perform_publish.go new file mode 100644 index 000000000..d7863620a --- /dev/null +++ b/roomserver/internal/perform_publish.go @@ -0,0 +1,20 @@ +package internal + +import ( + "context" + + "github.com/matrix-org/dendrite/roomserver/api" +) + +func (r *RoomserverInternalAPI) PerformPublish( + ctx context.Context, + req *api.PerformPublishRequest, + res *api.PerformPublishResponse, +) { + err := r.DB.PublishRoom(ctx, req.RoomID, req.Visibility == "public") + if err != nil { + res.Error = &api.PerformError{ + Msg: err.Error(), + } + } +} diff --git a/roomserver/internal/query.go b/roomserver/internal/query.go index 4fc8e4c25..7fa3247a6 100644 --- a/roomserver/internal/query.go +++ b/roomserver/internal/query.go @@ -225,13 +225,18 @@ func (r *RoomserverInternalAPI) QueryMembershipForUser( } response.IsInRoom = stillInRoom - eventIDMap, err := r.DB.EventIDs(ctx, []types.EventNID{membershipEventNID}) + + evs, err := r.DB.Events(ctx, []types.EventNID{membershipEventNID}) if err != nil { return err } + if len(evs) != 1 { + return fmt.Errorf("failed to load membership event for event NID %d", membershipEventNID) + } - response.EventID = eventIDMap[membershipEventNID] - return nil + response.EventID = evs[0].EventID() + response.Membership, err = evs[0].Membership() + return err } // QueryMembershipsForRoom implements api.RoomserverInternalAPI @@ -925,3 +930,16 @@ func (r *RoomserverInternalAPI) QueryRoomVersionForRoom( r.Cache.StoreRoomVersion(request.RoomID, response.RoomVersion) return nil } + +func (r *RoomserverInternalAPI) QueryPublishedRooms( + ctx context.Context, + req *api.QueryPublishedRoomsRequest, + res *api.QueryPublishedRoomsResponse, +) error { + rooms, err := r.DB.GetPublishedRooms(ctx) + if err != nil { + return err + } + res.RoomIDs = rooms + return nil +} diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index 8a2b1204c..ad24af4ad 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -29,6 +29,7 @@ const ( RoomserverPerformJoinPath = "/roomserver/performJoin" RoomserverPerformLeavePath = "/roomserver/performLeave" RoomserverPerformBackfillPath = "/roomserver/performBackfill" + RoomserverPerformPublishPath = "/roomserver/performPublish" // Query operations RoomserverQueryLatestEventsAndStatePath = "/roomserver/queryLatestEventsAndState" @@ -41,6 +42,7 @@ const ( RoomserverQueryStateAndAuthChainPath = "/roomserver/queryStateAndAuthChain" RoomserverQueryRoomVersionCapabilitiesPath = "/roomserver/queryRoomVersionCapabilities" RoomserverQueryRoomVersionForRoomPath = "/roomserver/queryRoomVersionForRoom" + RoomserverQueryPublishedRoomsPath = "/roomserver/queryPublishedRooms" ) type httpRoomserverInternalAPI struct { @@ -194,6 +196,23 @@ func (h *httpRoomserverInternalAPI) PerformLeave( return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } +func (h *httpRoomserverInternalAPI) PerformPublish( + ctx context.Context, + req *api.PerformPublishRequest, + res *api.PerformPublishResponse, +) { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformPublish") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverPerformPublishPath + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) + if err != nil { + res.Error = &api.PerformError{ + Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err), + } + } +} + // QueryLatestEventsAndState implements RoomserverQueryAPI func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState( ctx context.Context, @@ -233,6 +252,18 @@ func (h *httpRoomserverInternalAPI) QueryEventsByID( return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } +func (h *httpRoomserverInternalAPI) QueryPublishedRooms( + ctx context.Context, + request *api.QueryPublishedRoomsRequest, + response *api.QueryPublishedRoomsResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryPublishedRooms") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryPublishedRoomsPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + // QueryMembershipForUser implements RoomserverQueryAPI func (h *httpRoomserverInternalAPI) QueryMembershipForUser( ctx context.Context, diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go index 1c47e87e2..bb54abf9c 100644 --- a/roomserver/inthttp/server.go +++ b/roomserver/inthttp/server.go @@ -61,6 +61,31 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(RoomserverPerformPublishPath, + httputil.MakeInternalAPI("performPublish", func(req *http.Request) util.JSONResponse { + var request api.PerformPublishRequest + var response api.PerformPublishResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + r.PerformPublish(req.Context(), &request, &response) + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverQueryPublishedRoomsPath, + httputil.MakeInternalAPI("queryPublishedRooms", func(req *http.Request) util.JSONResponse { + var request api.QueryPublishedRoomsRequest + var response api.QueryPublishedRoomsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryPublishedRooms(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) internalAPIMux.Handle( RoomserverQueryLatestEventsAndStatePath, httputil.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 52e6a96b7..5c916f294 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -102,9 +102,9 @@ type Database interface { // Returns an error if there was a problem talking to the database. LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error) // Look up the active invites targeting a user in a room and return the - // numeric state key IDs for the user IDs who sent them. + // numeric state key IDs for the user IDs who sent them along with the event IDs for the invites. // Returns an error if there was a problem talking to the database. - GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, err error) + GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, err error) // Save a given room alias with the room ID it refers to. // Returns an error if there was a problem talking to the database. SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error @@ -139,4 +139,8 @@ type Database interface { EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) // Look up the room version for a given room. GetRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) + // Publish or unpublish a room from the room directory. + PublishRoom(ctx context.Context, roomID string, publish bool) error + // Returns a list of room IDs for rooms which are published. + GetPublishedRooms(ctx context.Context) ([]string, error) } diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go index 048a094dc..bb7195164 100644 --- a/roomserver/storage/postgres/invite_table.go +++ b/roomserver/storage/postgres/invite_table.go @@ -62,7 +62,7 @@ const insertInviteEventSQL = "" + " ON CONFLICT DO NOTHING" const selectInviteActiveForUserInRoomSQL = "" + - "SELECT sender_nid FROM roomserver_invites" + + "SELECT invite_event_id, sender_nid FROM roomserver_invites" + " WHERE target_nid = $1 AND room_nid = $2" + " AND NOT retired" @@ -141,21 +141,24 @@ func (s *inviteStatements) UpdateInviteRetired( func (s *inviteStatements) SelectInviteActiveForUserInRoom( ctx context.Context, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID, -) ([]types.EventStateKeyNID, error) { +) ([]types.EventStateKeyNID, []string, error) { rows, err := s.selectInviteActiveForUserInRoomStmt.QueryContext( ctx, targetUserNID, roomNID, ) if err != nil { - return nil, err + return nil, nil, err } defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") var result []types.EventStateKeyNID + var eventIDs []string for rows.Next() { + var inviteEventID string var senderUserNID int64 - if err := rows.Scan(&senderUserNID); err != nil { - return nil, err + if err := rows.Scan(&inviteEventID, &senderUserNID); err != nil { + return nil, nil, err } result = append(result, types.EventStateKeyNID(senderUserNID)) + eventIDs = append(eventIDs, inviteEventID) } - return result, rows.Err() + return result, eventIDs, rows.Err() } diff --git a/roomserver/storage/postgres/published_table.go b/roomserver/storage/postgres/published_table.go new file mode 100644 index 000000000..23a9b067e --- /dev/null +++ b/roomserver/storage/postgres/published_table.go @@ -0,0 +1,101 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" + "github.com/matrix-org/dendrite/roomserver/storage/tables" +) + +const publishedSchema = ` +-- Stores which rooms are published in the room directory +CREATE TABLE IF NOT EXISTS roomserver_published ( + -- The room ID of the room + room_id TEXT NOT NULL PRIMARY KEY, + -- Whether it is published or not + published BOOLEAN NOT NULL DEFAULT false +); +` + +const upsertPublishedSQL = "" + + "INSERT INTO roomserver_published (room_id, published) VALUES ($1, $2) " + + "ON CONFLICT (room_id) DO UPDATE SET published=$2" + +const selectAllPublishedSQL = "" + + "SELECT room_id FROM roomserver_published WHERE published = $1 ORDER BY room_id ASC" + +const selectPublishedSQL = "" + + "SELECT published FROM roomserver_published WHERE room_id = $1" + +type publishedStatements struct { + upsertPublishedStmt *sql.Stmt + selectAllPublishedStmt *sql.Stmt + selectPublishedStmt *sql.Stmt +} + +func NewPostgresPublishedTable(db *sql.DB) (tables.Published, error) { + s := &publishedStatements{} + _, err := db.Exec(publishedSchema) + if err != nil { + return nil, err + } + return s, shared.StatementList{ + {&s.upsertPublishedStmt, upsertPublishedSQL}, + {&s.selectAllPublishedStmt, selectAllPublishedSQL}, + {&s.selectPublishedStmt, selectPublishedSQL}, + }.Prepare(db) +} + +func (s *publishedStatements) UpsertRoomPublished( + ctx context.Context, roomID string, published bool, +) (err error) { + _, err = s.upsertPublishedStmt.ExecContext(ctx, roomID, published) + return +} + +func (s *publishedStatements) SelectPublishedFromRoomID( + ctx context.Context, roomID string, +) (published bool, err error) { + err = s.selectPublishedStmt.QueryRowContext(ctx, roomID).Scan(&published) + if err == sql.ErrNoRows { + return false, nil + } + return +} + +func (s *publishedStatements) SelectAllPublishedRooms( + ctx context.Context, published bool, +) ([]string, error) { + rows, err := s.selectAllPublishedStmt.QueryContext(ctx, published) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectAllPublishedStmt: rows.close() failed") + + var roomIDs []string + for rows.Next() { + var roomID string + if err = rows.Scan(&roomID); err != nil { + return nil, err + } + + roomIDs = append(roomIDs, roomID) + } + return roomIDs, rows.Err() +} diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index d76ee0a92..23d078e4a 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -87,6 +87,10 @@ func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, if err != nil { return nil, err } + published, err := NewPostgresPublishedTable(db) + if err != nil { + return nil, err + } d.Database = shared.Database{ DB: db, EventTypesTable: eventTypes, @@ -101,6 +105,7 @@ func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, RoomAliasesTable: roomAliases, InvitesTable: invites, MembershipTable: membership, + PublishedTable: published, } return &d, nil } diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 2751cc557..166822d0c 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -26,6 +26,7 @@ type Database struct { PrevEventsTable tables.PreviousEvents InvitesTable tables.Invites MembershipTable tables.Membership + PublishedTable tables.Published } func (d *Database) EventTypeNIDs( @@ -265,7 +266,7 @@ func (d *Database) GetInvitesForUser( ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, -) (senderUserIDs []types.EventStateKeyNID, err error) { +) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, err error) { return d.InvitesTable.SelectInviteActiveForUserInRoom(ctx, targetUserNID, roomNID) } @@ -420,6 +421,14 @@ func (d *Database) StoreEvent( }, nil } +func (d *Database) PublishRoom(ctx context.Context, roomID string, publish bool) error { + return d.PublishedTable.UpsertRoomPublished(ctx, roomID, publish) +} + +func (d *Database) GetPublishedRooms(ctx context.Context) ([]string, error) { + return d.PublishedTable.SelectAllPublishedRooms(ctx, true) +} + func (d *Database) assignRoomNID( ctx context.Context, txn *sql.Tx, roomID string, roomVersion gomatrixserverlib.RoomVersion, diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index 21745d1b0..8b6cbe3fc 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -45,7 +45,7 @@ const insertInviteEventSQL = "" + " ON CONFLICT DO NOTHING" const selectInviteActiveForUserInRoomSQL = "" + - "SELECT sender_nid FROM roomserver_invites" + + "SELECT invite_event_id, sender_nid FROM roomserver_invites" + " WHERE target_nid = $1 AND room_nid = $2" + " AND NOT retired" @@ -133,21 +133,24 @@ func (s *inviteStatements) UpdateInviteRetired( func (s *inviteStatements) SelectInviteActiveForUserInRoom( ctx context.Context, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID, -) ([]types.EventStateKeyNID, error) { +) ([]types.EventStateKeyNID, []string, error) { rows, err := s.selectInviteActiveForUserInRoomStmt.QueryContext( ctx, targetUserNID, roomNID, ) if err != nil { - return nil, err + return nil, nil, err } defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") var result []types.EventStateKeyNID + var eventIDs []string for rows.Next() { + var eventID string var senderUserNID int64 - if err := rows.Scan(&senderUserNID); err != nil { - return nil, err + if err := rows.Scan(&eventID, &senderUserNID); err != nil { + return nil, nil, err } result = append(result, types.EventStateKeyNID(senderUserNID)) + eventIDs = append(eventIDs, eventID) } - return result, nil + return result, eventIDs, nil } diff --git a/roomserver/storage/sqlite3/published_table.go b/roomserver/storage/sqlite3/published_table.go new file mode 100644 index 000000000..9995fff6d --- /dev/null +++ b/roomserver/storage/sqlite3/published_table.go @@ -0,0 +1,100 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" + "github.com/matrix-org/dendrite/roomserver/storage/tables" +) + +const publishedSchema = ` +-- Stores which rooms are published in the room directory +CREATE TABLE IF NOT EXISTS roomserver_published ( + -- The room ID of the room + room_id TEXT NOT NULL PRIMARY KEY, + -- Whether it is published or not + published BOOLEAN NOT NULL DEFAULT false +); +` + +const upsertPublishedSQL = "" + + "INSERT OR REPLACE INTO roomserver_published (room_id, published) VALUES ($1, $2)" + +const selectAllPublishedSQL = "" + + "SELECT room_id FROM roomserver_published WHERE published = $1 ORDER BY room_id ASC" + +const selectPublishedSQL = "" + + "SELECT published FROM roomserver_published WHERE room_id = $1" + +type publishedStatements struct { + upsertPublishedStmt *sql.Stmt + selectAllPublishedStmt *sql.Stmt + selectPublishedStmt *sql.Stmt +} + +func NewSqlitePublishedTable(db *sql.DB) (tables.Published, error) { + s := &publishedStatements{} + _, err := db.Exec(publishedSchema) + if err != nil { + return nil, err + } + return s, shared.StatementList{ + {&s.upsertPublishedStmt, upsertPublishedSQL}, + {&s.selectAllPublishedStmt, selectAllPublishedSQL}, + {&s.selectPublishedStmt, selectPublishedSQL}, + }.Prepare(db) +} + +func (s *publishedStatements) UpsertRoomPublished( + ctx context.Context, roomID string, published bool, +) (err error) { + _, err = s.upsertPublishedStmt.ExecContext(ctx, roomID, published) + return +} + +func (s *publishedStatements) SelectPublishedFromRoomID( + ctx context.Context, roomID string, +) (published bool, err error) { + err = s.selectPublishedStmt.QueryRowContext(ctx, roomID).Scan(&published) + if err == sql.ErrNoRows { + return false, nil + } + return +} + +func (s *publishedStatements) SelectAllPublishedRooms( + ctx context.Context, published bool, +) ([]string, error) { + rows, err := s.selectAllPublishedStmt.QueryContext(ctx, published) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectAllPublishedStmt: rows.close() failed") + + var roomIDs []string + for rows.Next() { + var roomID string + if err = rows.Scan(&roomID); err != nil { + return nil, err + } + + roomIDs = append(roomIDs, roomID) + } + return roomIDs, rows.Err() +} diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 8e9352192..767b13ce0 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -110,6 +110,10 @@ func Open(dataSourceName string) (*Database, error) { if err != nil { return nil, err } + published, err := NewSqlitePublishedTable(d.db) + if err != nil { + return nil, err + } d.Database = shared.Database{ DB: d.db, EventsTable: d.events, @@ -124,6 +128,7 @@ func Open(dataSourceName string) (*Database, error) { RoomAliasesTable: roomAliases, InvitesTable: d.invites, MembershipTable: d.membership, + PublishedTable: published, } return &d, nil } diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index 11cff8a8b..7499089ca 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -100,8 +100,8 @@ type PreviousEvents interface { type Invites interface { InsertInviteEvent(ctx context.Context, txn *sql.Tx, inviteEventID string, roomNID types.RoomNID, targetUserNID, senderUserNID types.EventStateKeyNID, inviteEventJSON []byte) (bool, error) UpdateInviteRetired(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) ([]string, error) - // SelectInviteActiveForUserInRoom returns a list of sender state key NIDs - SelectInviteActiveForUserInRoom(ctx context.Context, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID) ([]types.EventStateKeyNID, error) + // SelectInviteActiveForUserInRoom returns a list of sender state key NIDs and invite event IDs matching those nids. + SelectInviteActiveForUserInRoom(ctx context.Context, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID) ([]types.EventStateKeyNID, []string, error) } type MembershipState int64 @@ -120,3 +120,9 @@ type Membership interface { SelectMembershipsFromRoomAndMembership(ctx context.Context, roomNID types.RoomNID, membership MembershipState, localOnly bool) (eventNIDs []types.EventNID, err error) UpdateMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, senderUserNID types.EventStateKeyNID, membership MembershipState, eventNID types.EventNID) error } + +type Published interface { + UpsertRoomPublished(ctx context.Context, roomID string, published bool) (err error) + SelectPublishedFromRoomID(ctx context.Context, roomID string) (published bool, err error) + SelectAllPublishedRooms(ctx context.Context, published bool) ([]string, error) +} diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 98be5bb73..af7f612b3 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -157,7 +157,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent( func (s *OutputRoomEventConsumer) onRetireInviteEvent( ctx context.Context, msg api.OutputRetireInviteEvent, ) error { - err := s.db.RetireInviteEvent(ctx, msg.EventID) + sp, err := s.db.RetireInviteEvent(ctx, msg.EventID) if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ @@ -166,8 +166,9 @@ func (s *OutputRoomEventConsumer) onRetireInviteEvent( }).Panicf("roomserver output log: remove invite failure") return nil } - // TODO: Notify any active sync requests that the invite has been retired. - // s.notifier.OnNewEvent(nil, msg.TargetUserID, syncStreamPos) + // Notify any active sync requests that the invite has been retired. + // Invites share the same stream counter as PDUs + s.notifier.OnNewEvent(nil, "", []string{msg.TargetUserID}, types.NewStreamToken(sp, 0)) return nil } diff --git a/clientapi/routing/filter.go b/syncapi/routing/filter.go similarity index 65% rename from clientapi/routing/filter.go rename to syncapi/routing/filter.go index 6520e6e40..baa4d841c 100644 --- a/clientapi/routing/filter.go +++ b/syncapi/routing/filter.go @@ -15,19 +15,22 @@ package routing import ( + "encoding/json" + "io/ioutil" "net/http" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/sync" "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/tidwall/gjson" ) // GetFilter implements GET /_matrix/client/r0/user/{userId}/filter/{filterId} func GetFilter( - req *http.Request, device *api.Device, accountDB accounts.Database, userID string, filterID string, + req *http.Request, device *api.Device, syncDB storage.Database, userID string, filterID string, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -41,7 +44,7 @@ func GetFilter( return jsonerror.InternalServerError() } - filter, err := accountDB.GetFilter(req.Context(), localpart, filterID) + filter, err := syncDB.GetFilter(req.Context(), localpart, filterID) if err != nil { //TODO better error handling. This error message is *probably* right, // but if there are obscure db errors, this will also be returned, @@ -64,7 +67,7 @@ type filterResponse struct { //PutFilter implements POST /_matrix/client/r0/user/{userId}/filter func PutFilter( - req *http.Request, device *api.Device, accountDB accounts.Database, userID string, + req *http.Request, device *api.Device, syncDB storage.Database, userID string, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -81,8 +84,27 @@ func PutFilter( var filter gomatrixserverlib.Filter - if reqErr := httputil.UnmarshalJSONRequest(req, &filter); reqErr != nil { - return *reqErr + defer req.Body.Close() // nolint:errcheck + body, err := ioutil.ReadAll(req.Body) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("The request body could not be read. " + err.Error()), + } + } + + if err = json.Unmarshal(body, &filter); err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("The request body could not be decoded into valid JSON. " + err.Error()), + } + } + // the filter `limit` is `int` which defaults to 0 if not set which is not what we want. We want to use the default + // limit if it is unset, which is what this does. + limitRes := gjson.GetBytes(body, "room.timeline.limit") + if !limitRes.Exists() { + util.GetLogger(req.Context()).Infof("missing timeline limit, using default") + filter.Room.Timeline.Limit = sync.DefaultTimelineLimit } // Validate generates a user-friendly error @@ -93,9 +115,9 @@ func PutFilter( } } - filterID, err := accountDB.PutFilter(req.Context(), localpart, &filter) + filterID, err := syncDB.PutFilter(req.Context(), localpart, &filter) if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.PutFilter failed") + util.GetLogger(req.Context()).WithError(err).Error("syncDB.PutFilter failed") return jsonerror.InternalServerError() } diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 5744de05a..a98955c57 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -55,4 +55,24 @@ func Setup( } return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], federation, rsAPI, cfg) })).Methods(http.MethodGet, http.MethodOptions) + + r0mux.Handle("/user/{userId}/filter", + httputil.MakeAuthAPI("put_filter", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return PutFilter(req, device, syncDB, vars["userId"]) + }), + ).Methods(http.MethodPost, http.MethodOptions) + + r0mux.Handle("/user/{userId}/filter/{filterId}", + httputil.MakeAuthAPI("get_filter", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetFilter(req, device, syncDB, vars["userId"], vars["filterId"]) + }), + ).Methods(http.MethodGet, http.MethodOptions) } diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 7b3bd6785..c4dae4d09 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -78,9 +78,9 @@ type Database interface { // If the invite was successfully stored this returns the stream ID it was stored at. // Returns an error if there was a problem communicating with the database. AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent) (types.StreamPosition, error) - // RetireInviteEvent removes an old invite event from the database. + // RetireInviteEvent removes an old invite event from the database. Returns the new position of the retired invite. // Returns an error if there was a problem communicating with the database. - RetireInviteEvent(ctx context.Context, inviteEventID string) error + RetireInviteEvent(ctx context.Context, inviteEventID string) (types.StreamPosition, error) // SetTypingTimeoutCallback sets a callback function that is called right after // a user is removed from the typing user list due to timeout. SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) @@ -128,4 +128,12 @@ type Database interface { CleanSendToDeviceUpdates(ctx context.Context, toUpdate, toDelete []types.SendToDeviceNID, token types.StreamingToken) (err error) // SendToDeviceUpdatesWaiting returns true if there are send-to-device updates waiting to be sent. SendToDeviceUpdatesWaiting(ctx context.Context, userID, deviceID string) (bool, error) + // GetFilter looks up the filter associated with a given local user and filter ID. + // Returns a filter structure. Otherwise returns an error if no such filter exists + // or if there was an error talking to the database. + GetFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error) + // PutFilter puts the passed filter into the database. + // Returns the filterID as a string. Otherwise returns an error if something + // goes wrong. + PutFilter(ctx context.Context, localpart string, filter *gomatrixserverlib.Filter) (string, error) } diff --git a/userapi/storage/accounts/postgres/filter_table.go b/syncapi/storage/postgres/filter_table.go similarity index 83% rename from userapi/storage/accounts/postgres/filter_table.go rename to syncapi/storage/postgres/filter_table.go index c54e4bc42..beeb864ba 100644 --- a/userapi/storage/accounts/postgres/filter_table.go +++ b/syncapi/storage/postgres/filter_table.go @@ -19,12 +19,13 @@ import ( "database/sql" "encoding/json" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/gomatrixserverlib" ) const filterSchema = ` -- Stores data about filters -CREATE TABLE IF NOT EXISTS account_filter ( +CREATE TABLE IF NOT EXISTS syncapi_filter ( -- The filter filter TEXT NOT NULL, -- The ID @@ -35,17 +36,17 @@ CREATE TABLE IF NOT EXISTS account_filter ( PRIMARY KEY(id, localpart) ); -CREATE INDEX IF NOT EXISTS account_filter_localpart ON account_filter(localpart); +CREATE INDEX IF NOT EXISTS syncapi_filter_localpart ON syncapi_filter(localpart); ` const selectFilterSQL = "" + - "SELECT filter FROM account_filter WHERE localpart = $1 AND id = $2" + "SELECT filter FROM syncapi_filter WHERE localpart = $1 AND id = $2" const selectFilterIDByContentSQL = "" + - "SELECT id FROM account_filter WHERE localpart = $1 AND filter = $2" + "SELECT id FROM syncapi_filter WHERE localpart = $1 AND filter = $2" const insertFilterSQL = "" + - "INSERT INTO account_filter (filter, id, localpart) VALUES ($1, DEFAULT, $2) RETURNING id" + "INSERT INTO syncapi_filter (filter, id, localpart) VALUES ($1, DEFAULT, $2) RETURNING id" type filterStatements struct { selectFilterStmt *sql.Stmt @@ -53,24 +54,25 @@ type filterStatements struct { insertFilterStmt *sql.Stmt } -func (s *filterStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(filterSchema) +func NewPostgresFilterTable(db *sql.DB) (tables.Filter, error) { + _, err := db.Exec(filterSchema) if err != nil { - return + return nil, err } + s := &filterStatements{} if s.selectFilterStmt, err = db.Prepare(selectFilterSQL); err != nil { - return + return nil, err } if s.selectFilterIDByContentStmt, err = db.Prepare(selectFilterIDByContentSQL); err != nil { - return + return nil, err } if s.insertFilterStmt, err = db.Prepare(insertFilterSQL); err != nil { - return + return nil, err } - return + return s, nil } -func (s *filterStatements) selectFilter( +func (s *filterStatements) SelectFilter( ctx context.Context, localpart string, filterID string, ) (*gomatrixserverlib.Filter, error) { // Retrieve filter from database (stored as canonical JSON) @@ -88,7 +90,7 @@ func (s *filterStatements) selectFilter( return &filter, nil } -func (s *filterStatements) insertFilter( +func (s *filterStatements) InsertFilter( ctx context.Context, filter *gomatrixserverlib.Filter, localpart string, ) (filterID string, err error) { var existingFilterID string diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go index 5031d64e5..530dc6452 100644 --- a/syncapi/storage/postgres/invites_table.go +++ b/syncapi/storage/postgres/invites_table.go @@ -33,7 +33,8 @@ CREATE TABLE IF NOT EXISTS syncapi_invite_events ( event_id TEXT NOT NULL, room_id TEXT NOT NULL, target_user_id TEXT NOT NULL, - headered_event_json TEXT NOT NULL + headered_event_json TEXT NOT NULL, + deleted BOOL NOT NULL ); -- For looking up the invites for a given user. @@ -47,14 +48,14 @@ CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx const insertInviteEventSQL = "" + "INSERT INTO syncapi_invite_events (" + - " room_id, event_id, target_user_id, headered_event_json" + - ") VALUES ($1, $2, $3, $4) RETURNING id" + " room_id, event_id, target_user_id, headered_event_json, deleted" + + ") VALUES ($1, $2, $3, $4, FALSE) RETURNING id" const deleteInviteEventSQL = "" + - "DELETE FROM syncapi_invite_events WHERE event_id = $1" + "UPDATE syncapi_invite_events SET deleted=TRUE, id=nextval('syncapi_stream_id') WHERE event_id = $1 RETURNING id" const selectInviteEventsInRangeSQL = "" + - "SELECT room_id, headered_event_json FROM syncapi_invite_events" + + "SELECT room_id, headered_event_json, deleted FROM syncapi_invite_events" + " WHERE target_user_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC" @@ -110,40 +111,46 @@ func (s *inviteEventsStatements) InsertInviteEvent( func (s *inviteEventsStatements) DeleteInviteEvent( ctx context.Context, inviteEventID string, -) error { - _, err := s.deleteInviteEventStmt.ExecContext(ctx, inviteEventID) - return err +) (sp types.StreamPosition, err error) { + err = s.deleteInviteEventStmt.QueryRowContext(ctx, inviteEventID).Scan(&sp) + return } // selectInviteEventsInRange returns a map of room ID to invite event for the // active invites for the target user ID in the supplied range. func (s *inviteEventsStatements) SelectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range, -) (map[string]gomatrixserverlib.HeaderedEvent, error) { +) (map[string]gomatrixserverlib.HeaderedEvent, map[string]gomatrixserverlib.HeaderedEvent, error) { stmt := sqlutil.TxStmt(txn, s.selectInviteEventsInRangeStmt) rows, err := stmt.QueryContext(ctx, targetUserID, r.Low(), r.High()) if err != nil { - return nil, err + return nil, nil, err } defer internal.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") result := map[string]gomatrixserverlib.HeaderedEvent{} + retired := map[string]gomatrixserverlib.HeaderedEvent{} for rows.Next() { var ( roomID string eventJSON []byte + deleted bool ) - if err = rows.Scan(&roomID, &eventJSON); err != nil { - return nil, err + if err = rows.Scan(&roomID, &eventJSON, &deleted); err != nil { + return nil, nil, err } var event gomatrixserverlib.HeaderedEvent if err := json.Unmarshal(eventJSON, &event); err != nil { - return nil, err + return nil, nil, err } - result[roomID] = event + if deleted { + retired[roomID] = event + } else { + result[roomID] = event + } } - return result, rows.Err() + return result, retired, rows.Err() } func (s *inviteEventsStatements) SelectMaxInviteID( diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index f01b2eabd..c7c4dc63b 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -301,21 +301,21 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int, chronologicalOrder bool, onlySyncEvents bool, -) ([]types.StreamEvent, error) { +) ([]types.StreamEvent, bool, error) { var stmt *sql.Stmt if onlySyncEvents { stmt = sqlutil.TxStmt(txn, s.selectRecentEventsForSyncStmt) } else { stmt = sqlutil.TxStmt(txn, s.selectRecentEventsStmt) } - rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) + rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit+1) if err != nil { - return nil, err + return nil, false, err } defer internal.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { - return nil, err + return nil, false, err } if chronologicalOrder { // The events need to be returned from oldest to latest, which isn't @@ -325,7 +325,19 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( return events[i].StreamPosition < events[j].StreamPosition }) } - return events, nil + // we queried for 1 more than the limit, so if we returned one more mark limited=true + limited := false + if len(events) > limit { + limited = true + // re-slice the extra (oldest) event out: in chronological order this is the first entry, else the last. + if chronologicalOrder { + events = events[1:] + } else { + events = events[:len(events)-1] + } + } + + return events, limited, nil } // selectEarlyEvents returns the earliest events in the given room, starting diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 573586cc7..10c1b37c7 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -71,6 +71,10 @@ func NewDatabase(dbDataSourceName string, dbProperties sqlutil.DbProperties) (*S if err != nil { return nil, err } + filter, err := NewPostgresFilterTable(d.db) + if err != nil { + return nil, err + } d.Database = shared.Database{ DB: d.db, Invites: invites, @@ -79,6 +83,7 @@ func NewDatabase(dbDataSourceName string, dbProperties sqlutil.DbProperties) (*S Topology: topology, CurrentRoomState: currState, BackwardExtremities: backwardExtremities, + Filter: filter, SendToDevice: sendToDevice, SendToDeviceWriter: sqlutil.NewTransactionWriter(), EDUCache: cache.New(), diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 74ae3eabd..01362ddd6 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -43,6 +43,7 @@ type Database struct { CurrentRoomState tables.CurrentRoomState BackwardExtremities tables.BackwardsExtremities SendToDevice tables.SendToDevice + Filter tables.Filter SendToDeviceWriter *sqlutil.TransactionWriter EDUCache *cache.EDUCache } @@ -78,7 +79,7 @@ func (d *Database) GetEventsInStreamingRange( } if backwardOrdering { // When using backward ordering, we want the most recent events first. - if events, err = d.OutputEvents.SelectRecentEvents( + if events, _, err = d.OutputEvents.SelectRecentEvents( ctx, nil, roomID, r, limit, false, false, ); err != nil { return @@ -180,11 +181,8 @@ func (d *Database) AddInviteEvent( // Returns an error if there was a problem communicating with the database. func (d *Database) RetireInviteEvent( ctx context.Context, inviteEventID string, -) error { - // TODO: Record that invite has been retired in a stream so that we can - // notify the user in an incremental sync. - err := d.Invites.DeleteInviteEvent(ctx, inviteEventID) - return err +) (types.StreamPosition, error) { + return d.Invites.DeleteInviteEvent(ctx, inviteEventID) } // GetAccountDataInRange returns all account data for a given user inserted or @@ -548,6 +546,18 @@ func (d *Database) addEDUDeltaToResponse( return } +func (d *Database) GetFilter( + ctx context.Context, localpart string, filterID string, +) (*gomatrixserverlib.Filter, error) { + return d.Filter.SelectFilter(ctx, localpart, filterID) +} + +func (d *Database) PutFilter( + ctx context.Context, localpart string, filter *gomatrixserverlib.Filter, +) (string, error) { + return d.Filter.InsertFilter(ctx, filter, localpart) +} + func (d *Database) IncrementalSync( ctx context.Context, res *types.Response, device userapi.Device, @@ -645,7 +655,8 @@ func (d *Database) getResponseWithPDUsForCompleteSync( // TODO: When filters are added, we may need to call this multiple times to get enough events. // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 var recentStreamEvents []types.StreamEvent - recentStreamEvents, err = d.OutputEvents.SelectRecentEvents( + var limited bool + recentStreamEvents, limited, err = d.OutputEvents.SelectRecentEvents( ctx, txn, roomID, r, numRecentEventsPerRoom, true, true, ) if err != nil { @@ -673,7 +684,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync( jr := types.NewJoinResponse() jr.Timeline.PrevBatch = prevBatchStr jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) - jr.Timeline.Limited = true + jr.Timeline.Limited = limited jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[roomID] = *jr } @@ -724,7 +735,7 @@ func (d *Database) addInvitesToResponse( r types.Range, res *types.Response, ) error { - invites, err := d.Invites.SelectInviteEventsInRange( + invites, retiredInvites, err := d.Invites.SelectInviteEventsInRange( ctx, txn, userID, r, ) if err != nil { @@ -734,6 +745,10 @@ func (d *Database) addInvitesToResponse( ir := types.NewInviteResponse(inviteEvent) res.Rooms.Invite[roomID] = *ir } + for roomID := range retiredInvites { + lr := types.NewLeaveResponse() + res.Rooms.Leave[roomID] = *lr + } return nil } @@ -775,7 +790,7 @@ func (d *Database) addRoomDeltaToResponse( // This is all "okay" assuming history_visibility == "shared" which it is by default. r.To = delta.membershipPos } - recentStreamEvents, err := d.OutputEvents.SelectRecentEvents( + recentStreamEvents, limited, err := d.OutputEvents.SelectRecentEvents( ctx, txn, delta.roomID, r, numRecentEventsPerRoom, true, true, ) @@ -795,7 +810,7 @@ func (d *Database) addRoomDeltaToResponse( jr.Timeline.PrevBatch = prevBatch.String() jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) - jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true + jr.Timeline.Limited = limited jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[delta.roomID] = *jr case gomatrixserverlib.Leave: diff --git a/userapi/storage/accounts/sqlite3/filter_table.go b/syncapi/storage/sqlite3/filter_table.go similarity index 83% rename from userapi/storage/accounts/sqlite3/filter_table.go rename to syncapi/storage/sqlite3/filter_table.go index 7f1a0c249..8b26759dc 100644 --- a/userapi/storage/accounts/sqlite3/filter_table.go +++ b/syncapi/storage/sqlite3/filter_table.go @@ -20,12 +20,13 @@ import ( "encoding/json" "fmt" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/gomatrixserverlib" ) const filterSchema = ` -- Stores data about filters -CREATE TABLE IF NOT EXISTS account_filter ( +CREATE TABLE IF NOT EXISTS syncapi_filter ( -- The filter filter TEXT NOT NULL, -- The ID @@ -36,17 +37,17 @@ CREATE TABLE IF NOT EXISTS account_filter ( UNIQUE (id, localpart) ); -CREATE INDEX IF NOT EXISTS account_filter_localpart ON account_filter(localpart); +CREATE INDEX IF NOT EXISTS syncapi_filter_localpart ON syncapi_filter(localpart); ` const selectFilterSQL = "" + - "SELECT filter FROM account_filter WHERE localpart = $1 AND id = $2" + "SELECT filter FROM syncapi_filter WHERE localpart = $1 AND id = $2" const selectFilterIDByContentSQL = "" + - "SELECT id FROM account_filter WHERE localpart = $1 AND filter = $2" + "SELECT id FROM syncapi_filter WHERE localpart = $1 AND filter = $2" const insertFilterSQL = "" + - "INSERT INTO account_filter (filter, localpart) VALUES ($1, $2)" + "INSERT INTO syncapi_filter (filter, localpart) VALUES ($1, $2)" type filterStatements struct { selectFilterStmt *sql.Stmt @@ -54,24 +55,25 @@ type filterStatements struct { insertFilterStmt *sql.Stmt } -func (s *filterStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(filterSchema) +func NewSqliteFilterTable(db *sql.DB) (tables.Filter, error) { + _, err := db.Exec(filterSchema) if err != nil { - return + return nil, err } + s := &filterStatements{} if s.selectFilterStmt, err = db.Prepare(selectFilterSQL); err != nil { - return + return nil, err } if s.selectFilterIDByContentStmt, err = db.Prepare(selectFilterIDByContentSQL); err != nil { - return + return nil, err } if s.insertFilterStmt, err = db.Prepare(insertFilterSQL); err != nil { - return + return nil, err } - return + return s, nil } -func (s *filterStatements) selectFilter( +func (s *filterStatements) SelectFilter( ctx context.Context, localpart string, filterID string, ) (*gomatrixserverlib.Filter, error) { // Retrieve filter from database (stored as canonical JSON) @@ -89,7 +91,7 @@ func (s *filterStatements) selectFilter( return &filter, nil } -func (s *filterStatements) insertFilter( +func (s *filterStatements) InsertFilter( ctx context.Context, filter *gomatrixserverlib.Filter, localpart string, ) (filterID string, err error) { var existingFilterID string diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index bb58e3456..aa0513888 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -33,7 +33,8 @@ CREATE TABLE IF NOT EXISTS syncapi_invite_events ( event_id TEXT NOT NULL, room_id TEXT NOT NULL, target_user_id TEXT NOT NULL, - headered_event_json TEXT NOT NULL + headered_event_json TEXT NOT NULL, + deleted BOOL NOT NULL ); CREATE INDEX IF NOT EXISTS syncapi_invites_target_user_id_idx ON syncapi_invite_events (target_user_id, id); @@ -42,14 +43,14 @@ CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx ON syncapi_invite_events const insertInviteEventSQL = "" + "INSERT INTO syncapi_invite_events" + - " (id, room_id, event_id, target_user_id, headered_event_json)" + - " VALUES ($1, $2, $3, $4, $5)" + " (id, room_id, event_id, target_user_id, headered_event_json, deleted)" + + " VALUES ($1, $2, $3, $4, $5, false)" const deleteInviteEventSQL = "" + - "DELETE FROM syncapi_invite_events WHERE event_id = $1" + "UPDATE syncapi_invite_events SET deleted=true, id=$1 WHERE event_id = $2" const selectInviteEventsInRangeSQL = "" + - "SELECT room_id, headered_event_json FROM syncapi_invite_events" + + "SELECT room_id, headered_event_json, deleted FROM syncapi_invite_events" + " WHERE target_user_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC" @@ -114,40 +115,49 @@ func (s *inviteEventsStatements) InsertInviteEvent( func (s *inviteEventsStatements) DeleteInviteEvent( ctx context.Context, inviteEventID string, -) error { - _, err := s.deleteInviteEventStmt.ExecContext(ctx, inviteEventID) - return err +) (types.StreamPosition, error) { + streamPos, err := s.streamIDStatements.nextStreamID(ctx, nil) + if err != nil { + return streamPos, err + } + _, err = s.deleteInviteEventStmt.ExecContext(ctx, streamPos, inviteEventID) + return streamPos, err } // selectInviteEventsInRange returns a map of room ID to invite event for the // active invites for the target user ID in the supplied range. func (s *inviteEventsStatements) SelectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range, -) (map[string]gomatrixserverlib.HeaderedEvent, error) { +) (map[string]gomatrixserverlib.HeaderedEvent, map[string]gomatrixserverlib.HeaderedEvent, error) { stmt := sqlutil.TxStmt(txn, s.selectInviteEventsInRangeStmt) rows, err := stmt.QueryContext(ctx, targetUserID, r.Low(), r.High()) if err != nil { - return nil, err + return nil, nil, err } defer internal.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") result := map[string]gomatrixserverlib.HeaderedEvent{} + retired := map[string]gomatrixserverlib.HeaderedEvent{} for rows.Next() { var ( roomID string eventJSON []byte + deleted bool ) - if err = rows.Scan(&roomID, &eventJSON); err != nil { - return nil, err + if err = rows.Scan(&roomID, &eventJSON, &deleted); err != nil { + return nil, nil, err } var event gomatrixserverlib.HeaderedEvent if err := json.Unmarshal(eventJSON, &event); err != nil { - return nil, err + return nil, nil, err + } + if deleted { + retired[roomID] = event + } else { + result[roomID] = event } - - result[roomID] = event } - return result, nil + return result, retired, nil } func (s *inviteEventsStatements) SelectMaxInviteID( diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 367ab3c9a..0c909cc4d 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -311,7 +311,7 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int, chronologicalOrder bool, onlySyncEvents bool, -) ([]types.StreamEvent, error) { +) ([]types.StreamEvent, bool, error) { var stmt *sql.Stmt if onlySyncEvents { stmt = sqlutil.TxStmt(txn, s.selectRecentEventsForSyncStmt) @@ -319,14 +319,14 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( stmt = sqlutil.TxStmt(txn, s.selectRecentEventsStmt) } - rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) + rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit+1) if err != nil { - return nil, err + return nil, false, err } defer internal.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { - return nil, err + return nil, false, err } if chronologicalOrder { // The events need to be returned from oldest to latest, which isn't @@ -336,7 +336,18 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( return events[i].StreamPosition < events[j].StreamPosition }) } - return events, nil + // we queried for 1 more than the limit, so if we returned one more mark limited=true + limited := false + if len(events) > limit { + limited = true + // re-slice the extra (oldest) event out: in chronological order this is the first entry, else the last. + if chronologicalOrder { + events = events[1:] + } else { + events = events[:len(events)-1] + } + } + return events, limited, nil } func (s *outputRoomEventsStatements) SelectEarlyEvents( diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 51cdbe325..c85db5a4f 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -87,6 +87,10 @@ func (d *SyncServerDatasource) prepare() (err error) { if err != nil { return err } + filter, err := NewSqliteFilterTable(d.db) + if err != nil { + return err + } d.Database = shared.Database{ DB: d.db, Invites: invites, @@ -95,6 +99,7 @@ func (d *SyncServerDatasource) prepare() (err error) { BackwardExtremities: bwExtrem, CurrentRoomState: roomState, Topology: topology, + Filter: filter, SendToDevice: sendToDevice, SendToDeviceWriter: sqlutil.NewTransactionWriter(), EDUCache: cache.New(), diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 85084facb..feacbc18c 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -601,6 +601,83 @@ func TestSendToDeviceBehaviour(t *testing.T) { } } +func TestInviteBehaviour(t *testing.T) { + db := MustCreateDatabase(t) + inviteRoom1 := "!inviteRoom1:somewhere" + inviteEvent1 := MustCreateEvent(t, inviteRoom1, nil, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"invite"}`)), + Type: "m.room.member", + StateKey: &testUserIDA, + Sender: "@inviteUser1:somewhere", + }) + inviteRoom2 := "!inviteRoom2:somewhere" + inviteEvent2 := MustCreateEvent(t, inviteRoom2, nil, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"invite"}`)), + Type: "m.room.member", + StateKey: &testUserIDA, + Sender: "@inviteUser2:somewhere", + }) + for _, ev := range []gomatrixserverlib.HeaderedEvent{inviteEvent1, inviteEvent2} { + _, err := db.AddInviteEvent(ctx, ev) + if err != nil { + t.Fatalf("Failed to AddInviteEvent: %s", err) + } + } + latest, err := db.SyncPosition(ctx) + if err != nil { + t.Fatalf("failed to get SyncPosition: %s", err) + } + // both invite events should appear in a new sync + beforeRetireRes := types.NewResponse() + beforeRetireRes, err = db.IncrementalSync(ctx, beforeRetireRes, testUserDeviceA, types.NewStreamToken(0, 0), latest, 0, false) + if err != nil { + t.Fatalf("IncrementalSync failed: %s", err) + } + assertInvitedToRooms(t, beforeRetireRes, []string{inviteRoom1, inviteRoom2}) + + // retire one event: a fresh sync should just return 1 invite room + if _, err = db.RetireInviteEvent(ctx, inviteEvent1.EventID()); err != nil { + t.Fatalf("Failed to RetireInviteEvent: %s", err) + } + latest, err = db.SyncPosition(ctx) + if err != nil { + t.Fatalf("failed to get SyncPosition: %s", err) + } + res := types.NewResponse() + res, err = db.IncrementalSync(ctx, res, testUserDeviceA, types.NewStreamToken(0, 0), latest, 0, false) + if err != nil { + t.Fatalf("IncrementalSync failed: %s", err) + } + assertInvitedToRooms(t, res, []string{inviteRoom2}) + + // a sync after we have received both invites should result in a leave for the retired room + beforeRetireTok, err := types.NewStreamTokenFromString(beforeRetireRes.NextBatch) + if err != nil { + t.Fatalf("NewStreamTokenFromString cannot parse next batch '%s' : %s", beforeRetireRes.NextBatch, err) + } + res = types.NewResponse() + res, err = db.IncrementalSync(ctx, res, testUserDeviceA, beforeRetireTok, latest, 0, false) + if err != nil { + t.Fatalf("IncrementalSync failed: %s", err) + } + assertInvitedToRooms(t, res, []string{}) + if _, ok := res.Rooms.Leave[inviteRoom1]; !ok { + t.Fatalf("IncrementalSync: expected to see room left after it was retired but it wasn't") + } +} + +func assertInvitedToRooms(t *testing.T, res *types.Response, roomIDs []string) { + t.Helper() + if len(res.Rooms.Invite) != len(roomIDs) { + t.Fatalf("got %d invited rooms, want %d", len(res.Rooms.Invite), len(roomIDs)) + } + for _, roomID := range roomIDs { + if _, ok := res.Rooms.Invite[roomID]; !ok { + t.Fatalf("missing room ID %s", roomID) + } + } +} + func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatrixserverlib.ClientEvent, wants []gomatrixserverlib.HeaderedEvent) { if len(gots) != len(wants) { t.Fatalf("%s response returned %d events, want %d", msg, len(gots), len(wants)) diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 0b7d15951..4ac0be4ec 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -32,9 +32,9 @@ type AccountData interface { type Invites interface { InsertInviteEvent(ctx context.Context, txn *sql.Tx, inviteEvent gomatrixserverlib.HeaderedEvent) (streamPos types.StreamPosition, err error) - DeleteInviteEvent(ctx context.Context, inviteEventID string) error + DeleteInviteEvent(ctx context.Context, inviteEventID string) (types.StreamPosition, error) // SelectInviteEventsInRange returns a map of room ID to invite events. - SelectInviteEventsInRange(ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range) (map[string]gomatrixserverlib.HeaderedEvent, error) + SelectInviteEventsInRange(ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range) (invites map[string]gomatrixserverlib.HeaderedEvent, retired map[string]gomatrixserverlib.HeaderedEvent, err error) SelectMaxInviteID(ctx context.Context, txn *sql.Tx) (id int64, err error) } @@ -44,8 +44,8 @@ type Events interface { InsertEvent(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, transactionID *api.TransactionID, excludeFromSync bool) (streamPos types.StreamPosition, err error) // SelectRecentEvents returns events between the two stream positions: exclusive of low and inclusive of high. // If onlySyncEvents has a value of true, only returns the events that aren't marked as to exclude from sync. - // Returns up to `limit` events. - SelectRecentEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, error) + // Returns up to `limit` events. Returns `limited=true` if there are more events in this range but we hit the `limit`. + SelectRecentEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) // SelectEarlyEvents returns the earliest events in the given room. SelectEarlyEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int) ([]types.StreamEvent, error) SelectEvents(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]types.StreamEvent, error) @@ -133,3 +133,8 @@ type SendToDevice interface { DeleteSendToDeviceMessages(ctx context.Context, txn *sql.Tx, nids []types.SendToDeviceNID) (err error) CountSendToDeviceMessages(ctx context.Context, txn *sql.Tx, userID, deviceID string) (count int, err error) } + +type Filter interface { + SelectFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error) + InsertFilter(ctx context.Context, filter *gomatrixserverlib.Filter, localpart string) (filterID string, err error) +} diff --git a/syncapi/sync/notifier_test.go b/syncapi/sync/notifier_test.go index ecc4fcbfc..f2a368ec2 100644 --- a/syncapi/sync/notifier_test.go +++ b/syncapi/sync/notifier_test.go @@ -363,7 +363,7 @@ func newTestSyncRequest(userID, deviceID string, since types.StreamingToken) syn timeout: 1 * time.Minute, since: &since, wantFullState: false, - limit: defaultTimelineLimit, + limit: DefaultTimelineLimit, log: util.GetLogger(context.TODO()), ctx: context.TODO(), } diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index 5dd92c853..41b18aa10 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -21,14 +21,16 @@ import ( "strconv" "time" + "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" log "github.com/sirupsen/logrus" ) const defaultSyncTimeout = time.Duration(0) -const defaultTimelineLimit = 20 +const DefaultTimelineLimit = 20 type filter struct { Room struct { @@ -49,7 +51,7 @@ type syncRequest struct { log *log.Entry } -func newSyncRequest(req *http.Request, device userapi.Device) (*syncRequest, error) { +func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Database) (*syncRequest, error) { timeout := getTimeout(req.URL.Query().Get("timeout")) fullState := req.URL.Query().Get("full_state") wantFullState := fullState != "" && fullState != "false" @@ -66,15 +68,28 @@ func newSyncRequest(req *http.Request, device userapi.Device) (*syncRequest, err tok := types.NewStreamToken(0, 0) since = &tok } - timelineLimit := defaultTimelineLimit + timelineLimit := DefaultTimelineLimit // TODO: read from stored filters too filterQuery := req.URL.Query().Get("filter") - if filterQuery != "" && filterQuery[0] == '{' { - // attempt to parse the timeline limit at least - var f filter - err := json.Unmarshal([]byte(filterQuery), &f) - if err == nil && f.Room.Timeline.Limit != nil { - timelineLimit = *f.Room.Timeline.Limit + if filterQuery != "" { + if filterQuery[0] == '{' { + // attempt to parse the timeline limit at least + var f filter + err := json.Unmarshal([]byte(filterQuery), &f) + if err == nil && f.Room.Timeline.Limit != nil { + timelineLimit = *f.Room.Timeline.Limit + } + } else { + // attempt to load the filter ID + localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return nil, err + } + f, err := syncDB.GetFilter(req.Context(), localpart, filterQuery) + if err == nil { + timelineLimit = f.Room.Timeline.Limit + } } } // TODO: Additional query params: set_presence, filter diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 743c63a62..196d446a2 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -49,7 +49,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. var syncData *types.Response // Extract values from request - syncReq, err := newSyncRequest(req, *device) + syncReq, err := newSyncRequest(req, *device, rp.db) if err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 1094416a1..019f2e69b 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -290,10 +290,10 @@ type Response struct { NextBatch string `json:"next_batch"` AccountData struct { Events []gomatrixserverlib.ClientEvent `json:"events"` - } `json:"account_data"` + } `json:"account_data,omitempty"` Presence struct { Events []gomatrixserverlib.ClientEvent `json:"events"` - } `json:"presence"` + } `json:"presence,omitempty"` Rooms struct { Join map[string]JoinResponse `json:"join"` Invite map[string]InviteResponse `json:"invite"` diff --git a/sytest-blacklist b/sytest-blacklist index 9f140ed1c..65e6c1b16 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -45,6 +45,9 @@ Can recv device messages over federation Device messages over federation wake up /sync Wildcard device messages over federation wake up /sync +# See https://github.com/matrix-org/sytest/pull/901 +Remote invited user can see room metadata + # We don't implement soft-failed events yet, but because the /send response is vague, # this test thinks it's all fine... Inbound federation accepts a second soft-failed event diff --git a/sytest-whitelist b/sytest-whitelist index 0036d60ea..85517cf9a 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -181,7 +181,11 @@ Outbound federation can query profile data /event/ on joined room works /event/ does not allow access to events before the user joined Federation key API allows unsigned requests for keys +GET /publicRooms lists rooms +GET /publicRooms includes avatar URLs Can paginate public room list +GET /publicRooms lists newly-created room +Name/topic keys are correct GET /directory/room/:room_alias yields room ID PUT /directory/room/:room_alias creates alias Room aliases can contain Unicode @@ -243,6 +247,9 @@ User can invite local user to room with version 2 Remote user can backfill in a room with version 2 Inbound federation accepts attempts to join v2 rooms from servers with support Outbound federation can send invites via v2 API +Outbound federation can send invites via v1 API +Inbound federation can receive invites via v1 API +Inbound federation can receive invites via v2 API User can create and send/receive messages in a room with version 3 local user can join room with version 3 Remote user can backfill in a room with version 3 @@ -357,9 +364,36 @@ Getting state checks the events requested belong to the room Getting state IDs checks the events requested belong to the room Can invite users to invite-only rooms Uninvited users cannot join the room +Users cannot invite themselves to a room +Users cannot invite a user that is already in the room Invited user can reject invite Invited user can reject invite for empty room Invited user can reject local invite after originator leaves +PUT /rooms/:room_id/typing/:user_id sets typing notification Typing notification sent to local room members Typing notifications also sent to remote room members Typing can be explicitly stopped +Banned user is kicked and may not rejoin until unbanned +Inbound federation rejects attempts to join v1 rooms from servers without v1 support +Inbound federation rejects attempts to join v2 rooms from servers lacking version support +Inbound federation rejects attempts to join v2 rooms from servers only supporting v1 +Outbound federation passes make_join failures through to the client +Outbound federation correctly handles unsupported room versions +Remote users may not join unfederated rooms +Guest users denied access over federation if guest access prohibited +Non-numeric ports in server names are rejected +Invited user can reject invite over federation +Invited user can reject invite over federation for empty room +Can reject invites over federation for rooms with version 1 +Can reject invites over federation for rooms with version 2 +Can reject invites over federation for rooms with version 3 +Can reject invites over federation for rooms with version 4 +Can reject invites over federation for rooms with version 5 +Can reject invites over federation for rooms with version 6 +Event size limits +Can sync a room with a single message +Can sync a room with a message with a transaction id +A full_state incremental update returns only recent timeline +A prev_batch token can be used in the v1 messages API +We don't send redundant membership state across incremental syncs by default +Typing notifications don't leak diff --git a/userapi/internal/api.go b/userapi/internal/api.go index b081eca49..1d10d1d8b 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -85,6 +85,11 @@ func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.P } return nil } + + if err = a.AccountDB.SetDisplayName(ctx, req.Localpart, req.Localpart); err != nil { + return err + } + res.AccountCreated = true res.Account = acc return nil diff --git a/userapi/storage/accounts/interface.go b/userapi/storage/accounts/interface.go index c6692879b..6f6caf111 100644 --- a/userapi/storage/accounts/interface.go +++ b/userapi/storage/accounts/interface.go @@ -22,7 +22,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" ) type Database interface { @@ -36,10 +35,6 @@ type Database interface { // account already exists, it will return nil, ErrUserExists. CreateAccount(ctx context.Context, localpart, plaintextPassword, appserviceID string) (*api.Account, error) CreateGuestAccount(ctx context.Context) (*api.Account, error) - UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error - GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error) - GetRoomIDsByLocalPart(ctx context.Context, localpart string) ([]string, error) - GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error) SaveAccountData(ctx context.Context, localpart, roomID, dataType string, content json.RawMessage) error GetAccountData(ctx context.Context, localpart string) (global map[string]json.RawMessage, rooms map[string]map[string]json.RawMessage, err error) // GetAccountDataByType returns account data matching a given @@ -52,8 +47,6 @@ type Database interface { RemoveThreePIDAssociation(ctx context.Context, threepid string, medium string) (err error) GetLocalpartForThreePID(ctx context.Context, threepid string, medium string) (localpart string, err error) GetThreePIDsForLocalpart(ctx context.Context, localpart string) (threepids []authtypes.ThreePID, err error) - GetFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error) - PutFilter(ctx context.Context, localpart string, filter *gomatrixserverlib.Filter) (string, error) CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) GetAccountByLocalpart(ctx context.Context, localpart string) (*api.Account, error) } diff --git a/userapi/storage/accounts/postgres/membership_table.go b/userapi/storage/accounts/postgres/membership_table.go deleted file mode 100644 index 623530acc..000000000 --- a/userapi/storage/accounts/postgres/membership_table.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "context" - "database/sql" - - "github.com/lib/pq" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/internal" -) - -const membershipSchema = ` --- Stores data about users memberships to rooms. -CREATE TABLE IF NOT EXISTS account_memberships ( - -- The Matrix user ID localpart for the member - localpart TEXT NOT NULL, - -- The room this user is a member of - room_id TEXT NOT NULL, - -- The ID of the join membership event - event_id TEXT NOT NULL, - - -- A user can only be member of a room once - PRIMARY KEY (localpart, room_id) -); - --- Use index to process deletion by ID more efficiently -CREATE UNIQUE INDEX IF NOT EXISTS account_membership_event_id ON account_memberships(event_id); -` - -const insertMembershipSQL = ` - INSERT INTO account_memberships(localpart, room_id, event_id) VALUES ($1, $2, $3) - ON CONFLICT (localpart, room_id) DO UPDATE SET event_id = EXCLUDED.event_id -` - -const selectMembershipsByLocalpartSQL = "" + - "SELECT room_id, event_id FROM account_memberships WHERE localpart = $1" - -const selectMembershipInRoomByLocalpartSQL = "" + - "SELECT event_id FROM account_memberships WHERE localpart = $1 AND room_id = $2" - -const selectRoomIDsByLocalPartSQL = "" + - "SELECT room_id FROM account_memberships WHERE localpart = $1" - -const deleteMembershipsByEventIDsSQL = "" + - "DELETE FROM account_memberships WHERE event_id = ANY($1)" - -type membershipStatements struct { - deleteMembershipsByEventIDsStmt *sql.Stmt - insertMembershipStmt *sql.Stmt - selectMembershipInRoomByLocalpartStmt *sql.Stmt - selectMembershipsByLocalpartStmt *sql.Stmt - selectRoomIDsByLocalPartStmt *sql.Stmt -} - -func (s *membershipStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(membershipSchema) - if err != nil { - return - } - if s.deleteMembershipsByEventIDsStmt, err = db.Prepare(deleteMembershipsByEventIDsSQL); err != nil { - return - } - if s.insertMembershipStmt, err = db.Prepare(insertMembershipSQL); err != nil { - return - } - if s.selectMembershipInRoomByLocalpartStmt, err = db.Prepare(selectMembershipInRoomByLocalpartSQL); err != nil { - return - } - if s.selectMembershipsByLocalpartStmt, err = db.Prepare(selectMembershipsByLocalpartSQL); err != nil { - return - } - if s.selectRoomIDsByLocalPartStmt, err = db.Prepare(selectRoomIDsByLocalPartSQL); err != nil { - return - } - return -} - -func (s *membershipStatements) insertMembership( - ctx context.Context, txn *sql.Tx, localpart, roomID, eventID string, -) (err error) { - stmt := txn.Stmt(s.insertMembershipStmt) - _, err = stmt.ExecContext(ctx, localpart, roomID, eventID) - return -} - -func (s *membershipStatements) deleteMembershipsByEventIDs( - ctx context.Context, txn *sql.Tx, eventIDs []string, -) (err error) { - stmt := txn.Stmt(s.deleteMembershipsByEventIDsStmt) - _, err = stmt.ExecContext(ctx, pq.StringArray(eventIDs)) - return -} - -func (s *membershipStatements) selectMembershipInRoomByLocalpart( - ctx context.Context, localpart, roomID string, -) (authtypes.Membership, error) { - membership := authtypes.Membership{Localpart: localpart, RoomID: roomID} - stmt := s.selectMembershipInRoomByLocalpartStmt - err := stmt.QueryRowContext(ctx, localpart, roomID).Scan(&membership.EventID) - - return membership, err -} - -func (s *membershipStatements) selectMembershipsByLocalpart( - ctx context.Context, localpart string, -) (memberships []authtypes.Membership, err error) { - stmt := s.selectMembershipsByLocalpartStmt - rows, err := stmt.QueryContext(ctx, localpart) - if err != nil { - return - } - - memberships = []authtypes.Membership{} - - defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsByLocalpart: rows.close() failed") - for rows.Next() { - var m authtypes.Membership - m.Localpart = localpart - if err = rows.Scan(&m.RoomID, &m.EventID); err != nil { - return - } - memberships = append(memberships, m) - } - return memberships, rows.Err() -} - -func (s *membershipStatements) selectRoomIDsByLocalPart( - ctx context.Context, localPart string, -) ([]string, error) { - stmt := s.selectRoomIDsByLocalPartStmt - rows, err := stmt.QueryContext(ctx, localPart) - if err != nil { - return nil, err - } - roomIDs := []string{} - defer rows.Close() // nolint: errcheck - for rows.Next() { - var roomID string - if err = rows.Scan(&roomID); err != nil { - return nil, err - } - roomIDs = append(roomIDs, roomID) - } - return roomIDs, rows.Err() -} diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go index e55099800..c76b92f10 100644 --- a/userapi/storage/accounts/postgres/storage.go +++ b/userapi/storage/accounts/postgres/storage.go @@ -37,10 +37,8 @@ type Database struct { sqlutil.PartitionOffsetStatements accounts accountsStatements profiles profilesStatements - memberships membershipStatements accountDatas accountDataStatements threepids threepidStatements - filter filterStatements serverName gomatrixserverlib.ServerName } @@ -63,10 +61,6 @@ func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serve if err = p.prepare(db); err != nil { return nil, err } - m := membershipStatements{} - if err = m.prepare(db); err != nil { - return nil, err - } ac := accountDataStatements{} if err = ac.prepare(db); err != nil { return nil, err @@ -75,11 +69,7 @@ func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serve if err = t.prepare(db); err != nil { return nil, err } - f := filterStatements{} - if err = f.prepare(db); err != nil { - return nil, err - } - return &Database{db, partitions, a, p, m, ac, t, f, serverName}, nil + return &Database{db, partitions, a, p, ac, t, serverName}, nil } // GetAccountByPassword returns the account associated with the given localpart and password. @@ -184,112 +174,6 @@ func (d *Database) createAccount( return d.accounts.insertAccount(ctx, txn, localpart, hash, appserviceID) } -// SaveMembership saves the user matching a given localpart as a member of a given -// room. It also stores the ID of the membership event. -// If a membership already exists between the user and the room, or if the -// insert fails, returns the SQL error -func (d *Database) saveMembership( - ctx context.Context, txn *sql.Tx, localpart, roomID, eventID string, -) error { - return d.memberships.insertMembership(ctx, txn, localpart, roomID, eventID) -} - -// removeMembershipsByEventIDs removes the memberships corresponding to the -// `join` membership events IDs in the eventIDs slice. -// If the removal fails, or if there is no membership to remove, returns an error -func (d *Database) removeMembershipsByEventIDs( - ctx context.Context, txn *sql.Tx, eventIDs []string, -) error { - return d.memberships.deleteMembershipsByEventIDs(ctx, txn, eventIDs) -} - -// UpdateMemberships adds the "join" membership events included in a given state -// events array, and removes those which ID is included in a given array of events -// IDs. All of the process is run in a transaction, which commits only once/if every -// insertion and deletion has been successfully processed. -// Returns a SQL error if there was an issue with any part of the process -func (d *Database) UpdateMemberships( - ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string, -) error { - return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { - if err := d.removeMembershipsByEventIDs(ctx, txn, idsToRemove); err != nil { - return err - } - - for _, event := range eventsToAdd { - if err := d.newMembership(ctx, txn, event); err != nil { - return err - } - } - - return nil - }) -} - -// GetMembershipInRoomByLocalpart returns the membership for an user -// matching the given localpart if he is a member of the room matching roomID, -// if not sql.ErrNoRows is returned. -// If there was an issue during the retrieval, returns the SQL error -func (d *Database) GetMembershipInRoomByLocalpart( - ctx context.Context, localpart, roomID string, -) (authtypes.Membership, error) { - return d.memberships.selectMembershipInRoomByLocalpart(ctx, localpart, roomID) -} - -// GetRoomIDsByLocalPart returns an array containing the room ids of all -// the rooms a user matching a given localpart is a member of -// If no membership match the given localpart, returns an empty array -// If there was an issue during the retrieval, returns the SQL error -func (d *Database) GetRoomIDsByLocalPart( - ctx context.Context, localpart string, -) ([]string, error) { - return d.memberships.selectRoomIDsByLocalPart(ctx, localpart) -} - -// GetMembershipsByLocalpart returns an array containing the memberships for all -// the rooms a user matching a given localpart is a member of -// If no membership match the given localpart, returns an empty array -// If there was an issue during the retrieval, returns the SQL error -func (d *Database) GetMembershipsByLocalpart( - ctx context.Context, localpart string, -) (memberships []authtypes.Membership, err error) { - return d.memberships.selectMembershipsByLocalpart(ctx, localpart) -} - -// newMembership saves a new membership in the database. -// If the event isn't a valid m.room.member event with type `join`, does nothing. -// If an error occurred, returns the SQL error -func (d *Database) newMembership( - ctx context.Context, txn *sql.Tx, ev gomatrixserverlib.Event, -) error { - if ev.Type() == "m.room.member" && ev.StateKey() != nil { - localpart, serverName, err := gomatrixserverlib.SplitID('@', *ev.StateKey()) - if err != nil { - return err - } - - // We only want state events from local users - if string(serverName) != string(d.serverName) { - return nil - } - - eventID := ev.EventID() - roomID := ev.RoomID() - membership, err := ev.Membership() - if err != nil { - return err - } - - // Only "join" membership events can be considered as new memberships - if membership == gomatrixserverlib.Join { - if err := d.saveMembership(ctx, txn, localpart, roomID, eventID); err != nil { - return err - } - } - } - return nil -} - // SaveAccountData saves new account data for a given user and a given room. // If the account data is not specific to a room, the room ID should be an empty string // If an account data already exists for a given set (user, room, data type), it will @@ -396,24 +280,6 @@ func (d *Database) GetThreePIDsForLocalpart( return d.threepids.selectThreePIDsForLocalpart(ctx, localpart) } -// GetFilter looks up the filter associated with a given local user and filter ID. -// Returns a filter structure. Otherwise returns an error if no such filter exists -// or if there was an error talking to the database. -func (d *Database) GetFilter( - ctx context.Context, localpart string, filterID string, -) (*gomatrixserverlib.Filter, error) { - return d.filter.selectFilter(ctx, localpart, filterID) -} - -// PutFilter puts the passed filter into the database. -// Returns the filterID as a string. Otherwise returns an error if something -// goes wrong. -func (d *Database) PutFilter( - ctx context.Context, localpart string, filter *gomatrixserverlib.Filter, -) (string, error) { - return d.filter.insertFilter(ctx, filter, localpart) -} - // CheckAccountAvailability checks if the username/localpart is already present // in the database. // If the DB returns sql.ErrNoRows the Localpart isn't taken. diff --git a/userapi/storage/accounts/sqlite3/membership_table.go b/userapi/storage/accounts/sqlite3/membership_table.go deleted file mode 100644 index 67958f27d..000000000 --- a/userapi/storage/accounts/sqlite3/membership_table.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "context" - "database/sql" - "strings" - - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/sqlutil" -) - -const membershipSchema = ` --- Stores data about users memberships to rooms. -CREATE TABLE IF NOT EXISTS account_memberships ( - -- The Matrix user ID localpart for the member - localpart TEXT NOT NULL, - -- The room this user is a member of - room_id TEXT NOT NULL, - -- The ID of the join membership event - event_id TEXT NOT NULL, - - -- A user can only be member of a room once - PRIMARY KEY (localpart, room_id), - - UNIQUE (event_id) -); -` - -const insertMembershipSQL = ` - INSERT INTO account_memberships(localpart, room_id, event_id) VALUES ($1, $2, $3) - ON CONFLICT (localpart, room_id) DO UPDATE SET event_id = EXCLUDED.event_id -` - -const selectMembershipsByLocalpartSQL = "" + - "SELECT room_id, event_id FROM account_memberships WHERE localpart = $1" - -const selectMembershipInRoomByLocalpartSQL = "" + - "SELECT event_id FROM account_memberships WHERE localpart = $1 AND room_id = $2" - -const selectRoomIDsByLocalPartSQL = "" + - "SELECT room_id FROM account_memberships WHERE localpart = $1" - -const deleteMembershipsByEventIDsSQL = "" + - "DELETE FROM account_memberships WHERE event_id IN ($1)" - -type membershipStatements struct { - insertMembershipStmt *sql.Stmt - selectMembershipInRoomByLocalpartStmt *sql.Stmt - selectMembershipsByLocalpartStmt *sql.Stmt - selectRoomIDsByLocalPartStmt *sql.Stmt -} - -func (s *membershipStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(membershipSchema) - if err != nil { - return - } - if s.insertMembershipStmt, err = db.Prepare(insertMembershipSQL); err != nil { - return - } - if s.selectMembershipInRoomByLocalpartStmt, err = db.Prepare(selectMembershipInRoomByLocalpartSQL); err != nil { - return - } - if s.selectMembershipsByLocalpartStmt, err = db.Prepare(selectMembershipsByLocalpartSQL); err != nil { - return - } - if s.selectRoomIDsByLocalPartStmt, err = db.Prepare(selectRoomIDsByLocalPartSQL); err != nil { - return - } - return -} - -func (s *membershipStatements) insertMembership( - ctx context.Context, txn *sql.Tx, localpart, roomID, eventID string, -) (err error) { - stmt := txn.Stmt(s.insertMembershipStmt) - _, err = stmt.ExecContext(ctx, localpart, roomID, eventID) - return -} - -func (s *membershipStatements) deleteMembershipsByEventIDs( - ctx context.Context, txn *sql.Tx, eventIDs []string, -) (err error) { - sqlStr := strings.Replace(deleteMembershipsByEventIDsSQL, "($1)", sqlutil.QueryVariadic(len(eventIDs)), 1) - iEventIDs := make([]interface{}, len(eventIDs)) - for i, e := range eventIDs { - iEventIDs[i] = e - } - _, err = txn.ExecContext(ctx, sqlStr, iEventIDs...) - return -} - -func (s *membershipStatements) selectMembershipInRoomByLocalpart( - ctx context.Context, localpart, roomID string, -) (authtypes.Membership, error) { - membership := authtypes.Membership{Localpart: localpart, RoomID: roomID} - stmt := s.selectMembershipInRoomByLocalpartStmt - err := stmt.QueryRowContext(ctx, localpart, roomID).Scan(&membership.EventID) - - return membership, err -} - -func (s *membershipStatements) selectMembershipsByLocalpart( - ctx context.Context, localpart string, -) (memberships []authtypes.Membership, err error) { - stmt := s.selectMembershipsByLocalpartStmt - rows, err := stmt.QueryContext(ctx, localpart) - if err != nil { - return - } - - memberships = []authtypes.Membership{} - - defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsByLocalpart: rows.close() failed") - for rows.Next() { - var m authtypes.Membership - m.Localpart = localpart - if err := rows.Scan(&m.RoomID, &m.EventID); err != nil { - return nil, err - } - memberships = append(memberships, m) - } - - return -} -func (s *membershipStatements) selectRoomIDsByLocalPart( - ctx context.Context, localPart string, -) ([]string, error) { - stmt := s.selectRoomIDsByLocalPartStmt - rows, err := stmt.QueryContext(ctx, localPart) - if err != nil { - return nil, err - } - roomIDs := []string{} - defer rows.Close() // nolint: errcheck - for rows.Next() { - var roomID string - if err = rows.Scan(&roomID); err != nil { - return nil, err - } - roomIDs = append(roomIDs, roomID) - } - return roomIDs, rows.Err() -} diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go index dbf6606c3..72b27c8bf 100644 --- a/userapi/storage/accounts/sqlite3/storage.go +++ b/userapi/storage/accounts/sqlite3/storage.go @@ -36,13 +36,11 @@ type Database struct { sqlutil.PartitionOffsetStatements accounts accountsStatements profiles profilesStatements - memberships membershipStatements accountDatas accountDataStatements threepids threepidStatements - filter filterStatements serverName gomatrixserverlib.ServerName - createGuestAccountMu sync.Mutex + createAccountMu sync.Mutex } // NewDatabase creates a new accounts and profiles database @@ -68,10 +66,6 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) if err = p.prepare(db); err != nil { return nil, err } - m := membershipStatements{} - if err = m.prepare(db); err != nil { - return nil, err - } ac := accountDataStatements{} if err = ac.prepare(db); err != nil { return nil, err @@ -80,11 +74,7 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) if err = t.prepare(db); err != nil { return nil, err } - f := filterStatements{} - if err = f.prepare(db); err != nil { - return nil, err - } - return &Database{db, partitions, a, p, m, ac, t, f, serverName, sync.Mutex{}}, nil + return &Database{db, partitions, a, p, ac, t, serverName, sync.Mutex{}}, nil } // GetAccountByPassword returns the account associated with the given localpart and password. @@ -129,14 +119,14 @@ func (d *Database) SetDisplayName( // CreateGuestAccount makes a new guest account and creates an empty profile // for this account. func (d *Database) CreateGuestAccount(ctx context.Context) (acc *api.Account, err error) { + // We need to lock so we sequentially create numeric localparts. If we don't, two calls to + // this function will cause the same number to be selected and one will fail with 'database is locked' + // when the first txn upgrades to a write txn. We also need to lock the account creation else we can + // race with CreateAccount + // We know we'll be the only process since this is sqlite ;) so a lock here will be all that is needed. + d.createAccountMu.Lock() + defer d.createAccountMu.Unlock() err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { - // We need to lock so we sequentially create numeric localparts. If we don't, two calls to - // this function will cause the same number to be selected and one will fail with 'database is locked' - // when the first txn upgrades to a write txn. - // We know we'll be the only process since this is sqlite ;) so a lock here will be all that is needed. - d.createGuestAccountMu.Lock() - defer d.createGuestAccountMu.Unlock() - var numLocalpart int64 numLocalpart, err = d.accounts.selectNewNumericLocalpart(ctx, txn) if err != nil { @@ -155,6 +145,9 @@ func (d *Database) CreateGuestAccount(ctx context.Context) (acc *api.Account, er func (d *Database) CreateAccount( ctx context.Context, localpart, plaintextPassword, appserviceID string, ) (acc *api.Account, err error) { + // Create one account at a time else we can get 'database is locked'. + d.createAccountMu.Lock() + defer d.createAccountMu.Unlock() err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { acc, err = d.createAccount(ctx, txn, localpart, plaintextPassword, appserviceID) return err @@ -195,112 +188,6 @@ func (d *Database) createAccount( return d.accounts.insertAccount(ctx, txn, localpart, hash, appserviceID) } -// SaveMembership saves the user matching a given localpart as a member of a given -// room. It also stores the ID of the membership event. -// If a membership already exists between the user and the room, or if the -// insert fails, returns the SQL error -func (d *Database) saveMembership( - ctx context.Context, txn *sql.Tx, localpart, roomID, eventID string, -) error { - return d.memberships.insertMembership(ctx, txn, localpart, roomID, eventID) -} - -// removeMembershipsByEventIDs removes the memberships corresponding to the -// `join` membership events IDs in the eventIDs slice. -// If the removal fails, or if there is no membership to remove, returns an error -func (d *Database) removeMembershipsByEventIDs( - ctx context.Context, txn *sql.Tx, eventIDs []string, -) error { - return d.memberships.deleteMembershipsByEventIDs(ctx, txn, eventIDs) -} - -// UpdateMemberships adds the "join" membership events included in a given state -// events array, and removes those which ID is included in a given array of events -// IDs. All of the process is run in a transaction, which commits only once/if every -// insertion and deletion has been successfully processed. -// Returns a SQL error if there was an issue with any part of the process -func (d *Database) UpdateMemberships( - ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string, -) error { - return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { - if err := d.removeMembershipsByEventIDs(ctx, txn, idsToRemove); err != nil { - return err - } - - for _, event := range eventsToAdd { - if err := d.newMembership(ctx, txn, event); err != nil { - return err - } - } - - return nil - }) -} - -// GetMembershipInRoomByLocalpart returns the membership for an user -// matching the given localpart if he is a member of the room matching roomID, -// if not sql.ErrNoRows is returned. -// If there was an issue during the retrieval, returns the SQL error -func (d *Database) GetMembershipInRoomByLocalpart( - ctx context.Context, localpart, roomID string, -) (authtypes.Membership, error) { - return d.memberships.selectMembershipInRoomByLocalpart(ctx, localpart, roomID) -} - -// GetMembershipsByLocalpart returns an array containing the memberships for all -// the rooms a user matching a given localpart is a member of -// If no membership match the given localpart, returns an empty array -// If there was an issue during the retrieval, returns the SQL error -func (d *Database) GetMembershipsByLocalpart( - ctx context.Context, localpart string, -) (memberships []authtypes.Membership, err error) { - return d.memberships.selectMembershipsByLocalpart(ctx, localpart) -} - -// GetRoomIDsByLocalPart returns an array containing the room ids of all -// the rooms a user matching a given localpart is a member of -// If no membership match the given localpart, returns an empty array -// If there was an issue during the retrieval, returns the SQL error -func (d *Database) GetRoomIDsByLocalPart( - ctx context.Context, localpart string, -) ([]string, error) { - return d.memberships.selectRoomIDsByLocalPart(ctx, localpart) -} - -// newMembership saves a new membership in the database. -// If the event isn't a valid m.room.member event with type `join`, does nothing. -// If an error occurred, returns the SQL error -func (d *Database) newMembership( - ctx context.Context, txn *sql.Tx, ev gomatrixserverlib.Event, -) error { - if ev.Type() == "m.room.member" && ev.StateKey() != nil { - localpart, serverName, err := gomatrixserverlib.SplitID('@', *ev.StateKey()) - if err != nil { - return err - } - - // We only want state events from local users - if string(serverName) != string(d.serverName) { - return nil - } - - eventID := ev.EventID() - roomID := ev.RoomID() - membership, err := ev.Membership() - if err != nil { - return err - } - - // Only "join" membership events can be considered as new memberships - if membership == gomatrixserverlib.Join { - if err := d.saveMembership(ctx, txn, localpart, roomID, eventID); err != nil { - return err - } - } - } - return nil -} - // SaveAccountData saves new account data for a given user and a given room. // If the account data is not specific to a room, the room ID should be an empty string // If an account data already exists for a given set (user, room, data type), it will @@ -407,24 +294,6 @@ func (d *Database) GetThreePIDsForLocalpart( return d.threepids.selectThreePIDsForLocalpart(ctx, localpart) } -// GetFilter looks up the filter associated with a given local user and filter ID. -// Returns a filter structure. Otherwise returns an error if no such filter exists -// or if there was an error talking to the database. -func (d *Database) GetFilter( - ctx context.Context, localpart string, filterID string, -) (*gomatrixserverlib.Filter, error) { - return d.filter.selectFilter(ctx, localpart, filterID) -} - -// PutFilter puts the passed filter into the database. -// Returns the filterID as a string. Otherwise returns an error if something -// goes wrong. -func (d *Database) PutFilter( - ctx context.Context, localpart string, filter *gomatrixserverlib.Filter, -) (string, error) { - return d.filter.insertFilter(ctx, filter, localpart) -} - // CheckAccountAvailability checks if the username/localpart is already present // in the database. // If the DB returns sql.ErrNoRows the Localpart isn't taken.