mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2025-12-21 05:43:09 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into tmp-file-delete

Browse source
This commit is contained in:
Neil Alexander 2020-07-03 10:29:08 +01:00 committed by GitHub
parent 27d21c9ee4 9dffeb9b67
commit a6721427b3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
148 changed files with 5027 additions and 3521 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
are-we-synapse-yet.list
View file

@ -91,6 +91,7 @@ snd PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id
get GET /rooms/:room_id/messages returns a message
get GET /rooms/:room_id/messages lazy loads members correctly
typ PUT /rooms/:room_id/typing/:user_id sets typing notification
typ Typing notifications don't leak (3 subtests)
rst GET /rooms/:room_id/state/m.room.power_levels can fetch levels
rst PUT /rooms/:room_id/state/m.room.power_levels can set levels
rst PUT power_levels should not explode if the old power levels were empty
@ -854,3 +855,7 @@ fbk Outbound federation rejects backfill containing invalid JSON for events in r
jso Invalid JSON integers
jso Invalid JSON floats
jso Invalid JSON special values
inv Can invite users to invite-only rooms (2 subtests)
plv setting 'm.room.name' respects room powerlevel (2 subtests)
psh Messages that notify from another user increment notification_count
psh Messages that org.matrix.msc2625.mark_unread from another user increment org.matrix.msc2625.unread_count

2
are-we-synapse-yet.py
View file

@ -159,6 +159,8 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
total_tests = 0
for gid, tests in gid_to_tests.items():
group_total = len(tests)
if group_total == 0:
continue
group_passing = 0
test_names_and_marks = []
for name, passing in tests.items():

4
build/docker/config/dendrite-config.yaml
View file

@ -98,7 +98,7 @@ database:
room_server: "postgres://dendrite:itsasecret@postgres/dendrite_roomserver?sslmode=disable"
server_key: "postgres://dendrite:itsasecret@postgres/dendrite_serverkey?sslmode=disable"
federation_sender: "postgres://dendrite:itsasecret@postgres/dendrite_federationsender?sslmode=disable"
public_rooms_api: "postgres://dendrite:itsasecret@postgres/dendrite_publicroomsapi?sslmode=disable"
current_state: "postgres://dendrite:itsasecret@postgres/dendrite_currentstate?sslmode=disable"
appservice: "postgres://dendrite:itsasecret@postgres/dendrite_appservice?sslmode=disable"
# If using naffka you need to specify a naffka database
#naffka: "postgres://dendrite:itsasecret@postgres/dendrite_naffka?sslmode=disable"
@ -113,7 +113,7 @@ listen:
server_key_api: "server_key_api:7778"
sync_api: "sync_api:7773"
media_api: "media_api:7774"
public_rooms_api: "public_rooms_api:7775"
current_state_server: "current_state_server:7775"
federation_sender: "federation_sender:7776"
edu_server: "edu_server:7777"
key_server: "key_server:7779"

10
build/docker/docker-compose.polylith.yml
View file

@ -7,8 +7,7 @@ services:
"--bind-address=:8008",
"--client-api-server-url=http://client_api:7771",
"--sync-api-server-url=http://sync_api:7773",
"--media-api-server-url=http://media_api:7774",
"--public-rooms-api-server-url=http://public_rooms_api:7775"
"--media-api-server-url=http://media_api:7774"
]
volumes:
- ./config:/etc/dendrite
@ -18,7 +17,6 @@ services:
- sync_api
- client_api
- media_api
- public_rooms_api
ports:
- "8008:8008"
@ -45,9 +43,9 @@ services:
networks:
- internal
public_rooms_api:
hostname: public_rooms_api
image: matrixdotorg/dendrite:publicroomsapi
current_state_server:
hostname: current_state_server
image: matrixdotorg/dendrite:currentstateserver
command: [
"--config=dendrite.yaml"
]

2
build/docker/images-build.sh
View file

@ -15,7 +15,7 @@ docker build -t matrixdotorg/dendrite:federationsender --build-arg component=de
docker build -t matrixdotorg/dendrite:federationproxy --build-arg component=federation-api-proxy -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:keyserver --build-arg component=dendrite-key-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:publicroomsapi --build-arg component=dendrite-public-rooms-api-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:currentstateserver --build-arg component=dendrite-current-state-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:serverkeyapi --build-arg component=dendrite-server-key-api-server -f build/docker/Dockerfile.component .

2
build/docker/images-pull.sh
View file

@ -11,7 +11,7 @@ docker pull matrixdotorg/dendrite:federationsender
docker pull matrixdotorg/dendrite:federationproxy
docker pull matrixdotorg/dendrite:keyserver
docker pull matrixdotorg/dendrite:mediaapi
docker pull matrixdotorg/dendrite:publicroomsapi
docker pull matrixdotorg/dendrite:currentstateserver
docker pull matrixdotorg/dendrite:roomserver
docker pull matrixdotorg/dendrite:syncapi
docker pull matrixdotorg/dendrite:userapi

2
build/docker/images-push.sh
View file

@ -11,7 +11,7 @@ docker push matrixdotorg/dendrite:federationsender
docker push matrixdotorg/dendrite:federationproxy
docker push matrixdotorg/dendrite:keyserver
docker push matrixdotorg/dendrite:mediaapi
docker push matrixdotorg/dendrite:publicroomsapi
docker push matrixdotorg/dendrite:currentstateserver
docker push matrixdotorg/dendrite:roomserver
docker push matrixdotorg/dendrite:syncapi
docker push matrixdotorg/dendrite:serverkeyapi

2
build/docker/postgres/create_db.sh
View file

@ -1,5 +1,5 @@
#!/bin/bash
for db in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi appservice naffka; do
for db in account device mediaapi syncapi roomserver serverkey federationsender currentstate appservice naffka; do
createdb -U dendrite -O dendrite dendrite_$db
done

2
build/gobind/build.sh
View file

@ -1,6 +1,6 @@
#!/bin/sh
gomobile bind -v \
-ldflags "-X $github.com/yggdrasil-network/yggdrasil-go/src/version.buildName=riot-ios-p2p" \
-ldflags "-X github.com/yggdrasil-network/yggdrasil-go/src/version.buildName=dendrite" \
-target ios \
github.com/matrix-org/dendrite/build/gobind

18
build/gobind/monolith.go
View file

@ -11,13 +11,13 @@ import (
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn"
"github.com/matrix-org/dendrite/currentstateserver"
"github.com/matrix-org/dendrite/eduserver"
"github.com/matrix-org/dendrite/eduserver/cache"
"github.com/matrix-org/dendrite/federationsender"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/userapi"
"github.com/matrix-org/gomatrixserverlib"
@ -25,6 +25,7 @@ import (
)
type DendriteMonolith struct {
YggdrasilNode *yggconn.Node
StorageDirectory string
listener net.Listener
}
@ -33,6 +34,10 @@ func (m *DendriteMonolith) BaseURL() string {
return fmt.Sprintf("http://%s", m.listener.Addr().String())
}
func (m *DendriteMonolith) PeerCount() int {
return m.YggdrasilNode.PeerCount()
}
func (m *DendriteMonolith) Start() {
logger := logrus.Logger{
Out: BindLogger{},
@ -49,6 +54,7 @@ func (m *DendriteMonolith) Start() {
if err != nil {
panic(err)
}
m.YggdrasilNode = ygg
cfg := &config.Dendrite{}
cfg.SetDefaults()
@ -68,7 +74,7 @@ func (m *DendriteMonolith) Start() {
cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s/dendrite-serverkey.db", m.StorageDirectory))
cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s/dendrite-federationsender.db", m.StorageDirectory))
cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s/dendrite-appservice.db", m.StorageDirectory))
cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-publicroomsa.db", m.StorageDirectory))
cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s/dendrite-currentstate.db", m.StorageDirectory))
cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s/dendrite-naffka.db", m.StorageDirectory))
if err = cfg.Derive(); err != nil {
panic(err)
@ -103,10 +109,7 @@ func (m *DendriteMonolith) Start() {
// This is different to rsAPI which can be the http client which doesn't need this dependency
rsAPI.SetFederationSenderAPI(fsAPI)
publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to public rooms db")
}
stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
monolith := setup.Monolith{
Config: base.Cfg,
@ -123,9 +126,8 @@ func (m *DendriteMonolith) Start() {
FederationSenderAPI: fsAPI,
RoomserverAPI: rsAPI,
UserAPI: userAPI,
StateAPI: stateAPI,
//ServerKeyAPI: serverKeyAPI,
PublicRoomsDB: publicRoomsDB,
}
monolith.AddAllPublicRoutes(base.PublicAPIMux)

6
publicroomsapi/types/types.go → clientapi/api/api.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,10 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package types
package api
// ExternalPublicRoomsProvider provides a list of homeservers who should be queried
// periodically for a list of public rooms on their server.
type ExternalPublicRoomsProvider interface {
// The list of homeserver domains to query. These servers will receive a request
// via this API: https://matrix.org/docs/spec/server_server/latest#public-room-directory

16
clientapi/clientapi.go
View file

@ -18,9 +18,10 @@ import (
"github.com/Shopify/sarama"
"github.com/gorilla/mux"
appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/clientapi/consumers"
"github.com/matrix-org/dendrite/clientapi/api"
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/dendrite/clientapi/routing"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
@ -30,14 +31,12 @@ import (
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/dendrite/userapi/storage/devices"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
)
// AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component.
func AddPublicRoutes(
router *mux.Router,
cfg *config.Dendrite,
consumer sarama.Consumer,
producer sarama.SyncProducer,
deviceDB devices.Database,
accountsDB accounts.Database,
@ -45,25 +44,20 @@ func AddPublicRoutes(
rsAPI roomserverAPI.RoomserverInternalAPI,
eduInputAPI eduServerAPI.EDUServerInputAPI,
asAPI appserviceAPI.AppServiceQueryAPI,
stateAPI currentstateAPI.CurrentStateInternalAPI,
transactionsCache *transactions.Cache,
fsAPI federationSenderAPI.FederationSenderInternalAPI,
userAPI userapi.UserInternalAPI,
extRoomsProvider api.ExternalPublicRoomsProvider,
) {
syncProducer := &producers.SyncAPIProducer{
Producer: producer,
Topic: string(cfg.Kafka.Topics.OutputClientData),
}
roomEventConsumer := consumers.NewOutputRoomEventConsumer(
cfg, consumer, accountsDB, rsAPI,
)
if err := roomEventConsumer.Start(); err != nil {
logrus.WithError(err).Panicf("failed to start room server consumer")
}
routing.Setup(
router, cfg, eduInputAPI, rsAPI, asAPI,
accountsDB, deviceDB, userAPI, federation,
syncProducer, transactionsCache, fsAPI,
syncProducer, transactionsCache, fsAPI, stateAPI, extRoomsProvider,
)
}

92
clientapi/consumers/roomserver.go
View file

@ -1,92 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package consumers
import (
"context"
"encoding/json"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/gomatrixserverlib"
"github.com/Shopify/sarama"
log "github.com/sirupsen/logrus"
)
// OutputRoomEventConsumer consumes events that originated in the room server.
type OutputRoomEventConsumer struct {
rsAPI api.RoomserverInternalAPI
rsConsumer *internal.ContinualConsumer
db accounts.Database
serverName string
}
// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
func NewOutputRoomEventConsumer(
cfg *config.Dendrite,
kafkaConsumer sarama.Consumer,
store accounts.Database,
rsAPI api.RoomserverInternalAPI,
) *OutputRoomEventConsumer {
consumer := internal.ContinualConsumer{
Topic: string(cfg.Kafka.Topics.OutputRoomEvent),
Consumer: kafkaConsumer,
PartitionStore: store,
}
s := &OutputRoomEventConsumer{
rsConsumer: &consumer,
db: store,
rsAPI: rsAPI,
serverName: string(cfg.Matrix.ServerName),
}
consumer.ProcessMessage = s.onMessage
return s
}
// Start consuming from room servers
func (s *OutputRoomEventConsumer) Start() error {
return s.rsConsumer.Start()
}
// onMessage is called when the sync server receives a new event from the room server output log.
// It is not safe for this function to be called from multiple goroutines, or else the
// sync stream position may race and be incorrectly calculated.
func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
// Parse out the event JSON
var output api.OutputEvent
if err := json.Unmarshal(msg.Value, &output); err != nil {
// If the message was invalid, log it and move on to the next message in the stream
log.WithError(err).Errorf("roomserver output log: message parse failure")
return nil
}
if output.Type != api.OutputTypeNewRoomEvent {
log.WithField("type", output.Type).Debug(
"roomserver output log: ignoring unknown output type",
)
return nil
}
return s.db.UpdateMemberships(
context.TODO(),
gomatrixserverlib.UnwrapEventHeaders(output.NewRoomEvent.AddsState()),
output.NewRoomEvent.RemovesStateEventIDs,
)
}

14
clientapi/jsonerror/jsonerror.go
View file

@ -125,10 +125,20 @@ func GuestAccessForbidden(msg string) *MatrixError {
return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg}
}
type IncompatibleRoomVersionError struct {
RoomVersion string `json:"room_version"`
Error string `json:"error"`
Code string `json:"errcode"`
}
// IncompatibleRoomVersion is an error which is returned when the client
// requests a room with a version that is unsupported.
func IncompatibleRoomVersion(roomVersion gomatrixserverlib.RoomVersion) *MatrixError {
return &MatrixError{"M_INCOMPATIBLE_ROOM_VERSION", string(roomVersion)}
func IncompatibleRoomVersion(roomVersion gomatrixserverlib.RoomVersion) *IncompatibleRoomVersionError {
return &IncompatibleRoomVersionError{
Code: "M_INCOMPATIBLE_ROOM_VERSION",
RoomVersion: string(roomVersion),
Error: "Your homeserver does not support the features required to join this room",
}
}
// UnsupportedRoomVersion is an error which is returned when the client

20
clientapi/routing/createroom.go
View file

@ -28,7 +28,6 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/threepid"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
@ -373,13 +372,9 @@ func createRoom(
// If this is a direct message then we should invite the participants.
for _, invitee := range r.Invite {
// Build the membership request.
body := threepid.MembershipRequest{
UserID: invitee,
}
// Build the invite event.
inviteEvent, err := buildMembershipEvent(
req.Context(), body, accountDB, device, gomatrixserverlib.Invite,
req.Context(), invitee, "", accountDB, device, gomatrixserverlib.Invite,
roomID, true, cfg, evTime, rsAPI, asAPI,
)
if err != nil {
@ -415,6 +410,19 @@ func createRoom(
}
}
if r.Visibility == "public" {
// expose this room in the published room list
var pubRes roomserverAPI.PerformPublishResponse
rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
RoomID: roomID,
Visibility: "public",
}, &pubRes)
if pubRes.Error != nil {
// treat as non-fatal since the room is already made by this point
util.GetLogger(req.Context()).WithError(pubRes.Error).Error("failed to visibility:public")
}
}
response := createRoomResponse{
RoomID: roomID,
RoomAlias: roomAlias,

90
clientapi/routing/directory.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -20,10 +20,12 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/userapi/api"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
@ -232,3 +234,89 @@ func RemoveLocalAlias(
JSON: struct{}{},
}
}
type roomVisibility struct {
Visibility string `json:"visibility"`
}
// GetVisibility implements GET /directory/list/room/{roomID}
func GetVisibility(
req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI,
roomID string,
) util.JSONResponse {
var res roomserverAPI.QueryPublishedRoomsResponse
err := rsAPI.QueryPublishedRooms(req.Context(), &roomserverAPI.QueryPublishedRoomsRequest{
RoomID: roomID,
}, &res)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("QueryPublishedRooms failed")
return jsonerror.InternalServerError()
}
var v roomVisibility
if len(res.RoomIDs) == 1 {
v.Visibility = gomatrixserverlib.Public
} else {
v.Visibility = "private"
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: v,
}
}
// SetVisibility implements PUT /directory/list/room/{roomID}
// TODO: Allow admin users to edit the room visibility
func SetVisibility(
req *http.Request, stateAPI currentstateAPI.CurrentStateInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, dev *userapi.Device,
roomID string,
) util.JSONResponse {
resErr := checkMemberInRoom(req.Context(), stateAPI, dev.UserID, roomID)
if resErr != nil {
return *resErr
}
queryEventsReq := roomserverAPI.QueryLatestEventsAndStateRequest{
RoomID: roomID,
StateToFetch: []gomatrixserverlib.StateKeyTuple{{
EventType: gomatrixserverlib.MRoomPowerLevels,
StateKey: "",
}},
}
var queryEventsRes roomserverAPI.QueryLatestEventsAndStateResponse
err := rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes)
if err != nil || len(queryEventsRes.StateEvents) == 0 {
util.GetLogger(req.Context()).WithError(err).Error("could not query events from room")
return jsonerror.InternalServerError()
}
// NOTSPEC: Check if the user's power is greater than power required to change m.room.aliases event
power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event)
if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomAliases, true) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("userID doesn't have power level to change visibility"),
}
}
var v roomVisibility
if reqErr := httputil.UnmarshalJSONRequest(req, &v); reqErr != nil {
return *reqErr
}
var publishRes roomserverAPI.PerformPublishResponse
rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
RoomID: roomID,
Visibility: v.Visibility,
}, &publishRes)
if publishRes.Error != nil {
util.GetLogger(req.Context()).WithError(publishRes.Error).Error("PerformPublish failed")
return publishRes.Error.JSONResponse()
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
}
}

185
publicroomsapi/directory/public_rooms.go → clientapi/routing/directory_public.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package directory
package routing
import (
"context"
@ -20,13 +20,15 @@ import (
"net/http"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/matrix-org/dendrite/clientapi/api"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/matrix-org/dendrite/publicroomsapi/types"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
@ -43,13 +45,13 @@ type filter struct {
// GetPostPublicRooms implements GET and POST /publicRooms
func GetPostPublicRooms(
req *http.Request, publicRoomDatabase storage.Database,
req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI,
) util.JSONResponse {
var request PublicRoomReq
if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil {
return *fillErr
}
response, err := publicRooms(req.Context(), request, publicRoomDatabase)
response, err := publicRooms(req.Context(), request, rsAPI, stateAPI)
if err != nil {
return jsonerror.InternalServerError()
}
@ -61,14 +63,14 @@ func GetPostPublicRooms(
// GetPostPublicRoomsWithExternal is the same as GetPostPublicRooms but also mixes in public rooms from the provider supplied.
func GetPostPublicRoomsWithExternal(
req *http.Request, publicRoomDatabase storage.Database, fedClient *gomatrixserverlib.FederationClient,
extRoomsProvider types.ExternalPublicRoomsProvider,
req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI,
fedClient *gomatrixserverlib.FederationClient, extRoomsProvider api.ExternalPublicRoomsProvider,
) util.JSONResponse {
var request PublicRoomReq
if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil {
return *fillErr
}
response, err := publicRooms(req.Context(), request, publicRoomDatabase)
response, err := publicRooms(req.Context(), request, rsAPI, stateAPI)
if err != nil {
return jsonerror.InternalServerError()
}
@ -197,11 +199,16 @@ FanIn:
return publicRooms
}
func publicRooms(ctx context.Context, request PublicRoomReq, publicRoomDatabase storage.Database) (*gomatrixserverlib.RespPublicRooms, error) {
func publicRooms(ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.RoomserverInternalAPI,
stateAPI currentstateAPI.CurrentStateInternalAPI) (*gomatrixserverlib.RespPublicRooms, error) {
var response gomatrixserverlib.RespPublicRooms
var limit int16
var offset int64
limit = request.Limit
if limit == 0 {
limit = 50
}
offset, err := strconv.ParseInt(request.Since, 10, 64)
// ParseInt returns 0 and an error when trying to parse an empty string
// In that case, we want to assign 0 so we ignore the error
@ -210,53 +217,157 @@ func publicRooms(ctx context.Context, request PublicRoomReq, publicRoomDatabase
return nil, err
}
est, err := publicRoomDatabase.CountPublicRooms(ctx)
var queryRes roomserverAPI.QueryPublishedRoomsResponse
err = rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("publicRoomDatabase.CountPublicRooms failed")
util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed")
return nil, err
}
response.TotalRoomCountEstimate = int(est)
response.TotalRoomCountEstimate = len(queryRes.RoomIDs)
if offset > 0 {
response.PrevBatch = strconv.Itoa(int(offset) - 1)
roomIDs, prev, next := sliceInto(queryRes.RoomIDs, offset, limit)
if prev >= 0 {
response.PrevBatch = "T" + strconv.Itoa(prev)
}
nextIndex := int(offset) + int(limit)
if response.TotalRoomCountEstimate > nextIndex {
response.NextBatch = strconv.Itoa(nextIndex)
if next >= 0 {
response.NextBatch = "T" + strconv.Itoa(next)
}
if response.Chunk, err = publicRoomDatabase.GetPublicRooms(
ctx, offset, limit, request.Filter.SearchTerms,
); err != nil {
util.GetLogger(ctx).WithError(err).Error("publicRoomDatabase.GetPublicRooms failed")
return nil, err
}
return &response, nil
response.Chunk, err = fillInRooms(ctx, roomIDs, stateAPI)
return &response, err
}
// fillPublicRoomsReq fills the Limit, Since and Filter attributes of a GET or POST request
// on /publicRooms by parsing the incoming HTTP request
// Filter is only filled for POST requests
func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSONResponse {
if httpReq.Method == http.MethodGet {
if httpReq.Method != "GET" && httpReq.Method != "POST" {
return &util.JSONResponse{
Code: http.StatusMethodNotAllowed,
JSON: jsonerror.NotFound("Bad method"),
}
}
if httpReq.Method == "GET" {
limit, err := strconv.Atoi(httpReq.FormValue("limit"))
// Atoi returns 0 and an error when trying to parse an empty string
// In that case, we want to assign 0 so we ignore the error
if err != nil && len(httpReq.FormValue("limit")) > 0 {
util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed")
reqErr := jsonerror.InternalServerError()
return &reqErr
return &util.JSONResponse{
Code: 400,
JSON: jsonerror.BadJSON("limit param is not a number"),
}
}
request.Limit = int16(limit)
request.Since = httpReq.FormValue("since")
return nil
} else if httpReq.Method == http.MethodPost {
return httputil.UnmarshalJSONRequest(httpReq, request)
} else {
resErr := httputil.UnmarshalJSONRequest(httpReq, request)
if resErr != nil {
return resErr
}
}
return &util.JSONResponse{
Code: http.StatusMethodNotAllowed,
JSON: jsonerror.NotFound("Bad method"),
}
// strip the 'T' which is only required because when sytest does pagination tests it stops
// iterating when !prev_batch which then fails if prev_batch==0, so add arbitrary text to
// make it truthy not falsey.
request.Since = strings.TrimPrefix(request.Since, "T")
return nil
}
// due to lots of switches
// nolint:gocyclo
func fillInRooms(ctx context.Context, roomIDs []string, stateAPI currentstateAPI.CurrentStateInternalAPI) ([]gomatrixserverlib.PublicRoom, error) {
avatarTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.avatar", StateKey: ""}
nameTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.name", StateKey: ""}
canonicalTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCanonicalAlias, StateKey: ""}
topicTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.topic", StateKey: ""}
guestTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.guest_access", StateKey: ""}
visibilityTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomHistoryVisibility, StateKey: ""}
joinRuleTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""}
var stateRes currentstateAPI.QueryBulkStateContentResponse
err := stateAPI.QueryBulkStateContent(ctx, &currentstateAPI.QueryBulkStateContentRequest{
RoomIDs: roomIDs,
AllowWildcards: true,
StateTuples: []gomatrixserverlib.StateKeyTuple{
nameTuple, canonicalTuple, topicTuple, guestTuple, visibilityTuple, joinRuleTuple, avatarTuple,
{EventType: gomatrixserverlib.MRoomMember, StateKey: "*"},
},
}, &stateRes)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("QueryBulkStateContent failed")
return nil, err
}
chunk := make([]gomatrixserverlib.PublicRoom, len(roomIDs))
i := 0
for roomID, data := range stateRes.Rooms {
pub := gomatrixserverlib.PublicRoom{
RoomID: roomID,
}
joinCount := 0
var joinRule, guestAccess string
for tuple, contentVal := range data {
if tuple.EventType == gomatrixserverlib.MRoomMember && contentVal == "join" {
joinCount++
continue
}
switch tuple {
case avatarTuple:
pub.AvatarURL = contentVal
case nameTuple:
pub.Name = contentVal
case topicTuple:
pub.Topic = contentVal
case canonicalTuple:
pub.CanonicalAlias = contentVal
case visibilityTuple:
pub.WorldReadable = contentVal == "world_readable"
// need both of these to determine whether guests can join
case joinRuleTuple:
joinRule = contentVal
case guestTuple:
guestAccess = contentVal
}
}
if joinRule == gomatrixserverlib.Public && guestAccess == "can_join" {
pub.GuestCanJoin = true
}
pub.JoinedMembersCount = joinCount
chunk[i] = pub
i++
}
return chunk, nil
}
// sliceInto returns a subslice of `slice` which honours the since/limit values given.
//
// 0 1 2 3 4 5 6 index
// [A, B, C, D, E, F, G] slice
//
// limit=3 => A,B,C (prev='', next='3')
// limit=3&since=3 => D,E,F (prev='0', next='6')
// limit=3&since=6 => G (prev='3', next='')
//
// A value of '-1' for prev/next indicates no position.
func sliceInto(slice []string, since int64, limit int16) (subset []string, prev, next int) {
prev = -1
next = -1
if since > 0 {
prev = int(since) - int(limit)
}
nextIndex := int(since) + int(limit)
if len(slice) > nextIndex { // there are more rooms ahead of us
next = nextIndex
}
// apply sanity caps
if since < 0 {
since = 0
}
if nextIndex > len(slice) {
nextIndex = len(slice)
}
subset = slice[since:nextIndex]
return
}

48
clientapi/routing/directory_public_test.go Normal file
View file

@ -0,0 +1,48 @@
package routing
import (
"reflect"
"testing"
)
func TestSliceInto(t *testing.T) {
slice := []string{"a", "b", "c", "d", "e", "f", "g"}
limit := int16(3)
testCases := []struct {
since int64
wantPrev int
wantNext int
wantSubset []string
}{
{
since: 0,
wantPrev: -1,
wantNext: 3,
wantSubset: slice[0:3],
},
{
since: 3,
wantPrev: 0,
wantNext: 6,
wantSubset: slice[3:6],
},
{
since: 6,
wantPrev: 3,
wantNext: -1,
wantSubset: slice[6:7],
},
}
for _, tc := range testCases {
subset, prev, next := sliceInto(slice, tc.since, limit)
if !reflect.DeepEqual(subset, tc.wantSubset) {
t.Errorf("returned subset is wrong, got %v want %v", subset, tc.wantSubset)
}
if prev != tc.wantPrev {
t.Errorf("returned prev is wrong, got %d want %d", prev, tc.wantPrev)
}
if next != tc.wantNext {
t.Errorf("returned next is wrong, got %d want %d", next, tc.wantNext)
}
}
}

270
clientapi/routing/membership.go
View file

@ -25,6 +25,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/threepid"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/roomserver/api"
@ -38,40 +39,141 @@ import (
var errMissingUserID = errors.New("'user_id' must be supplied")
// SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite)
// by building a m.room.member event then sending it to the room server
// TODO: Can we improve the cyclo count here? Separate code paths for invites?
// nolint:gocyclo
func SendMembership(
func SendBan(
req *http.Request, accountDB accounts.Database, device *userapi.Device,
roomID string, membership string, cfg *config.Dendrite,
roomID string, cfg *config.Dendrite,
rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
) util.JSONResponse {
verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
verRes := api.QueryRoomVersionForRoomResponse{}
if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
if reqErr != nil {
return *reqErr
}
return sendMembership(req.Context(), accountDB, device, roomID, "ban", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
}
func sendMembership(ctx context.Context, accountDB accounts.Database, device *userapi.Device,
roomID, membership, reason string, cfg *config.Dendrite, targetUserID string, evTime time.Time,
roomVer gomatrixserverlib.RoomVersion,
rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI) util.JSONResponse {
event, err := buildMembershipEvent(
ctx, targetUserID, reason, accountDB, device, membership,
roomID, false, cfg, evTime, rsAPI, asAPI,
)
if err == errMissingUserID {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
JSON: jsonerror.BadJSON(err.Error()),
}
} else if err == eventutil.ErrRoomNoExists {
return util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound(err.Error()),
}
} else if err != nil {
util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
return jsonerror.InternalServerError()
}
_, err = roomserverAPI.SendEvents(
ctx, rsAPI,
[]gomatrixserverlib.HeaderedEvent{event.Headered(roomVer)},
cfg.Matrix.ServerName,
nil,
)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("SendEvents failed")
return jsonerror.InternalServerError()
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
}
}
func SendKick(
req *http.Request, accountDB accounts.Database, device *userapi.Device,
roomID string, cfg *config.Dendrite,
rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
) util.JSONResponse {
body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
if reqErr != nil {
return *reqErr
}
if body.UserID == "" {
return util.JSONResponse{
Code: 400,
JSON: jsonerror.BadJSON("missing user_id"),
}
}
var body threepid.MembershipRequest
if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
var queryRes roomserverAPI.QueryMembershipForUserResponse
err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
RoomID: roomID,
UserID: body.UserID,
}, &queryRes)
if err != nil {
return util.ErrorResponse(err)
}
// kick is only valid if the user is not currently banned
if queryRes.Membership == "ban" {
return util.JSONResponse{
Code: 403,
JSON: jsonerror.Unknown("cannot /kick banned users"),
}
}
// TODO: should we be using SendLeave instead?
return sendMembership(req.Context(), accountDB, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
}
func SendUnban(
req *http.Request, accountDB accounts.Database, device *userapi.Device,
roomID string, cfg *config.Dendrite,
rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
) util.JSONResponse {
body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
if reqErr != nil {
return *reqErr
}
if body.UserID == "" {
return util.JSONResponse{
Code: 400,
JSON: jsonerror.BadJSON("missing user_id"),
}
}
var queryRes roomserverAPI.QueryMembershipForUserResponse
err := rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{
RoomID: roomID,
UserID: body.UserID,
}, &queryRes)
if err != nil {
return util.ErrorResponse(err)
}
// unban is only valid if the user is currently banned
if queryRes.Membership != "ban" {
return util.JSONResponse{
Code: 400,
JSON: jsonerror.Unknown("can only /unban users that are banned"),
}
}
// TODO: should we be using SendLeave instead?
return sendMembership(req.Context(), accountDB, device, roomID, "leave", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
}
func SendInvite(
req *http.Request, accountDB accounts.Database, device *userapi.Device,
roomID string, cfg *config.Dendrite,
rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
) util.JSONResponse {
body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
if reqErr != nil {
return *reqErr
}
evTime, err := httputil.ParseTSParam(req)
if err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.InvalidArgumentValue(err.Error()),
}
}
inviteStored, jsonErrResp := checkAndProcessThreepid(
req, device, &body, cfg, rsAPI, accountDB,
membership, roomID, evTime,
req, device, body, cfg, rsAPI, accountDB, roomID, evTime,
)
if jsonErrResp != nil {
return *jsonErrResp
@ -88,7 +190,7 @@ func SendMembership(
}
event, err := buildMembershipEvent(
req.Context(), body, accountDB, device, membership,
req.Context(), body.UserID, body.Reason, accountDB, device, "invite",
roomID, false, cfg, evTime, rsAPI, asAPI,
)
if err == errMissingUserID {
@ -106,14 +208,9 @@ func SendMembership(
return jsonerror.InternalServerError()
}
var returnData interface{} = struct{}{}
switch membership {
case gomatrixserverlib.Invite:
// Invites need to be handled specially
perr := roomserverAPI.SendInvite(
req.Context(), rsAPI,
event.Headered(verRes.RoomVersion),
event.Headered(roomVer),
nil, // ask the roomserver to draw up invite room state for us
cfg.Matrix.ServerName,
nil,
@ -122,45 +219,21 @@ func SendMembership(
util.GetLogger(req.Context()).WithError(perr).Error("producer.SendInvite failed")
return perr.JSONResponse()
}
case gomatrixserverlib.Join:
// The join membership requires the room id to be sent in the response
returnData = struct {
RoomID string `json:"room_id"`
}{roomID}
fallthrough
default:
_, err = roomserverAPI.SendEvents(
req.Context(), rsAPI,
[]gomatrixserverlib.HeaderedEvent{event.Headered(verRes.RoomVersion)},
cfg.Matrix.ServerName,
nil,
)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
return jsonerror.InternalServerError()
}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: returnData,
JSON: struct{}{},
}
}
func buildMembershipEvent(
ctx context.Context,
body threepid.MembershipRequest, accountDB accounts.Database,
targetUserID, reason string, accountDB accounts.Database,
device *userapi.Device,
membership, roomID string, isDirect bool,
cfg *config.Dendrite, evTime time.Time,
rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
) (*gomatrixserverlib.Event, error) {
stateKey, reason, err := getMembershipStateKey(body, device, membership)
if err != nil {
return nil, err
}
profile, err := loadProfile(ctx, stateKey, cfg, accountDB, asAPI)
profile, err := loadProfile(ctx, targetUserID, cfg, accountDB, asAPI)
if err != nil {
return nil, err
}
@ -169,12 +242,7 @@ func buildMembershipEvent(
Sender: device.UserID,
RoomID: roomID,
Type: "m.room.member",
StateKey: &stateKey,
}
// "unban" or "kick" isn't a valid membership value, change it to "leave"
if membership == "unban" || membership == "kick" {
membership = gomatrixserverlib.Leave
StateKey: &targetUserID,
}
content := gomatrixserverlib.MemberContent{
@ -218,29 +286,33 @@ func loadProfile(
return profile, err
}
// getMembershipStateKey extracts the target user ID of a membership change.
// For "join" and "leave" this will be the ID of the user making the change.
// For "ban", "unban", "kick" and "invite" the target user ID will be in the JSON request body.
// In the latter case, if there was an issue retrieving the user ID from the request body,
// returns a JSONResponse with a corresponding error code and message.
func getMembershipStateKey(
body threepid.MembershipRequest, device *userapi.Device, membership string,
) (stateKey string, reason string, err error) {
if membership == gomatrixserverlib.Ban || membership == "unban" || membership == "kick" || membership == gomatrixserverlib.Invite {
// If we're in this case, the state key is contained in the request body,
// possibly along with a reason (for "kick" and "ban") so we need to parse
// it
if body.UserID == "" {
err = errMissingUserID
func extractRequestData(req *http.Request, roomID string, rsAPI api.RoomserverInternalAPI) (
body *threepid.MembershipRequest, evTime time.Time, roomVer gomatrixserverlib.RoomVersion, resErr *util.JSONResponse,
) {
verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
verRes := api.QueryRoomVersionForRoomResponse{}
if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil {
resErr = &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.UnsupportedRoomVersion(err.Error()),
}
return
}
roomVer = verRes.RoomVersion
if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
resErr = reqErr
return
}
stateKey = body.UserID
reason = body.Reason
} else {
stateKey = device.UserID
evTime, err := httputil.ParseTSParam(req)
if err != nil {
resErr = &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.InvalidArgumentValue(err.Error()),
}
return
}
return
}
@ -251,13 +323,13 @@ func checkAndProcessThreepid(
cfg *config.Dendrite,
rsAPI roomserverAPI.RoomserverInternalAPI,
accountDB accounts.Database,
membership, roomID string,
roomID string,
evTime time.Time,
) (inviteStored bool, errRes *util.JSONResponse) {
inviteStored, err := threepid.CheckAndProcessInvite(
req.Context(), device, body, cfg, rsAPI, accountDB,
membership, roomID, evTime,
roomID, evTime,
)
if err == threepid.ErrMissingParameter {
return inviteStored, &util.JSONResponse{
@ -287,3 +359,35 @@ func checkAndProcessThreepid(
}
return
}
func checkMemberInRoom(ctx context.Context, stateAPI currentstateAPI.CurrentStateInternalAPI, userID, roomID string) *util.JSONResponse {
tuple := gomatrixserverlib.StateKeyTuple{
EventType: gomatrixserverlib.MRoomMember,
StateKey: userID,
}
var membershipRes currentstateAPI.QueryCurrentStateResponse
err := stateAPI.QueryCurrentState(ctx, &currentstateAPI.QueryCurrentStateRequest{
RoomID: roomID,
StateTuples: []gomatrixserverlib.StateKeyTuple{tuple},
}, &membershipRes)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("QueryCurrentState: could not query membership for user")
e := jsonerror.InternalServerError()
return &e
}
ev, ok := membershipRes.StateEvents[tuple]
if !ok {
return &util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("user does not belong to room"),
}
}
membership, err := ev.Membership()
if err != nil || membership != "join" {
return &util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("user does not belong to room"),
}
}
return nil
}

20
clientapi/routing/memberships.go
View file

@ -18,9 +18,8 @@ import (
"encoding/json"
"net/http"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/roomserver/api"
userapi "github.com/matrix-org/dendrite/userapi/api"
@ -95,20 +94,19 @@ func GetMemberships(
func GetJoinedRooms(
req *http.Request,
device *userapi.Device,
accountsDB accounts.Database,
stateAPI currentstateAPI.CurrentStateInternalAPI,
) util.JSONResponse {
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
var res currentstateAPI.QueryRoomsForUserResponse
err := stateAPI.QueryRoomsForUser(req.Context(), &currentstateAPI.QueryRoomsForUserRequest{
UserID: device.UserID,
WantMembership: "join",
}, &res)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
return jsonerror.InternalServerError()
}
joinedRooms, err := accountsDB.GetRoomIDsByLocalPart(req.Context(), localpart)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("accountsDB.GetRoomIDsByLocalPart failed")
util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed")
return jsonerror.InternalServerError()
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: getJoinedRoomsResponse{joinedRooms},
JSON: getJoinedRoomsResponse{res.RoomIDs},
}
}

37
clientapi/routing/profile.go
View file

@ -23,6 +23,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/roomserver/api"
@ -93,8 +94,8 @@ func GetAvatarURL(
// SetAvatarURL implements PUT /profile/{userID}/avatar_url
// nolint:gocyclo
func SetAvatarURL(
req *http.Request, accountDB accounts.Database, device *userapi.Device,
userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI,
req *http.Request, accountDB accounts.Database, stateAPI currentstateAPI.CurrentStateInternalAPI,
device *userapi.Device, userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI,
) util.JSONResponse {
if userID != device.UserID {
return util.JSONResponse{
@ -139,9 +140,13 @@ func SetAvatarURL(
return jsonerror.InternalServerError()
}
memberships, err := accountDB.GetMembershipsByLocalpart(req.Context(), localpart)
var res currentstateAPI.QueryRoomsForUserResponse
err = stateAPI.QueryRoomsForUser(req.Context(), &currentstateAPI.QueryRoomsForUserRequest{
UserID: device.UserID,
WantMembership: "join",
}, &res)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetMembershipsByLocalpart failed")
util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed")
return jsonerror.InternalServerError()
}
@ -152,7 +157,7 @@ func SetAvatarURL(
}
events, err := buildMembershipEvents(
req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI,
req.Context(), res.RoomIDs, newProfile, userID, cfg, evTime, rsAPI,
)
switch e := err.(type) {
case nil:
@ -207,8 +212,8 @@ func GetDisplayName(
// SetDisplayName implements PUT /profile/{userID}/displayname
// nolint:gocyclo
func SetDisplayName(
req *http.Request, accountDB accounts.Database, device *userapi.Device,
userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI,
req *http.Request, accountDB accounts.Database, stateAPI currentstateAPI.CurrentStateInternalAPI,
device *userapi.Device, userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI,
) util.JSONResponse {
if userID != device.UserID {
return util.JSONResponse{
@ -253,9 +258,13 @@ func SetDisplayName(
return jsonerror.InternalServerError()
}
memberships, err := accountDB.GetMembershipsByLocalpart(req.Context(), localpart)
var res currentstateAPI.QueryRoomsForUserResponse
err = stateAPI.QueryRoomsForUser(req.Context(), &currentstateAPI.QueryRoomsForUserRequest{
UserID: device.UserID,
WantMembership: "join",
}, &res)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetMembershipsByLocalpart failed")
util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed")
return jsonerror.InternalServerError()
}
@ -266,7 +275,7 @@ func SetDisplayName(
}
events, err := buildMembershipEvents(
req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI,
req.Context(), res.RoomIDs, newProfile, userID, cfg, evTime, rsAPI,
)
switch e := err.(type) {
case nil:
@ -335,14 +344,14 @@ func getProfile(
func buildMembershipEvents(
ctx context.Context,
memberships []authtypes.Membership,
roomIDs []string,
newProfile authtypes.Profile, userID string, cfg *config.Dendrite,
evTime time.Time, rsAPI api.RoomserverInternalAPI,
) ([]gomatrixserverlib.HeaderedEvent, error) {
evs := []gomatrixserverlib.HeaderedEvent{}
for _, membership := range memberships {
verReq := api.QueryRoomVersionForRoomRequest{RoomID: membership.RoomID}
for _, roomID := range roomIDs {
verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
verRes := api.QueryRoomVersionForRoomResponse{}
if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil {
return []gomatrixserverlib.HeaderedEvent{}, err
@ -350,7 +359,7 @@ func buildMembershipEvents(
builder := gomatrixserverlib.EventBuilder{
Sender: userID,
RoomID: membership.RoomID,
RoomID: roomID,
Type: "m.room.member",
StateKey: &userID,
}

192
clientapi/routing/routing.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -21,15 +21,17 @@ import (
"github.com/gorilla/mux"
appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/clientapi/api"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/producers"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/transactions"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/userapi/api"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/dendrite/userapi/storage/devices"
"github.com/matrix-org/gomatrixserverlib"
@ -53,11 +55,13 @@ func Setup(
asAPI appserviceAPI.AppServiceQueryAPI,
accountDB accounts.Database,
deviceDB devices.Database,
userAPI api.UserInternalAPI,
userAPI userapi.UserInternalAPI,
federation *gomatrixserverlib.FederationClient,
syncProducer *producers.SyncAPIProducer,
transactionsCache *transactions.Cache,
federationSender federationSenderAPI.FederationSenderInternalAPI,
stateAPI currentstateAPI.CurrentStateInternalAPI,
extRoomsProvider api.ExternalPublicRoomsProvider,
) {
publicAPIMux.Handle("/client/versions",
@ -81,12 +85,12 @@ func Setup(
unstableMux := publicAPIMux.PathPrefix(pathPrefixUnstable).Subrouter()
r0mux.Handle("/createRoom",
httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return CreateRoom(req, device, cfg, accountDB, rsAPI, asAPI)
}),
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/join/{roomIDOrAlias}",
httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -97,12 +101,23 @@ func Setup(
}),
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/joined_rooms",
httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
return GetJoinedRooms(req, device, accountDB)
httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return GetJoinedRooms(req, device, stateAPI)
}),
).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/join",
httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return JoinRoomByIDOrAlias(
req, device, rsAPI, accountDB, vars["roomID"],
)
}),
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/leave",
httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -112,17 +127,44 @@ func Setup(
)
}),
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|invite)}",
httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
r0mux.Handle("/rooms/{roomID}/ban",
httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, rsAPI, asAPI)
return SendBan(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
}),
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/invite",
httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return SendInvite(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
}),
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/kick",
httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return SendKick(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
}),
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/unban",
httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return SendUnban(req, accountDB, device, vars["roomID"], cfg, rsAPI, asAPI)
}),
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/send/{eventType}",
httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -131,7 +173,7 @@ func Setup(
}),
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}",
httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -142,7 +184,7 @@ func Setup(
}),
).Methods(http.MethodPut, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/event/{eventID}",
httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -151,7 +193,7 @@ func Setup(
}),
).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -159,7 +201,7 @@ func Setup(
return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"])
})).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/state/{type:[^/]+/?}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
r0mux.Handle("/rooms/{roomID}/state/{type:[^/]+/?}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -173,7 +215,7 @@ func Setup(
return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], eventType, "", eventFormat)
})).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -183,7 +225,7 @@ func Setup(
})).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}",
httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -199,7 +241,7 @@ func Setup(
).Methods(http.MethodPut, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}",
httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -232,7 +274,7 @@ func Setup(
).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/directory/room/{roomAlias}",
httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -242,7 +284,7 @@ func Setup(
).Methods(http.MethodPut, http.MethodOptions)
r0mux.Handle("/directory/room/{roomAlias}",
httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -250,31 +292,59 @@ func Setup(
return RemoveLocalAlias(req, device, vars["roomAlias"], rsAPI)
}),
).Methods(http.MethodDelete, http.MethodOptions)
r0mux.Handle("/directory/list/room/{roomID}",
httputil.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return GetVisibility(req, rsAPI, vars["roomID"])
}),
).Methods(http.MethodGet, http.MethodOptions)
// TODO: Add AS support
r0mux.Handle("/directory/list/room/{roomID}",
httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return SetVisibility(req, stateAPI, rsAPI, device, vars["roomID"])
}),
).Methods(http.MethodPut, http.MethodOptions)
r0mux.Handle("/publicRooms",
httputil.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse {
/* TODO:
if extRoomsProvider != nil {
return GetPostPublicRoomsWithExternal(req, stateAPI, fedClient, extRoomsProvider)
} */
return GetPostPublicRooms(req, rsAPI, stateAPI)
}),
).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
r0mux.Handle("/logout",
httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return Logout(req, deviceDB, device)
}),
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/logout/all",
httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return LogoutAll(req, deviceDB, device)
}),
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/typing/{userID}",
httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, eduAPI)
return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, eduAPI, stateAPI)
}),
).Methods(http.MethodPut, http.MethodOptions)
r0mux.Handle("/sendToDevice/{eventType}/{txnID}",
httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -288,7 +358,7 @@ func Setup(
// rather than r0. It's an exact duplicate of the above handler.
// TODO: Remove this if/when sytest is fixed!
unstableMux.Handle("/sendToDevice/{eventType}/{txnID}",
httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -299,7 +369,7 @@ func Setup(
).Methods(http.MethodPut, http.MethodOptions)
r0mux.Handle("/account/whoami",
httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return Whoami(req, device)
}),
).Methods(http.MethodGet, http.MethodOptions)
@ -338,26 +408,6 @@ func Setup(
}),
).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/user/{userId}/filter",
httputil.MakeAuthAPI("put_filter", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return PutFilter(req, device, accountDB, vars["userId"])
}),
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/user/{userId}/filter/{filterId}",
httputil.MakeAuthAPI("get_filter", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return GetFilter(req, device, accountDB, vars["userId"], vars["filterId"])
}),
).Methods(http.MethodGet, http.MethodOptions)
// Riot user settings
r0mux.Handle("/profile/{userID}",
@ -381,12 +431,12 @@ func Setup(
).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/profile/{userID}/avatar_url",
httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return SetAvatarURL(req, accountDB, device, vars["userID"], cfg, rsAPI)
return SetAvatarURL(req, accountDB, stateAPI, device, vars["userID"], cfg, rsAPI)
}),
).Methods(http.MethodPut, http.MethodOptions)
// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
@ -403,31 +453,31 @@ func Setup(
).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/profile/{userID}/displayname",
httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return SetDisplayName(req, accountDB, device, vars["userID"], cfg, rsAPI)
return SetDisplayName(req, accountDB, stateAPI, device, vars["userID"], cfg, rsAPI)
}),
).Methods(http.MethodPut, http.MethodOptions)
// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
// PUT requests, so we need to allow this method
r0mux.Handle("/account/3pid",
httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return GetAssociated3PIDs(req, accountDB, device)
}),
).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/account/3pid",
httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return CheckAndSave3PIDAssociation(req, accountDB, device, cfg)
}),
).Methods(http.MethodPost, http.MethodOptions)
unstableMux.Handle("/account/3pid/delete",
httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return Forget3PID(req, accountDB)
}),
).Methods(http.MethodPost, http.MethodOptions)
@ -450,7 +500,7 @@ func Setup(
).Methods(http.MethodPut, http.MethodOptions)
r0mux.Handle("/voip/turnServer",
httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return RequestTurnServer(req, device, cfg)
}),
).Methods(http.MethodGet, http.MethodOptions)
@ -476,7 +526,7 @@ func Setup(
).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/user/{userID}/account_data/{type}",
httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -486,7 +536,7 @@ func Setup(
).Methods(http.MethodPut, http.MethodOptions)
r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -496,7 +546,7 @@ func Setup(
).Methods(http.MethodPut, http.MethodOptions)
r0mux.Handle("/user/{userID}/account_data/{type}",
httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -506,7 +556,7 @@ func Setup(
).Methods(http.MethodGet)
r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -516,7 +566,7 @@ func Setup(
).Methods(http.MethodGet)
r0mux.Handle("/rooms/{roomID}/members",
httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -526,7 +576,7 @@ func Setup(
).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/joined_members",
httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -543,13 +593,13 @@ func Setup(
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/devices",
httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return GetDevicesByLocalpart(req, deviceDB, device)
}),
).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/devices/{deviceID}",
httputil.MakeAuthAPI("get_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("get_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -559,7 +609,7 @@ func Setup(
).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/devices/{deviceID}",
httputil.MakeAuthAPI("device_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("device_data", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -569,7 +619,7 @@ func Setup(
).Methods(http.MethodPut, http.MethodOptions)
r0mux.Handle("/devices/{deviceID}",
httputil.MakeAuthAPI("delete_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("delete_device", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -579,7 +629,7 @@ func Setup(
).Methods(http.MethodDelete, http.MethodOptions)
r0mux.Handle("/delete_devices",
httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return DeleteDevices(req, deviceDB, device)
}),
).Methods(http.MethodPost, http.MethodOptions)
@ -604,7 +654,7 @@ func Setup(
).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/user/{userId}/rooms/{roomId}/tags",
httputil.MakeAuthAPI("get_tags", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("get_tags", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -614,7 +664,7 @@ func Setup(
).Methods(http.MethodGet, http.MethodOptions)
r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}",
httputil.MakeAuthAPI("put_tag", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("put_tag", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -624,7 +674,7 @@ func Setup(
).Methods(http.MethodPut, http.MethodOptions)
r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}",
httputil.MakeAuthAPI("delete_tag", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("delete_tag", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
@ -634,7 +684,7 @@ func Setup(
).Methods(http.MethodDelete, http.MethodOptions)
r0mux.Handle("/capabilities",
httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse {
httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return GetCapabilities(req, rsAPI)
}),
).Methods(http.MethodGet)

11
clientapi/routing/sendevent.go
View file

@ -157,6 +157,17 @@ func generateSendEvent(
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON(e.Error()),
}
} else if e, ok := err.(gomatrixserverlib.EventValidationError); ok {
if e.Code == gomatrixserverlib.EventValidationTooLarge {
return nil, &util.JSONResponse{
Code: http.StatusRequestEntityTooLarge,
JSON: jsonerror.BadJSON(e.Error()),
}
}
return nil, &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON(e.Error()),
}
} else if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("eventutil.BuildEvent failed")
resErr := jsonerror.InternalServerError()

36
clientapi/routing/sendtyping.go
View file

@ -13,15 +13,15 @@
package routing
import (
"database/sql"
"net/http"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/userutil"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
"github.com/matrix-org/dendrite/eduserver/api"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
@ -36,6 +36,7 @@ func SendTyping(
req *http.Request, device *userapi.Device, roomID string,
userID string, accountDB accounts.Database,
eduAPI api.EDUServerInputAPI,
stateAPI currentstateAPI.CurrentStateInternalAPI,
) util.JSONResponse {
if device.UserID != userID {
return util.JSONResponse{
@ -44,23 +45,38 @@ func SendTyping(
}
}
localpart, err := userutil.ParseUsernameParam(userID, nil)
// Verify that the user is a member of this room
tuple := gomatrixserverlib.StateKeyTuple{
EventType: gomatrixserverlib.MRoomMember,
StateKey: userID,
}
var res currentstateAPI.QueryCurrentStateResponse
err := stateAPI.QueryCurrentState(req.Context(), &currentstateAPI.QueryCurrentStateRequest{
RoomID: roomID,
StateTuples: []gomatrixserverlib.StateKeyTuple{tuple},
}, &res)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("userutil.ParseUsernameParam failed")
util.GetLogger(req.Context()).WithError(err).Error("QueryCurrentState failed")
return jsonerror.InternalServerError()
}
// Verify that the user is a member of this room
_, err = accountDB.GetMembershipInRoomByLocalpart(req.Context(), localpart, roomID)
if err == sql.ErrNoRows {
ev := res.StateEvents[tuple]
if ev == nil {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("User not in this room"),
}
} else if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetMembershipInRoomByLocalPart failed")
}
membership, err := ev.Membership()
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("Member event isn't valid")
return jsonerror.InternalServerError()
}
if membership != gomatrixserverlib.Join {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("User not in this room"),
}
}
// parse the incoming http request
var r typingContentJSON

4
clientapi/threepid/invites.go
View file

@ -88,10 +88,10 @@ func CheckAndProcessInvite(
ctx context.Context,
device *userapi.Device, body *MembershipRequest, cfg *config.Dendrite,
rsAPI api.RoomserverInternalAPI, db accounts.Database,
membership string, roomID string,
roomID string,
evTime time.Time,
) (inviteStoredOnIDServer bool, err error) {
if membership != gomatrixserverlib.Invite || (body.Address == "" && body.IDServer == "" && body.Medium == "") {
if body.Address == "" && body.IDServer == "" && body.Medium == "" {
// If none of the 3PID-specific fields are supplied, it's a standard invite
// so return nil for it to be processed as such
return

21
cmd/client-api-proxy/main.go
View file

@ -51,16 +51,14 @@ var (
syncServerURL = flag.String("sync-api-server-url", "", "The base URL of the listening 'dendrite-sync-api-server' process. E.g. 'http://localhost:4200'")
clientAPIURL = flag.String("client-api-server-url", "", "The base URL of the listening 'dendrite-client-api-server' process. E.g. 'http://localhost:4321'")
mediaAPIURL = flag.String("media-api-server-url", "", "The base URL of the listening 'dendrite-media-api-server' process. E.g. 'http://localhost:7779'")
publicRoomsAPIURL = flag.String("public-rooms-api-server-url", "", "The base URL of the listening 'dendrite-public-rooms-api-server' process. E.g. 'http://localhost:7775'")
bindAddress = flag.String("bind-address", ":8008", "The listening port for the proxy.")
certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS")
keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS")
)
func makeProxy(targetURL string) (*httputil.ReverseProxy, error) {
if !strings.HasSuffix(targetURL, "/") {
targetURL += "/"
}
targetURL = strings.TrimSuffix(targetURL, "/")
// Check that we can parse the URL.
_, err := url.Parse(targetURL)
if err != nil {
@ -122,13 +120,6 @@ func main() {
fmt.Fprintln(os.Stderr, "no --media-api-server-url specified.")
os.Exit(1)
}
if *publicRoomsAPIURL == "" {
flag.Usage()
fmt.Fprintln(os.Stderr, "no --public-rooms-api-server-url specified.")
os.Exit(1)
}
syncProxy, err := makeProxy(*syncServerURL)
if err != nil {
panic(err)
@ -141,14 +132,8 @@ func main() {
if err != nil {
panic(err)
}
publicRoomsProxy, err := makeProxy(*publicRoomsAPIURL)
if err != nil {
panic(err)
}
http.Handle("/_matrix/client/r0/sync", syncProxy)
http.Handle("/_matrix/client/r0/directory/list/", publicRoomsProxy)
http.Handle("/_matrix/client/r0/publicRooms", publicRoomsProxy)
http.Handle("/_matrix/media/v1/", mediaProxy)
http.Handle("/", clientProxy)
@ -160,8 +145,6 @@ func main() {
fmt.Println("Proxying requests to:")
fmt.Println(" /_matrix/client/r0/sync => ", *syncServerURL+"/api/_matrix/client/r0/sync")
fmt.Println(" /_matrix/client/r0/directory/list => ", *publicRoomsAPIURL+"/_matrix/client/r0/directory/list")
fmt.Println(" /_matrix/client/r0/publicRooms => ", *publicRoomsAPIURL+"/_matrix/media/client/r0/publicRooms")
fmt.Println(" /_matrix/media/v1 => ", *mediaAPIURL+"/api/_matrix/media/v1")
fmt.Println(" /* => ", *clientAPIURL+"/api/*")
fmt.Println("Listening on ", *bindAddress)

5
cmd/dendrite-client-api-server/main.go
View file

@ -35,10 +35,11 @@ func main() {
fsAPI := base.FederationSenderHTTPClient()
eduInputAPI := base.EDUServerClient()
userAPI := base.UserAPIClient()
stateAPI := base.CurrentStateAPIClient()
clientapi.AddPublicRoutes(
base.PublicAPIMux, base.Cfg, base.KafkaConsumer, base.KafkaProducer, deviceDB, accountDB, federation,
rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI,
base.PublicAPIMux, base.Cfg, base.KafkaProducer, deviceDB, accountDB, federation,
rsAPI, eduInputAPI, asQuery, stateAPI, transactions.New(), fsAPI, userAPI, nil,
)
base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI))

33
cmd/dendrite-current-state-server/main.go Normal file
View file

@ -0,0 +1,33 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"github.com/matrix-org/dendrite/currentstateserver"
"github.com/matrix-org/dendrite/internal/setup"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "CurrentStateServer", true)
defer base.Close() // nolint: errcheck
stateAPI := currentstateserver.NewInternalAPI(cfg, base.KafkaConsumer)
currentstateserver.AddInternalRoutes(base.InternalAPIMux, stateAPI)
base.SetupAndServeHTTP(string(base.Cfg.Bind.CurrentState), string(base.Cfg.Listen.CurrentState))
}

10
cmd/dendrite-demo-libp2p/main.go
View file

@ -29,7 +29,7 @@ import (
p2phttp "github.com/libp2p/go-libp2p-http"
p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery"
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage"
"github.com/matrix-org/dendrite/currentstateserver"
"github.com/matrix-org/dendrite/eduserver"
"github.com/matrix-org/dendrite/federationsender"
"github.com/matrix-org/dendrite/internal/config"
@ -129,8 +129,8 @@ func main() {
cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName))
cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s-publicroomsa.db", *instanceName))
cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName))
if err = cfg.Derive(); err != nil {
panic(err)
}
@ -162,10 +162,13 @@ func main() {
&base.Base, federation, rsAPI, keyRing,
)
rsAPI.SetFederationSenderAPI(fsAPI)
/* TODO:
publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub, cfg.Matrix.ServerName)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to public rooms db")
}
*/
stateAPI := currentstateserver.NewInternalAPI(base.Base.Cfg, base.Base.KafkaConsumer)
monolith := setup.Monolith{
Config: base.Base.Cfg,
@ -182,9 +185,8 @@ func main() {
FederationSenderAPI: fsAPI,
RoomserverAPI: rsAPI,
ServerKeyAPI: serverKeyAPI,
StateAPI: stateAPI,
UserAPI: userAPI,
PublicRoomsDB: publicRoomsDB,
}
monolith.AddAllPublicRoutes(base.Base.PublicAPIMux)

164
cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go
View file

@ -1,164 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgreswithdht
import (
"context"
"encoding/json"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/matrix-org/dendrite/publicroomsapi/storage/postgres"
"github.com/matrix-org/gomatrixserverlib"
dht "github.com/libp2p/go-libp2p-kad-dht"
)
const DHTInterval = time.Second * 10
// PublicRoomsServerDatabase represents a public rooms server database.
type PublicRoomsServerDatabase struct {
dht *dht.IpfsDHT
postgres.PublicRoomsServerDatabase
ourRoomsContext context.Context // our current value in the DHT
ourRoomsCancel context.CancelFunc // cancel when we want to expire our value
foundRooms map[string]gomatrixserverlib.PublicRoom // additional rooms we have learned about from the DHT
foundRoomsMutex sync.RWMutex // protects foundRooms
maintenanceTimer *time.Timer //
roomsAdvertised atomic.Value // stores int
roomsDiscovered atomic.Value // stores int
}
// NewPublicRoomsServerDatabase creates a new public rooms server database.
func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) {
pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil, localServerName)
if err != nil {
return nil, err
}
provider := PublicRoomsServerDatabase{
dht: dht,
PublicRoomsServerDatabase: *pg,
}
go provider.ResetDHTMaintenance()
provider.roomsAdvertised.Store(0)
provider.roomsDiscovered.Store(0)
return &provider, nil
}
func (d *PublicRoomsServerDatabase) GetRoomVisibility(ctx context.Context, roomID string) (bool, error) {
return d.PublicRoomsServerDatabase.GetRoomVisibility(ctx, roomID)
}
func (d *PublicRoomsServerDatabase) SetRoomVisibility(ctx context.Context, visible bool, roomID string) error {
d.ResetDHTMaintenance()
return d.PublicRoomsServerDatabase.SetRoomVisibility(ctx, visible, roomID)
}
func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) {
count, err := d.PublicRoomsServerDatabase.CountPublicRooms(ctx)
if err != nil {
return 0, err
}
d.foundRoomsMutex.RLock()
defer d.foundRoomsMutex.RUnlock()
return count + int64(len(d.foundRooms)), nil
}
func (d *PublicRoomsServerDatabase) GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) {
realfilter := filter
if realfilter == "__local__" {
realfilter = ""
}
rooms, err := d.PublicRoomsServerDatabase.GetPublicRooms(ctx, offset, limit, realfilter)
if err != nil {
return []gomatrixserverlib.PublicRoom{}, err
}
if filter != "__local__" {
d.foundRoomsMutex.RLock()
defer d.foundRoomsMutex.RUnlock()
for _, room := range d.foundRooms {
rooms = append(rooms, room)
}
}
return rooms, nil
}
func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error {
return d.PublicRoomsServerDatabase.UpdateRoomFromEvents(ctx, eventsToAdd, eventsToRemove)
}
func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error {
return d.PublicRoomsServerDatabase.UpdateRoomFromEvent(ctx, event)
}
func (d *PublicRoomsServerDatabase) ResetDHTMaintenance() {
if d.maintenanceTimer != nil && !d.maintenanceTimer.Stop() {
<-d.maintenanceTimer.C
}
d.Interval()
}
func (d *PublicRoomsServerDatabase) Interval() {
if err := d.AdvertiseRoomsIntoDHT(); err != nil {
// fmt.Println("Failed to advertise room in DHT:", err)
}
if err := d.FindRoomsInDHT(); err != nil {
// fmt.Println("Failed to find rooms in DHT:", err)
}
fmt.Println("Found", d.roomsDiscovered.Load(), "room(s), advertised", d.roomsAdvertised.Load(), "room(s)")
d.maintenanceTimer = time.AfterFunc(DHTInterval, d.Interval)
}
func (d *PublicRoomsServerDatabase) AdvertiseRoomsIntoDHT() error {
dbCtx, dbCancel := context.WithTimeout(context.Background(), 3*time.Second)
_ = dbCancel
ourRooms, err := d.GetPublicRooms(dbCtx, 0, 1024, "__local__")
if err != nil {
return err
}
if j, err := json.Marshal(ourRooms); err == nil {
d.roomsAdvertised.Store(len(ourRooms))
d.ourRoomsContext, d.ourRoomsCancel = context.WithCancel(context.Background())
if err := d.dht.PutValue(d.ourRoomsContext, "/matrix/publicRooms", j); err != nil {
return err
}
}
return nil
}
func (d *PublicRoomsServerDatabase) FindRoomsInDHT() error {
d.foundRoomsMutex.Lock()
searchCtx, searchCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer searchCancel()
defer d.foundRoomsMutex.Unlock()
results, err := d.dht.GetValues(searchCtx, "/matrix/publicRooms", 1024)
if err != nil {
return err
}
d.foundRooms = make(map[string]gomatrixserverlib.PublicRoom)
for _, result := range results {
var received []gomatrixserverlib.PublicRoom
if err := json.Unmarshal(result.Val, &received); err != nil {
return err
}
for _, room := range received {
d.foundRooms[room.RoomID] = room
}
}
d.roomsDiscovered.Store(len(d.foundRooms))
return nil
}

179
cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go
View file

@ -1,179 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgreswithpubsub
import (
"context"
"encoding/json"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/matrix-org/dendrite/publicroomsapi/storage/postgres"
"github.com/matrix-org/gomatrixserverlib"
pubsub "github.com/libp2p/go-libp2p-pubsub"
)
const MaintenanceInterval = time.Second * 10
type discoveredRoom struct {
time time.Time
room gomatrixserverlib.PublicRoom
}
// PublicRoomsServerDatabase represents a public rooms server database.
type PublicRoomsServerDatabase struct {
postgres.PublicRoomsServerDatabase //
pubsub *pubsub.PubSub //
subscription *pubsub.Subscription //
foundRooms map[string]discoveredRoom // additional rooms we have learned about from the DHT
foundRoomsMutex sync.RWMutex // protects foundRooms
maintenanceTimer *time.Timer //
roomsAdvertised atomic.Value // stores int
}
// NewPublicRoomsServerDatabase creates a new public rooms server database.
func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) {
pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil, localServerName)
if err != nil {
return nil, err
}
provider := PublicRoomsServerDatabase{
pubsub: pubsub,
PublicRoomsServerDatabase: *pg,
foundRooms: make(map[string]discoveredRoom),
}
if topic, err := pubsub.Join("/matrix/publicRooms"); err != nil {
return nil, err
} else if sub, err := topic.Subscribe(); err == nil {
provider.subscription = sub
go provider.MaintenanceTimer()
go provider.FindRooms()
provider.roomsAdvertised.Store(0)
return &provider, nil
} else {
return nil, err
}
}
func (d *PublicRoomsServerDatabase) GetRoomVisibility(ctx context.Context, roomID string) (bool, error) {
return d.PublicRoomsServerDatabase.GetRoomVisibility(ctx, roomID)
}
func (d *PublicRoomsServerDatabase) SetRoomVisibility(ctx context.Context, visible bool, roomID string) error {
d.MaintenanceTimer()
return d.PublicRoomsServerDatabase.SetRoomVisibility(ctx, visible, roomID)
}
func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) {
d.foundRoomsMutex.RLock()
defer d.foundRoomsMutex.RUnlock()
return int64(len(d.foundRooms)), nil
}
func (d *PublicRoomsServerDatabase) GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) {
var rooms []gomatrixserverlib.PublicRoom
if filter == "__local__" {
if r, err := d.PublicRoomsServerDatabase.GetPublicRooms(ctx, offset, limit, ""); err == nil {
rooms = append(rooms, r...)
} else {
return []gomatrixserverlib.PublicRoom{}, err
}
} else {
d.foundRoomsMutex.RLock()
defer d.foundRoomsMutex.RUnlock()
for _, room := range d.foundRooms {
rooms = append(rooms, room.room)
}
}
return rooms, nil
}
func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error {
return d.PublicRoomsServerDatabase.UpdateRoomFromEvents(ctx, eventsToAdd, eventsToRemove)
}
func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error {
return d.PublicRoomsServerDatabase.UpdateRoomFromEvent(ctx, event)
}
func (d *PublicRoomsServerDatabase) MaintenanceTimer() {
if d.maintenanceTimer != nil && !d.maintenanceTimer.Stop() {
<-d.maintenanceTimer.C
}
d.Interval()
}
func (d *PublicRoomsServerDatabase) Interval() {
d.foundRoomsMutex.Lock()
for k, v := range d.foundRooms {
if time.Since(v.time) > time.Minute {
delete(d.foundRooms, k)
}
}
d.foundRoomsMutex.Unlock()
if err := d.AdvertiseRooms(); err != nil {
fmt.Println("Failed to advertise room in DHT:", err)
}
d.foundRoomsMutex.RLock()
defer d.foundRoomsMutex.RUnlock()
fmt.Println("Found", len(d.foundRooms), "room(s), advertised", d.roomsAdvertised.Load(), "room(s)")
d.maintenanceTimer = time.AfterFunc(MaintenanceInterval, d.Interval)
}
func (d *PublicRoomsServerDatabase) AdvertiseRooms() error {
dbCtx, dbCancel := context.WithTimeout(context.Background(), 3*time.Second)
_ = dbCancel
ourRooms, err := d.GetPublicRooms(dbCtx, 0, 1024, "__local__")
if err != nil {
return err
}
advertised := 0
for _, room := range ourRooms {
if j, err := json.Marshal(room); err == nil {
if topic, err := d.pubsub.Join("/matrix/publicRooms"); err != nil {
fmt.Println("Failed to subscribe to topic:", err)
} else if err := topic.Publish(context.TODO(), j); err != nil {
fmt.Println("Failed to publish public room:", err)
} else {
advertised++
}
}
}
d.roomsAdvertised.Store(advertised)
return nil
}
func (d *PublicRoomsServerDatabase) FindRooms() {
for {
msg, err := d.subscription.Next(context.Background())
if err != nil {
continue
}
received := discoveredRoom{
time: time.Now(),
}
if err := json.Unmarshal(msg.Data, &received.room); err != nil {
fmt.Println("Unmarshal error:", err)
continue
}
d.foundRoomsMutex.Lock()
d.foundRooms[received.room.RoomID] = received
d.foundRoomsMutex.Unlock()
}
}

62
cmd/dendrite-demo-libp2p/storage/storage.go
View file

@ -1,62 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"net/url"
dht "github.com/libp2p/go-libp2p-kad-dht"
pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage/postgreswithdht"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3"
"github.com/matrix-org/gomatrixserverlib"
)
const schemePostgres = "postgres"
const schemeFile = "file"
// NewPublicRoomsServerDatabase opens a database connection.
func NewPublicRoomsServerDatabaseWithDHT(dataSourceName string, dht *dht.IpfsDHT, localServerName gomatrixserverlib.ServerName) (storage.Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName)
}
switch uri.Scheme {
case schemePostgres:
return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName)
case schemeFile:
return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName)
default:
return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName)
}
}
// NewPublicRoomsServerDatabase opens a database connection.
func NewPublicRoomsServerDatabaseWithPubSub(dataSourceName string, pubsub *pubsub.PubSub, localServerName gomatrixserverlib.ServerName) (storage.Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName)
}
switch uri.Scheme {
case schemePostgres:
return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName)
case schemeFile:
return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName)
default:
return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName)
}
}

16
cmd/dendrite-demo-yggdrasil/main.go
View file

@ -27,13 +27,14 @@ import (
"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn"
"github.com/matrix-org/dendrite/currentstateserver"
"github.com/matrix-org/dendrite/eduserver"
"github.com/matrix-org/dendrite/eduserver/cache"
"github.com/matrix-org/dendrite/federationsender"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/userapi"
"github.com/matrix-org/gomatrixserverlib"
@ -50,6 +51,7 @@ var (
// nolint:gocyclo
func main() {
flag.Parse()
internal.SetupPprof()
ygg, err := yggconn.Setup(*instanceName, *instancePeer, ".")
if err != nil {
@ -73,7 +75,7 @@ func main() {
cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName))
cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s-publicroomsa.db", *instanceName))
cfg.Database.CurrentState = config.DataSource(fmt.Sprintf("file:%s-currentstate.db", *instanceName))
cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
if err = cfg.Derive(); err != nil {
panic(err)
@ -108,13 +110,10 @@ func main() {
rsComponent.SetFederationSenderAPI(fsAPI)
publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to public rooms db")
}
embed.Embed(base.BaseMux, *instancePort, "Yggdrasil Demo")
stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
monolith := setup.Monolith{
Config: base.Cfg,
AccountDB: accountDB,
@ -130,9 +129,8 @@ func main() {
FederationSenderAPI: fsAPI,
RoomserverAPI: rsAPI,
UserAPI: userAPI,
StateAPI: stateAPI,
//ServerKeyAPI: serverKeyAPI,
PublicRoomsDB: publicRoomsDB,
}
monolith.AddAllPublicRoutes(base.PublicAPIMux)

55
cmd/dendrite-demo-yggdrasil/yggconn/node.go
View file

@ -17,6 +17,7 @@ package yggconn
import (
"context"
"crypto/ed25519"
"crypto/tls"
"encoding/hex"
"encoding/json"
"fmt"
@ -26,10 +27,11 @@ import (
"os"
"strings"
"sync"
"time"
"github.com/lucas-clemente/quic-go"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert"
"github.com/libp2p/go-yamux"
yggdrasiladmin "github.com/yggdrasil-network/yggdrasil-go/src/admin"
yggdrasilconfig "github.com/yggdrasil-network/yggdrasil-go/src/config"
yggdrasilmulticast "github.com/yggdrasil-network/yggdrasil-go/src/multicast"
@ -45,10 +47,20 @@ type Node struct {
admin *yggdrasiladmin.AdminSocket
multicast *yggdrasilmulticast.Multicast
log *gologme.Logger
listener *yggdrasil.Listener
dialer *yggdrasil.Dialer
sessions sync.Map // string -> yamux.Session
incoming chan *yamux.Stream
packetConn *yggdrasil.PacketConn
listener quic.Listener
tlsConfig *tls.Config
quicConfig *quic.Config
sessions sync.Map // string -> quic.Session
incoming chan QUICStream
}
func (n *Node) BuildName() string {
return "dendrite"
}
func (n *Node) BuildVersion() string {
return "dev"
}
func (n *Node) Dialer(_, address string) (net.Conn, error) {
@ -74,8 +86,9 @@ func Setup(instanceName, instancePeer, storageDirectory string) (*Node, error) {
admin: &yggdrasiladmin.AdminSocket{},
multicast: &yggdrasilmulticast.Multicast{},
log: gologme.New(os.Stdout, "YGG ", log.Flags()),
incoming: make(chan *yamux.Stream),
incoming: make(chan QUICStream),
}
n.core.SetBuildInfo(n)
yggfile := fmt.Sprintf("%s/%s-yggdrasil.conf", storageDirectory, instanceName)
if _, err := os.Stat(yggfile); !os.IsNotExist(err) {
@ -114,29 +127,21 @@ func Setup(instanceName, instancePeer, storageDirectory string) (*Node, error) {
panic(err)
}
}
/*
if err = n.admin.Init(n.core, n.state, n.log, nil); err != nil {
panic(err)
}
if err = n.admin.Start(); err != nil {
panic(err)
}
*/
if err = n.multicast.Init(n.core, n.state, n.log, nil); err != nil {
panic(err)
}
if err = n.multicast.Start(); err != nil {
panic(err)
}
//n.admin.SetupAdminHandlers(n.admin)
//n.multicast.SetupAdminHandlers(n.admin)
n.listener, err = n.core.ConnListen()
if err != nil {
panic(err)
}
n.dialer, err = n.core.ConnDialer()
if err != nil {
panic(err)
n.packetConn = n.core.PacketConn()
n.tlsConfig = n.generateTLSConfig()
n.quicConfig = &quic.Config{
MaxIncomingStreams: 0,
MaxIncomingUniStreams: 0,
KeepAlive: true,
MaxIdleTimeout: time.Second * 120,
HandshakeTimeout: time.Second * 30,
}
n.log.Println("Public curve25519:", n.core.EncryptionPublicKey())
@ -174,3 +179,7 @@ func (n *Node) SigningPrivateKey() ed25519.PrivateKey {
privBytes, _ := hex.DecodeString(n.config.SigningPrivateKey)
return ed25519.PrivateKey(privBytes)
}
func (n *Node) PeerCount() int {
return len(n.core.GetPeers()) - 1
}

122
cmd/dendrite-demo-yggdrasil/yggconn/session.go
View file

@ -16,60 +16,52 @@ package yggconn
import (
"context"
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/pem"
"errors"
"math/big"
"net"
"strings"
"time"
"github.com/libp2p/go-yamux"
"github.com/lucas-clemente/quic-go"
"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
)
func (n *Node) yamuxConfig() *yamux.Config {
cfg := yamux.DefaultConfig()
cfg.EnableKeepAlive = false
cfg.ConnectionWriteTimeout = time.Second * 15
cfg.MaxMessageSize = 65535
cfg.ReadBufSize = 655350
return cfg
}
func (n *Node) listenFromYgg() {
var err error
n.listener, err = quic.Listen(
n.packetConn, // yggdrasil.PacketConn
n.tlsConfig, // TLS config
n.quicConfig, // QUIC config
)
if err != nil {
panic(err)
}
for {
conn, err := n.listener.Accept()
session, err := n.listener.Accept(context.TODO())
if err != nil {
n.log.Println("n.listener.Accept:", err)
return
}
var session *yamux.Session
// If the remote address is lower than ours then we'll be the
// server. Otherwse we'll be the client.
if strings.Compare(conn.RemoteAddr().String(), n.DerivedSessionName()) < 0 {
session, err = yamux.Server(conn, n.yamuxConfig())
} else {
session, err = yamux.Client(conn, n.yamuxConfig())
}
if err != nil {
return
}
go n.listenFromYggConn(session)
go n.listenFromQUIC(session)
}
}
func (n *Node) listenFromYggConn(session *yamux.Session) {
func (n *Node) listenFromQUIC(session quic.Session) {
n.sessions.Store(session.RemoteAddr().String(), session)
defer n.sessions.Delete(session.RemoteAddr())
defer func() {
if err := session.Close(); err != nil {
n.log.Println("session.Close:", err)
}
}()
for {
st, err := session.AcceptStream()
st, err := session.AcceptStream(context.TODO())
if err != nil {
n.log.Println("session.AcceptStream:", err)
return
}
n.incoming <- st
n.incoming <- QUICStream{st, session}
}
}
@ -96,29 +88,63 @@ func (n *Node) Dial(network, address string) (net.Conn, error) {
// Implements http.Transport.DialContext
func (n *Node) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
s, ok1 := n.sessions.Load(address)
session, ok2 := s.(*yamux.Session)
if !ok1 || !ok2 || (ok1 && ok2 && session.IsClosed()) {
conn, err := n.dialer.DialContext(ctx, network, address)
session, ok2 := s.(quic.Session)
if !ok1 || !ok2 || (ok1 && ok2 && session.ConnectionState().HandshakeComplete) {
dest, err := hex.DecodeString(address)
if err != nil {
return nil, err
}
if len(dest) != crypto.BoxPubKeyLen {
return nil, errors.New("invalid key length supplied")
}
var pubKey crypto.BoxPubKey
copy(pubKey[:], dest)
session, err = quic.Dial(
n.packetConn, // yggdrasil.PacketConn
&pubKey, // dial address
address, // dial SNI
n.tlsConfig, // TLS config
n.quicConfig, // QUIC config
)
if err != nil {
n.log.Println("n.dialer.DialContext:", err)
return nil, err
}
// If the remote address is lower than ours then we will be the
// server. Otherwise we'll be the client.
if strings.Compare(conn.RemoteAddr().String(), n.DerivedSessionName()) < 0 {
session, err = yamux.Server(conn, n.yamuxConfig())
} else {
session, err = yamux.Client(conn, n.yamuxConfig())
}
if err != nil {
return nil, err
}
go n.listenFromYggConn(session)
go n.listenFromQUIC(session)
}
st, err := session.OpenStream()
if err != nil {
n.log.Println("session.OpenStream:", err)
return nil, err
}
return st, nil
return QUICStream{st, session}, nil
}
func (n *Node) generateTLSConfig() *tls.Config {
key, err := rsa.GenerateKey(rand.Reader, 1024)
if err != nil {
panic(err)
}
template := x509.Certificate{
SerialNumber: big.NewInt(1),
NotAfter: time.Now().Add(time.Hour * 24 * 365),
DNSNames: []string{n.DerivedSessionName()},
}
certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
if err != nil {
panic(err)
}
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
if err != nil {
panic(err)
}
return &tls.Config{
Certificates: []tls.Certificate{tlsCert},
NextProtos: []string{"quic-matrix-ygg"},
InsecureSkipVerify: true,
}
}

20
cmd/dendrite-demo-yggdrasil/yggconn/stream.go Normal file
View file

@ -0,0 +1,20 @@
package yggconn
import (
"net"
"github.com/lucas-clemente/quic-go"
)
type QUICStream struct {
quic.Stream
session quic.Session
}
func (s QUICStream) LocalAddr() net.Addr {
return s.session.LocalAddr()
}
func (s QUICStream) RemoteAddr() net.Addr {
return s.session.RemoteAddr()
}

2
cmd/dendrite-federation-api-server/main.go
View file

@ -33,7 +33,7 @@ func main() {
federationapi.AddPublicRoutes(
base.PublicAPIMux, base.Cfg, userAPI, federation, keyRing,
rsAPI, fsAPI, base.EDUServerClient(),
rsAPI, fsAPI, base.EDUServerClient(), base.CurrentStateAPIClient(),
)
base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI))

10
cmd/dendrite-monolith-server/main.go
View file

@ -20,13 +20,13 @@ import (
"os"
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/currentstateserver"
"github.com/matrix-org/dendrite/eduserver"
"github.com/matrix-org/dendrite/eduserver/cache"
"github.com/matrix-org/dendrite/federationsender"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/serverkeyapi"
@ -117,10 +117,7 @@ func main() {
// This is different to rsAPI which can be the http client which doesn't need this dependency
rsImpl.SetFederationSenderAPI(fsAPI)
publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to public rooms db")
}
stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
monolith := setup.Monolith{
Config: base.Cfg,
@ -137,9 +134,8 @@ func main() {
FederationSenderAPI: fsAPI,
RoomserverAPI: rsAPI,
ServerKeyAPI: serverKeyAPI,
StateAPI: stateAPI,
UserAPI: userAPI,
PublicRoomsDB: publicRoomsDB,
}
monolith.AddAllPublicRoutes(base.PublicAPIMux)

41
cmd/dendrite-public-rooms-api-server/main.go
View file

@ -1,41 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/publicroomsapi"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/sirupsen/logrus"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "PublicRoomsAPI", true)
defer base.Close() // nolint: errcheck
userAPI := base.UserAPIClient()
rsAPI := base.RoomserverHTTPClient()
publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to public rooms db")
}
publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, base.KafkaConsumer, userAPI, publicRoomsDB, rsAPI, nil, nil)
base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI))
}

12
cmd/dendritejs/main.go
View file

@ -22,13 +22,13 @@ import (
"syscall/js"
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/currentstateserver"
"github.com/matrix-org/dendrite/eduserver"
"github.com/matrix-org/dendrite/eduserver/cache"
"github.com/matrix-org/dendrite/federationsender"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/userapi"
go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p"
@ -168,10 +168,10 @@ func main() {
cfg.Database.FederationSender = "file:/idb/dendritejs_fedsender.db"
cfg.Database.MediaAPI = "file:/idb/dendritejs_mediaapi.db"
cfg.Database.Naffka = "file:/idb/dendritejs_naffka.db"
cfg.Database.PublicRoomsAPI = "file:/idb/dendritejs_publicrooms.db"
cfg.Database.RoomServer = "file:/idb/dendritejs_roomserver.db"
cfg.Database.ServerKey = "file:/idb/dendritejs_serverkey.db"
cfg.Database.SyncAPI = "file:/idb/dendritejs_syncapi.db"
cfg.Database.CurrentState = "file:/idb/dendritejs_currentstate.db"
cfg.Kafka.Topics.OutputTypingEvent = "output_typing_event"
cfg.Kafka.Topics.OutputSendToDeviceEvent = "output_send_to_device_event"
cfg.Kafka.Topics.OutputClientData = "output_client_data"
@ -213,10 +213,7 @@ func main() {
rsAPI.SetFederationSenderAPI(fedSenderAPI)
p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI)
publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), cfg.Matrix.ServerName)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to public rooms db")
}
stateAPI := currentstateserver.NewInternalAPI(base.Cfg, base.KafkaConsumer)
monolith := setup.Monolith{
Config: base.Cfg,
@ -232,10 +229,9 @@ func main() {
EDUInternalAPI: eduInputAPI,
FederationSenderAPI: fedSenderAPI,
RoomserverAPI: rsAPI,
StateAPI: stateAPI,
UserAPI: userAPI,
//ServerKeyAPI: serverKeyAPI,
PublicRoomsDB: publicRoomsDB,
ExtPublicRoomsProvider: p2pPublicRoomProvider,
}
monolith.AddAllPublicRoutes(base.PublicAPIMux)

104
currentstateserver/api/api.go Normal file
View file

@ -0,0 +1,104 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/matrix-org/gomatrixserverlib"
)
type CurrentStateInternalAPI interface {
// QueryCurrentState retrieves the requested state events. If state events are not found, they will be missing from
// the response.
QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error
// QueryRoomsForUser retrieves a list of room IDs matching the given query.
QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error
// QueryBulkStateContent does a bulk query for state event content in the given rooms.
QueryBulkStateContent(ctx context.Context, req *QueryBulkStateContentRequest, res *QueryBulkStateContentResponse) error
}
type QueryRoomsForUserRequest struct {
UserID string
// The desired membership of the user. If this is the empty string then no rooms are returned.
WantMembership string
}
type QueryRoomsForUserResponse struct {
RoomIDs []string
}
type QueryBulkStateContentRequest struct {
// Returns state events in these rooms
RoomIDs []string
// If true, treats the '*' StateKey as "all state events of this type" rather than a literal value of '*'
AllowWildcards bool
// The state events to return. Only a small subset of tuples are allowed in this request as only certain events
// have their content fields extracted. Specifically, the tuple Type must be one of:
// m.room.avatar
// m.room.create
// m.room.canonical_alias
// m.room.guest_access
// m.room.history_visibility
// m.room.join_rules
// m.room.member
// m.room.name
// m.room.topic
// Any other tuple type will result in the query failing.
StateTuples []gomatrixserverlib.StateKeyTuple
}
type QueryBulkStateContentResponse struct {
// map of room ID -> tuple -> content_value
Rooms map[string]map[gomatrixserverlib.StateKeyTuple]string
}
type QueryCurrentStateRequest struct {
RoomID string
StateTuples []gomatrixserverlib.StateKeyTuple
}
type QueryCurrentStateResponse struct {
StateEvents map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent
}
// MarshalJSON stringifies the StateKeyTuple keys so they can be sent over the wire in HTTP API mode.
func (r *QueryCurrentStateResponse) MarshalJSON() ([]byte, error) {
se := make(map[string]*gomatrixserverlib.HeaderedEvent, len(r.StateEvents))
for k, v := range r.StateEvents {
// use 0x1F (unit separator) as the delimiter between type/state key,
se[fmt.Sprintf("%s\x1F%s", k.EventType, k.StateKey)] = v
}
return json.Marshal(se)
}
func (r *QueryCurrentStateResponse) UnmarshalJSON(data []byte) error {
res := make(map[string]*gomatrixserverlib.HeaderedEvent)
err := json.Unmarshal(data, &res)
if err != nil {
return err
}
r.StateEvents = make(map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent, len(res))
for k, v := range res {
fields := strings.Split(k, "\x1F")
r.StateEvents[gomatrixserverlib.StateKeyTuple{
EventType: fields[0],
StateKey: fields[1],
}] = v
}
return nil
}

140
currentstateserver/consumers/roomserver.go Normal file
View file

@ -0,0 +1,140 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package consumers
import (
"context"
"encoding/json"
"github.com/Shopify/sarama"
"github.com/matrix-org/dendrite/currentstateserver/storage"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/syncapi/types"
"github.com/matrix-org/gomatrixserverlib"
log "github.com/sirupsen/logrus"
)
type OutputRoomEventConsumer struct {
rsConsumer *internal.ContinualConsumer
db storage.Database
}
func NewOutputRoomEventConsumer(topicName string, kafkaConsumer sarama.Consumer, store storage.Database) *OutputRoomEventConsumer {
consumer := &internal.ContinualConsumer{
Topic: topicName,
Consumer: kafkaConsumer,
PartitionStore: store,
}
s := &OutputRoomEventConsumer{
rsConsumer: consumer,
db: store,
}
consumer.ProcessMessage = s.onMessage
return s
}
func (c *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
// Parse out the event JSON
var output api.OutputEvent
if err := json.Unmarshal(msg.Value, &output); err != nil {
// If the message was invalid, log it and move on to the next message in the stream
log.WithError(err).Errorf("roomserver output log: message parse failure")
return nil
}
switch output.Type {
case api.OutputTypeNewRoomEvent:
return c.onNewRoomEvent(context.TODO(), *output.NewRoomEvent)
case api.OutputTypeNewInviteEvent:
case api.OutputTypeRetireInviteEvent:
default:
log.WithField("type", output.Type).Debug(
"roomserver output log: ignoring unknown output type",
)
}
return nil
}
func (c *OutputRoomEventConsumer) onNewRoomEvent(
ctx context.Context, msg api.OutputNewRoomEvent,
) error {
ev := msg.Event
addsStateEvents := msg.AddsState()
ev, err := c.updateStateEvent(ev)
if err != nil {
return err
}
for i := range addsStateEvents {
addsStateEvents[i], err = c.updateStateEvent(addsStateEvents[i])
if err != nil {
return err
}
}
err = c.db.StoreStateEvents(
ctx,
addsStateEvents,
msg.RemovesStateEventIDs,
)
if err != nil {
// panic rather than continue with an inconsistent database
log.WithFields(log.Fields{
"event": string(ev.JSON()),
log.ErrorKey: err,
"add": msg.AddsStateEventIDs,
"del": msg.RemovesStateEventIDs,
}).Panicf("roomserver output log: write event failure")
}
return nil
}
// Start consuming from room servers
func (c *OutputRoomEventConsumer) Start() error {
return c.rsConsumer.Start()
}
func (c *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.HeaderedEvent) (gomatrixserverlib.HeaderedEvent, error) {
var stateKey string
if event.StateKey() == nil {
stateKey = ""
} else {
stateKey = *event.StateKey()
}
prevEvent, err := c.db.GetStateEvent(
context.TODO(), event.RoomID(), event.Type(), stateKey,
)
if err != nil {
return event, err
}
if prevEvent == nil {
return event, nil
}
prev := types.PrevEventRef{
PrevContent: prevEvent.Content(),
ReplacesState: prevEvent.EventID(),
PrevSender: prevEvent.Sender(),
}
event.Event, err = event.SetUnsigned(prev)
return event, err
}

51
currentstateserver/currentstateserver.go Normal file
View file

@ -0,0 +1,51 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package currentstateserver
import (
"github.com/Shopify/sarama"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/currentstateserver/api"
"github.com/matrix-org/dendrite/currentstateserver/consumers"
"github.com/matrix-org/dendrite/currentstateserver/internal"
"github.com/matrix-org/dendrite/currentstateserver/inthttp"
"github.com/matrix-org/dendrite/currentstateserver/storage"
"github.com/matrix-org/dendrite/internal/config"
"github.com/sirupsen/logrus"
)
// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
// on the given input API.
func AddInternalRoutes(router *mux.Router, intAPI api.CurrentStateInternalAPI) {
inthttp.AddRoutes(router, intAPI)
}
// NewInternalAPI returns a concrete implementation of the internal API. Callers
// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
func NewInternalAPI(cfg *config.Dendrite, consumer sarama.Consumer) api.CurrentStateInternalAPI {
csDB, err := storage.NewDatabase(string(cfg.Database.CurrentState), cfg.DbProperties())
if err != nil {
logrus.WithError(err).Panicf("failed to open database")
}
roomConsumer := consumers.NewOutputRoomEventConsumer(
string(cfg.Kafka.Topics.OutputRoomEvent), consumer, csDB,
)
if err = roomConsumer.Start(); err != nil {
logrus.WithError(err).Panicf("failed to start room server consumer")
}
return &internal.CurrentStateInternalAPI{
DB: csDB,
}
}

180
currentstateserver/currentstateserver_test.go Normal file
View file

@ -0,0 +1,180 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package currentstateserver
import (
"context"
"encoding/json"
"net/http"
"reflect"
"testing"
"time"
"github.com/Shopify/sarama"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/currentstateserver/api"
"github.com/matrix-org/dendrite/currentstateserver/inthttp"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/internal/test"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/naffka"
)
var (
testRoomVersion = gomatrixserverlib.RoomVersionV1
testData = []json.RawMessage{
[]byte(`{"auth_events":[],"content":{"creator":"@userid:kaer.morhen"},"depth":0,"event_id":"$0ok8ynDp7kjc95e3:kaer.morhen","hashes":{"sha256":"17kPoH+h0Dk4Omn7Sus0qMb6+oGcf+CZFEgDhv7UKWs"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"jP4a04f5/F10Pw95FPpdCyKAO44JOwUQ/MZOOeA/RTU1Dn+AHPMzGSaZnuGjRr/xQuADt+I3ctb5ZQfLKNzHDw"}},"state_key":"","type":"m.room.create"}`),
[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}]],"content":{"membership":"join"},"depth":1,"event_id":"$LEwEu0kxrtu5fOiS:kaer.morhen","hashes":{"sha256":"B7M88PhXf3vd1LaFtjQutFu4x/w7fHD28XKZ4sAsJTo"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"p2vqmuJn7ZBRImctSaKbXCAxCcBlIjPH9JHte1ouIUGy84gpu4eLipOvSBCLL26hXfC0Zrm4WUto6Hr+ohdrCg"}},"state_key":"@userid:kaer.morhen","type":"m.room.member"}`),
[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"join_rule":"public"},"depth":2,"event_id":"$SMHlqUrNhhBBRLeN:kaer.morhen","hashes":{"sha256":"vIuJQvmMjrGxshAkj1SXe0C4RqvMbv4ZADDw9pFCWqQ"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"hBMsb3Qppo3RaqqAl4JyTgaiWEbW5hlckATky6PrHun+F3YM203TzG7w9clwuQU5F5pZoB1a6nw+to0hN90FAw"}},"state_key":"","type":"m.room.join_rules"}`),
[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"history_visibility":"shared"},"depth":3,"event_id":"$6F1yGIbO0J7TM93h:kaer.morhen","hashes":{"sha256":"Mr23GKSlZW7UCCYLgOWawI2Sg6KIoMjUWO2TDenuOgw"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$SMHlqUrNhhBBRLeN:kaer.morhen",{"sha256":"SylzE8U02I+6eyEHgL+FlU0L5YdqrVp8OOlxKS9VQW0"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"sHLKrFI3hKGrEJfpMVZSDS3LvLasQsy50CTsOwru9XTVxgRsPo6wozNtRVjxo1J3Rk18RC9JppovmQ5VR5EcDw"}},"state_key":"","type":"m.room.history_visibility"}`),
[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"ban":50,"events":null,"events_default":0,"invite":0,"kick":50,"redact":50,"state_default":50,"users":null,"users_default":0},"depth":4,"event_id":"$UKNe10XzYzG0TeA9:kaer.morhen","hashes":{"sha256":"ngbP3yja9U5dlckKerUs/fSOhtKxZMCVvsfhPURSS28"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$6F1yGIbO0J7TM93h:kaer.morhen",{"sha256":"A4CucrKSoWX4IaJXhq02mBg1sxIyZEftbC+5p3fZAvk"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"zOmwlP01QL3yFchzuR9WHvogOoBZA3oVtNIF3lM0ZfDnqlSYZB9sns27G/4HVq0k7alaK7ZE3oGoCrVnMkPNCw"}},"state_key":"","type":"m.room.power_levels"}`),
// messages
[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":5,"event_id":"$gl2T9l3qm0kUbiIJ:kaer.morhen","hashes":{"sha256":"Qx3nRMHLDPSL5hBAzuX84FiSSP0K0Kju2iFoBWH4Za8"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$UKNe10XzYzG0TeA9:kaer.morhen",{"sha256":"KtSRyMjt0ZSjsv2koixTRCxIRCGoOp6QrKscsW97XRo"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"sqDgv3EG7ml5VREzmT9aZeBpS4gAPNIaIeJOwqjDhY0GPU/BcpX5wY4R7hYLrNe5cChgV+eFy/GWm1Zfg5FfDg"}},"type":"m.room.message"}`),
[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":6,"event_id":"$MYSbs8m4rEbsCWXD:kaer.morhen","hashes":{"sha256":"kgbYM7v4Ud2YaBsjBTolM4ySg6rHcJNYI6nWhMSdFUA"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$gl2T9l3qm0kUbiIJ:kaer.morhen",{"sha256":"C/rD04h9wGxRdN2G/IBfrgoE1UovzLZ+uskwaKZ37/Q"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"x0UoKh968jj/F5l1/R7Ew0T6CTKuew3PLNHASNxqck/bkNe8yYQiDHXRr+kZxObeqPZZTpaF1+EI+bLU9W8GDQ"}},"type":"m.room.message"}`),
[]byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":7,"event_id":"$N5x9WJkl9ClPrAEg:kaer.morhen","hashes":{"sha256":"FWM8oz4yquTunRZ67qlW2gzPDzdWfBP6RPHXhK1I/x8"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$MYSbs8m4rEbsCWXD:kaer.morhen",{"sha256":"fatqgW+SE8mb2wFn3UN+drmluoD4UJ/EcSrL6Ur9q1M"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"Y+LX/xcyufoXMOIoqQBNOzy6lZfUGB1ffgXIrSugk6obMiyAsiRejHQN/pciZXsHKxMJLYRFAz4zSJoS/LGPAA"}},"type":"m.room.message"}`),
}
testEvents = []gomatrixserverlib.HeaderedEvent{}
testStateEvents = make(map[gomatrixserverlib.StateKeyTuple]gomatrixserverlib.HeaderedEvent)
kafkaTopic = "room_events"
)
func init() {
for _, j := range testData {
e, err := gomatrixserverlib.NewEventFromTrustedJSON(j, false, testRoomVersion)
if err != nil {
panic("cannot load test data: " + err.Error())
}
h := e.Headered(testRoomVersion)
testEvents = append(testEvents, h)
if e.StateKey() != nil {
testStateEvents[gomatrixserverlib.StateKeyTuple{
EventType: e.Type(),
StateKey: *e.StateKey(),
}] = h
}
}
}
func MustWriteOutputEvent(t *testing.T, producer sarama.SyncProducer, out *roomserverAPI.OutputNewRoomEvent) error {
value, err := json.Marshal(roomserverAPI.OutputEvent{
Type: roomserverAPI.OutputTypeNewRoomEvent,
NewRoomEvent: out,
})
if err != nil {
t.Fatalf("failed to marshal output event: %s", err)
}
_, _, err = producer.SendMessage(&sarama.ProducerMessage{
Topic: kafkaTopic,
Key: sarama.StringEncoder(out.Event.RoomID()),
Value: sarama.ByteEncoder(value),
})
if err != nil {
t.Fatalf("failed to send message: %s", err)
}
return nil
}
func MustMakeInternalAPI(t *testing.T) (api.CurrentStateInternalAPI, sarama.SyncProducer) {
cfg := &config.Dendrite{}
cfg.Kafka.Topics.OutputRoomEvent = config.Topic(kafkaTopic)
cfg.Database.CurrentState = config.DataSource("file::memory:")
db, err := sqlutil.Open(sqlutil.SQLiteDriverName(), "file::memory:", nil)
if err != nil {
t.Fatalf("Failed to open naffka database: %s", err)
}
naffkaDB, err := naffka.NewSqliteDatabase(db)
if err != nil {
t.Fatalf("Failed to setup naffka database: %s", err)
}
naff, err := naffka.New(naffkaDB)
if err != nil {
t.Fatalf("Failed to create naffka consumer: %s", err)
}
return NewInternalAPI(cfg, naff), naff
}
func TestQueryCurrentState(t *testing.T) {
currStateAPI, producer := MustMakeInternalAPI(t)
plTuple := gomatrixserverlib.StateKeyTuple{
EventType: "m.room.power_levels",
StateKey: "",
}
plEvent := testEvents[4]
MustWriteOutputEvent(t, producer, &roomserverAPI.OutputNewRoomEvent{
Event: plEvent,
AddsStateEventIDs: []string{plEvent.EventID()},
})
// we have no good way to know /when/ the server has consumed the event
time.Sleep(100 * time.Millisecond)
testCases := []struct {
req api.QueryCurrentStateRequest
wantRes api.QueryCurrentStateResponse
wantErr error
}{
{
req: api.QueryCurrentStateRequest{
RoomID: plEvent.RoomID(),
StateTuples: []gomatrixserverlib.StateKeyTuple{
plTuple,
},
},
wantRes: api.QueryCurrentStateResponse{
StateEvents: map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent{
plTuple: &plEvent,
},
},
},
}
runCases := func(testAPI api.CurrentStateInternalAPI) {
for _, tc := range testCases {
var gotRes api.QueryCurrentStateResponse
gotErr := testAPI.QueryCurrentState(context.TODO(), &tc.req, &gotRes)
if tc.wantErr == nil && gotErr != nil || tc.wantErr != nil && gotErr == nil {
t.Errorf("QueryCurrentState error, got %s want %s", gotErr, tc.wantErr)
continue
}
for tuple, wantEvent := range tc.wantRes.StateEvents {
gotEvent, ok := gotRes.StateEvents[tuple]
if !ok {
t.Errorf("QueryCurrentState want tuple %+v but it is missing from the response", tuple)
continue
}
if !reflect.DeepEqual(gotEvent.JSON(), wantEvent.JSON()) {
t.Errorf("QueryCurrentState tuple %+v got event JSON %s want %s", tuple, string(gotEvent.JSON()), string(wantEvent.JSON()))
}
}
}
}
t.Run("HTTP API", func(t *testing.T) {
router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter()
AddInternalRoutes(router, currStateAPI)
apiURL, cancel := test.ListenAndServe(t, router, false)
defer cancel()
httpAPI, err := inthttp.NewCurrentStateAPIClient(apiURL, &http.Client{})
if err != nil {
t.Fatalf("failed to create HTTP client")
}
runCases(httpAPI)
})
t.Run("Monolith", func(t *testing.T) {
runCases(currStateAPI)
})
}

70
currentstateserver/internal/api.go Normal file
View file

@ -0,0 +1,70 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package internal
import (
"context"
"github.com/matrix-org/dendrite/currentstateserver/api"
"github.com/matrix-org/dendrite/currentstateserver/storage"
"github.com/matrix-org/gomatrixserverlib"
)
type CurrentStateInternalAPI struct {
DB storage.Database
}
func (a *CurrentStateInternalAPI) QueryCurrentState(ctx context.Context, req *api.QueryCurrentStateRequest, res *api.QueryCurrentStateResponse) error {
res.StateEvents = make(map[gomatrixserverlib.StateKeyTuple]*gomatrixserverlib.HeaderedEvent)
for _, tuple := range req.StateTuples {
ev, err := a.DB.GetStateEvent(ctx, req.RoomID, tuple.EventType, tuple.StateKey)
if err != nil {
return err
}
if ev != nil {
res.StateEvents[tuple] = ev
}
}
return nil
}
func (a *CurrentStateInternalAPI) QueryRoomsForUser(ctx context.Context, req *api.QueryRoomsForUserRequest, res *api.QueryRoomsForUserResponse) error {
roomIDs, err := a.DB.GetRoomsByMembership(ctx, req.UserID, req.WantMembership)
if err != nil {
return err
}
res.RoomIDs = roomIDs
return nil
}
func (a *CurrentStateInternalAPI) QueryBulkStateContent(ctx context.Context, req *api.QueryBulkStateContentRequest, res *api.QueryBulkStateContentResponse) error {
events, err := a.DB.GetBulkStateContent(ctx, req.RoomIDs, req.StateTuples, req.AllowWildcards)
if err != nil {
return err
}
res.Rooms = make(map[string]map[gomatrixserverlib.StateKeyTuple]string)
for _, ev := range events {
if res.Rooms[ev.RoomID] == nil {
res.Rooms[ev.RoomID] = make(map[gomatrixserverlib.StateKeyTuple]string)
}
room := res.Rooms[ev.RoomID]
room[gomatrixserverlib.StateKeyTuple{
EventType: ev.EventType,
StateKey: ev.StateKey,
}] = ev.ContentValue
res.Rooms[ev.RoomID] = room
}
return nil
}

88
currentstateserver/inthttp/client.go Normal file
View file

@ -0,0 +1,88 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package inthttp
import (
"context"
"errors"
"net/http"
"github.com/matrix-org/dendrite/currentstateserver/api"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/opentracing/opentracing-go"
)
// HTTP paths for the internal HTTP APIs
const (
QueryCurrentStatePath = "/currentstateserver/queryCurrentState"
QueryRoomsForUserPath = "/currentstateserver/queryRoomsForUser"
QueryBulkStateContentPath = "/currentstateserver/queryBulkStateContent"
)
// NewCurrentStateAPIClient creates a CurrentStateInternalAPI implemented by talking to a HTTP POST API.
// If httpClient is nil an error is returned
func NewCurrentStateAPIClient(
apiURL string,
httpClient *http.Client,
) (api.CurrentStateInternalAPI, error) {
if httpClient == nil {
return nil, errors.New("NewCurrentStateAPIClient: httpClient is <nil>")
}
return &httpCurrentStateInternalAPI{
apiURL: apiURL,
httpClient: httpClient,
}, nil
}
type httpCurrentStateInternalAPI struct {
apiURL string
httpClient *http.Client
}
func (h *httpCurrentStateInternalAPI) QueryCurrentState(
ctx context.Context,
request *api.QueryCurrentStateRequest,
response *api.QueryCurrentStateResponse,
) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "QueryCurrentState")
defer span.Finish()
apiURL := h.apiURL + QueryCurrentStatePath
return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
}
func (h *httpCurrentStateInternalAPI) QueryRoomsForUser(
ctx context.Context,
request *api.QueryRoomsForUserRequest,
response *api.QueryRoomsForUserResponse,
) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRoomsForUser")
defer span.Finish()
apiURL := h.apiURL + QueryRoomsForUserPath
return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
}
func (h *httpCurrentStateInternalAPI) QueryBulkStateContent(
ctx context.Context,
request *api.QueryBulkStateContentRequest,
response *api.QueryBulkStateContentResponse,
) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "QueryBulkStateContent")
defer span.Finish()
apiURL := h.apiURL + QueryBulkStateContentPath
return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
}

67
currentstateserver/inthttp/server.go Normal file
View file

@ -0,0 +1,67 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package inthttp
import (
"encoding/json"
"net/http"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/currentstateserver/api"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/util"
)
func AddRoutes(internalAPIMux *mux.Router, intAPI api.CurrentStateInternalAPI) {
internalAPIMux.Handle(QueryCurrentStatePath,
httputil.MakeInternalAPI("queryCurrentState", func(req *http.Request) util.JSONResponse {
request := api.QueryCurrentStateRequest{}
response := api.QueryCurrentStateResponse{}
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
return util.MessageResponse(http.StatusBadRequest, err.Error())
}
if err := intAPI.QueryCurrentState(req.Context(), &request, &response); err != nil {
return util.ErrorResponse(err)
}
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
)
internalAPIMux.Handle(QueryRoomsForUserPath,
httputil.MakeInternalAPI("queryRoomsForUser", func(req *http.Request) util.JSONResponse {
request := api.QueryRoomsForUserRequest{}
response := api.QueryRoomsForUserResponse{}
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
return util.MessageResponse(http.StatusBadRequest, err.Error())
}
if err := intAPI.QueryRoomsForUser(req.Context(), &request, &response); err != nil {
return util.ErrorResponse(err)
}
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
)
internalAPIMux.Handle(QueryBulkStateContentPath,
httputil.MakeInternalAPI("queryBulkStateContent", func(req *http.Request) util.JSONResponse {
request := api.QueryBulkStateContentRequest{}
response := api.QueryBulkStateContentResponse{}
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
return util.MessageResponse(http.StatusBadRequest, err.Error())
}
if err := intAPI.QueryBulkStateContent(req.Context(), &request, &response); err != nil {
return util.ErrorResponse(err)
}
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
)
}

38
currentstateserver/storage/interface.go Normal file
View file

@ -0,0 +1,38 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"context"
"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/gomatrixserverlib"
)
type Database interface {
internal.PartitionStorer
// StoreStateEvents updates the database with new events from the roomserver.
StoreStateEvents(ctx context.Context, addStateEvents []gomatrixserverlib.HeaderedEvent, removeStateEventIDs []string) error
// GetStateEvent returns the state event of a given type for a given room with a given state key
// If no event could be found, returns nil
// If there was an issue during the retrieval, returns an error
GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error)
// GetRoomsByMembership returns a list of room IDs matching the provided membership and user ID (as state_key).
GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error)
// GetBulkStateContent returns all state events which match a given room ID and a given state key tuple. Both must be satisfied for a match.
// If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned.
GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error)
}

272
currentstateserver/storage/postgres/current_room_state_table.go Normal file
View file

@ -0,0 +1,272 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgres
import (
"context"
"database/sql"
"encoding/json"
"github.com/lib/pq"
"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/gomatrixserverlib"
)
var currentRoomStateSchema = `
-- Stores the current room state for every room.
CREATE TABLE IF NOT EXISTS currentstate_current_room_state (
-- The 'room_id' key for the state event.
room_id TEXT NOT NULL,
-- The state event ID
event_id TEXT NOT NULL,
-- The state event type e.g 'm.room.member'
type TEXT NOT NULL,
-- The 'sender' property of the event.
sender TEXT NOT NULL,
-- The state_key value for this state event e.g ''
state_key TEXT NOT NULL,
-- The JSON for the event. Stored as TEXT because this should be valid UTF-8.
headered_event_json TEXT NOT NULL,
-- A piece of extracted content e.g membership for m.room.member events
content_value TEXT NOT NULL DEFAULT '',
-- Clobber based on 3-uple of room_id, type and state_key
CONSTRAINT currentstate_current_room_state_unique UNIQUE (room_id, type, state_key)
);
-- for event deletion
CREATE UNIQUE INDEX IF NOT EXISTS currentstate_event_id_idx ON currentstate_current_room_state(event_id, room_id, type, sender);
-- for querying membership states of users
CREATE INDEX IF NOT EXISTS currentstate_membership_idx ON currentstate_current_room_state(type, state_key, content_value)
WHERE type='m.room.member' AND content_value IS NOT NULL AND content_value != 'leave';
`
const upsertRoomStateSQL = "" +
"INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, content_value)" +
" VALUES ($1, $2, $3, $4, $5, $6, $7)" +
" ON CONFLICT ON CONSTRAINT currentstate_current_room_state_unique" +
" DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, content_value = $7"
const deleteRoomStateByEventIDSQL = "" +
"DELETE FROM currentstate_current_room_state WHERE event_id = $1"
const selectRoomIDsWithMembershipSQL = "" +
"SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND content_value = $2"
const selectStateEventSQL = "" +
"SELECT headered_event_json FROM currentstate_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3"
const selectEventsWithEventIDsSQL = "" +
"SELECT headered_event_json FROM currentstate_current_room_state WHERE event_id = ANY($1)"
const selectBulkStateContentSQL = "" +
"SELECT room_id, type, state_key, content_value FROM currentstate_current_room_state WHERE room_id = ANY($1) AND type = ANY($2) AND state_key = ANY($3)"
const selectBulkStateContentWildSQL = "" +
"SELECT room_id, type, state_key, content_value FROM currentstate_current_room_state WHERE room_id = ANY($1) AND type = ANY($2)"
type currentRoomStateStatements struct {
upsertRoomStateStmt *sql.Stmt
deleteRoomStateByEventIDStmt *sql.Stmt
selectRoomIDsWithMembershipStmt *sql.Stmt
selectEventsWithEventIDsStmt *sql.Stmt
selectStateEventStmt *sql.Stmt
selectBulkStateContentStmt *sql.Stmt
selectBulkStateContentWildStmt *sql.Stmt
}
func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) {
s := &currentRoomStateStatements{}
_, err := db.Exec(currentRoomStateSchema)
if err != nil {
return nil, err
}
if s.upsertRoomStateStmt, err = db.Prepare(upsertRoomStateSQL); err != nil {
return nil, err
}
if s.deleteRoomStateByEventIDStmt, err = db.Prepare(deleteRoomStateByEventIDSQL); err != nil {
return nil, err
}
if s.selectRoomIDsWithMembershipStmt, err = db.Prepare(selectRoomIDsWithMembershipSQL); err != nil {
return nil, err
}
if s.selectEventsWithEventIDsStmt, err = db.Prepare(selectEventsWithEventIDsSQL); err != nil {
return nil, err
}
if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil {
return nil, err
}
if s.selectBulkStateContentStmt, err = db.Prepare(selectBulkStateContentSQL); err != nil {
return nil, err
}
if s.selectBulkStateContentWildStmt, err = db.Prepare(selectBulkStateContentWildSQL); err != nil {
return nil, err
}
return s, nil
}
// SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state.
func (s *currentRoomStateStatements) SelectRoomIDsWithMembership(
ctx context.Context,
txn *sql.Tx,
userID string,
contentVal string,
) ([]string, error) {
stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt)
rows, err := stmt.QueryContext(ctx, userID, contentVal)
if err != nil {
return nil, err
}
defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithMembership: rows.close() failed")
var result []string
for rows.Next() {
var roomID string
if err := rows.Scan(&roomID); err != nil {
return nil, err
}
result = append(result, roomID)
}
return result, rows.Err()
}
func (s *currentRoomStateStatements) DeleteRoomStateByEventID(
ctx context.Context, txn *sql.Tx, eventID string,
) error {
stmt := sqlutil.TxStmt(txn, s.deleteRoomStateByEventIDStmt)
_, err := stmt.ExecContext(ctx, eventID)
return err
}
func (s *currentRoomStateStatements) UpsertRoomState(
ctx context.Context, txn *sql.Tx,
event gomatrixserverlib.HeaderedEvent, contentVal string,
) error {
headeredJSON, err := json.Marshal(event)
if err != nil {
return err
}
// upsert state event
stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt)
_, err = stmt.ExecContext(
ctx,
event.RoomID(),
event.EventID(),
event.Type(),
event.Sender(),
*event.StateKey(),
headeredJSON,
contentVal,
)
return err
}
func (s *currentRoomStateStatements) SelectEventsWithEventIDs(
ctx context.Context, txn *sql.Tx, eventIDs []string,
) ([]gomatrixserverlib.HeaderedEvent, error) {
stmt := sqlutil.TxStmt(txn, s.selectEventsWithEventIDsStmt)
rows, err := stmt.QueryContext(ctx, pq.StringArray(eventIDs))
if err != nil {
return nil, err
}
defer internal.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed")
result := []gomatrixserverlib.HeaderedEvent{}
for rows.Next() {
var eventBytes []byte
if err := rows.Scan(&eventBytes); err != nil {
return nil, err
}
// TODO: Handle redacted events
var ev gomatrixserverlib.HeaderedEvent
if err := json.Unmarshal(eventBytes, &ev); err != nil {
return nil, err
}
result = append(result, ev)
}
return result, rows.Err()
}
func (s *currentRoomStateStatements) SelectStateEvent(
ctx context.Context, roomID, evType, stateKey string,
) (*gomatrixserverlib.HeaderedEvent, error) {
stmt := s.selectStateEventStmt
var res []byte
err := stmt.QueryRowContext(ctx, roomID, evType, stateKey).Scan(&res)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, err
}
var ev gomatrixserverlib.HeaderedEvent
if err = json.Unmarshal(res, &ev); err != nil {
return nil, err
}
return &ev, err
}
func (s *currentRoomStateStatements) SelectBulkStateContent(
ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool,
) ([]tables.StrippedEvent, error) {
hasWildcards := false
eventTypeSet := make(map[string]bool)
stateKeySet := make(map[string]bool)
var eventTypes []string
var stateKeys []string
for _, tuple := range tuples {
if !eventTypeSet[tuple.EventType] {
eventTypeSet[tuple.EventType] = true
eventTypes = append(eventTypes, tuple.EventType)
}
if !stateKeySet[tuple.StateKey] {
stateKeySet[tuple.StateKey] = true
stateKeys = append(stateKeys, tuple.StateKey)
}
if tuple.StateKey == "*" {
hasWildcards = true
}
}
var rows *sql.Rows
var err error
if hasWildcards && allowWildcards {
rows, err = s.selectBulkStateContentWildStmt.QueryContext(ctx, pq.StringArray(roomIDs), pq.StringArray(eventTypes))
} else {
rows, err = s.selectBulkStateContentStmt.QueryContext(
ctx, pq.StringArray(roomIDs), pq.StringArray(eventTypes), pq.StringArray(stateKeys),
)
}
if err != nil {
return nil, err
}
strippedEvents := []tables.StrippedEvent{}
defer internal.CloseAndLogIfError(ctx, rows, "SelectBulkStateContent: rows.close() failed")
for rows.Next() {
var roomID string
var eventType string
var stateKey string
var contentVal string
if err = rows.Scan(&roomID, &eventType, &stateKey, &contentVal); err != nil {
return nil, err
}
strippedEvents = append(strippedEvents, tables.StrippedEvent{
RoomID: roomID,
ContentValue: contentVal,
EventType: eventType,
StateKey: stateKey,
})
}
return strippedEvents, rows.Err()
}

35
currentstateserver/storage/postgres/storage.go Normal file
View file

@ -0,0 +1,35 @@
package postgres
import (
"database/sql"
"github.com/matrix-org/dendrite/currentstateserver/storage/shared"
"github.com/matrix-org/dendrite/internal/sqlutil"
)
type Database struct {
shared.Database
db *sql.DB
sqlutil.PartitionOffsetStatements
}
// NewDatabase creates a new sync server database
func NewDatabase(dbDataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) {
var d Database
var err error
if d.db, err = sqlutil.Open("postgres", dbDataSourceName, dbProperties); err != nil {
return nil, err
}
if err = d.PartitionOffsetStatements.Prepare(d.db, "currentstate"); err != nil {
return nil, err
}
currRoomState, err := NewPostgresCurrentRoomStateTable(d.db)
if err != nil {
return nil, err
}
d.Database = shared.Database{
DB: d.db,
CurrentRoomState: currRoomState,
}
return &d, nil
}

66
currentstateserver/storage/shared/storage.go Normal file
View file

@ -0,0 +1,66 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package shared
import (
"context"
"database/sql"
"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/gomatrixserverlib"
)
type Database struct {
DB *sql.DB
CurrentRoomState tables.CurrentRoomState
}
func (d *Database) GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) {
return d.CurrentRoomState.SelectStateEvent(ctx, roomID, evType, stateKey)
}
func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error) {
return d.CurrentRoomState.SelectBulkStateContent(ctx, roomIDs, tuples, allowWildcards)
}
func (d *Database) StoreStateEvents(ctx context.Context, addStateEvents []gomatrixserverlib.HeaderedEvent,
removeStateEventIDs []string) error {
return sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error {
// remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add.
for _, eventID := range removeStateEventIDs {
if err := d.CurrentRoomState.DeleteRoomStateByEventID(ctx, txn, eventID); err != nil {
return err
}
}
for _, event := range addStateEvents {
if event.StateKey() == nil {
// ignore non state events
continue
}
contentVal := tables.ExtractContentValue(&event)
if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, contentVal); err != nil {
return err
}
}
return nil
})
}
func (d *Database) GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error) {
return d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, nil, userID, membership)
}

276
currentstateserver/storage/sqlite3/current_room_state_table.go Normal file
View file

@ -0,0 +1,276 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sqlite3
import (
"context"
"database/sql"
"encoding/json"
"strings"
"github.com/matrix-org/dendrite/currentstateserver/storage/tables"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/gomatrixserverlib"
)
const currentRoomStateSchema = `
-- Stores the current room state for every room.
CREATE TABLE IF NOT EXISTS currentstate_current_room_state (
room_id TEXT NOT NULL,
event_id TEXT NOT NULL,
type TEXT NOT NULL,
sender TEXT NOT NULL,
state_key TEXT NOT NULL,
headered_event_json TEXT NOT NULL,
content_value TEXT NOT NULL DEFAULT '',
UNIQUE (room_id, type, state_key)
);
-- for event deletion
CREATE UNIQUE INDEX IF NOT EXISTS currentstate_event_id_idx ON currentstate_current_room_state(event_id, room_id, type, sender);
`
const upsertRoomStateSQL = "" +
"INSERT INTO currentstate_current_room_state (room_id, event_id, type, sender, state_key, headered_event_json, content_value)" +
" VALUES ($1, $2, $3, $4, $5, $6, $7)" +
" ON CONFLICT (event_id, room_id, type, sender)" +
" DO UPDATE SET event_id = $2, sender=$4, headered_event_json = $6, content_value = $7"
const deleteRoomStateByEventIDSQL = "" +
"DELETE FROM currentstate_current_room_state WHERE event_id = $1"
const selectRoomIDsWithMembershipSQL = "" +
"SELECT room_id FROM currentstate_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND content_value = $2"
const selectStateEventSQL = "" +
"SELECT headered_event_json FROM currentstate_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3"
const selectEventsWithEventIDsSQL = "" +
"SELECT headered_event_json FROM currentstate_current_room_state WHERE event_id IN ($1)"
const selectBulkStateContentSQL = "" +
"SELECT room_id, type, state_key, content_value FROM currentstate_current_room_state WHERE room_id IN ($1) AND type IN ($2) AND state_key IN ($3)"
const selectBulkStateContentWildSQL = "" +
"SELECT room_id, type, state_key, content_value FROM currentstate_current_room_state WHERE room_id IN ($1) AND type IN ($2)"
type currentRoomStateStatements struct {
db *sql.DB
upsertRoomStateStmt *sql.Stmt
deleteRoomStateByEventIDStmt *sql.Stmt
selectRoomIDsWithMembershipStmt *sql.Stmt
selectStateEventStmt *sql.Stmt
}
func NewSqliteCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) {
s := &currentRoomStateStatements{
db: db,
}
_, err := db.Exec(currentRoomStateSchema)
if err != nil {
return nil, err
}
if s.upsertRoomStateStmt, err = db.Prepare(upsertRoomStateSQL); err != nil {
return nil, err
}
if s.deleteRoomStateByEventIDStmt, err = db.Prepare(deleteRoomStateByEventIDSQL); err != nil {
return nil, err
}
if s.selectRoomIDsWithMembershipStmt, err = db.Prepare(selectRoomIDsWithMembershipSQL); err != nil {
return nil, err
}
if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil {
return nil, err
}
return s, nil
}
// SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state.
func (s *currentRoomStateStatements) SelectRoomIDsWithMembership(
ctx context.Context,
txn *sql.Tx,
userID string,
membership string,
) ([]string, error) {
stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt)
rows, err := stmt.QueryContext(ctx, userID, membership)
if err != nil {
return nil, err
}
defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithMembership: rows.close() failed")
var result []string
for rows.Next() {
var roomID string
if err := rows.Scan(&roomID); err != nil {
return nil, err
}
result = append(result, roomID)
}
return result, nil
}
func (s *currentRoomStateStatements) DeleteRoomStateByEventID(
ctx context.Context, txn *sql.Tx, eventID string,
) error {
stmt := sqlutil.TxStmt(txn, s.deleteRoomStateByEventIDStmt)
_, err := stmt.ExecContext(ctx, eventID)
return err
}
func (s *currentRoomStateStatements) UpsertRoomState(
ctx context.Context, txn *sql.Tx,
event gomatrixserverlib.HeaderedEvent, contentVal string,
) error {
headeredJSON, err := json.Marshal(event)
if err != nil {
return err
}
// upsert state event
stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt)
_, err = stmt.ExecContext(
ctx,
event.RoomID(),
event.EventID(),
event.Type(),
event.Sender(),
*event.StateKey(),
headeredJSON,
contentVal,
)
return err
}
func (s *currentRoomStateStatements) SelectEventsWithEventIDs(
ctx context.Context, txn *sql.Tx, eventIDs []string,
) ([]gomatrixserverlib.HeaderedEvent, error) {
iEventIDs := make([]interface{}, len(eventIDs))
for k, v := range eventIDs {
iEventIDs[k] = v
}
query := strings.Replace(selectEventsWithEventIDsSQL, "($1)", sqlutil.QueryVariadic(len(iEventIDs)), 1)
rows, err := txn.QueryContext(ctx, query, iEventIDs...)
if err != nil {
return nil, err
}
defer internal.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed")
result := []gomatrixserverlib.HeaderedEvent{}
for rows.Next() {
var eventBytes []byte
if err := rows.Scan(&eventBytes); err != nil {
return nil, err
}
// TODO: Handle redacted events
var ev gomatrixserverlib.HeaderedEvent
if err := json.Unmarshal(eventBytes, &ev); err != nil {
return nil, err
}
result = append(result, ev)
}
return result, nil
}
func (s *currentRoomStateStatements) SelectStateEvent(
ctx context.Context, roomID, evType, stateKey string,
) (*gomatrixserverlib.HeaderedEvent, error) {
stmt := s.selectStateEventStmt
var res []byte
err := stmt.QueryRowContext(ctx, roomID, evType, stateKey).Scan(&res)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, err
}
var ev gomatrixserverlib.HeaderedEvent
if err = json.Unmarshal(res, &ev); err != nil {
return nil, err
}
return &ev, err
}
func (s *currentRoomStateStatements) SelectBulkStateContent(
ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool,
) ([]tables.StrippedEvent, error) {
hasWildcards := false
eventTypeSet := make(map[string]bool)
stateKeySet := make(map[string]bool)
var eventTypes []string
var stateKeys []string
for _, tuple := range tuples {
if !eventTypeSet[tuple.EventType] {
eventTypeSet[tuple.EventType] = true
eventTypes = append(eventTypes, tuple.EventType)
}
if !stateKeySet[tuple.StateKey] {
stateKeySet[tuple.StateKey] = true
stateKeys = append(stateKeys, tuple.StateKey)
}
if tuple.StateKey == "*" {
hasWildcards = true
}
}
iRoomIDs := make([]interface{}, len(roomIDs))
for i, v := range roomIDs {
iRoomIDs[i] = v
}
iEventTypes := make([]interface{}, len(eventTypes))
for i, v := range eventTypes {
iEventTypes[i] = v
}
iStateKeys := make([]interface{}, len(stateKeys))
for i, v := range stateKeys {
iStateKeys[i] = v
}
var query string
var args []interface{}
if hasWildcards && allowWildcards {
query = strings.Replace(selectBulkStateContentWildSQL, "($1)", sqlutil.QueryVariadic(len(iRoomIDs)), 1)
query = strings.Replace(query, "($2)", sqlutil.QueryVariadicOffset(len(iEventTypes), len(iRoomIDs)), 1)
args = append(iRoomIDs, iEventTypes...)
} else {
query = strings.Replace(selectBulkStateContentSQL, "($1)", sqlutil.QueryVariadic(len(iRoomIDs)), 1)
query = strings.Replace(query, "($2)", sqlutil.QueryVariadicOffset(len(iEventTypes), len(iRoomIDs)), 1)
query = strings.Replace(query, "($3)", sqlutil.QueryVariadicOffset(len(iStateKeys), len(iEventTypes)+len(iRoomIDs)), 1)
args = append(iRoomIDs, iEventTypes...)
args = append(args, iStateKeys...)
}
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
strippedEvents := []tables.StrippedEvent{}
defer internal.CloseAndLogIfError(ctx, rows, "SelectBulkStateContent: rows.close() failed")
for rows.Next() {
var roomID string
var eventType string
var stateKey string
var contentVal string
if err = rows.Scan(&roomID, &eventType, &stateKey, &contentVal); err != nil {
return nil, err
}
strippedEvents = append(strippedEvents, tables.StrippedEvent{
RoomID: roomID,
ContentValue: contentVal,
EventType: eventType,
StateKey: stateKey,
})
}
return strippedEvents, rows.Err()
}

39
currentstateserver/storage/sqlite3/storage.go Normal file
View file

@ -0,0 +1,39 @@
package sqlite3
import (
"database/sql"
"github.com/matrix-org/dendrite/currentstateserver/storage/shared"
"github.com/matrix-org/dendrite/internal/sqlutil"
)
type Database struct {
shared.Database
db *sql.DB
sqlutil.PartitionOffsetStatements
}
// NewDatabase creates a new sync server database
// nolint: gocyclo
func NewDatabase(dataSourceName string) (*Database, error) {
var d Database
cs, err := sqlutil.ParseFileURI(dataSourceName)
if err != nil {
return nil, err
}
if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil {
return nil, err
}
if err = d.PartitionOffsetStatements.Prepare(d.db, "currentstate"); err != nil {
return nil, err
}
currRoomState, err := NewSqliteCurrentRoomStateTable(d.db)
if err != nil {
return nil, err
}
d.Database = shared.Database{
DB: d.db,
CurrentRoomState: currRoomState,
}
return &d, nil
}

41
currentstateserver/storage/storage.go Normal file
View file

@ -0,0 +1,41 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !wasm
package storage
import (
"net/url"
"github.com/matrix-org/dendrite/currentstateserver/storage/postgres"
"github.com/matrix-org/dendrite/currentstateserver/storage/sqlite3"
"github.com/matrix-org/dendrite/internal/sqlutil"
)
// NewDatabase opens a database connection.
func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return postgres.NewDatabase(dataSourceName, dbProperties)
}
switch uri.Scheme {
case "postgres":
return postgres.NewDatabase(dataSourceName, dbProperties)
case "file":
return sqlite3.NewDatabase(dataSourceName)
default:
return postgres.NewDatabase(dataSourceName, dbProperties)
}
}

15
publicroomsapi/storage/storage_wasm.go → currentstateserver/storage/storage_wasm.go
View file

@ -18,21 +18,24 @@ import (
"fmt"
"net/url"
"github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/currentstateserver/storage/sqlite3"
"github.com/matrix-org/dendrite/internal/sqlutil"
)
// NewPublicRoomsServerDatabase opens a database connection.
func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatrixserverlib.ServerName) (Database, error) {
// NewDatabase opens a database connection.
func NewDatabase(
dataSourceName string,
dbProperties sqlutil.DbProperties, // nolint:unparam
) (Database, error) {
uri, err := url.Parse(dataSourceName)
if err != nil {
return nil, err
return nil, fmt.Errorf("Cannot use postgres implementation")
}
switch uri.Scheme {
case "postgres":
return nil, fmt.Errorf("Cannot use postgres implementation")
case "file":
return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName)
return sqlite3.NewDatabase(dataSourceName)
default:
return nil, fmt.Errorf("Cannot use postgres implementation")
}

79
currentstateserver/storage/tables/interface.go Normal file
View file

@ -0,0 +1,79 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tables
import (
"context"
"database/sql"
"github.com/matrix-org/gomatrixserverlib"
"github.com/tidwall/gjson"
)
type CurrentRoomState interface {
SelectStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error)
SelectEventsWithEventIDs(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error)
// UpsertRoomState stores the given event in the database, along with an extracted piece of content.
// The piece of content will vary depending on the event type, and table implementations may use this information to optimise
// lookups e.g membership lookups. The mapped value of `contentVal` is outlined in ExtractContentValue. An empty `contentVal`
// means there is nothing to store for this field.
UpsertRoomState(ctx context.Context, txn *sql.Tx, event gomatrixserverlib.HeaderedEvent, contentVal string) error
DeleteRoomStateByEventID(ctx context.Context, txn *sql.Tx, eventID string) error
// SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state.
SelectRoomIDsWithMembership(ctx context.Context, txn *sql.Tx, userID string, membership string) ([]string, error)
SelectBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]StrippedEvent, error)
}
// StrippedEvent represents a stripped event for returning extracted content values.
type StrippedEvent struct {
RoomID string
EventType string
StateKey string
ContentValue string
}
// ExtractContentValue from the given state event. For example, given an m.room.name event with:
// content: { name: "Foo" }
// this returns "Foo".
func ExtractContentValue(ev *gomatrixserverlib.HeaderedEvent) string {
content := ev.Content()
key := ""
switch ev.Type() {
case gomatrixserverlib.MRoomCreate:
key = "creator"
case gomatrixserverlib.MRoomCanonicalAlias:
key = "alias"
case gomatrixserverlib.MRoomHistoryVisibility:
key = "history_visibility"
case gomatrixserverlib.MRoomJoinRules:
key = "join_rule"
case gomatrixserverlib.MRoomMember:
key = "membership"
case gomatrixserverlib.MRoomName:
key = "name"
case "m.room.avatar":
key = "url"
case "m.room.topic":
key = "topic"
case "m.room.guest_access":
key = "guest_access"
}
result := gjson.GetBytes(content, key)
if !result.Exists() {
return ""
}
// this returns the empty string if this is not a string type
return result.Str
}

4
dendrite-config.yaml
View file

@ -120,7 +120,7 @@ database:
server_key: "postgres://dendrite:itsasecret@localhost/dendrite_serverkey?sslmode=disable"
federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable"
appservice: "postgres://dendrite:itsasecret@localhost/dendrite_appservice?sslmode=disable"
public_rooms_api: "postgres://dendrite:itsasecret@localhost/dendrite_publicroomsapi?sslmode=disable"
current_state: "postgres://dendrite:itsasecret@localhost/dendrite_currentstate?sslmode=disable"
max_open_conns: 100
max_idle_conns: 2
conn_max_lifetime: -1
@ -136,13 +136,13 @@ listen:
federation_api: "localhost:7772"
sync_api: "localhost:7773"
media_api: "localhost:7774"
public_rooms_api: "localhost:7775"
federation_sender: "localhost:7776"
appservice_api: "localhost:7777"
edu_server: "localhost:7778"
key_server: "localhost:7779"
server_key_api: "localhost:7780"
user_api: "localhost:7781"
current_state_server: "localhost:7782"
# The configuration for tracing the dendrite components.
tracing:

32
docs/INSTALL.md
View file

@ -108,7 +108,7 @@ Assuming that Postgres 9.5 (or later) is installed:
* Create the component databases:
```bash
for i in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi appservice naffka; do
for i in account device mediaapi syncapi roomserver serverkey federationsender currentstate appservice naffka; do
sudo -u postgres createdb -O dendrite dendrite_$i
done
```
@ -176,17 +176,17 @@ The following contains scripts which will run all the required processes in orde
| | :7774
| |
| |
| | /directory +----------------------------------+
| | +--------->| dendrite-public-rooms-api-server |<========++
| | | +----------------------------------+ ||
| | | :7775 | ||
| | | +<-----------+ ||
| | | | ||
| | | /sync +--------------------------+ ||
| |
| |
| |
| |
| |
| |
| | /sync +--------------------------+
| | +--------->| dendrite-sync-api-server |<================++
| | | | +--------------------------+ ||
| | | | :7773 | ^^ ||
Matrix +------------------+ | | | | || client_data ||
| | | +--------------------------+ ||
| | | :7773 | ^^ ||
Matrix +------------------+ | | | || client_data ||
Clients --->| client-api-proxy |-------+ +<-----------+ ++=============++ ||
+------------------+ | | | || ||
:8008 | | CS API +----------------------------+ || ||
@ -232,7 +232,6 @@ your client at `http://localhost:8008`.
--client-api-server-url "http://localhost:7771" \
--sync-api-server-url "http://localhost:7773" \
--media-api-server-url "http://localhost:7774" \
--public-rooms-api-server-url "http://localhost:7775" \
```
### Federation proxy
@ -282,15 +281,6 @@ order to upload and retrieve media.
./bin/dendrite-media-api-server --config dendrite.yaml
```
### Public room server
This implements `/directory` requests. Clients talk to this via the proxy
in order to retrieve room directory listings.
```bash
./bin/dendrite-public-rooms-api-server --config dendrite.yaml
```
### Federation API server
This implements federation requests. Servers talk to this via the proxy in

50
docs/WIRING-Current.md
View file

@ -9,23 +9,22 @@ a request/response model like HTTP or RPC. Therefore, components can expose "int
Note in Monolith mode these are actually direct function calls and are not serialised HTTP requests.
```
Tier 1 Sync PublicRooms FederationAPI ClientAPI MediaAPI
Public Facing | .-----1------` | | | | | | | | |
2 | .-------3-----------------` | | | `--------|-|-|-|--11--------------------.
| | | .--------4----------------------------------` | | | |
| | | | .---5-----------` | | | | | |
| | | | | .---6----------------------------` | | |
| | | | | | | .-----7----------` | |
| | | | | | 8 | | 10 |
| | | | | | | | `---9----. | |
V V V V V V V V V V V
Tier 1 Sync FederationAPI ClientAPI MediaAPI
Public Facing | | | | | | | | | |
2 .-------3-----------------` | | | `--------|-|-|-|--11--------------------.
| | .--------4----------------------------------` | | | |
| | | .---5-----------` | | | | | |
| | | | .---6----------------------------` | | |
| | | | | | .-----7----------` | |
| | | | | 8 | | 10 |
| | | | | | | `---9----. | |
V V V V V V V V V V
Tier 2 Roomserver EDUServer FedSender AppService KeyServer ServerKeyAPI
Internal only | `------------------------12----------^ ^
`------------------------------------------------------------13----------`
Client ---> Server
```
- 1 (PublicRooms -> Roomserver): Calculating current auth for changing visibility
- 2 (Sync -> Roomserver): When making backfill requests
- 3 (FedAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing backfill/state/state_ids requests
- 4 (ClientAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing /state requests
@ -46,20 +45,20 @@ In addition to this, all public facing components (Tier 1) talk to the `UserAPI`
```
.----1--------------------------------------------.
V |
Tier 1 Sync PublicRooms FederationAPI ClientAPI MediaAPI
Public Facing ^ ^ ^ ^
| | | |
2 | | |
Tier 1 Sync FederationAPI ClientAPI MediaAPI
Public Facing ^ ^ ^
| | |
2 | |
| `-3------------. |
| | | |
| | | |
| .------4------` | |
| | .--------5-----|------------------------------`
| | | |
| | |
| | |
| | |
| .--------4-----|------------------------------`
| | |
Tier 2 Roomserver EDUServer FedSender AppService KeyServer ServerKeyAPI
Internal only | | ^ ^
| `-----6----------` |
`--------------------7--------`
| `-----5----------` |
`--------------------6--------`
Producer ----> Consumer
@ -67,7 +66,6 @@ Producer ----> Consumer
- 1 (ClientAPI -> Sync): For tracking account data
- 2 (Roomserver -> Sync): For all data to send to clients
- 3 (EDUServer -> Sync): For typing/send-to-device data to send to clients
- 4 (Roomserver -> PublicRooms): For tracking the current room name/topic/joined count/etc.
- 5 (Roomserver -> ClientAPI): For tracking memberships for profile updates.
- 6 (EDUServer -> FedSender): For sending EDUs over federation
- 7 (Roomserver -> FedSender): For sending PDUs over federation, for tracking joined hosts.
- 4 (Roomserver -> ClientAPI): For tracking memberships for profile updates.
- 5 (EDUServer -> FedSender): For sending EDUs over federation
- 6 (Roomserver -> FedSender): For sending PDUs over federation, for tracking joined hosts.

4
federationapi/federationapi.go
View file

@ -16,6 +16,7 @@ package federationapi
import (
"github.com/gorilla/mux"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
@ -36,11 +37,12 @@ func AddPublicRoutes(
rsAPI roomserverAPI.RoomserverInternalAPI,
federationSenderAPI federationSenderAPI.FederationSenderInternalAPI,
eduAPI eduserverAPI.EDUServerInputAPI,
stateAPI currentstateAPI.CurrentStateInternalAPI,
) {
routing.Setup(
router, cfg, rsAPI,
eduAPI, federationSenderAPI, keyRing,
federation, userAPI,
federation, userAPI, stateAPI,
)
}

2
federationapi/federationapi_test.go
View file

@ -31,7 +31,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
fsAPI := base.FederationSenderHTTPClient()
// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil)
federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil, nil)
httputil.SetupHTTPAPI(
base.BaseMux,
base.PublicAPIMux,

67
federationapi/routing/invite.go
View file

@ -15,6 +15,7 @@
package routing
import (
"context"
"encoding/json"
"fmt"
"net/http"
@ -27,8 +28,8 @@ import (
"github.com/matrix-org/util"
)
// Invite implements /_matrix/federation/v2/invite/{roomID}/{eventID}
func Invite(
// InviteV2 implements /_matrix/federation/v2/invite/{roomID}/{eventID}
func InviteV2(
httpReq *http.Request,
request *gomatrixserverlib.FederationRequest,
roomID string,
@ -44,14 +45,58 @@ func Invite(
JSON: jsonerror.NotJSON("The request body could not be decoded into an invite request. " + err.Error()),
}
}
event := inviteReq.Event()
return processInvite(
httpReq.Context(), inviteReq.Event(), inviteReq.RoomVersion(), inviteReq.InviteRoomState(), roomID, eventID, cfg, rsAPI, keys,
)
}
// InviteV1 implements /_matrix/federation/v1/invite/{roomID}/{eventID}
func InviteV1(
httpReq *http.Request,
request *gomatrixserverlib.FederationRequest,
roomID string,
eventID string,
cfg *config.Dendrite,
rsAPI api.RoomserverInternalAPI,
keys gomatrixserverlib.JSONVerifier,
) util.JSONResponse {
roomVer := gomatrixserverlib.RoomVersionV1
body := request.Content()
event, err := gomatrixserverlib.NewEventFromTrustedJSON(body, false, roomVer)
if err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.NotJSON("The request body could not be decoded into an invite v1 request: " + err.Error()),
}
}
var strippedState []gomatrixserverlib.InviteV2StrippedState
if err := json.Unmarshal(event.Unsigned(), &strippedState); err != nil {
// just warn, they may not have added any.
util.GetLogger(httpReq.Context()).Warnf("failed to extract stripped state from invite event")
}
return processInvite(
httpReq.Context(), event, roomVer, strippedState, roomID, eventID, cfg, rsAPI, keys,
)
}
func processInvite(
ctx context.Context,
event gomatrixserverlib.Event,
roomVer gomatrixserverlib.RoomVersion,
strippedState []gomatrixserverlib.InviteV2StrippedState,
roomID string,
eventID string,
cfg *config.Dendrite,
rsAPI api.RoomserverInternalAPI,
keys gomatrixserverlib.JSONVerifier,
) util.JSONResponse {
// Check that we can accept invites for this room version.
if _, err := roomserverVersion.SupportedRoomVersion(inviteReq.RoomVersion()); err != nil {
if _, err := roomserverVersion.SupportedRoomVersion(roomVer); err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.UnsupportedRoomVersion(
fmt.Sprintf("Room version %q is not supported by this server.", inviteReq.RoomVersion()),
fmt.Sprintf("Room version %q is not supported by this server.", roomVer),
),
}
}
@ -80,9 +125,9 @@ func Invite(
AtTS: event.OriginServerTS(),
StrictValidityChecking: true,
}}
verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests)
verifyResults, err := keys.VerifyJSONs(ctx, verifyRequests)
if err != nil {
util.GetLogger(httpReq.Context()).WithError(err).Error("keys.VerifyJSONs failed")
util.GetLogger(ctx).WithError(err).Error("keys.VerifyJSONs failed")
return jsonerror.InternalServerError()
}
if verifyResults[0].Error != nil {
@ -99,13 +144,9 @@ func Invite(
// Add the invite event to the roomserver.
if perr := api.SendInvite(
httpReq.Context(), rsAPI,
signedEvent.Headered(inviteReq.RoomVersion()),
inviteReq.InviteRoomState(),
event.Origin(),
nil,
ctx, rsAPI, signedEvent.Headered(roomVer), strippedState, event.Origin(), nil,
); perr != nil {
util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendInvite failed")
util.GetLogger(ctx).WithError(err).Error("producer.SendInvite failed")
return perr.JSONResponse()
}

178
federationapi/routing/publicrooms.go Normal file
View file

@ -0,0 +1,178 @@
package routing
import (
"context"
"net/http"
"strconv"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
type PublicRoomReq struct {
Since string `json:"since,omitempty"`
Limit int16 `json:"limit,omitempty"`
Filter filter `json:"filter,omitempty"`
}
type filter struct {
SearchTerms string `json:"generic_search_term,omitempty"`
}
// GetPostPublicRooms implements GET and POST /publicRooms
func GetPostPublicRooms(req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, stateAPI currentstateAPI.CurrentStateInternalAPI) util.JSONResponse {
var request PublicRoomReq
if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil {
return *fillErr
}
if request.Limit == 0 {
request.Limit = 50
}
response, err := publicRooms(req.Context(), request, rsAPI, stateAPI)
if err != nil {
return jsonerror.InternalServerError()
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: response,
}
}
func publicRooms(ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.RoomserverInternalAPI,
stateAPI currentstateAPI.CurrentStateInternalAPI) (*gomatrixserverlib.RespPublicRooms, error) {
var response gomatrixserverlib.RespPublicRooms
var limit int16
var offset int64
limit = request.Limit
offset, err := strconv.ParseInt(request.Since, 10, 64)
// ParseInt returns 0 and an error when trying to parse an empty string
// In that case, we want to assign 0 so we ignore the error
if err != nil && len(request.Since) > 0 {
util.GetLogger(ctx).WithError(err).Error("strconv.ParseInt failed")
return nil, err
}
var queryRes roomserverAPI.QueryPublishedRoomsResponse
err = rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed")
return nil, err
}
response.TotalRoomCountEstimate = len(queryRes.RoomIDs)
if offset > 0 {
response.PrevBatch = strconv.Itoa(int(offset) - 1)
}
nextIndex := int(offset) + int(limit)
if response.TotalRoomCountEstimate > nextIndex {
response.NextBatch = strconv.Itoa(nextIndex)
}
if offset < 0 {
offset = 0
}
if nextIndex > len(queryRes.RoomIDs) {
nextIndex = len(queryRes.RoomIDs)
}
roomIDs := queryRes.RoomIDs[offset:nextIndex]
response.Chunk, err = fillInRooms(ctx, roomIDs, stateAPI)
return &response, err
}
// fillPublicRoomsReq fills the Limit, Since and Filter attributes of a GET or POST request
// on /publicRooms by parsing the incoming HTTP request
// Filter is only filled for POST requests
func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSONResponse {
if httpReq.Method == http.MethodGet {
limit, err := strconv.Atoi(httpReq.FormValue("limit"))
// Atoi returns 0 and an error when trying to parse an empty string
// In that case, we want to assign 0 so we ignore the error
if err != nil && len(httpReq.FormValue("limit")) > 0 {
util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed")
reqErr := jsonerror.InternalServerError()
return &reqErr
}
request.Limit = int16(limit)
request.Since = httpReq.FormValue("since")
return nil
} else if httpReq.Method == http.MethodPost {
return httputil.UnmarshalJSONRequest(httpReq, request)
}
return &util.JSONResponse{
Code: http.StatusMethodNotAllowed,
JSON: jsonerror.NotFound("Bad method"),
}
}
// due to lots of switches
// nolint:gocyclo
func fillInRooms(ctx context.Context, roomIDs []string, stateAPI currentstateAPI.CurrentStateInternalAPI) ([]gomatrixserverlib.PublicRoom, error) {
avatarTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.avatar", StateKey: ""}
nameTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.name", StateKey: ""}
canonicalTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomCanonicalAlias, StateKey: ""}
topicTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.topic", StateKey: ""}
guestTuple := gomatrixserverlib.StateKeyTuple{EventType: "m.room.guest_access", StateKey: ""}
visibilityTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomHistoryVisibility, StateKey: ""}
joinRuleTuple := gomatrixserverlib.StateKeyTuple{EventType: gomatrixserverlib.MRoomJoinRules, StateKey: ""}
var stateRes currentstateAPI.QueryBulkStateContentResponse
err := stateAPI.QueryBulkStateContent(ctx, &currentstateAPI.QueryBulkStateContentRequest{
RoomIDs: roomIDs,
AllowWildcards: true,
StateTuples: []gomatrixserverlib.StateKeyTuple{
nameTuple, canonicalTuple, topicTuple, guestTuple, visibilityTuple, joinRuleTuple, avatarTuple,
{EventType: gomatrixserverlib.MRoomMember, StateKey: "*"},
},
}, &stateRes)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("QueryBulkStateContent failed")
return nil, err
}
util.GetLogger(ctx).Infof("room IDs: %+v", roomIDs)
util.GetLogger(ctx).Infof("State res: %+v", stateRes.Rooms)
chunk := make([]gomatrixserverlib.PublicRoom, len(roomIDs))
i := 0
for roomID, data := range stateRes.Rooms {
pub := gomatrixserverlib.PublicRoom{
RoomID: roomID,
}
joinCount := 0
var joinRule, guestAccess string
for tuple, contentVal := range data {
if tuple.EventType == gomatrixserverlib.MRoomMember && contentVal == "join" {
joinCount++
continue
}
switch tuple {
case avatarTuple:
pub.AvatarURL = contentVal
case nameTuple:
pub.Name = contentVal
case topicTuple:
pub.Topic = contentVal
case canonicalTuple:
pub.CanonicalAlias = contentVal
case visibilityTuple:
pub.WorldReadable = contentVal == "world_readable"
// need both of these to determine whether guests can join
case joinRuleTuple:
joinRule = contentVal
case guestTuple:
guestAccess = contentVal
}
}
if joinRule == gomatrixserverlib.Public && guestAccess == "can_join" {
pub.GuestCanJoin = true
}
pub.JoinedMembersCount = joinCount
chunk[i] = pub
i++
}
return chunk, nil
}

26
federationapi/routing/routing.go
View file

@ -19,6 +19,7 @@ import (
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
@ -52,6 +53,7 @@ func Setup(
keys gomatrixserverlib.JSONVerifier,
federation *gomatrixserverlib.FederationClient,
userAPI userapi.UserInternalAPI,
stateAPI currentstateAPI.CurrentStateInternalAPI,
) {
v2keysmux := publicAPIMux.PathPrefix(pathPrefixV2Keys).Subrouter()
v1fedmux := publicAPIMux.PathPrefix(pathPrefixV1Federation).Subrouter()
@ -83,10 +85,26 @@ func Setup(
},
)).Methods(http.MethodPut, http.MethodOptions)
v1fedmux.Handle("/invite/{roomID}/{eventID}", httputil.MakeFedAPI(
"federation_invite", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
res := InviteV1(
httpReq, request, vars["roomID"], vars["eventID"],
cfg, rsAPI, keys,
)
return util.JSONResponse{
Code: res.Code,
JSON: []interface{}{
res.Code, res.JSON,
},
}
},
)).Methods(http.MethodPut, http.MethodOptions)
v2fedmux.Handle("/invite/{roomID}/{eventID}", httputil.MakeFedAPI(
"federation_invite", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
return Invite(
return InviteV2(
httpReq, request, vars["roomID"], vars["eventID"],
cfg, rsAPI, keys,
)
@ -275,4 +293,10 @@ func Setup(
return Backfill(httpReq, request, rsAPI, vars["roomID"], cfg)
},
)).Methods(http.MethodGet)
v1fedmux.Handle("/publicRooms",
httputil.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse {
return GetPostPublicRooms(req, rsAPI, stateAPI)
}),
).Methods(http.MethodGet)
}

27
federationapi/routing/send.go
View file

@ -404,8 +404,7 @@ func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event, roomVer
// at this point we know we're going to have a gap: we need to work out the room state at the new backwards extremity.
// security: we have to do state resolution on the new backwards extremity (TODO: WHY)
// Therefore, we cannot just query /state_ids with this event to get the state before. Instead, we need to query
// the state AFTER all the prev_events for this event, then mix in our current room state and apply state resolution
// to that to get the state before the event.
// the state AFTER all the prev_events for this event, then apply state resolution to that to get the state before the event.
var states []*gomatrixserverlib.RespState
needed := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{*backwardsExtremity}).Tuples()
for _, prevEventID := range backwardsExtremity.PrevEventIDs() {
@ -417,13 +416,6 @@ func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event, roomVer
}
states = append(states, prevState)
}
// mix in the current room state
currState, err := t.lookupCurrentState(backwardsExtremity)
if err != nil {
util.GetLogger(t.context).WithError(err).Errorf("Failed to lookup current room state")
return err
}
states = append(states, currState)
resolvedState, err := t.resolveStatesAndCheck(roomVersion, states, backwardsExtremity)
if err != nil {
util.GetLogger(t.context).WithError(err).Errorf("Failed to resolve state conflicts for event %s", backwardsExtremity.EventID())
@ -526,23 +518,6 @@ func (t *txnReq) lookupStateAfterEventLocally(roomID, eventID string, needed []g
}
}
func (t *txnReq) lookupCurrentState(newEvent *gomatrixserverlib.Event) (*gomatrixserverlib.RespState, error) {
// Ask the roomserver for information about this room
queryReq := api.QueryLatestEventsAndStateRequest{
RoomID: newEvent.RoomID(),
StateToFetch: gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{*newEvent}).Tuples(),
}
var queryRes api.QueryLatestEventsAndStateResponse
if err := t.rsAPI.QueryLatestEventsAndState(t.context, &queryReq, &queryRes); err != nil {
return nil, fmt.Errorf("lookupCurrentState rsAPI.QueryLatestEventsAndState: %w", err)
}
evs := gomatrixserverlib.UnwrapEventHeaders(queryRes.StateEvents)
return &gomatrixserverlib.RespState{
StateEvents: evs,
AuthEvents: evs,
}, nil
}
// lookuptStateBeforeEvent returns the room state before the event e, which is just /state_ids and/or /state depending on what
// the server supports.
func (t *txnReq) lookupStateBeforeEvent(roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (

15
federationapi/routing/send_test.go
View file

@ -111,6 +111,13 @@ func (t *testRoomserverAPI) PerformJoin(
) {
}
func (t *testRoomserverAPI) PerformPublish(
ctx context.Context,
req *api.PerformPublishRequest,
res *api.PerformPublishResponse,
) {
}
func (t *testRoomserverAPI) PerformLeave(
ctx context.Context,
req *api.PerformLeaveRequest,
@ -168,6 +175,14 @@ func (t *testRoomserverAPI) QueryMembershipForUser(
return fmt.Errorf("not implemented")
}
func (t *testRoomserverAPI) QueryPublishedRooms(
ctx context.Context,
request *api.QueryPublishedRoomsRequest,
response *api.QueryPublishedRoomsResponse,
) error {
return fmt.Errorf("not implemented")
}
// Query a list of membership events for a room
func (t *testRoomserverAPI) QueryMembershipsForRoom(
ctx context.Context,

4
federationsender/api/api.go
View file

@ -4,6 +4,7 @@ import (
"context"
"github.com/matrix-org/dendrite/federationsender/types"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/gomatrixserverlib"
)
@ -28,7 +29,7 @@ type FederationSenderInternalAPI interface {
ctx context.Context,
request *PerformJoinRequest,
response *PerformJoinResponse,
) error
)
// Handle an instruction to make_leave & send_leave with a remote server.
PerformLeave(
ctx context.Context,
@ -62,6 +63,7 @@ type PerformJoinRequest struct {
}
type PerformJoinResponse struct {
LastError *gomatrix.HTTPError
}
type PerformLeaveRequest struct {

3
federationsender/federationsender.go
View file

@ -50,7 +50,8 @@ func NewInternalAPI(
statistics := &types.Statistics{}
queues := queue.NewOutgoingQueues(
base.Cfg.Matrix.ServerName, federation, rsAPI, statistics, &queue.SigningInfo{
federationSenderDB, base.Cfg.Matrix.ServerName, federation, rsAPI, statistics,
&queue.SigningInfo{
KeyID: base.Cfg.Matrix.KeyID,
PrivateKey: base.Cfg.Matrix.PrivateKey,
ServerName: base.Cfg.Matrix.ServerName,

28
federationsender/internal/perform.go
View file

@ -2,6 +2,7 @@ package internal
import (
"context"
"errors"
"fmt"
"time"
@ -9,6 +10,7 @@ import (
"github.com/matrix-org/dendrite/federationsender/internal/perform"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/roomserver/version"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/sirupsen/logrus"
@ -40,7 +42,7 @@ func (r *FederationSenderInternalAPI) PerformJoin(
ctx context.Context,
request *api.PerformJoinRequest,
response *api.PerformJoinResponse,
) (err error) {
) {
// Look up the supported room versions.
var supportedVersions []gomatrixserverlib.RoomVersion
for version := range version.SupportedRoomVersions() {
@ -63,6 +65,7 @@ func (r *FederationSenderInternalAPI) PerformJoin(
// Try each server that we were provided until we land on one that
// successfully completes the make-join send-join dance.
var lastErr error
for _, serverName := range request.ServerNames {
if err := r.performJoinUsingServer(
ctx,
@ -76,17 +79,32 @@ func (r *FederationSenderInternalAPI) PerformJoin(
"server_name": serverName,
"room_id": request.RoomID,
}).Warnf("Failed to join room through server")
lastErr = err
continue
}
// We're all good.
return nil
return
}
// If we reach here then we didn't complete a join for some reason.
return fmt.Errorf(
"failed to join user %q to room %q through %d server(s)",
request.UserID, request.RoomID, len(request.ServerNames),
var httpErr gomatrix.HTTPError
if ok := errors.As(lastErr, &httpErr); ok {
httpErr.Message = string(httpErr.Contents)
// Clear the wrapped error, else serialising to JSON (in polylith mode) will fail
httpErr.WrappedError = nil
response.LastError = &httpErr
} else {
response.LastError = &gomatrix.HTTPError{
Code: 0,
WrappedError: nil,
Message: lastErr.Error(),
}
}
logrus.Errorf(
"failed to join user %q to room %q through %d server(s): last error %s",
request.UserID, request.RoomID, len(request.ServerNames), lastErr,
)
}

12
federationsender/inthttp/client.go
View file

@ -7,6 +7,7 @@ import (
"github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/gomatrix"
"github.com/opentracing/opentracing-go"
)
@ -77,12 +78,19 @@ func (h *httpFederationSenderInternalAPI) PerformJoin(
ctx context.Context,
request *api.PerformJoinRequest,
response *api.PerformJoinResponse,
) error {
) {
span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoinRequest")
defer span.Finish()
apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath
return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
if err != nil {
response.LastError = &gomatrix.HTTPError{
Message: err.Error(),
Code: 0,
WrappedError: err,
}
}
}
// Handle an instruction to make_join & send_join with a remote server.

4
federationsender/inthttp/server.go
View file

@ -33,9 +33,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
return util.MessageResponse(http.StatusBadRequest, err.Error())
}
if err := intAPI.PerformJoin(req.Context(), &request, &response); err != nil {
return util.ErrorResponse(err)
}
intAPI.PerformJoin(req.Context(), &request, &response)
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
)

238
federationsender/queue/destinationqueue.go
View file

@ -18,8 +18,10 @@ import (
"context"
"encoding/json"
"fmt"
"sync"
"time"
"github.com/matrix-org/dendrite/federationsender/storage"
"github.com/matrix-org/dendrite/federationsender/types"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrix"
@ -29,11 +31,14 @@ import (
"go.uber.org/atomic"
)
const maxPDUsPerTransaction = 50
// destinationQueue is a queue of events for a single destination.
// It is responsible for sending the events to the destination and
// ensures that only one request is in flight to a given destination
// at a time.
type destinationQueue struct {
db storage.Database
signing *SigningInfo
rsAPI api.RoomserverInternalAPI
client *gomatrixserverlib.FederationClient // federation client
@ -42,13 +47,15 @@ type destinationQueue struct {
running atomic.Bool // is the queue worker running?
backingOff atomic.Bool // true if we're backing off
statistics *types.ServerStatistics // statistics about this remote server
incomingPDUs chan *gomatrixserverlib.HeaderedEvent // PDUs to send
incomingEDUs chan *gomatrixserverlib.EDU // EDUs to send
incomingInvites chan *gomatrixserverlib.InviteV2Request // invites to send
lastTransactionIDs []gomatrixserverlib.TransactionID // last transaction ID
pendingPDUs []*gomatrixserverlib.HeaderedEvent // owned by backgroundSend
incomingEDUs chan *gomatrixserverlib.EDU // EDUs to send
transactionIDMutex sync.Mutex // protects transactionID
transactionID gomatrixserverlib.TransactionID // last transaction ID
transactionCount atomic.Int32 // how many events in this transaction so far
pendingPDUs atomic.Int64 // how many PDUs are waiting to be sent
pendingEDUs []*gomatrixserverlib.EDU // owned by backgroundSend
pendingInvites []*gomatrixserverlib.InviteV2Request // owned by backgroundSend
wakeServerCh chan bool // interrupts idle wait
retryServerCh chan bool // interrupts backoff
}
@ -79,15 +86,44 @@ func (oq *destinationQueue) retry() {
// Send event adds the event to the pending queue for the destination.
// If the queue is empty then it starts a background goroutine to
// start sending events to that destination.
func (oq *destinationQueue) sendEvent(ev *gomatrixserverlib.HeaderedEvent) {
func (oq *destinationQueue) sendEvent(nid int64) {
if oq.statistics.Blacklisted() {
// If the destination is blacklisted then drop the event.
return
}
if !oq.running.Load() {
go oq.backgroundSend()
oq.wakeQueueIfNeeded()
// Create a transaction ID. We'll either do this if we don't have
// one made up yet, or if we've exceeded the number of maximum
// events allowed in a single tranaction. We'll reset the counter
// when we do.
oq.transactionIDMutex.Lock()
if oq.transactionID == "" || oq.transactionCount.Load() >= maxPDUsPerTransaction {
now := gomatrixserverlib.AsTimestamp(time.Now())
oq.transactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, oq.statistics.SuccessCount()))
oq.transactionCount.Store(0)
}
oq.transactionIDMutex.Unlock()
// Create a database entry that associates the given PDU NID with
// this destination queue. We'll then be able to retrieve the PDU
// later.
if err := oq.db.AssociatePDUWithDestination(
context.TODO(),
oq.transactionID, // the current transaction ID
oq.destination, // the destination server name
[]int64{nid}, // NID from federationsender_queue_json table
); err != nil {
log.WithError(err).Errorf("failed to associate PDU NID %d with destination %q", nid, oq.destination)
return
}
// We've successfully added a PDU to the transaction so increase
// the counter.
oq.transactionCount.Add(1)
// Signal that we've sent a new PDU. This will cause the queue to
// wake up if it's asleep. The return to the Add function will only
// be 1 if the previous value was 0, e.g. nothing was waiting before.
if oq.pendingPDUs.Add(1) == 1 {
oq.wakeServerCh <- true
}
oq.incomingPDUs <- ev
}
// sendEDU adds the EDU event to the pending queue for the destination.
@ -98,9 +134,7 @@ func (oq *destinationQueue) sendEDU(ev *gomatrixserverlib.EDU) {
// If the destination is blacklisted then drop the event.
return
}
if !oq.running.Load() {
go oq.backgroundSend()
}
oq.wakeQueueIfNeeded()
oq.incomingEDUs <- ev
}
@ -112,10 +146,30 @@ func (oq *destinationQueue) sendInvite(ev *gomatrixserverlib.InviteV2Request) {
// If the destination is blacklisted then drop the event.
return
}
oq.wakeQueueIfNeeded()
oq.incomingInvites <- ev
}
func (oq *destinationQueue) wakeQueueIfNeeded() {
if !oq.running.Load() {
// Look up how many events are pending in this queue. We need
// to do this so that the queue thinks it has work to do.
count, err := oq.db.GetPendingPDUCount(
context.TODO(),
oq.destination,
)
if err == nil {
oq.pendingPDUs.Store(count)
log.Printf("Destination queue %q has %d pending PDUs", oq.destination, count)
} else {
log.WithError(err).Errorf("Can't get pending PDU count for %q destination queue", oq.destination)
}
if count > 0 {
oq.wakeServerCh <- true
}
// Then start the queue.
go oq.backgroundSend()
}
oq.incomingInvites <- ev
}
// backgroundSend is the worker goroutine for sending events.
@ -129,26 +183,16 @@ func (oq *destinationQueue) backgroundSend() {
defer oq.running.Store(false)
for {
// Wait either for incoming events, or until we hit an
// idle timeout.
// If we have nothing to do then wait either for incoming events, or
// until we hit an idle timeout.
select {
case pdu := <-oq.incomingPDUs:
// Ordering of PDUs is important so we add them to the end
// of the queue and they will all be added to transactions
// in order.
oq.pendingPDUs = append(oq.pendingPDUs, pdu)
// If there are any more things waiting in the channel queue
// then read them. This is safe because we guarantee only
// having one goroutine per destination queue, so the channel
// isn't being consumed anywhere else.
for len(oq.incomingPDUs) > 0 {
oq.pendingPDUs = append(oq.pendingPDUs, <-oq.incomingPDUs)
}
case <-oq.wakeServerCh:
// We were woken up because there are new PDUs waiting in the
// database.
case edu := <-oq.incomingEDUs:
// Likewise for EDUs, although we should probably not try
// too hard with some EDUs (like typing notifications) after
// a certain amount of time has passed.
// TODO: think about EDU expiry some more
// EDUs are handled in-memory for now. We will try to keep
// the ordering intact.
// TODO: Certain EDU types need persistence, e.g. send-to-device
oq.pendingEDUs = append(oq.pendingEDUs, edu)
// If there are any more things waiting in the channel queue
// then read them. This is safe because we guarantee only
@ -175,9 +219,9 @@ func (oq *destinationQueue) backgroundSend() {
oq.pendingInvites = append(oq.pendingInvites, <-oq.incomingInvites)
}
case <-time.After(time.Second * 30):
// The worker is idle so stop the goroutine. It'll
// get restarted automatically the next time we
// get an event.
// The worker is idle so stop the goroutine. It'll get
// restarted automatically the next time we have an event to
// send.
return
}
@ -193,47 +237,31 @@ func (oq *destinationQueue) backgroundSend() {
oq.backingOff.Store(false)
}
// How many things do we have waiting?
numPDUs := len(oq.pendingPDUs)
numEDUs := len(oq.pendingEDUs)
numInvites := len(oq.pendingInvites)
// If we have pending PDUs or EDUs then construct a transaction.
if numPDUs > 0 || numEDUs > 0 {
if oq.pendingPDUs.Load() > 0 || len(oq.pendingEDUs) > 0 {
// Try sending the next transaction and see what happens.
transaction, terr := oq.nextTransaction(oq.pendingPDUs, oq.pendingEDUs, oq.statistics.SuccessCount())
transaction, terr := oq.nextTransaction(oq.pendingEDUs)
if terr != nil {
// We failed to send the transaction.
if giveUp := oq.statistics.Failure(); giveUp {
// It's been suggested that we should give up because
// the backoff has exceeded a maximum allowable value.
// It's been suggested that we should give up because the backoff
// has exceeded a maximum allowable value. Clean up the in-memory
// buffers at this point. The PDU clean-up is already on a defer.
oq.cleanPendingEDUs()
oq.cleanPendingInvites()
return
}
} else if transaction {
// If we successfully sent the transaction then clear out
// the pending events and EDUs.
// the pending events and EDUs, and wipe our transaction ID.
oq.statistics.Success()
// Reallocate so that the underlying arrays can be GC'd, as
// opposed to growing forever.
for i := 0; i < numPDUs; i++ {
oq.pendingPDUs[i] = nil
}
for i := 0; i < numEDUs; i++ {
oq.pendingEDUs[i] = nil
}
oq.pendingPDUs = append(
[]*gomatrixserverlib.HeaderedEvent{},
oq.pendingPDUs[numPDUs:]...,
)
oq.pendingEDUs = append(
[]*gomatrixserverlib.EDU{},
oq.pendingEDUs[numEDUs:]...,
)
// Clean up the in-memory buffers.
oq.cleanPendingEDUs()
}
}
// Try sending the next invite and see what happens.
if numInvites > 0 {
if len(oq.pendingInvites) > 0 {
sent, ierr := oq.nextInvites(oq.pendingInvites)
if ierr != nil {
// We failed to send the transaction so increase the
@ -249,59 +277,119 @@ func (oq *destinationQueue) backgroundSend() {
oq.statistics.Success()
// Reallocate so that the underlying array can be GC'd, as
// opposed to growing forever.
oq.pendingInvites = append(
[]*gomatrixserverlib.InviteV2Request{},
oq.pendingInvites[sent:]...,
)
oq.cleanPendingInvites()
}
}
}
}
// cleanPendingEDUs cleans out the pending EDU buffer, removing
// all references so that the underlying objects can be GC'd.
func (oq *destinationQueue) cleanPendingEDUs() {
for i := 0; i < len(oq.pendingEDUs); i++ {
oq.pendingEDUs[i] = nil
}
oq.pendingEDUs = []*gomatrixserverlib.EDU{}
}
// cleanPendingInvites cleans out the pending invite buffer,
// removing all references so that the underlying objects can
// be GC'd.
func (oq *destinationQueue) cleanPendingInvites() {
for i := 0; i < len(oq.pendingInvites); i++ {
oq.pendingInvites[i] = nil
}
oq.pendingInvites = []*gomatrixserverlib.InviteV2Request{}
}
// nextTransaction creates a new transaction from the pending event
// queue and sends it. Returns true if a transaction was sent or
// false otherwise.
func (oq *destinationQueue) nextTransaction(
pendingPDUs []*gomatrixserverlib.HeaderedEvent,
pendingEDUs []*gomatrixserverlib.EDU,
sentCounter uint32,
) (bool, error) {
// Before we do anything, we need to roll over the transaction
// ID that is being used to coalesce events into the next TX.
// Otherwise it's possible that we'll pick up an incomplete
// transaction and end up nuking the rest of the events at the
// cleanup stage.
oq.transactionIDMutex.Lock()
oq.transactionID = ""
oq.transactionIDMutex.Unlock()
oq.transactionCount.Store(0)
// Create the transaction.
t := gomatrixserverlib.Transaction{
PDUs: []json.RawMessage{},
EDUs: []gomatrixserverlib.EDU{},
}
now := gomatrixserverlib.AsTimestamp(time.Now())
t.TransactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, sentCounter))
t.Origin = oq.origin
t.Destination = oq.destination
t.OriginServerTS = now
t.PreviousIDs = oq.lastTransactionIDs
if t.PreviousIDs == nil {
t.PreviousIDs = []gomatrixserverlib.TransactionID{}
t.OriginServerTS = gomatrixserverlib.AsTimestamp(time.Now())
// Ask the database for any pending PDUs from the next transaction.
// maxPDUsPerTransaction is an upper limit but we probably won't
// actually retrieve that many events.
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
txid, pdus, err := oq.db.GetNextTransactionPDUs(
ctx, // context
oq.destination, // server name
maxPDUsPerTransaction, // max events to retrieve
)
if err != nil {
log.WithError(err).Errorf("failed to get next transaction PDUs for server %q", oq.destination)
return false, fmt.Errorf("oq.db.GetNextTransactionPDUs: %w", err)
}
oq.lastTransactionIDs = []gomatrixserverlib.TransactionID{t.TransactionID}
// If we didn't get anything from the database and there are no
// pending EDUs then there's nothing to do - stop here.
if len(pdus) == 0 && len(pendingEDUs) == 0 {
return false, nil
}
for _, pdu := range pendingPDUs {
// Pick out the transaction ID from the database. If we didn't
// get a transaction ID (i.e. because there are no PDUs but only
// EDUs) then generate a transaction ID.
t.TransactionID = txid
if t.TransactionID == "" {
now := gomatrixserverlib.AsTimestamp(time.Now())
t.TransactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, oq.statistics.SuccessCount()))
}
// Go through PDUs that we retrieved from the database, if any,
// and add them into the transaction.
for _, pdu := range pdus {
// Append the JSON of the event, since this is a json.RawMessage type in the
// gomatrixserverlib.Transaction struct
t.PDUs = append(t.PDUs, (*pdu).JSON())
}
// Do the same for pending EDUS in the queue.
for _, edu := range pendingEDUs {
t.EDUs = append(t.EDUs, *edu)
}
logrus.WithField("server_name", oq.destination).Infof("Sending transaction %q containing %d PDUs, %d EDUs", t.TransactionID, len(t.PDUs), len(t.EDUs))
// Try to send the transaction to the destination server.
// TODO: we should check for 500-ish fails vs 400-ish here,
// since we shouldn't queue things indefinitely in response
// to a 400-ish error
_, err := oq.client.SendTransaction(context.TODO(), t)
_, err = oq.client.SendTransaction(context.TODO(), t)
switch e := err.(type) {
case nil:
// No error was returned so the transaction looks to have
// been successfully sent.
oq.pendingPDUs.Sub(int64(len(t.PDUs)))
// Clean up the transaction in the database.
if err = oq.db.CleanTransactionPDUs(
context.TODO(),
t.Destination,
t.TransactionID,
); err != nil {
log.WithError(err).Errorf("failed to clean transaction %q for server %q", t.TransactionID, t.Destination)
}
return true, nil
case gomatrix.HTTPError:
// We received a HTTP error back. In this instance we only

21
federationsender/queue/queue.go
View file

@ -15,10 +15,13 @@
package queue
import (
"context"
"crypto/ed25519"
"encoding/json"
"fmt"
"sync"
"github.com/matrix-org/dendrite/federationsender/storage"
"github.com/matrix-org/dendrite/federationsender/types"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib"
@ -29,6 +32,7 @@ import (
// OutgoingQueues is a collection of queues for sending transactions to other
// matrix servers
type OutgoingQueues struct {
db storage.Database
rsAPI api.RoomserverInternalAPI
origin gomatrixserverlib.ServerName
client *gomatrixserverlib.FederationClient
@ -40,6 +44,7 @@ type OutgoingQueues struct {
// NewOutgoingQueues makes a new OutgoingQueues
func NewOutgoingQueues(
db storage.Database,
origin gomatrixserverlib.ServerName,
client *gomatrixserverlib.FederationClient,
rsAPI api.RoomserverInternalAPI,
@ -47,6 +52,7 @@ func NewOutgoingQueues(
signing *SigningInfo,
) *OutgoingQueues {
return &OutgoingQueues{
db: db,
rsAPI: rsAPI,
origin: origin,
client: client,
@ -76,14 +82,15 @@ func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *d
oq := oqs.queues[destination]
if oq == nil {
oq = &destinationQueue{
db: oqs.db,
rsAPI: oqs.rsAPI,
origin: oqs.origin,
destination: destination,
client: oqs.client,
statistics: oqs.statistics.ForServer(destination),
incomingPDUs: make(chan *gomatrixserverlib.HeaderedEvent, 128),
incomingEDUs: make(chan *gomatrixserverlib.EDU, 128),
incomingInvites: make(chan *gomatrixserverlib.InviteV2Request, 128),
wakeServerCh: make(chan bool, 128),
retryServerCh: make(chan bool),
signing: oqs.signing,
}
@ -115,8 +122,18 @@ func (oqs *OutgoingQueues) SendEvent(
"destinations": destinations, "event": ev.EventID(),
}).Info("Sending event")
headeredJSON, err := json.Marshal(ev)
if err != nil {
return fmt.Errorf("json.Marshal: %w", err)
}
nid, err := oqs.db.StoreJSON(context.TODO(), string(headeredJSON))
if err != nil {
return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err)
}
for _, destination := range destinations {
oqs.getQueue(destination).sendEvent(ev)
oqs.getQueue(destination).sendEvent(nid)
}
return nil

6
federationsender/storage/interface.go
View file

@ -19,10 +19,16 @@ import (
"github.com/matrix-org/dendrite/federationsender/types"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/gomatrixserverlib"
)
type Database interface {
internal.PartitionStorer
UpdateRoom(ctx context.Context, roomID, oldEventID, newEventID string, addHosts []types.JoinedHost, removeHosts []string) (joinedHosts []types.JoinedHost, err error)
GetJoinedHosts(ctx context.Context, roomID string) ([]types.JoinedHost, error)
StoreJSON(ctx context.Context, js string) (int64, error)
AssociatePDUWithDestination(ctx context.Context, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, nids []int64) error
GetNextTransactionPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (gomatrixserverlib.TransactionID, []*gomatrixserverlib.HeaderedEvent, error)
CleanTransactionPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, transactionID gomatrixserverlib.TransactionID) error
GetPendingPDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error)
}

111
federationsender/storage/postgres/queue_json_table.go Normal file
View file

@ -0,0 +1,111 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgres
import (
"context"
"database/sql"
"github.com/lib/pq"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/sqlutil"
)
const queueJSONSchema = `
-- The federationsender_queue_json table contains event contents that
-- we failed to send.
CREATE TABLE IF NOT EXISTS federationsender_queue_json (
-- The JSON NID. This allows the federationsender_queue_retry table to
-- cross-reference to find the JSON blob.
json_nid BIGSERIAL,
-- The JSON body. Text so that we preserve UTF-8.
json_body TEXT NOT NULL
);
`
const insertJSONSQL = "" +
"INSERT INTO federationsender_queue_json (json_body)" +
" VALUES ($1)" +
" RETURNING json_nid"
const deleteJSONSQL = "" +
"DELETE FROM federationsender_queue_json WHERE json_nid = ANY($1)"
const selectJSONSQL = "" +
"SELECT json_nid, json_body FROM federationsender_queue_json" +
" WHERE json_nid = ANY($1)"
type queueJSONStatements struct {
insertJSONStmt *sql.Stmt
deleteJSONStmt *sql.Stmt
selectJSONStmt *sql.Stmt
}
func (s *queueJSONStatements) prepare(db *sql.DB) (err error) {
_, err = db.Exec(queueJSONSchema)
if err != nil {
return
}
if s.insertJSONStmt, err = db.Prepare(insertJSONSQL); err != nil {
return
}
if s.deleteJSONStmt, err = db.Prepare(deleteJSONSQL); err != nil {
return
}
if s.selectJSONStmt, err = db.Prepare(selectJSONSQL); err != nil {
return
}
return
}
func (s *queueJSONStatements) insertQueueJSON(
ctx context.Context, txn *sql.Tx, json string,
) (int64, error) {
stmt := sqlutil.TxStmt(txn, s.insertJSONStmt)
var lastid int64
if err := stmt.QueryRowContext(ctx, json).Scan(&lastid); err != nil {
return 0, err
}
return lastid, nil
}
func (s *queueJSONStatements) deleteQueueJSON(
ctx context.Context, txn *sql.Tx, nids []int64,
) error {
stmt := sqlutil.TxStmt(txn, s.deleteJSONStmt)
_, err := stmt.ExecContext(ctx, pq.Int64Array(nids))
return err
}
func (s *queueJSONStatements) selectQueueJSON(
ctx context.Context, txn *sql.Tx, jsonNIDs []int64,
) (map[int64][]byte, error) {
blobs := map[int64][]byte{}
stmt := sqlutil.TxStmt(txn, s.selectJSONStmt)
rows, err := stmt.QueryContext(ctx, pq.Int64Array(jsonNIDs))
if err != nil {
return nil, err
}
defer internal.CloseAndLogIfError(ctx, rows, "selectJSON: rows.close() failed")
for rows.Next() {
var nid int64
var blob []byte
if err = rows.Scan(&nid, &blob); err != nil {
return nil, err
}
blobs[nid] = blob
}
return blobs, err
}

192
federationsender/storage/postgres/queue_pdus_table.go Normal file
View file

@ -0,0 +1,192 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgres
import (
"context"
"database/sql"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/gomatrixserverlib"
)
const queuePDUsSchema = `
CREATE TABLE IF NOT EXISTS federationsender_queue_pdus (
-- The transaction ID that was generated before persisting the event.
transaction_id TEXT NOT NULL,
-- The destination server that we will send the event to.
server_name TEXT NOT NULL,
-- The JSON NID from the federationsender_queue_pdus_json table.
json_nid BIGINT NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_pdus_pdus_json_nid_idx
ON federationsender_queue_pdus (json_nid, server_name);
`
const insertQueuePDUSQL = "" +
"INSERT INTO federationsender_queue_pdus (transaction_id, server_name, json_nid)" +
" VALUES ($1, $2, $3)"
const deleteQueueTransactionPDUsSQL = "" +
"DELETE FROM federationsender_queue_pdus WHERE server_name = $1 AND transaction_id = $2"
const selectQueueNextTransactionIDSQL = "" +
"SELECT transaction_id FROM federationsender_queue_pdus" +
" WHERE server_name = $1" +
" ORDER BY transaction_id ASC" +
" LIMIT 1"
const selectQueuePDUsByTransactionSQL = "" +
"SELECT json_nid FROM federationsender_queue_pdus" +
" WHERE server_name = $1 AND transaction_id = $2" +
" LIMIT $3"
const selectQueueReferenceJSONCountSQL = "" +
"SELECT COUNT(*) FROM federationsender_queue_pdus" +
" WHERE json_nid = $1"
const selectQueuePDUsCountSQL = "" +
"SELECT COUNT(*) FROM federationsender_queue_pdus" +
" WHERE server_name = $1"
type queuePDUsStatements struct {
insertQueuePDUStmt *sql.Stmt
deleteQueueTransactionPDUsStmt *sql.Stmt
selectQueueNextTransactionIDStmt *sql.Stmt
selectQueuePDUsByTransactionStmt *sql.Stmt
selectQueueReferenceJSONCountStmt *sql.Stmt
selectQueuePDUsCountStmt *sql.Stmt
}
func (s *queuePDUsStatements) prepare(db *sql.DB) (err error) {
_, err = db.Exec(queuePDUsSchema)
if err != nil {
return
}
if s.insertQueuePDUStmt, err = db.Prepare(insertQueuePDUSQL); err != nil {
return
}
if s.deleteQueueTransactionPDUsStmt, err = db.Prepare(deleteQueueTransactionPDUsSQL); err != nil {
return
}
if s.selectQueueNextTransactionIDStmt, err = db.Prepare(selectQueueNextTransactionIDSQL); err != nil {
return
}
if s.selectQueuePDUsByTransactionStmt, err = db.Prepare(selectQueuePDUsByTransactionSQL); err != nil {
return
}
if s.selectQueueReferenceJSONCountStmt, err = db.Prepare(selectQueueReferenceJSONCountSQL); err != nil {
return
}
if s.selectQueuePDUsCountStmt, err = db.Prepare(selectQueuePDUsCountSQL); err != nil {
return
}
return
}
func (s *queuePDUsStatements) insertQueuePDU(
ctx context.Context,
txn *sql.Tx,
transactionID gomatrixserverlib.TransactionID,
serverName gomatrixserverlib.ServerName,
nid int64,
) error {
stmt := sqlutil.TxStmt(txn, s.insertQueuePDUStmt)
_, err := stmt.ExecContext(
ctx,
transactionID, // the transaction ID that we initially attempted
serverName, // destination server name
nid, // JSON blob NID
)
return err
}
func (s *queuePDUsStatements) deleteQueueTransaction(
ctx context.Context, txn *sql.Tx,
serverName gomatrixserverlib.ServerName,
transactionID gomatrixserverlib.TransactionID,
) error {
stmt := sqlutil.TxStmt(txn, s.deleteQueueTransactionPDUsStmt)
_, err := stmt.ExecContext(ctx, serverName, transactionID)
return err
}
func (s *queuePDUsStatements) selectQueueNextTransactionID(
ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
) (gomatrixserverlib.TransactionID, error) {
var transactionID gomatrixserverlib.TransactionID
stmt := sqlutil.TxStmt(txn, s.selectQueueNextTransactionIDStmt)
err := stmt.QueryRowContext(ctx, serverName).Scan(&transactionID)
if err == sql.ErrNoRows {
return "", nil
}
return transactionID, err
}
func (s *queuePDUsStatements) selectQueueReferenceJSONCount(
ctx context.Context, txn *sql.Tx, jsonNID int64,
) (int64, error) {
var count int64
stmt := sqlutil.TxStmt(txn, s.selectQueueReferenceJSONCountStmt)
err := stmt.QueryRowContext(ctx, jsonNID).Scan(&count)
if err == sql.ErrNoRows {
// It's acceptable for there to be no rows referencing a given
// JSON NID but it's not an error condition. Just return as if
// there's a zero count.
return 0, nil
}
return count, err
}
func (s *queuePDUsStatements) selectQueuePDUCount(
ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
) (int64, error) {
var count int64
stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsCountStmt)
err := stmt.QueryRowContext(ctx, serverName).Scan(&count)
if err == sql.ErrNoRows {
// It's acceptable for there to be no rows referencing a given
// JSON NID but it's not an error condition. Just return as if
// there's a zero count.
return 0, nil
}
return count, err
}
func (s *queuePDUsStatements) selectQueuePDUs(
ctx context.Context, txn *sql.Tx,
serverName gomatrixserverlib.ServerName,
transactionID gomatrixserverlib.TransactionID,
limit int,
) ([]int64, error) {
stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsByTransactionStmt)
rows, err := stmt.QueryContext(ctx, serverName, transactionID, limit)
if err != nil {
return nil, err
}
defer internal.CloseAndLogIfError(ctx, rows, "queueFromStmt: rows.close() failed")
var result []int64
for rows.Next() {
var nid int64
if err = rows.Scan(&nid); err != nil {
return nil, err
}
result = append(result, nid)
}
return result, rows.Err()
}

144
federationsender/storage/postgres/storage.go
View file

@ -18,15 +18,20 @@ package postgres
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"github.com/matrix-org/dendrite/federationsender/types"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/gomatrixserverlib"
)
// Database stores information needed by the federation sender
type Database struct {
joinedHostsStatements
roomStatements
queuePDUsStatements
queueJSONStatements
sqlutil.PartitionOffsetStatements
db *sql.DB
}
@ -55,6 +60,14 @@ func (d *Database) prepare() error {
return err
}
if err = d.queuePDUsStatements.prepare(d.db); err != nil {
return err
}
if err = d.queueJSONStatements.prepare(d.db); err != nil {
return err
}
return d.PartitionOffsetStatements.Prepare(d.db, "federationsender")
}
@ -120,3 +133,134 @@ func (d *Database) GetJoinedHosts(
) ([]types.JoinedHost, error) {
return d.selectJoinedHosts(ctx, roomID)
}
// StoreJSON adds a JSON blob into the queue JSON table and returns
// a NID. The NID will then be used when inserting the per-destination
// metadata entries.
func (d *Database) StoreJSON(
ctx context.Context, js string,
) (int64, error) {
nid, err := d.insertQueueJSON(ctx, nil, js)
if err != nil {
return 0, fmt.Errorf("d.insertQueueJSON: %w", err)
}
return nid, nil
}
// AssociatePDUWithDestination creates an association that the
// destination queues will use to determine which JSON blobs to send
// to which servers.
func (d *Database) AssociatePDUWithDestination(
ctx context.Context,
transactionID gomatrixserverlib.TransactionID,
serverName gomatrixserverlib.ServerName,
nids []int64,
) error {
return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
for _, nid := range nids {
if err := d.insertQueuePDU(
ctx, // context
txn, // SQL transaction
transactionID, // transaction ID
serverName, // destination server name
nid, // NID from the federationsender_queue_json table
); err != nil {
return fmt.Errorf("d.insertQueueRetryStmt.ExecContext: %w", err)
}
}
return nil
})
}
// GetNextTransactionPDUs retrieves events from the database for
// the next pending transaction, up to the limit specified.
func (d *Database) GetNextTransactionPDUs(
ctx context.Context,
serverName gomatrixserverlib.ServerName,
limit int,
) (
transactionID gomatrixserverlib.TransactionID,
events []*gomatrixserverlib.HeaderedEvent,
err error,
) {
err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
transactionID, err = d.selectQueueNextTransactionID(ctx, txn, serverName)
if err != nil {
return fmt.Errorf("d.selectQueueNextTransactionID: %w", err)
}
if transactionID == "" {
return nil
}
nids, err := d.selectQueuePDUs(ctx, txn, serverName, transactionID, limit)
if err != nil {
return fmt.Errorf("d.selectQueuePDUs: %w", err)
}
blobs, err := d.selectQueueJSON(ctx, txn, nids)
if err != nil {
return fmt.Errorf("d.selectJSON: %w", err)
}
for _, blob := range blobs {
var event gomatrixserverlib.HeaderedEvent
if err := json.Unmarshal(blob, &event); err != nil {
return fmt.Errorf("json.Unmarshal: %w", err)
}
events = append(events, &event)
}
return nil
})
return
}
// CleanTransactionPDUs cleans up all associated events for a
// given transaction. This is done when the transaction was sent
// successfully.
func (d *Database) CleanTransactionPDUs(
ctx context.Context,
serverName gomatrixserverlib.ServerName,
transactionID gomatrixserverlib.TransactionID,
) error {
return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
nids, err := d.selectQueuePDUs(ctx, txn, serverName, transactionID, 50)
if err != nil {
return fmt.Errorf("d.selectQueuePDUs: %w", err)
}
if err = d.deleteQueueTransaction(ctx, txn, serverName, transactionID); err != nil {
return fmt.Errorf("d.deleteQueueTransaction: %w", err)
}
var count int64
var deleteNIDs []int64
for _, nid := range nids {
count, err = d.selectQueueReferenceJSONCount(ctx, txn, nid)
if err != nil {
return fmt.Errorf("d.selectQueueReferenceJSONCount: %w", err)
}
if count == 0 {
deleteNIDs = append(deleteNIDs, nid)
}
}
if len(deleteNIDs) > 0 {
if err = d.deleteQueueJSON(ctx, txn, deleteNIDs); err != nil {
return fmt.Errorf("d.deleteQueueJSON: %w", err)
}
}
return nil
})
}
// GetPendingPDUCount returns the number of PDUs waiting to be
// sent for a given servername.
func (d *Database) GetPendingPDUCount(
ctx context.Context,
serverName gomatrixserverlib.ServerName,
) (int64, error) {
return d.selectQueuePDUCount(ctx, nil, serverName)
}

132
federationsender/storage/sqlite3/queue_json_table.go Normal file
View file

@ -0,0 +1,132 @@
// Copyright 2017-2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sqlite3
import (
"context"
"database/sql"
"fmt"
"strings"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/sqlutil"
)
const queueJSONSchema = `
-- The queue_retry_json table contains event contents that
-- we failed to send.
CREATE TABLE IF NOT EXISTS federationsender_queue_json (
-- The JSON NID. This allows the federationsender_queue_retry table to
-- cross-reference to find the JSON blob.
json_nid INTEGER PRIMARY KEY AUTOINCREMENT,
-- The JSON body. Text so that we preserve UTF-8.
json_body TEXT NOT NULL
);
`
const insertJSONSQL = "" +
"INSERT INTO federationsender_queue_json (json_body)" +
" VALUES ($1)"
const deleteJSONSQL = "" +
"DELETE FROM federationsender_queue_json WHERE json_nid IN ($1)"
const selectJSONSQL = "" +
"SELECT json_nid, json_body FROM federationsender_queue_json" +
" WHERE json_nid IN ($1)"
type queueJSONStatements struct {
insertJSONStmt *sql.Stmt
//deleteJSONStmt *sql.Stmt - prepared at runtime due to variadic
//selectJSONStmt *sql.Stmt - prepared at runtime due to variadic
}
func (s *queueJSONStatements) prepare(db *sql.DB) (err error) {
_, err = db.Exec(queueJSONSchema)
if err != nil {
return
}
if s.insertJSONStmt, err = db.Prepare(insertJSONSQL); err != nil {
return
}
return
}
func (s *queueJSONStatements) insertQueueJSON(
ctx context.Context, txn *sql.Tx, json string,
) (int64, error) {
stmt := sqlutil.TxStmt(txn, s.insertJSONStmt)
res, err := stmt.ExecContext(ctx, json)
if err != nil {
return 0, fmt.Errorf("stmt.QueryContext: %w", err)
}
lastid, err := res.LastInsertId()
if err != nil {
return 0, fmt.Errorf("res.LastInsertId: %w", err)
}
return lastid, nil
}
func (s *queueJSONStatements) deleteQueueJSON(
ctx context.Context, txn *sql.Tx, nids []int64,
) error {
deleteSQL := strings.Replace(deleteJSONSQL, "($1)", sqlutil.QueryVariadic(len(nids)), 1)
deleteStmt, err := txn.Prepare(deleteSQL)
if err != nil {
return fmt.Errorf("s.deleteQueueJSON s.db.Prepare: %w", err)
}
iNIDs := make([]interface{}, len(nids))
for k, v := range nids {
iNIDs[k] = v
}
stmt := sqlutil.TxStmt(txn, deleteStmt)
_, err = stmt.ExecContext(ctx, iNIDs...)
return err
}
func (s *queueJSONStatements) selectQueueJSON(
ctx context.Context, txn *sql.Tx, jsonNIDs []int64,
) (map[int64][]byte, error) {
selectSQL := strings.Replace(selectJSONSQL, "($1)", sqlutil.QueryVariadic(len(jsonNIDs)), 1)
selectStmt, err := txn.Prepare(selectSQL)
if err != nil {
return nil, fmt.Errorf("s.selectQueueJSON s.db.Prepare: %w", err)
}
iNIDs := make([]interface{}, len(jsonNIDs))
for k, v := range jsonNIDs {
iNIDs[k] = v
}
blobs := map[int64][]byte{}
stmt := sqlutil.TxStmt(txn, selectStmt)
rows, err := stmt.QueryContext(ctx, iNIDs...)
if err != nil {
return nil, fmt.Errorf("s.selectQueueJSON stmt.QueryContext: %w", err)
}
defer internal.CloseAndLogIfError(ctx, rows, "selectJSON: rows.close() failed")
for rows.Next() {
var nid int64
var blob []byte
if err = rows.Scan(&nid, &blob); err != nil {
return nil, fmt.Errorf("s.selectQueueJSON rows.Scan: %w", err)
}
blobs[nid] = blob
}
return blobs, err
}

190
federationsender/storage/sqlite3/queue_pdus_table.go Normal file
View file

@ -0,0 +1,190 @@
// Copyright 2017-2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sqlite3
import (
"context"
"database/sql"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/gomatrixserverlib"
)
const queuePDUsSchema = `
CREATE TABLE IF NOT EXISTS federationsender_queue_pdus (
-- The transaction ID that was generated before persisting the event.
transaction_id TEXT NOT NULL,
-- The domain part of the user ID the m.room.member event is for.
server_name TEXT NOT NULL,
-- The JSON NID from the federationsender_queue_pdus_json table.
json_nid BIGINT NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS federationsender_queue_pdus_pdus_json_nid_idx
ON federationsender_queue_pdus (json_nid, server_name);
`
const insertQueuePDUSQL = "" +
"INSERT INTO federationsender_queue_pdus (transaction_id, server_name, json_nid)" +
" VALUES ($1, $2, $3)"
const deleteQueueTransactionPDUsSQL = "" +
"DELETE FROM federationsender_queue_pdus WHERE server_name = $1 AND transaction_id = $2"
const selectQueueNextTransactionIDSQL = "" +
"SELECT transaction_id FROM federationsender_queue_pdus" +
" WHERE server_name = $1" +
" ORDER BY transaction_id ASC" +
" LIMIT 1"
const selectQueuePDUsByTransactionSQL = "" +
"SELECT json_nid FROM federationsender_queue_pdus" +
" WHERE server_name = $1 AND transaction_id = $2" +
" LIMIT $3"
const selectQueueReferenceJSONCountSQL = "" +
"SELECT COUNT(*) FROM federationsender_queue_pdus" +
" WHERE json_nid = $1"
const selectQueuePDUsCountSQL = "" +
"SELECT COUNT(*) FROM federationsender_queue_pdus" +
" WHERE server_name = $1"
type queuePDUsStatements struct {
insertQueuePDUStmt *sql.Stmt
deleteQueueTransactionPDUsStmt *sql.Stmt
selectQueueNextTransactionIDStmt *sql.Stmt
selectQueuePDUsByTransactionStmt *sql.Stmt
selectQueueReferenceJSONCountStmt *sql.Stmt
selectQueuePDUsCountStmt *sql.Stmt
}
func (s *queuePDUsStatements) prepare(db *sql.DB) (err error) {
_, err = db.Exec(queuePDUsSchema)
if err != nil {
return
}
if s.insertQueuePDUStmt, err = db.Prepare(insertQueuePDUSQL); err != nil {
return
}
if s.deleteQueueTransactionPDUsStmt, err = db.Prepare(deleteQueueTransactionPDUsSQL); err != nil {
return
}
if s.selectQueueNextTransactionIDStmt, err = db.Prepare(selectQueueNextTransactionIDSQL); err != nil {
return
}
if s.selectQueuePDUsByTransactionStmt, err = db.Prepare(selectQueuePDUsByTransactionSQL); err != nil {
return
}
if s.selectQueueReferenceJSONCountStmt, err = db.Prepare(selectQueueReferenceJSONCountSQL); err != nil {
return
}
if s.selectQueuePDUsCountStmt, err = db.Prepare(selectQueuePDUsCountSQL); err != nil {
return
}
return
}
func (s *queuePDUsStatements) insertQueuePDU(
ctx context.Context,
txn *sql.Tx,
transactionID gomatrixserverlib.TransactionID,
serverName gomatrixserverlib.ServerName,
nid int64,
) error {
stmt := sqlutil.TxStmt(txn, s.insertQueuePDUStmt)
_, err := stmt.ExecContext(
ctx,
transactionID, // the transaction ID that we initially attempted
serverName, // destination server name
nid, // JSON blob NID
)
return err
}
func (s *queuePDUsStatements) deleteQueueTransaction(
ctx context.Context, txn *sql.Tx,
serverName gomatrixserverlib.ServerName,
transactionID gomatrixserverlib.TransactionID,
) error {
stmt := sqlutil.TxStmt(txn, s.deleteQueueTransactionPDUsStmt)
_, err := stmt.ExecContext(ctx, serverName, transactionID)
return err
}
func (s *queuePDUsStatements) selectQueueNextTransactionID(
ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
) (gomatrixserverlib.TransactionID, error) {
var transactionID gomatrixserverlib.TransactionID
stmt := sqlutil.TxStmt(txn, s.selectQueueNextTransactionIDStmt)
err := stmt.QueryRowContext(ctx, serverName).Scan(&transactionID)
if err == sql.ErrNoRows {
return "", nil
}
return transactionID, err
}
func (s *queuePDUsStatements) selectQueueReferenceJSONCount(
ctx context.Context, txn *sql.Tx, jsonNID int64,
) (int64, error) {
var count int64
stmt := sqlutil.TxStmt(txn, s.selectQueueReferenceJSONCountStmt)
err := stmt.QueryRowContext(ctx, jsonNID).Scan(&count)
if err == sql.ErrNoRows {
return -1, nil
}
return count, err
}
func (s *queuePDUsStatements) selectQueuePDUCount(
ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName,
) (int64, error) {
var count int64
stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsCountStmt)
err := stmt.QueryRowContext(ctx, serverName).Scan(&count)
if err == sql.ErrNoRows {
// It's acceptable for there to be no rows referencing a given
// JSON NID but it's not an error condition. Just return as if
// there's a zero count.
return 0, nil
}
return count, err
}
func (s *queuePDUsStatements) selectQueuePDUs(
ctx context.Context, txn *sql.Tx,
serverName gomatrixserverlib.ServerName,
transactionID gomatrixserverlib.TransactionID,
limit int,
) ([]int64, error) {
stmt := sqlutil.TxStmt(txn, s.selectQueuePDUsByTransactionStmt)
rows, err := stmt.QueryContext(ctx, serverName, transactionID, limit)
if err != nil {
return nil, err
}
defer internal.CloseAndLogIfError(ctx, rows, "queueFromStmt: rows.close() failed")
var result []int64
for rows.Next() {
var nid int64
if err = rows.Scan(&nid); err != nil {
return nil, err
}
result = append(result, nid)
}
return result, rows.Err()
}

144
federationsender/storage/sqlite3/storage.go
View file

@ -18,17 +18,22 @@ package sqlite3
import (
"context"
"database/sql"
"encoding/json"
"fmt"
_ "github.com/mattn/go-sqlite3"
"github.com/matrix-org/dendrite/federationsender/types"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/gomatrixserverlib"
)
// Database stores information needed by the federation sender
type Database struct {
joinedHostsStatements
roomStatements
queuePDUsStatements
queueJSONStatements
sqlutil.PartitionOffsetStatements
db *sql.DB
}
@ -61,6 +66,14 @@ func (d *Database) prepare() error {
return err
}
if err = d.queuePDUsStatements.prepare(d.db); err != nil {
return err
}
if err = d.queueJSONStatements.prepare(d.db); err != nil {
return err
}
return d.PartitionOffsetStatements.Prepare(d.db, "federationsender")
}
@ -126,3 +139,134 @@ func (d *Database) GetJoinedHosts(
) ([]types.JoinedHost, error) {
return d.selectJoinedHosts(ctx, roomID)
}
// StoreJSON adds a JSON blob into the queue JSON table and returns
// a NID. The NID will then be used when inserting the per-destination
// metadata entries.
func (d *Database) StoreJSON(
ctx context.Context, js string,
) (int64, error) {
nid, err := d.insertQueueJSON(ctx, nil, js)
if err != nil {
return 0, fmt.Errorf("d.insertQueueJSON: %w", err)
}
return nid, nil
}
// AssociatePDUWithDestination creates an association that the
// destination queues will use to determine which JSON blobs to send
// to which servers.
func (d *Database) AssociatePDUWithDestination(
ctx context.Context,
transactionID gomatrixserverlib.TransactionID,
serverName gomatrixserverlib.ServerName,
nids []int64,
) error {
return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
for _, nid := range nids {
if err := d.insertQueuePDU(
ctx, // context
txn, // SQL transaction
transactionID, // transaction ID
serverName, // destination server name
nid, // NID from the federationsender_queue_json table
); err != nil {
return fmt.Errorf("d.insertQueueRetryStmt.ExecContext: %w", err)
}
}
return nil
})
}
// GetNextTransactionPDUs retrieves events from the database for
// the next pending transaction, up to the limit specified.
func (d *Database) GetNextTransactionPDUs(
ctx context.Context,
serverName gomatrixserverlib.ServerName,
limit int,
) (
transactionID gomatrixserverlib.TransactionID,
events []*gomatrixserverlib.HeaderedEvent,
err error,
) {
err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
transactionID, err = d.selectQueueNextTransactionID(ctx, txn, serverName)
if err != nil {
return fmt.Errorf("d.selectQueueNextTransactionID: %w", err)
}
if transactionID == "" {
return nil
}
nids, err := d.selectQueuePDUs(ctx, txn, serverName, transactionID, limit)
if err != nil {
return fmt.Errorf("d.selectQueuePDUs: %w", err)
}
blobs, err := d.selectQueueJSON(ctx, txn, nids)
if err != nil {
return fmt.Errorf("d.selectJSON: %w", err)
}
for _, blob := range blobs {
var event gomatrixserverlib.HeaderedEvent
if err := json.Unmarshal(blob, &event); err != nil {
return fmt.Errorf("json.Unmarshal: %w", err)
}
events = append(events, &event)
}
return nil
})
return
}
// CleanTransactionPDUs cleans up all associated events for a
// given transaction. This is done when the transaction was sent
// successfully.
func (d *Database) CleanTransactionPDUs(
ctx context.Context,
serverName gomatrixserverlib.ServerName,
transactionID gomatrixserverlib.TransactionID,
) error {
return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error {
nids, err := d.selectQueuePDUs(ctx, txn, serverName, transactionID, 50)
if err != nil {
return fmt.Errorf("d.selectQueuePDUs: %w", err)
}
if err = d.deleteQueueTransaction(ctx, txn, serverName, transactionID); err != nil {
return fmt.Errorf("d.deleteQueueTransaction: %w", err)
}
var count int64
var deleteNIDs []int64
for _, nid := range nids {
count, err = d.selectQueueReferenceJSONCount(ctx, txn, nid)
if err != nil {
return fmt.Errorf("d.selectQueueReferenceJSONCount: %w", err)
}
if count == 0 {
deleteNIDs = append(deleteNIDs, nid)
}
}
if len(deleteNIDs) > 0 {
if err = d.deleteQueueJSON(ctx, txn, deleteNIDs); err != nil {
return fmt.Errorf("d.deleteQueueJSON: %w", err)
}
}
return nil
})
}
// GetPendingPDUCount returns the number of PDUs waiting to be
// sent for a given servername.
func (d *Database) GetPendingPDUCount(
ctx context.Context,
serverName gomatrixserverlib.ServerName,
) (int64, error) {
return d.selectQueuePDUCount(ctx, nil, serverName)
}

11
go.mod
View file

@ -3,6 +3,8 @@ module github.com/matrix-org/dendrite
require (
github.com/Shopify/sarama v1.26.1
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/go-metrics v0.0.1 // indirect
github.com/gologme/log v1.2.0
github.com/gorilla/mux v1.7.3
github.com/hashicorp/golang-lru v0.5.4
@ -15,12 +17,13 @@ require (
github.com/libp2p/go-libp2p-kad-dht v0.5.0
github.com/libp2p/go-libp2p-pubsub v0.2.5
github.com/libp2p/go-libp2p-record v0.1.2
github.com/libp2p/go-yamux v1.3.7
github.com/libp2p/go-yamux v1.3.7 // indirect
github.com/lucas-clemente/quic-go v0.17.2
github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5
github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
github.com/matrix-org/gomatrixserverlib v0.0.0-20200623103809-13ff8109e137
github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe
github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f
github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
github.com/mattn/go-sqlite3 v2.0.2+incompatible
@ -35,9 +38,9 @@ require (
github.com/uber-go/atomic v1.3.0 // indirect
github.com/uber/jaeger-client-go v2.15.0+incompatible
github.com/uber/jaeger-lib v1.5.0
github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b
github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200702163833-11ecfa688d93
go.uber.org/atomic v1.4.0
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d
golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5
gopkg.in/h2non/bimg.v1 v1.0.18
gopkg.in/yaml.v2 v2.2.8
)

156
go.sum
View file

@ -1,4 +1,12 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0 h1:p3puK8Sl2xK+2FnnIvY/C0N1aqJo2kbEsdAzU+Tnv48=
@ -17,11 +25,13 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
@ -35,8 +45,11 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
github.com/cheggaaa/pb/v3 v3.0.4/go.mod h1:7rgWxLrAUcFMkvJuv09+DYi7mMUYi8nO9iOWcvGJPfw=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
@ -46,6 +59,7 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
@ -59,6 +73,10 @@ github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhY
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8=
github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
@ -67,14 +85,20 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k=
github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk=
github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
@ -88,18 +112,29 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg=
github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
github.com/gologme/log v1.2.0 h1:Ya5Ip/KD6FX7uH0S31QO87nCCSucKtF44TLbTtO7V4c=
github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
@ -107,13 +142,22 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
@ -185,11 +229,14 @@ github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr1
github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0/go.mod h1:rUi0/YffDo1oXBOGn1KRq7Fr07LX48XEBecQnmwjsAo=
@ -215,6 +262,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
@ -356,11 +404,18 @@ github.com/libp2p/go-yamux v1.3.0 h1:FsYzT16Wq2XqUGJsBbOxoz9g+dFklvNi7jN6YFPfl7U
github.com/libp2p/go-yamux v1.3.0/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
github.com/libp2p/go-yamux v1.3.7 h1:v40A1eSPJDIZwz2AvrV3cxpTZEGDP11QJbukmEhYyQI=
github.com/libp2p/go-yamux v1.3.7/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE=
github.com/lucas-clemente/quic-go v0.17.2 h1:4iQInIuNQkPNZmsy9rCnwuOzpH0qGnDo4jn0QfI/qE4=
github.com/lucas-clemente/quic-go v0.17.2/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE=
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
github.com/lxn/walk v0.0.0-20191128110447-55ccb3a9f5c1/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ=
github.com/lxn/win v0.0.0-20191128105842-2da648fda5b4/go.mod h1:ouWl4wViUNh8tPSIwxTVMuS014WakR1hqvBc2I0bMoA=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
github.com/marten-seemann/qtls v0.9.1 h1:O0YKQxNVPaiFgMng0suWEOY2Sb4LT2sRn9Qimq3Z1IQ=
github.com/marten-seemann/qtls v0.9.1/go.mod h1:T1MmAdDPyISzxlK6kjRr0pcZFBVd1OZbBb/j3cvzHhk=
github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY=
github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY=
github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg=
@ -371,8 +426,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf
github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4=
github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
github.com/matrix-org/gomatrixserverlib v0.0.0-20200623103809-13ff8109e137 h1:+eBh4L04+08IslvFM071TNrQTggU317GsQKzZ1SGEVo=
github.com/matrix-org/gomatrixserverlib v0.0.0-20200623103809-13ff8109e137/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe h1:rCjG+azihYsO+EIdm//Zx5gQ7hzeJVraeSukLsW1Mic=
github.com/matrix-org/gomatrixserverlib v0.0.0-20200630110352-4948932681fe/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y=
github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go=
github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=
@ -392,6 +447,7 @@ github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
github.com/miekg/dns v1.1.12 h1:WMhc1ik4LNkTg8U9l3hI1LvxKmIL+f1+WV/SZtCbDDA=
github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
@ -457,6 +513,8 @@ github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY=
github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 h1:evlcQnJY+v8XRRchV3hXzpHDl6GcEZeLXAhlH9Csdww=
@ -465,18 +523,21 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU=
github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg=
github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg=
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg=
github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
@ -487,30 +548,61 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8=
github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U=
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ=
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY=
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0=
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU=
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc=
@ -530,6 +622,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc=
github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
@ -546,6 +639,8 @@ github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW
github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo=
github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=
@ -565,8 +660,9 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhe
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yggdrasil-network/yggdrasil-extras v0.0.0-20200525205615-6c8a4a2e8855/go.mod h1:xQdsh08Io6nV4WRnOVTe6gI8/2iTvfLDQ0CYa5aMt+I=
github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b h1:ELOisSxFXCcptRs4LFub+Hz5fYUvV12wZrTps99Eb3E=
github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b/go.mod h1:d+Nz6SPeG6kmeSPFL0cvfWfgwEql75fUnZiAONgvyBE=
github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200702163833-11ecfa688d93 h1:DX2HXQHoejo9GqkvFuRS9iHrjhfv/9WgL3TjmUz/AaY=
github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200702163833-11ecfa688d93/go.mod h1:d+Nz6SPeG6kmeSPFL0cvfWfgwEql75fUnZiAONgvyBE=
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@ -578,14 +674,18 @@ go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@ -596,7 +696,10 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw=
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 h1:Q7tZBpemrlsc2I7IyODzhtallWRSm4Q0d09pL6XbQtU=
golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@ -604,10 +707,15 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
@ -619,6 +727,10 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -631,17 +743,20 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191003212358-c178f38b412c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -656,19 +771,26 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200301040627-c5d0d7b4ec88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3-0.20191230102452-929e72ca90de h1:aYKJLPSrddB2N7/6OKyFqJ337SXpo61bBuvO5p1+7iY=
golang.org/x/text v0.3.3-0.20191230102452-929e72ca90de/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
@ -676,14 +798,32 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
golang.zx2c4.com/wireguard v0.0.20200122-0.20200214175355-9cbcff10dd3e/go.mod h1:P2HsVp8SKwZEufsnezXZA4GRX/T49/HlU7DGuelXsU4=
golang.zx2c4.com/wireguard v0.0.20200320/go.mod h1:lDian4Sw4poJ04SgHh35nzMVwGSYlPumkdnHcucAQoY=
golang.zx2c4.com/wireguard/windows v0.1.0/go.mod h1:EK7CxrFnicmYJ0ZCF6crBh2/EMMeSxMlqgLlwN0Kv9s=
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
@ -699,6 +839,7 @@ gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4=
gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So=
gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM=
gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw=
gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM=
@ -723,5 +864,12 @@ gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099 h1:XJP7lxbSxWLOMNdBE4B/STaqVy6L73o0knwj2vIlxnw=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=

27
internal/config/config.go
View file

@ -160,10 +160,13 @@ type Dendrite struct {
// Postgres Config
Database struct {
// The Account database stores the login details and account information
// for local users. It is accessed by the ClientAPI.
// for local users. It is accessed by the UserAPI.
Account DataSource `yaml:"account"`
// The CurrentState database stores the current state of all rooms.
// It is accessed by the CurrentStateServer.
CurrentState DataSource `yaml:"current_state"`
// The Device database stores session information for the devices of logged
// in local users. It is accessed by the ClientAPI, the MediaAPI and the SyncAPI.
// in local users. It is accessed by the UserAPI.
Device DataSource `yaml:"device"`
// The MediaAPI database stores information about files uploaded and downloaded
// by local users. It is only accessed by the MediaAPI.
@ -183,9 +186,6 @@ type Dendrite struct {
// The AppServices database stores information used by the AppService component.
// It is only accessed by the AppService component.
AppService DataSource `yaml:"appservice"`
// The PublicRoomsAPI database stores information used to compute the public
// room directory. It is only accessed by the PublicRoomsAPI server.
PublicRoomsAPI DataSource `yaml:"public_rooms_api"`
// The Naffka database is used internally by the naffka library, if used.
Naffka DataSource `yaml:"naffka,omitempty"`
// Maximum open connections to the DB (0 = use default, negative means unlimited)
@ -222,6 +222,7 @@ type Dendrite struct {
Bind struct {
MediaAPI Address `yaml:"media_api"`
ClientAPI Address `yaml:"client_api"`
CurrentState Address `yaml:"current_state_server"`
FederationAPI Address `yaml:"federation_api"`
ServerKeyAPI Address `yaml:"server_key_api"`
AppServiceAPI Address `yaml:"appservice_api"`
@ -229,7 +230,6 @@ type Dendrite struct {
UserAPI Address `yaml:"user_api"`
RoomServer Address `yaml:"room_server"`
FederationSender Address `yaml:"federation_sender"`
PublicRoomsAPI Address `yaml:"public_rooms_api"`
EDUServer Address `yaml:"edu_server"`
KeyServer Address `yaml:"key_server"`
} `yaml:"bind"`
@ -238,6 +238,7 @@ type Dendrite struct {
Listen struct {
MediaAPI Address `yaml:"media_api"`
ClientAPI Address `yaml:"client_api"`
CurrentState Address `yaml:"current_state_server"`
FederationAPI Address `yaml:"federation_api"`
ServerKeyAPI Address `yaml:"server_key_api"`
AppServiceAPI Address `yaml:"appservice_api"`
@ -245,7 +246,6 @@ type Dendrite struct {
UserAPI Address `yaml:"user_api"`
RoomServer Address `yaml:"room_server"`
FederationSender Address `yaml:"federation_sender"`
PublicRoomsAPI Address `yaml:"public_rooms_api"`
EDUServer Address `yaml:"edu_server"`
KeyServer Address `yaml:"key_server"`
} `yaml:"listen"`
@ -601,6 +601,7 @@ func (config *Dendrite) checkDatabase(configErrs *configErrors) {
checkNotEmpty(configErrs, "database.media_api", string(config.Database.MediaAPI))
checkNotEmpty(configErrs, "database.sync_api", string(config.Database.SyncAPI))
checkNotEmpty(configErrs, "database.room_server", string(config.Database.RoomServer))
checkNotEmpty(configErrs, "database.current_state", string(config.Database.CurrentState))
}
// checkListen verifies the parameters listen.* are valid.
@ -613,6 +614,7 @@ func (config *Dendrite) checkListen(configErrs *configErrors) {
checkNotEmpty(configErrs, "listen.edu_server", string(config.Listen.EDUServer))
checkNotEmpty(configErrs, "listen.server_key_api", string(config.Listen.EDUServer))
checkNotEmpty(configErrs, "listen.user_api", string(config.Listen.UserAPI))
checkNotEmpty(configErrs, "listen.current_state_server", string(config.Listen.CurrentState))
}
// checkLogging verifies the parameters logging.* are valid.
@ -735,6 +737,15 @@ func (config *Dendrite) UserAPIURL() string {
return "http://" + string(config.Listen.UserAPI)
}
// CurrentStateAPIURL returns an HTTP URL for where the currentstateserver is listening.
func (config *Dendrite) CurrentStateAPIURL() string {
// Hard code the currentstateserver to talk HTTP for now.
// If we support HTTPS we need to think of a practical way to do certificate validation.
// People setting up servers shouldn't need to get a certificate valid for the public
// internet for an internal API.
return "http://" + string(config.Listen.CurrentState)
}
// EDUServerURL returns an HTTP URL for where the EDU server is listening.
func (config *Dendrite) EDUServerURL() string {
// Hard code the EDU server to talk HTTP for now.
@ -753,7 +764,7 @@ func (config *Dendrite) FederationSenderURL() string {
return "http://" + string(config.Listen.FederationSender)
}
// FederationSenderURL returns an HTTP URL for where the federation sender is listening.
// ServerKeyAPIURL returns an HTTP URL for where the federation sender is listening.
func (config *Dendrite) ServerKeyAPIURL() string {
// Hard code the server key API server to talk HTTP for now.
// If we support HTTPS we need to think of a practical way to do certificate validation.

2
internal/config/config_test.go
View file

@ -55,6 +55,7 @@ database:
sync_api: "postgresql:///syn_api"
room_server: "postgresql:///room_server"
appservice: "postgresql:///appservice"
current_state: "postgresql:///current_state"
listen:
room_server: "localhost:7770"
client_api: "localhost:7771"
@ -64,6 +65,7 @@ listen:
appservice_api: "localhost:7777"
edu_server: "localhost:7778"
user_api: "localhost:7779"
current_state_server: "localhost:7775"
logging:
- type: "file"
level: "info"

11
internal/setup/base.go
View file

@ -22,6 +22,7 @@ import (
"net/url"
"time"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/sqlutil"
@ -37,6 +38,7 @@ import (
appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
asinthttp "github.com/matrix-org/dendrite/appservice/inthttp"
currentstateinthttp "github.com/matrix-org/dendrite/currentstateserver/inthttp"
eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
eduinthttp "github.com/matrix-org/dendrite/eduserver/inthttp"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
@ -171,6 +173,15 @@ func (b *BaseDendrite) UserAPIClient() userapi.UserInternalAPI {
return userAPI
}
// CurrentStateAPIClient returns CurrentStateInternalAPI for hitting the currentstateserver over HTTP.
func (b *BaseDendrite) CurrentStateAPIClient() currentstateAPI.CurrentStateInternalAPI {
stateAPI, err := currentstateinthttp.NewCurrentStateAPIClient(b.Cfg.CurrentStateAPIURL(), b.httpClient)
if err != nil {
logrus.WithError(err).Panic("UserAPIClient failed", b.httpClient)
}
return stateAPI
}
// EDUServerClient returns EDUServerInputAPI for hitting the EDU server over HTTP
func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI {
e, err := eduinthttp.NewEDUServerClient(b.Cfg.EDUServerURL(), b.httpClient)

25
internal/setup/monolith.go
View file

@ -19,6 +19,8 @@ import (
"github.com/gorilla/mux"
appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/clientapi"
"github.com/matrix-org/dendrite/clientapi/api"
currentstateAPI "github.com/matrix-org/dendrite/currentstateserver/api"
eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
"github.com/matrix-org/dendrite/federationapi"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
@ -26,9 +28,6 @@ import (
"github.com/matrix-org/dendrite/internal/transactions"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/mediaapi"
"github.com/matrix-org/dendrite/publicroomsapi"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/matrix-org/dendrite/publicroomsapi/types"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api"
"github.com/matrix-org/dendrite/syncapi"
@ -56,36 +55,28 @@ type Monolith struct {
RoomserverAPI roomserverAPI.RoomserverInternalAPI
ServerKeyAPI serverKeyAPI.ServerKeyInternalAPI
UserAPI userapi.UserInternalAPI
// TODO: can we remove this? It's weird that we are required the database
// yet every other component can do that on its own. libp2p-demo uses a custom
// database though annoyingly.
PublicRoomsDB storage.Database
StateAPI currentstateAPI.CurrentStateInternalAPI
// Optional
ExtPublicRoomsProvider types.ExternalPublicRoomsProvider
ExtPublicRoomsProvider api.ExternalPublicRoomsProvider
}
// AddAllPublicRoutes attaches all public paths to the given router
func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) {
clientapi.AddPublicRoutes(
publicMux, m.Config, m.KafkaConsumer, m.KafkaProducer, m.DeviceDB, m.AccountDB,
publicMux, m.Config, m.KafkaProducer, m.DeviceDB, m.AccountDB,
m.FedClient, m.RoomserverAPI,
m.EDUInternalAPI, m.AppserviceAPI, transactions.New(),
m.FederationSenderAPI, m.UserAPI,
m.EDUInternalAPI, m.AppserviceAPI, m.StateAPI, transactions.New(),
m.FederationSenderAPI, m.UserAPI, m.ExtPublicRoomsProvider,
)
keyserver.AddPublicRoutes(publicMux, m.Config, m.UserAPI)
federationapi.AddPublicRoutes(
publicMux, m.Config, m.UserAPI, m.FedClient,
m.KeyRing, m.RoomserverAPI, m.FederationSenderAPI,
m.EDUInternalAPI,
m.EDUInternalAPI, m.StateAPI,
)
mediaapi.AddPublicRoutes(publicMux, m.Config, m.UserAPI, m.Client)
publicroomsapi.AddPublicRoutes(
publicMux, m.Config, m.KafkaConsumer, m.UserAPI, m.PublicRoomsDB, m.RoomserverAPI, m.FedClient,
m.ExtPublicRoomsProvider,
)
syncapi.AddPublicRoutes(
publicMux, m.KafkaConsumer, m.UserAPI, m.RoomserverAPI, m.FedClient, m.Config,
)

6
internal/test/config.go
View file

@ -96,7 +96,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con
cfg.Database.RoomServer = config.DataSource(database)
cfg.Database.ServerKey = config.DataSource(database)
cfg.Database.SyncAPI = config.DataSource(database)
cfg.Database.PublicRoomsAPI = config.DataSource(database)
cfg.Database.CurrentState = config.DataSource(database)
cfg.Listen.ClientAPI = assignAddress()
cfg.Listen.AppServiceAPI = assignAddress()
@ -104,7 +104,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con
cfg.Listen.MediaAPI = assignAddress()
cfg.Listen.RoomServer = assignAddress()
cfg.Listen.SyncAPI = assignAddress()
cfg.Listen.PublicRoomsAPI = assignAddress()
cfg.Listen.CurrentState = assignAddress()
cfg.Listen.EDUServer = assignAddress()
// Bind to the same address as the listen address
@ -115,7 +115,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con
cfg.Bind.MediaAPI = cfg.Listen.MediaAPI
cfg.Bind.RoomServer = cfg.Listen.RoomServer
cfg.Bind.SyncAPI = cfg.Listen.SyncAPI
cfg.Bind.PublicRoomsAPI = cfg.Listen.PublicRoomsAPI
cfg.Bind.CurrentState = cfg.Listen.CurrentState
cfg.Bind.EDUServer = cfg.Listen.EDUServer
return &cfg, port, nil

1
internal/test/server.go
View file

@ -99,7 +99,6 @@ func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) {
"--sync-api-server-url", "http://" + string(cfg.Listen.SyncAPI),
"--client-api-server-url", "http://" + string(cfg.Listen.ClientAPI),
"--media-api-server-url", "http://" + string(cfg.Listen.MediaAPI),
"--public-rooms-api-server-url", "http://" + string(cfg.Listen.PublicRoomsAPI),
"--tls-cert", "server.crt",
"--tls-key", "server.key",
}

5
publicroomsapi/README.md
View file

@ -1,5 +0,0 @@
# Public rooms API
This server is responsible for serving requests hitting `/publicRooms` and `/directory/list/room/{roomID}` as per:
https://matrix.org/docs/spec/client_server/r0.2.0.html#listing-rooms

99
publicroomsapi/consumers/roomserver.go
View file

@ -1,99 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package consumers
import (
"context"
"encoding/json"
"github.com/Shopify/sarama"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib"
log "github.com/sirupsen/logrus"
)
// OutputRoomEventConsumer consumes events that originated in the room server.
type OutputRoomEventConsumer struct {
rsAPI api.RoomserverInternalAPI
rsConsumer *internal.ContinualConsumer
db storage.Database
}
// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
func NewOutputRoomEventConsumer(
cfg *config.Dendrite,
kafkaConsumer sarama.Consumer,
store storage.Database,
rsAPI api.RoomserverInternalAPI,
) *OutputRoomEventConsumer {
consumer := internal.ContinualConsumer{
Topic: string(cfg.Kafka.Topics.OutputRoomEvent),
Consumer: kafkaConsumer,
PartitionStore: store,
}
s := &OutputRoomEventConsumer{
rsConsumer: &consumer,
db: store,
rsAPI: rsAPI,
}
consumer.ProcessMessage = s.onMessage
return s
}
// Start consuming from room servers
func (s *OutputRoomEventConsumer) Start() error {
return s.rsConsumer.Start()
}
// onMessage is called when the sync server receives a new event from the room server output log.
func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
// Parse out the event JSON
var output api.OutputEvent
if err := json.Unmarshal(msg.Value, &output); err != nil {
// If the message was invalid, log it and move on to the next message in the stream
log.WithError(err).Errorf("roomserver output log: message parse failure")
return nil
}
if output.Type != api.OutputTypeNewRoomEvent {
log.WithField("type", output.Type).Debug(
"roomserver output log: ignoring unknown output type",
)
return nil
}
var remQueryRes api.QueryEventsByIDResponse
if len(output.NewRoomEvent.RemovesStateEventIDs) > 0 {
remQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.RemovesStateEventIDs}
if err := s.rsAPI.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil {
log.Warn(err)
return err
}
}
var addQueryEvents, remQueryEvents []gomatrixserverlib.Event
for _, headeredEvent := range output.NewRoomEvent.AddsState() {
addQueryEvents = append(addQueryEvents, headeredEvent.Event)
}
for _, headeredEvent := range remQueryRes.Events {
remQueryEvents = append(remQueryEvents, headeredEvent.Event)
}
return s.db.UpdateRoomFromEvents(context.TODO(), addQueryEvents, remQueryEvents)
}

120
publicroomsapi/directory/directory.go
View file

@ -1,120 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package directory
import (
"net/http"
"github.com/matrix-org/dendrite/roomserver/api"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
type roomVisibility struct {
Visibility string `json:"visibility"`
}
// GetVisibility implements GET /directory/list/room/{roomID}
func GetVisibility(
req *http.Request, publicRoomsDatabase storage.Database,
roomID string,
) util.JSONResponse {
isPublic, err := publicRoomsDatabase.GetRoomVisibility(req.Context(), roomID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("publicRoomsDatabase.GetRoomVisibility failed")
return jsonerror.InternalServerError()
}
var v roomVisibility
if isPublic {
v.Visibility = gomatrixserverlib.Public
} else {
v.Visibility = "private"
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: v,
}
}
// SetVisibility implements PUT /directory/list/room/{roomID}
// TODO: Allow admin users to edit the room visibility
func SetVisibility(
req *http.Request, publicRoomsDatabase storage.Database, rsAPI api.RoomserverInternalAPI, dev *userapi.Device,
roomID string,
) util.JSONResponse {
queryMembershipReq := api.QueryMembershipForUserRequest{
RoomID: roomID,
UserID: dev.UserID,
}
var queryMembershipRes api.QueryMembershipForUserResponse
err := rsAPI.QueryMembershipForUser(req.Context(), &queryMembershipReq, &queryMembershipRes)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("could not query membership for user")
return jsonerror.InternalServerError()
}
// Check if user id is in room
if !queryMembershipRes.IsInRoom {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("user does not belong to room"),
}
}
queryEventsReq := api.QueryLatestEventsAndStateRequest{
RoomID: roomID,
StateToFetch: []gomatrixserverlib.StateKeyTuple{{
EventType: gomatrixserverlib.MRoomPowerLevels,
StateKey: "",
}},
}
var queryEventsRes api.QueryLatestEventsAndStateResponse
err = rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes)
if err != nil || len(queryEventsRes.StateEvents) == 0 {
util.GetLogger(req.Context()).WithError(err).Error("could not query events from room")
return jsonerror.InternalServerError()
}
// NOTSPEC: Check if the user's power is greater than power required to change m.room.aliases event
power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event)
if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomAliases, true) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("userID doesn't have power level to change visibility"),
}
}
var v roomVisibility
if reqErr := httputil.UnmarshalJSONRequest(req, &v); reqErr != nil {
return *reqErr
}
isPublic := v.Visibility == gomatrixserverlib.Public
if err := publicRoomsDatabase.SetRoomVisibility(req.Context(), isPublic, roomID); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("publicRoomsDatabase.SetRoomVisibility failed")
return jsonerror.InternalServerError()
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
}
}

51
publicroomsapi/publicroomsapi.go
View file

@ -1,51 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package publicroomsapi
import (
"github.com/Shopify/sarama"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/publicroomsapi/consumers"
"github.com/matrix-org/dendrite/publicroomsapi/routing"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/matrix-org/dendrite/publicroomsapi/types"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
)
// AddPublicRoutes sets up and registers HTTP handlers for the PublicRoomsAPI
// component.
func AddPublicRoutes(
router *mux.Router,
cfg *config.Dendrite,
consumer sarama.Consumer,
userAPI userapi.UserInternalAPI,
publicRoomsDB storage.Database,
rsAPI roomserverAPI.RoomserverInternalAPI,
fedClient *gomatrixserverlib.FederationClient,
extRoomsProvider types.ExternalPublicRoomsProvider,
) {
rsConsumer := consumers.NewOutputRoomEventConsumer(
cfg, consumer, publicRoomsDB, rsAPI,
)
if err := rsConsumer.Start(); err != nil {
logrus.WithError(err).Panic("failed to start public rooms server consumer")
}
routing.Setup(router, userAPI, publicRoomsDB, rsAPI, fedClient, extRoomsProvider)
}

79
publicroomsapi/routing/routing.go
View file

@ -1,79 +0,0 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package routing
import (
"net/http"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/roomserver/api"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/publicroomsapi/directory"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/matrix-org/dendrite/publicroomsapi/types"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
const pathPrefixR0 = "/client/r0"
// Setup configures the given mux with publicroomsapi server listeners
//
// Due to Setup being used to call many other functions, a gocyclo nolint is
// applied:
// nolint: gocyclo
func Setup(
publicAPIMux *mux.Router, userAPI userapi.UserInternalAPI, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI,
fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider,
) {
r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter()
r0mux.Handle("/directory/list/room/{roomID}",
httputil.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return directory.GetVisibility(req, publicRoomsDB, vars["roomID"])
}),
).Methods(http.MethodGet, http.MethodOptions)
// TODO: Add AS support
r0mux.Handle("/directory/list/room/{roomID}",
httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return directory.SetVisibility(req, publicRoomsDB, rsAPI, device, vars["roomID"])
}),
).Methods(http.MethodPut, http.MethodOptions)
r0mux.Handle("/publicRooms",
httputil.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse {
if extRoomsProvider != nil {
return directory.GetPostPublicRoomsWithExternal(req, publicRoomsDB, fedClient, extRoomsProvider)
}
return directory.GetPostPublicRooms(req, publicRoomsDB)
}),
).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
// Federation - TODO: should this live here or in federation API? It's sure easier if it's here so here it is.
publicAPIMux.Handle("/federation/v1/publicRooms",
httputil.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse {
return directory.GetPostPublicRooms(req, publicRoomsDB)
}),
).Methods(http.MethodGet)
}

32
publicroomsapi/storage/interface.go
View file

@ -1,32 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"context"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/gomatrixserverlib"
)
type Database interface {
internal.PartitionStorer
GetRoomVisibility(ctx context.Context, roomID string) (bool, error)
SetRoomVisibility(ctx context.Context, visible bool, roomID string) error
CountPublicRooms(ctx context.Context) (int64, error)
GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error)
UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error
UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error
}

36
publicroomsapi/storage/postgres/prepare.go
View file

@ -1,36 +0,0 @@
// Copyright 2017-2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgres
import (
"database/sql"
)
// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement.
type statementList []struct {
statement **sql.Stmt
sql string
}
// prepare the SQL for each statement in the list and assign the result to the prepared statement.
func (s statementList) prepare(db *sql.DB) (err error) {
for _, statement := range s {
if *statement.statement, err = db.Prepare(statement.sql); err != nil {
return
}
}
return
}

280
publicroomsapi/storage/postgres/public_rooms_table.go
View file

@ -1,280 +0,0 @@
// Copyright 2017-2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgres
import (
"context"
"database/sql"
"errors"
"fmt"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/gomatrixserverlib"
"github.com/lib/pq"
)
var editableAttributes = []string{
"aliases",
"canonical_alias",
"name",
"topic",
"world_readable",
"guest_can_join",
"avatar_url",
"visibility",
}
const publicRoomsSchema = `
-- Stores all of the rooms with data needed to create the server's room directory
CREATE TABLE IF NOT EXISTS publicroomsapi_public_rooms(
-- The room's ID
room_id TEXT NOT NULL PRIMARY KEY,
-- Number of joined members in the room
joined_members INTEGER NOT NULL DEFAULT 0,
-- Aliases of the room (empty array if none)
aliases TEXT[] NOT NULL DEFAULT '{}'::TEXT[],
-- Canonical alias of the room (empty string if none)
canonical_alias TEXT NOT NULL DEFAULT '',
-- Name of the room (empty string if none)
name TEXT NOT NULL DEFAULT '',
-- Topic of the room (empty string if none)
topic TEXT NOT NULL DEFAULT '',
-- Is the room world readable?
world_readable BOOLEAN NOT NULL DEFAULT false,
-- Can guest join the room?
guest_can_join BOOLEAN NOT NULL DEFAULT false,
-- URL of the room avatar (empty string if none)
avatar_url TEXT NOT NULL DEFAULT '',
-- Visibility of the room: true means the room is publicly visible, false
-- means the room is private
visibility BOOLEAN NOT NULL DEFAULT false
);
`
const countPublicRoomsSQL = "" +
"SELECT COUNT(*) FROM publicroomsapi_public_rooms" +
" WHERE visibility = true"
const selectPublicRoomsSQL = "" +
"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
" FROM publicroomsapi_public_rooms WHERE visibility = true" +
" ORDER BY joined_members DESC" +
" OFFSET $1"
const selectPublicRoomsWithLimitSQL = "" +
"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
" FROM publicroomsapi_public_rooms WHERE visibility = true" +
" ORDER BY joined_members DESC" +
" OFFSET $1 LIMIT $2"
const selectPublicRoomsWithFilterSQL = "" +
"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
" FROM publicroomsapi_public_rooms" +
" WHERE visibility = true" +
" AND (LOWER(name) LIKE LOWER($1)" +
" OR LOWER(topic) LIKE LOWER($1)" +
" OR LOWER(ARRAY_TO_STRING(aliases, ',')) LIKE LOWER($1))" +
" ORDER BY joined_members DESC" +
" OFFSET $2"
const selectPublicRoomsWithLimitAndFilterSQL = "" +
"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
" FROM publicroomsapi_public_rooms" +
" WHERE visibility = true" +
" AND (LOWER(name) LIKE LOWER($1)" +
" OR LOWER(topic) LIKE LOWER($1)" +
" OR LOWER(ARRAY_TO_STRING(aliases, ',')) LIKE LOWER($1))" +
" ORDER BY joined_members DESC" +
" OFFSET $2 LIMIT $3"
const selectRoomVisibilitySQL = "" +
"SELECT visibility FROM publicroomsapi_public_rooms" +
" WHERE room_id = $1"
const insertNewRoomSQL = "" +
"INSERT INTO publicroomsapi_public_rooms(room_id)" +
" VALUES ($1)"
const incrementJoinedMembersInRoomSQL = "" +
"UPDATE publicroomsapi_public_rooms" +
" SET joined_members = joined_members + 1" +
" WHERE room_id = $1"
const decrementJoinedMembersInRoomSQL = "" +
"UPDATE publicroomsapi_public_rooms" +
" SET joined_members = joined_members - 1" +
" WHERE room_id = $1"
const updateRoomAttributeSQL = "" +
"UPDATE publicroomsapi_public_rooms" +
" SET %s = $1" +
" WHERE room_id = $2"
type publicRoomsStatements struct {
countPublicRoomsStmt *sql.Stmt
selectPublicRoomsStmt *sql.Stmt
selectPublicRoomsWithLimitStmt *sql.Stmt
selectPublicRoomsWithFilterStmt *sql.Stmt
selectPublicRoomsWithLimitAndFilterStmt *sql.Stmt
selectRoomVisibilityStmt *sql.Stmt
insertNewRoomStmt *sql.Stmt
incrementJoinedMembersInRoomStmt *sql.Stmt
decrementJoinedMembersInRoomStmt *sql.Stmt
updateRoomAttributeStmts map[string]*sql.Stmt
}
func (s *publicRoomsStatements) prepare(db *sql.DB) (err error) {
_, err = db.Exec(publicRoomsSchema)
if err != nil {
return
}
stmts := statementList{
{&s.countPublicRoomsStmt, countPublicRoomsSQL},
{&s.selectPublicRoomsStmt, selectPublicRoomsSQL},
{&s.selectPublicRoomsWithLimitStmt, selectPublicRoomsWithLimitSQL},
{&s.selectPublicRoomsWithFilterStmt, selectPublicRoomsWithFilterSQL},
{&s.selectPublicRoomsWithLimitAndFilterStmt, selectPublicRoomsWithLimitAndFilterSQL},
{&s.selectRoomVisibilityStmt, selectRoomVisibilitySQL},
{&s.insertNewRoomStmt, insertNewRoomSQL},
{&s.incrementJoinedMembersInRoomStmt, incrementJoinedMembersInRoomSQL},
{&s.decrementJoinedMembersInRoomStmt, decrementJoinedMembersInRoomSQL},
}
if err = stmts.prepare(db); err != nil {
return
}
s.updateRoomAttributeStmts = make(map[string]*sql.Stmt)
for _, editable := range editableAttributes {
stmt := fmt.Sprintf(updateRoomAttributeSQL, editable)
if s.updateRoomAttributeStmts[editable], err = db.Prepare(stmt); err != nil {
return
}
}
return
}
func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64, err error) {
err = s.countPublicRoomsStmt.QueryRowContext(ctx).Scan(&nb)
return
}
func (s *publicRoomsStatements) selectPublicRooms(
ctx context.Context, offset int64, limit int16, filter string,
) ([]gomatrixserverlib.PublicRoom, error) {
var rows *sql.Rows
var err error
if len(filter) > 0 {
pattern := "%" + filter + "%"
if limit == 0 {
rows, err = s.selectPublicRoomsWithFilterStmt.QueryContext(
ctx, pattern, offset,
)
} else {
rows, err = s.selectPublicRoomsWithLimitAndFilterStmt.QueryContext(
ctx, pattern, offset, limit,
)
}
} else {
if limit == 0 {
rows, err = s.selectPublicRoomsStmt.QueryContext(ctx, offset)
} else {
rows, err = s.selectPublicRoomsWithLimitStmt.QueryContext(
ctx, offset, limit,
)
}
}
if err != nil {
return []gomatrixserverlib.PublicRoom{}, nil
}
defer internal.CloseAndLogIfError(ctx, rows, "selectPublicRooms: rows.close() failed")
rooms := []gomatrixserverlib.PublicRoom{}
for rows.Next() {
var r gomatrixserverlib.PublicRoom
var aliases pq.StringArray
err = rows.Scan(
&r.RoomID, &r.JoinedMembersCount, &aliases, &r.CanonicalAlias,
&r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL,
)
if err != nil {
return rooms, err
}
r.Aliases = aliases
rooms = append(rooms, r)
}
return rooms, rows.Err()
}
func (s *publicRoomsStatements) selectRoomVisibility(
ctx context.Context, roomID string,
) (v bool, err error) {
err = s.selectRoomVisibilityStmt.QueryRowContext(ctx, roomID).Scan(&v)
return
}
func (s *publicRoomsStatements) insertNewRoom(
ctx context.Context, roomID string,
) error {
_, err := s.insertNewRoomStmt.ExecContext(ctx, roomID)
return err
}
func (s *publicRoomsStatements) incrementJoinedMembersInRoom(
ctx context.Context, roomID string,
) error {
_, err := s.incrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID)
return err
}
func (s *publicRoomsStatements) decrementJoinedMembersInRoom(
ctx context.Context, roomID string,
) error {
_, err := s.decrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID)
return err
}
func (s *publicRoomsStatements) updateRoomAttribute(
ctx context.Context, attrName string, attrValue attributeValue, roomID string,
) error {
stmt, isEditable := s.updateRoomAttributeStmts[attrName]
if !isEditable {
return errors.New("Cannot edit " + attrName)
}
var value interface{}
switch v := attrValue.(type) {
case []string:
value = pq.StringArray(v)
case bool, string:
value = attrValue
default:
return errors.New("Unsupported attribute type, must be bool, string or []string")
}
_, err := stmt.ExecContext(ctx, value, roomID)
return err
}

259
publicroomsapi/storage/postgres/storage.go
View file

@ -1,259 +0,0 @@
// Copyright 2017-2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgres
import (
"context"
"database/sql"
"encoding/json"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/gomatrixserverlib"
)
// PublicRoomsServerDatabase represents a public rooms server database.
type PublicRoomsServerDatabase struct {
db *sql.DB
sqlutil.PartitionOffsetStatements
statements publicRoomsStatements
localServerName gomatrixserverlib.ServerName
}
type attributeValue interface{}
// NewPublicRoomsServerDatabase creates a new public rooms server database.
func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) {
var db *sql.DB
var err error
if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil {
return nil, err
}
storage := PublicRoomsServerDatabase{
db: db,
localServerName: localServerName,
}
if err = storage.PartitionOffsetStatements.Prepare(db, "publicroomsapi"); err != nil {
return nil, err
}
if err = storage.statements.prepare(db); err != nil {
return nil, err
}
return &storage, nil
}
// GetRoomVisibility returns the room visibility as a boolean: true if the room
// is publicly visible, false if not.
// Returns an error if the retrieval failed.
func (d *PublicRoomsServerDatabase) GetRoomVisibility(
ctx context.Context, roomID string,
) (bool, error) {
return d.statements.selectRoomVisibility(ctx, roomID)
}
// SetRoomVisibility updates the visibility attribute of a room. This attribute
// must be set to true if the room is publicly visible, false if not.
// Returns an error if the update failed.
func (d *PublicRoomsServerDatabase) SetRoomVisibility(
ctx context.Context, visible bool, roomID string,
) error {
return d.statements.updateRoomAttribute(ctx, "visibility", visible, roomID)
}
// CountPublicRooms returns the number of room set as publicly visible on the server.
// Returns an error if the retrieval failed.
func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) {
return d.statements.countPublicRooms(ctx)
}
// GetPublicRooms returns an array containing the local rooms set as publicly visible, ordered by their number
// of joined members. This array can be limited by a given number of elements, and offset by a given value.
// If the limit is 0, doesn't limit the number of results. If the offset is 0 too, the array contains all
// the rooms set as publicly visible on the server.
// Returns an error if the retrieval failed.
func (d *PublicRoomsServerDatabase) GetPublicRooms(
ctx context.Context, offset int64, limit int16, filter string,
) ([]gomatrixserverlib.PublicRoom, error) {
return d.statements.selectPublicRooms(ctx, offset, limit, filter)
}
// UpdateRoomFromEvents iterate over a slice of state events and call
// UpdateRoomFromEvent on each of them to update the database representation of
// the rooms updated by each event.
// The slice of events to remove is used to update the number of joined members
// for the room in the database.
// If the update triggered by one of the events failed, aborts the process and
// returns an error.
func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(
ctx context.Context,
eventsToAdd []gomatrixserverlib.Event,
eventsToRemove []gomatrixserverlib.Event,
) error {
for _, event := range eventsToAdd {
if err := d.UpdateRoomFromEvent(ctx, event); err != nil {
return err
}
}
for _, event := range eventsToRemove {
if event.Type() == "m.room.member" {
if err := d.updateNumJoinedUsers(ctx, event, true); err != nil {
return err
}
}
}
return nil
}
// UpdateRoomFromEvent updates the database representation of a room from a Matrix event, by
// checking the event's type to know which attribute to change and using the event's content
// to define the new value of the attribute.
// If the event doesn't match with any property used to compute the public room directory,
// does nothing.
// If something went wrong during the process, returns an error.
func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(
ctx context.Context, event gomatrixserverlib.Event,
) error {
// Process the event according to its type
switch event.Type() {
case "m.room.create":
return d.statements.insertNewRoom(ctx, event.RoomID())
case "m.room.member":
return d.updateNumJoinedUsers(ctx, event, false)
case "m.room.aliases":
return d.updateRoomAliases(ctx, event)
case "m.room.canonical_alias":
var content eventutil.CanonicalAliasContent
field := &(content.Alias)
attrName := "canonical_alias"
return d.updateStringAttribute(ctx, attrName, event, &content, field)
case "m.room.name":
var content eventutil.NameContent
field := &(content.Name)
attrName := "name"
return d.updateStringAttribute(ctx, attrName, event, &content, field)
case "m.room.topic":
var content eventutil.TopicContent
field := &(content.Topic)
attrName := "topic"
return d.updateStringAttribute(ctx, attrName, event, &content, field)
case "m.room.avatar":
var content eventutil.AvatarContent
field := &(content.URL)
attrName := "avatar_url"
return d.updateStringAttribute(ctx, attrName, event, &content, field)
case "m.room.history_visibility":
var content eventutil.HistoryVisibilityContent
field := &(content.HistoryVisibility)
attrName := "world_readable"
strForTrue := "world_readable"
return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue)
case "m.room.guest_access":
var content eventutil.GuestAccessContent
field := &(content.GuestAccess)
attrName := "guest_can_join"
strForTrue := "can_join"
return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue)
}
// If the event type didn't match, return with no error
return nil
}
// updateNumJoinedUsers updates the number of joined user in the database representation
// of a room using a given "m.room.member" Matrix event.
// If the membership property of the event isn't "join", ignores it and returs nil.
// If the remove parameter is set to false, increments the joined members counter in the
// database, if set to truem decrements it.
// Returns an error if the update failed.
func (d *PublicRoomsServerDatabase) updateNumJoinedUsers(
ctx context.Context, membershipEvent gomatrixserverlib.Event, remove bool,
) error {
membership, err := membershipEvent.Membership()
if err != nil {
return err
}
if membership != gomatrixserverlib.Join {
return nil
}
if remove {
return d.statements.decrementJoinedMembersInRoom(ctx, membershipEvent.RoomID())
}
return d.statements.incrementJoinedMembersInRoom(ctx, membershipEvent.RoomID())
}
// updateStringAttribute updates a given string attribute in the database
// representation of a room using a given string data field from content of the
// Matrix event triggering the update.
// Returns an error if decoding the Matrix event's content or updating the attribute
// failed.
func (d *PublicRoomsServerDatabase) updateStringAttribute(
ctx context.Context, attrName string, event gomatrixserverlib.Event,
content interface{}, field *string,
) error {
if err := json.Unmarshal(event.Content(), content); err != nil {
return err
}
return d.statements.updateRoomAttribute(ctx, attrName, *field, event.RoomID())
}
// updateBooleanAttribute updates a given boolean attribute in the database
// representation of a room using a given string data field from content of the
// Matrix event triggering the update.
// The attribute is set to true if the field matches a given string, false if not.
// Returns an error if decoding the Matrix event's content or updating the attribute
// failed.
func (d *PublicRoomsServerDatabase) updateBooleanAttribute(
ctx context.Context, attrName string, event gomatrixserverlib.Event,
content interface{}, field *string, strForTrue string,
) error {
if err := json.Unmarshal(event.Content(), content); err != nil {
return err
}
var attrValue bool
if *field == strForTrue {
attrValue = true
} else {
attrValue = false
}
return d.statements.updateRoomAttribute(ctx, attrName, attrValue, event.RoomID())
}
// updateRoomAliases decodes the content of a "m.room.aliases" Matrix event and update the list of aliases of
// a given room with it.
// Returns an error if decoding the Matrix event or updating the list failed.
func (d *PublicRoomsServerDatabase) updateRoomAliases(
ctx context.Context, aliasesEvent gomatrixserverlib.Event,
) error {
if aliasesEvent.StateKey() == nil || *aliasesEvent.StateKey() != string(d.localServerName) {
return nil // only store our own aliases
}
var content eventutil.AliasesContent
if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil {
return err
}
return d.statements.updateRoomAttribute(
ctx, "aliases", content.Aliases, aliasesEvent.RoomID(),
)
}

36
publicroomsapi/storage/sqlite3/prepare.go
View file

@ -1,36 +0,0 @@
// Copyright 2017-2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sqlite3
import (
"database/sql"
)
// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement.
type statementList []struct {
statement **sql.Stmt
sql string
}
// prepare the SQL for each statement in the list and assign the result to the prepared statement.
func (s statementList) prepare(db *sql.DB) (err error) {
for _, statement := range s {
if *statement.statement, err = db.Prepare(statement.sql); err != nil {
return
}
}
return
}

273
publicroomsapi/storage/sqlite3/public_rooms_table.go
View file

@ -1,273 +0,0 @@
// Copyright 2017-2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sqlite3
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/gomatrixserverlib"
)
var editableAttributes = []string{
"aliases",
"canonical_alias",
"name",
"topic",
"world_readable",
"guest_can_join",
"avatar_url",
"visibility",
}
const publicRoomsSchema = `
-- Stores all of the rooms with data needed to create the server's room directory
CREATE TABLE IF NOT EXISTS publicroomsapi_public_rooms(
room_id TEXT NOT NULL PRIMARY KEY,
joined_members INTEGER NOT NULL DEFAULT 0,
aliases TEXT NOT NULL DEFAULT '',
canonical_alias TEXT NOT NULL DEFAULT '',
name TEXT NOT NULL DEFAULT '',
topic TEXT NOT NULL DEFAULT '',
world_readable BOOLEAN NOT NULL DEFAULT false,
guest_can_join BOOLEAN NOT NULL DEFAULT false,
avatar_url TEXT NOT NULL DEFAULT '',
visibility BOOLEAN NOT NULL DEFAULT false
);
`
const countPublicRoomsSQL = "" +
"SELECT COUNT(*) FROM publicroomsapi_public_rooms" +
" WHERE visibility = true"
const selectPublicRoomsSQL = "" +
"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
" FROM publicroomsapi_public_rooms WHERE visibility = true" +
" ORDER BY joined_members DESC" +
" LIMIT 30 OFFSET $1"
const selectPublicRoomsWithLimitSQL = "" +
"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
" FROM publicroomsapi_public_rooms WHERE visibility = true" +
" ORDER BY joined_members DESC" +
" LIMIT $1 OFFSET $2"
const selectPublicRoomsWithFilterSQL = "" +
"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
" FROM publicroomsapi_public_rooms" +
" WHERE visibility = true" +
" AND (LOWER(name) LIKE LOWER($1)" +
" OR LOWER(topic) LIKE LOWER($1)" +
" OR LOWER(aliases) LIKE LOWER($1))" + // TODO: Is there a better way to search aliases?
" ORDER BY joined_members DESC" +
" LIMIT 30 OFFSET $2"
const selectPublicRoomsWithLimitAndFilterSQL = "" +
"SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" +
" FROM publicroomsapi_public_rooms" +
" WHERE visibility = true" +
" AND (LOWER(name) LIKE LOWER($1)" +
" OR LOWER(topic) LIKE LOWER($1)" +
" OR LOWER(aliases) LIKE LOWER($1))" + // TODO: Is there a better way to search aliases?
" ORDER BY joined_members DESC" +
" LIMIT $3 OFFSET $2"
const selectRoomVisibilitySQL = "" +
"SELECT visibility FROM publicroomsapi_public_rooms" +
" WHERE room_id = $1"
const insertNewRoomSQL = "" +
"INSERT INTO publicroomsapi_public_rooms(room_id)" +
" VALUES ($1)"
const incrementJoinedMembersInRoomSQL = "" +
"UPDATE publicroomsapi_public_rooms" +
" SET joined_members = joined_members + 1" +
" WHERE room_id = $1"
const decrementJoinedMembersInRoomSQL = "" +
"UPDATE publicroomsapi_public_rooms" +
" SET joined_members = joined_members - 1" +
" WHERE room_id = $1"
const updateRoomAttributeSQL = "" +
"UPDATE publicroomsapi_public_rooms" +
" SET %s = $1" +
" WHERE room_id = $2"
type publicRoomsStatements struct {
countPublicRoomsStmt *sql.Stmt
selectPublicRoomsStmt *sql.Stmt
selectPublicRoomsWithLimitStmt *sql.Stmt
selectPublicRoomsWithFilterStmt *sql.Stmt
selectPublicRoomsWithLimitAndFilterStmt *sql.Stmt
selectRoomVisibilityStmt *sql.Stmt
insertNewRoomStmt *sql.Stmt
incrementJoinedMembersInRoomStmt *sql.Stmt
decrementJoinedMembersInRoomStmt *sql.Stmt
updateRoomAttributeStmts map[string]*sql.Stmt
}
func (s *publicRoomsStatements) prepare(db *sql.DB) (err error) {
_, err = db.Exec(publicRoomsSchema)
if err != nil {
return
}
stmts := statementList{
{&s.countPublicRoomsStmt, countPublicRoomsSQL},
{&s.selectPublicRoomsStmt, selectPublicRoomsSQL},
{&s.selectPublicRoomsWithLimitStmt, selectPublicRoomsWithLimitSQL},
{&s.selectPublicRoomsWithFilterStmt, selectPublicRoomsWithFilterSQL},
{&s.selectPublicRoomsWithLimitAndFilterStmt, selectPublicRoomsWithLimitAndFilterSQL},
{&s.selectRoomVisibilityStmt, selectRoomVisibilitySQL},
{&s.insertNewRoomStmt, insertNewRoomSQL},
{&s.incrementJoinedMembersInRoomStmt, incrementJoinedMembersInRoomSQL},
{&s.decrementJoinedMembersInRoomStmt, decrementJoinedMembersInRoomSQL},
}
if err = stmts.prepare(db); err != nil {
return
}
s.updateRoomAttributeStmts = make(map[string]*sql.Stmt)
for _, editable := range editableAttributes {
stmt := fmt.Sprintf(updateRoomAttributeSQL, editable)
if s.updateRoomAttributeStmts[editable], err = db.Prepare(stmt); err != nil {
return
}
}
return
}
func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64, err error) {
err = s.countPublicRoomsStmt.QueryRowContext(ctx).Scan(&nb)
return
}
func (s *publicRoomsStatements) selectPublicRooms(
ctx context.Context, offset int64, limit int16, filter string,
) ([]gomatrixserverlib.PublicRoom, error) {
var rows *sql.Rows
var err error
if len(filter) > 0 {
pattern := "%" + filter + "%"
if limit == 0 {
rows, err = s.selectPublicRoomsWithFilterStmt.QueryContext(
ctx, pattern, offset,
)
} else {
rows, err = s.selectPublicRoomsWithLimitAndFilterStmt.QueryContext(
ctx, pattern, limit, offset,
)
}
} else {
if limit == 0 {
rows, err = s.selectPublicRoomsStmt.QueryContext(ctx, offset)
} else {
rows, err = s.selectPublicRoomsWithLimitStmt.QueryContext(
ctx, limit, offset,
)
}
}
if err != nil {
return []gomatrixserverlib.PublicRoom{}, nil
}
defer internal.CloseAndLogIfError(ctx, rows, "selectPublicRooms failed to close rows")
rooms := []gomatrixserverlib.PublicRoom{}
for rows.Next() {
var r gomatrixserverlib.PublicRoom
var aliasesJSON string
err = rows.Scan(
&r.RoomID, &r.JoinedMembersCount, &aliasesJSON, &r.CanonicalAlias,
&r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL,
)
if err != nil {
return rooms, err
}
if len(aliasesJSON) > 0 {
if err := json.Unmarshal([]byte(aliasesJSON), &r.Aliases); err != nil {
return rooms, err
}
}
rooms = append(rooms, r)
}
return rooms, nil
}
func (s *publicRoomsStatements) selectRoomVisibility(
ctx context.Context, roomID string,
) (v bool, err error) {
err = s.selectRoomVisibilityStmt.QueryRowContext(ctx, roomID).Scan(&v)
return
}
func (s *publicRoomsStatements) insertNewRoom(
ctx context.Context, roomID string,
) error {
_, err := s.insertNewRoomStmt.ExecContext(ctx, roomID)
return err
}
func (s *publicRoomsStatements) incrementJoinedMembersInRoom(
ctx context.Context, roomID string,
) error {
_, err := s.incrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID)
return err
}
func (s *publicRoomsStatements) decrementJoinedMembersInRoom(
ctx context.Context, roomID string,
) error {
_, err := s.decrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID)
return err
}
func (s *publicRoomsStatements) updateRoomAttribute(
ctx context.Context, attrName string, attrValue attributeValue, roomID string,
) error {
stmt, isEditable := s.updateRoomAttributeStmts[attrName]
if !isEditable {
return errors.New("Cannot edit " + attrName)
}
var value interface{}
switch v := attrValue.(type) {
case []string:
b, _ := json.Marshal(v)
value = string(b)
case bool, string:
value = attrValue
default:
return errors.New("Unsupported attribute type, must be bool, string or []string")
}
_, err := stmt.ExecContext(ctx, value, roomID)
return err
}

265
publicroomsapi/storage/sqlite3/storage.go
View file

@ -1,265 +0,0 @@
// Copyright 2017-2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sqlite3
import (
"context"
"database/sql"
"encoding/json"
_ "github.com/mattn/go-sqlite3"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/gomatrixserverlib"
)
// PublicRoomsServerDatabase represents a public rooms server database.
type PublicRoomsServerDatabase struct {
db *sql.DB
sqlutil.PartitionOffsetStatements
statements publicRoomsStatements
localServerName gomatrixserverlib.ServerName
}
type attributeValue interface{}
// NewPublicRoomsServerDatabase creates a new public rooms server database.
func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) {
var db *sql.DB
var err error
cs, err := sqlutil.ParseFileURI(dataSourceName)
if err != nil {
return nil, err
}
if db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil {
return nil, err
}
storage := PublicRoomsServerDatabase{
db: db,
localServerName: localServerName,
}
if err = storage.PartitionOffsetStatements.Prepare(db, "publicroomsapi"); err != nil {
return nil, err
}
if err = storage.statements.prepare(db); err != nil {
return nil, err
}
return &storage, nil
}
// GetRoomVisibility returns the room visibility as a boolean: true if the room
// is publicly visible, false if not.
// Returns an error if the retrieval failed.
func (d *PublicRoomsServerDatabase) GetRoomVisibility(
ctx context.Context, roomID string,
) (bool, error) {
return d.statements.selectRoomVisibility(ctx, roomID)
}
// SetRoomVisibility updates the visibility attribute of a room. This attribute
// must be set to true if the room is publicly visible, false if not.
// Returns an error if the update failed.
func (d *PublicRoomsServerDatabase) SetRoomVisibility(
ctx context.Context, visible bool, roomID string,
) error {
return d.statements.updateRoomAttribute(ctx, "visibility", visible, roomID)
}
// CountPublicRooms returns the number of room set as publicly visible on the server.
// Returns an error if the retrieval failed.
func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) {
return d.statements.countPublicRooms(ctx)
}
// GetPublicRooms returns an array containing the local rooms set as publicly visible, ordered by their number
// of joined members. This array can be limited by a given number of elements, and offset by a given value.
// If the limit is 0, doesn't limit the number of results. If the offset is 0 too, the array contains all
// the rooms set as publicly visible on the server.
// Returns an error if the retrieval failed.
func (d *PublicRoomsServerDatabase) GetPublicRooms(
ctx context.Context, offset int64, limit int16, filter string,
) ([]gomatrixserverlib.PublicRoom, error) {
return d.statements.selectPublicRooms(ctx, offset, limit, filter)
}
// UpdateRoomFromEvents iterate over a slice of state events and call
// UpdateRoomFromEvent on each of them to update the database representation of
// the rooms updated by each event.
// The slice of events to remove is used to update the number of joined members
// for the room in the database.
// If the update triggered by one of the events failed, aborts the process and
// returns an error.
func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(
ctx context.Context,
eventsToAdd []gomatrixserverlib.Event,
eventsToRemove []gomatrixserverlib.Event,
) error {
for _, event := range eventsToAdd {
if err := d.UpdateRoomFromEvent(ctx, event); err != nil {
return err
}
}
for _, event := range eventsToRemove {
if event.Type() == "m.room.member" {
if err := d.updateNumJoinedUsers(ctx, event, true); err != nil {
return err
}
}
}
return nil
}
// UpdateRoomFromEvent updates the database representation of a room from a Matrix event, by
// checking the event's type to know which attribute to change and using the event's content
// to define the new value of the attribute.
// If the event doesn't match with any property used to compute the public room directory,
// does nothing.
// If something went wrong during the process, returns an error.
func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(
ctx context.Context, event gomatrixserverlib.Event,
) error {
// Process the event according to its type
switch event.Type() {
case "m.room.create":
return d.statements.insertNewRoom(ctx, event.RoomID())
case "m.room.member":
return d.updateNumJoinedUsers(ctx, event, false)
case "m.room.aliases":
return d.updateRoomAliases(ctx, event)
case "m.room.canonical_alias":
var content eventutil.CanonicalAliasContent
field := &(content.Alias)
attrName := "canonical_alias"
return d.updateStringAttribute(ctx, attrName, event, &content, field)
case "m.room.name":
var content eventutil.NameContent
field := &(content.Name)
attrName := "name"
return d.updateStringAttribute(ctx, attrName, event, &content, field)
case "m.room.topic":
var content eventutil.TopicContent
field := &(content.Topic)
attrName := "topic"
return d.updateStringAttribute(ctx, attrName, event, &content, field)
case "m.room.avatar":
var content eventutil.AvatarContent
field := &(content.URL)
attrName := "avatar_url"
return d.updateStringAttribute(ctx, attrName, event, &content, field)
case "m.room.history_visibility":
var content eventutil.HistoryVisibilityContent
field := &(content.HistoryVisibility)
attrName := "world_readable"
strForTrue := "world_readable"
return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue)
case "m.room.guest_access":
var content eventutil.GuestAccessContent
field := &(content.GuestAccess)
attrName := "guest_can_join"
strForTrue := "can_join"
return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue)
}
// If the event type didn't match, return with no error
return nil
}
// updateNumJoinedUsers updates the number of joined user in the database representation
// of a room using a given "m.room.member" Matrix event.
// If the membership property of the event isn't "join", ignores it and returs nil.
// If the remove parameter is set to false, increments the joined members counter in the
// database, if set to truem decrements it.
// Returns an error if the update failed.
func (d *PublicRoomsServerDatabase) updateNumJoinedUsers(
ctx context.Context, membershipEvent gomatrixserverlib.Event, remove bool,
) error {
membership, err := membershipEvent.Membership()
if err != nil {
return err
}
if membership != gomatrixserverlib.Join {
return nil
}
if remove {
return d.statements.decrementJoinedMembersInRoom(ctx, membershipEvent.RoomID())
}
return d.statements.incrementJoinedMembersInRoom(ctx, membershipEvent.RoomID())
}
// updateStringAttribute updates a given string attribute in the database
// representation of a room using a given string data field from content of the
// Matrix event triggering the update.
// Returns an error if decoding the Matrix event's content or updating the attribute
// failed.
func (d *PublicRoomsServerDatabase) updateStringAttribute(
ctx context.Context, attrName string, event gomatrixserverlib.Event,
content interface{}, field *string,
) error {
if err := json.Unmarshal(event.Content(), content); err != nil {
return err
}
return d.statements.updateRoomAttribute(ctx, attrName, *field, event.RoomID())
}
// updateBooleanAttribute updates a given boolean attribute in the database
// representation of a room using a given string data field from content of the
// Matrix event triggering the update.
// The attribute is set to true if the field matches a given string, false if not.
// Returns an error if decoding the Matrix event's content or updating the attribute
// failed.
func (d *PublicRoomsServerDatabase) updateBooleanAttribute(
ctx context.Context, attrName string, event gomatrixserverlib.Event,
content interface{}, field *string, strForTrue string,
) error {
if err := json.Unmarshal(event.Content(), content); err != nil {
return err
}
var attrValue bool
if *field == strForTrue {
attrValue = true
} else {
attrValue = false
}
return d.statements.updateRoomAttribute(ctx, attrName, attrValue, event.RoomID())
}
// updateRoomAliases decodes the content of a "m.room.aliases" Matrix event and update the list of aliases of
// a given room with it.
// Returns an error if decoding the Matrix event or updating the list failed.
func (d *PublicRoomsServerDatabase) updateRoomAliases(
ctx context.Context, aliasesEvent gomatrixserverlib.Event,
) error {
if aliasesEvent.StateKey() == nil || *aliasesEvent.StateKey() != string(d.localServerName) {
return nil // only store our own aliases
}
var content eventutil.AliasesContent
if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil {
return err
}
return d.statements.updateRoomAttribute(
ctx, "aliases", content.Aliases, aliasesEvent.RoomID(),
)
}

Some files were not shown because too many files have changed in this diff Show more