Merge branch 'master' into tmp-file-delete

appservice/storage/postgres/storage.go

@@ -21,6 +21,7 @@ import (
 
 	// Import postgres database driver
 	_ "github.com/lib/pq"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -35,7 +36,7 @@ type Database struct {
 func NewDatabase(dataSourceName string) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sql.Open("postgres", dataSourceName); err != nil {
+	if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {

appservice/storage/sqlite3/storage.go

@@ -21,6 +21,7 @@ import (
 
 	// Import SQLite database driver
 	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 	_ "github.com/mattn/go-sqlite3"
 )
@@ -36,7 +37,7 @@ type Database struct {
 func NewDatabase(dataSourceName string) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {

are-we-synapse-yet.list

@@ -0,0 +1,830 @@
+reg GET /register yields a set of flows
+reg POST /register can create a user
+reg POST /register downcases capitals in usernames
+reg POST /register returns the same device_id as that in the request
+reg POST /register rejects registration of usernames with '!'
+reg POST /register rejects registration of usernames with '"'
+reg POST /register rejects registration of usernames with ':'
+reg POST /register rejects registration of usernames with '?'
+reg POST /register rejects registration of usernames with '\'
+reg POST /register rejects registration of usernames with '@'
+reg POST /register rejects registration of usernames with '['
+reg POST /register rejects registration of usernames with ']'
+reg POST /register rejects registration of usernames with '{'
+reg POST /register rejects registration of usernames with '|'
+reg POST /register rejects registration of usernames with '}'
+reg POST /register rejects registration of usernames with '£'
+reg POST /register rejects registration of usernames with 'é'
+reg POST /register rejects registration of usernames with '\n'
+reg POST /register rejects registration of usernames with '''
+reg POST /r0/admin/register with shared secret 
+reg POST /r0/admin/register admin with shared secret 
+reg POST /r0/admin/register with shared secret downcases capitals 
+reg POST /r0/admin/register with shared secret disallows symbols 
+reg POST rejects invalid utf-8 in JSON 
+log GET /login yields a set of flows
+log POST /login can log in as a user 
+log POST /login returns the same device_id as that in the request 
+log POST /login can log in as a user with just the local part of the id 
+log POST /login as non-existing user is rejected 
+log POST /login wrong password is rejected 
+v1s GET /events initially
+v1s GET /initialSync initially
+csa Version responds 200 OK with valid structure
+pro PUT /profile/:user_id/displayname sets my name
+pro GET /profile/:user_id/displayname publicly accessible
+pro PUT /profile/:user_id/avatar_url sets my avatar
+pro GET /profile/:user_id/avatar_url publicly accessible
+dev GET /device/{deviceId} 
+dev GET /device/{deviceId} gives a 404 for unknown devices
+dev GET /devices 
+dev PUT /device/{deviceId} updates device fields 
+dev PUT /device/{deviceId} gives a 404 for unknown devices
+dev DELETE /device/{deviceId} 
+dev DELETE /device/{deviceId} requires UI auth user to match device owner 
+dev DELETE /device/{deviceId} with no body gives a 401 
+dev The deleted device must be consistent through an interactive auth session 
+pre GET /presence/:user_id/status fetches initial status 
+pre PUT /presence/:user_id/status updates my presence 
+crm POST /createRoom makes a public room
+crm POST /createRoom makes a private room
+crm POST /createRoom makes a private room with invites
+crm POST /createRoom makes a room with a name
+crm POST /createRoom makes a room with a topic
+syn Can /sync newly created room
+crm POST /createRoom creates a room with the given version
+crm POST /createRoom rejects attempts to create rooms with numeric versions
+crm POST /createRoom rejects attempts to create rooms with unknown versions
+crm POST /createRoom ignores attempts to set the room version via creation_content
+mem GET /rooms/:room_id/state/m.room.member/:user_id fetches my membership
+mem GET /rooms/:room_id/state/m.room.member/:user_id?format=event fetches my membership event 
+rst GET /rooms/:room_id/state/m.room.power_levels fetches powerlevels
+mem GET /rooms/:room_id/joined_members fetches my membership 
+v1s GET /rooms/:room_id/initialSync fetches initial sync state 
+pub GET /publicRooms lists newly-created room 
+ali GET /directory/room/:room_alias yields room ID
+mem GET /joined_rooms lists newly-created room
+rst POST /rooms/:room_id/state/m.room.name sets name
+rst GET /rooms/:room_id/state/m.room.name gets name
+rst POST /rooms/:room_id/state/m.room.topic sets topic
+rst GET /rooms/:room_id/state/m.room.topic gets topic
+rst GET /rooms/:room_id/state fetches entire room state 
+crm POST /createRoom with creation content
+ali PUT /directory/room/:room_alias creates alias
+nsp GET /rooms/:room_id/aliases lists aliases 
+jon POST /rooms/:room_id/join can join a room
+jon POST /join/:room_alias can join a room
+jon POST /join/:room_id can join a room
+jon POST /join/:room_id can join a room with custom content
+jon POST /join/:room_alias can join a room with custom content
+lev POST /rooms/:room_id/leave can leave a room
+inv POST /rooms/:room_id/invite can send an invite
+ban POST /rooms/:room_id/ban can ban a user
+snd POST /rooms/:room_id/send/:event_type sends a message
+snd PUT /rooms/:room_id/send/:event_type/:txn_id sends a message
+snd PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id
+get GET /rooms/:room_id/messages returns a message
+get GET /rooms/:room_id/messages lazy loads members correctly 
+typ PUT /rooms/:room_id/typing/:user_id sets typing notification 
+rst GET /rooms/:room_id/state/m.room.power_levels can fetch levels
+rst PUT /rooms/:room_id/state/m.room.power_levels can set levels
+rst PUT power_levels should not explode if the old power levels were empty
+rst Both GET and PUT work
+rct POST /rooms/:room_id/receipt can create receipts 
+red POST /rooms/:room_id/read_markers can create read marker
+med POST /media/v1/upload can create an upload 
+med GET /media/v1/download can fetch the value again 
+cap GET /capabilities is present and well formed for registered user 
+cap GET /r0/capabilities is not public
+reg Register with a recaptcha 
+reg registration is idempotent, without username specified 
+reg registration is idempotent, with username specified 
+reg registration remembers parameters 
+reg registration accepts non-ascii passwords 
+reg registration with inhibit_login inhibits login 
+reg User signups are forbidden from starting with '_'
+reg Can register using an email address 
+log Can login with 3pid and password using m.login.password 
+log login types include SSO 
+log /login/cas/redirect redirects if the old m.login.cas login type is listed 
+log Can login with new user via CAS 
+lox Can logout current device 
+lox Can logout all devices 
+lox Request to logout with invalid an access token is rejected
+lox Request to logout without an access token is rejected
+log After changing password, can't log in with old password 
+log After changing password, can log in with new password 
+log After changing password, existing session still works 
+log After changing password, a different session no longer works by default 
+log After changing password, different sessions can optionally be kept 
+psh Pushers created with a different access token are deleted on password change 
+psh Pushers created with a the same access token are not deleted on password change 
+acc Can deactivate account 
+acc Can't deactivate account with wrong password 
+acc After deactivating account, can't log in with password 
+acc After deactivating account, can't log in with an email
+v1s initialSync sees my presence status 
+pre Presence change reports an event to myself 
+pre Friends presence changes reports events 
+crm Room creation reports m.room.create to myself
+crm Room creation reports m.room.member to myself
+rst Setting room topic reports m.room.topic to myself 
+v1s Global initialSync 
+v1s Global initialSync with limit=0 gives no messages 
+v1s Room initialSync 
+v1s Room initialSync with limit=0 gives no messages 
+rst Setting state twice is idempotent 
+jon Joining room twice is idempotent 
+syn New room members see their own join event
+v1s New room members see existing users' presence in room initialSync 
+syn Existing members see new members' join events
+syn Existing members see new members' presence 
+v1s All room members see all room members' presence in global initialSync 
+f,jon Remote users can join room by alias 
+syn New room members see their own join event 
+v1s New room members see existing members' presence in room initialSync 
+syn Existing members see new members' join events 
+syn Existing members see new member's presence 
+v1s New room members see first user's profile information in global initialSync 
+v1s New room members see first user's profile information in per-room initialSync 
+f,jon Remote users may not join unfederated rooms 
+syn Local room members see posted message events
+v1s Fetching eventstream a second time doesn't yield the message again
+syn Local non-members don't see posted message events
+get Local room members can get room messages 
+f,syn Remote room members also see posted message events
+f,get Remote room members can get room messages 
+get Message history can be paginated
+f,get Message history can be paginated over federation
+eph Ephemeral messages received from clients are correctly expired 
+ali Room aliases can contain Unicode
+f,ali Remote room alias queries can handle Unicode
+ali Canonical alias can be set 
+ali Canonical alias can include alt_aliases 
+ali Regular users can add and delete aliases in the default room configuration
+ali Regular users can add and delete aliases when m.room.aliases is restricted
+ali Deleting a non-existent alias should return a 404
+ali Users can't delete other's aliases
+ali Users with sufficient power-level can delete other's aliases 
+ali Can delete canonical alias 
+ali Alias creators can delete alias with no ops 
+ali Alias creators can delete canonical alias with no ops 
+ali Only room members can list aliases of a room 
+inv Can invite users to invite-only rooms 
+inv Uninvited users cannot join the room 
+inv Invited user can reject invite 
+f,inv Invited user can reject invite over federation 
+f,inv Invited user can reject invite over federation several times 
+inv Invited user can reject invite for empty room 
+f,inv Invited user can reject invite over federation for empty room 
+inv Invited user can reject local invite after originator leaves 
+inv Invited user can see room metadata 
+f,inv Remote invited user can see room metadata 
+inv Users cannot invite themselves to a room 
+inv Users cannot invite a user that is already in the room 
+ban Banned user is kicked and may not rejoin until unbanned 
+f,ban Remote banned user is kicked and may not rejoin until unbanned 
+ban 'ban' event respects room powerlevel 
+plv setting 'm.room.name' respects room powerlevel 
+plv setting 'm.room.power_levels' respects room powerlevel (2 subtests)
+plv Unprivileged users can set m.room.topic if it only needs level 0
+plv Users cannot set ban powerlevel higher than their own (2 subtests)
+plv Users cannot set kick powerlevel higher than their own (2 subtests)
+plv Users cannot set redact powerlevel higher than their own (2 subtests)
+v1s Check that event streams started after a client joined a room work (SYT-1) 
+v1s Event stream catches up fully after many messages 
+xxx POST /rooms/:room_id/redact/:event_id as power user redacts message 
+xxx POST /rooms/:room_id/redact/:event_id as original message sender redacts message 
+xxx POST /rooms/:room_id/redact/:event_id as random user does not redact message 
+xxx POST /redact disallows redaction of event in different room 
+xxx Redaction of a redaction redacts the redaction reason 
+v1s A departed room is still included in /initialSync (SPEC-216) 
+v1s Can get rooms/{roomId}/initialSync for a departed room (SPEC-216) 
+rst Can get rooms/{roomId}/state for a departed room (SPEC-216) 
+mem Can get rooms/{roomId}/members for a departed room (SPEC-216)
+get Can get rooms/{roomId}/messages for a departed room (SPEC-216) 
+rst Can get 'm.room.name' state for a departed room (SPEC-216) 
+syn Getting messages going forward is limited for a departed room (SPEC-216)
+3pd Can invite existing 3pid 
+3pd Can invite existing 3pid with no ops into a private room 
+3pd Can invite existing 3pid in createRoom 
+3pd Can invite unbound 3pid 
+f,3pd Can invite unbound 3pid over federation 
+3pd Can invite unbound 3pid with no ops into a private room 
+f,3pd Can invite unbound 3pid over federation with no ops into a private room 
+f,3pd Can invite unbound 3pid over federation with users from both servers 
+3pd Can accept unbound 3pid invite after inviter leaves 
+3pd Can accept third party invite with /join 
+3pd 3pid invite join with wrong but valid signature are rejected
+3pd 3pid invite join valid signature but revoked keys are rejected
+3pd 3pid invite join valid signature but unreachable ID server are rejected
+gst Guest user cannot call /events globally
+gst Guest users can join guest_access rooms
+gst Guest users can send messages to guest_access rooms if joined 
+gst Guest user calling /events doesn't tightloop 
+gst Guest users are kicked from guest_access rooms on revocation of guest_access 
+gst Guest user can set display names
+gst Guest users are kicked from guest_access rooms on revocation of guest_access over federation 
+gst Guest user can upgrade to fully featured user 
+gst Guest user cannot upgrade other users
+pub GET /publicRooms lists rooms 
+pub GET /publicRooms includes avatar URLs 
+gst Guest users can accept invites to private rooms over federation 
+gst Guest users denied access over federation if guest access prohibited 
+mem Room members can override their displayname on a room-specific basis 
+mem Room members can join a room with an overridden displayname
+mem Users cannot kick users from a room they are not in 
+mem Users cannot kick users who have already left a room 
+typ Typing notification sent to local room members 
+f,typ Typing notifications also sent to remote room members 
+typ Typing can be explicitly stopped 
+rct Read receipts are visible to /initialSync 
+rct Read receipts are sent as events 
+rct Receipts must be m.read 
+pro displayname updates affect room member events 
+pro avatar_url updates affect room member events 
+gst m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users
+gst m.room.history_visibility == "shared" allows/forbids appropriately for Guest users 
+gst m.room.history_visibility == "invited" allows/forbids appropriately for Guest users 
+gst m.room.history_visibility == "joined" allows/forbids appropriately for Guest users 
+gst m.room.history_visibility == "default" allows/forbids appropriately for Guest users 
+gst Guest non-joined user cannot call /events on shared room
+gst Guest non-joined user cannot call /events on invited room
+gst Guest non-joined user cannot call /events on joined room
+gst Guest non-joined user cannot call /events on default room
+gst Guest non-joined user can call /events on world_readable room 
+gst Guest non-joined users can get state for world_readable rooms
+gst Guest non-joined users can get individual state for world_readable rooms
+gst Guest non-joined users cannot room initalSync for non-world_readable rooms
+gst Guest non-joined users can room initialSync for world_readable rooms 
+gst Guest non-joined users can get individual state for world_readable rooms after leaving
+gst Guest non-joined users cannot send messages to guest_access rooms if not joined
+gst Guest users can sync from world_readable guest_access rooms if joined
+gst Guest users can sync from shared guest_access rooms if joined 
+gst Guest users can sync from invited guest_access rooms if joined 
+gst Guest users can sync from joined guest_access rooms if joined 
+gst Guest users can sync from default guest_access rooms if joined
+ath m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users
+ath m.room.history_visibility == "shared" allows/forbids appropriately for Real users 
+ath m.room.history_visibility == "invited" allows/forbids appropriately for Real users 
+ath m.room.history_visibility == "joined" allows/forbids appropriately for Real users 
+ath m.room.history_visibility == "default" allows/forbids appropriately for Real users 
+ath Real non-joined user cannot call /events on shared room
+ath Real non-joined user cannot call /events on invited room
+ath Real non-joined user cannot call /events on joined room
+ath Real non-joined user cannot call /events on default room
+ath Real non-joined user can call /events on world_readable room 
+ath Real non-joined users can get state for world_readable rooms
+ath Real non-joined users can get individual state for world_readable rooms
+ath Real non-joined users cannot room initalSync for non-world_readable rooms
+ath Real non-joined users can room initialSync for world_readable rooms 
+ath Real non-joined users can get individual state for world_readable rooms after leaving 
+ath Real non-joined users cannot send messages to guest_access rooms if not joined
+ath Real users can sync from world_readable guest_access rooms if joined
+ath Real users can sync from shared guest_access rooms if joined 
+ath Real users can sync from invited guest_access rooms if joined 
+ath Real users can sync from joined guest_access rooms if joined 
+ath Real users can sync from default guest_access rooms if joined
+ath Only see history_visibility changes on boundaries 
+f,ath Backfill works correctly with history visibility set to joined
+fgt Forgotten room messages cannot be paginated 
+fgt Forgetting room does not show up in v2 /sync 
+fgt Can forget room you've been kicked from 
+fgt Can't forget room you're still in
+mem Can re-join room if re-invited 
+ath Only original members of the room can see messages from erased users 
+mem /joined_rooms returns only joined rooms
+mem /joined_members return joined members 
+ctx /context/ on joined room works 
+ctx /context/ on non world readable room does not work 
+ctx /context/ returns correct number of events 
+ctx /context/ with lazy_load_members filter works 
+get /event/ on joined room works
+get /event/ on non world readable room does not work
+get /event/ does not allow access to events before the user joined
+mem Can get rooms/{roomId}/members
+mem Can get rooms/{roomId}/members at a given point 
+mem Can filter rooms/{roomId}/members 
+upg /upgrade creates a new room 
+upg /upgrade should preserve room visibility for public rooms 
+upg /upgrade should preserve room visibility for private rooms 
+upg /upgrade copies >100 power levels to the new room 
+upg /upgrade copies the power levels to the new room 
+upg /upgrade preserves the power level of the upgrading user in old and new rooms 
+upg /upgrade copies important state to the new room 
+upg /upgrade copies ban events to the new room 
+upg local user has push rules copied to upgraded room 
+f,upg remote user has push rules copied to upgraded room 
+upg /upgrade moves aliases to the new room 
+upg /upgrade moves remote aliases to the new room 
+upg /upgrade preserves direct room state 
+upg /upgrade preserves room federation ability 
+upg /upgrade restricts power levels in the old room 
+upg /upgrade restricts power levels in the old room when the old PLs are unusual 
+upg /upgrade to an unknown version is rejected 
+upg /upgrade is rejected if the user can't send state events 
+upg /upgrade of a bogus room fails gracefully 
+upg Cannot send tombstone event that points to the same room 
+f,upg Local and remote users' homeservers remove a room from their public directory on upgrade 
+rst Name/topic keys are correct 
+f,pub Can get remote public room list
+pub Can paginate public room list
+pub Can search public room list 
+syn Can create filter
+syn Can download filter
+syn Can sync
+syn Can sync a joined room
+syn Full state sync includes joined rooms
+syn Newly joined room is included in an incremental sync
+syn Newly joined room has correct timeline in incremental sync 
+syn Newly joined room includes presence in incremental sync 
+syn Get presence for newly joined members in incremental sync 
+syn Can sync a room with a single message 
+syn Can sync a room with a message with a transaction id 
+syn A message sent after an initial sync appears in the timeline of an incremental sync.
+syn A filtered timeline reaches its limit 
+syn Syncing a new room with a large timeline limit isn't limited 
+syn A full_state incremental update returns only recent timeline 
+syn A prev_batch token can be used in the v1 messages API 
+syn A next_batch token can be used in the v1 messages API 
+syn User sees their own presence in a sync 
+syn User is offline if they set_presence=offline in their sync
+syn User sees updates to presence from other users in the incremental sync. 
+syn State is included in the timeline in the initial sync 
+f,syn State from remote users is included in the state in the initial sync 
+syn Changes to state are included in an incremental sync
+syn Changes to state are included in an gapped incremental sync 
+f,syn State from remote users is included in the timeline in an incremental sync 
+syn A full_state incremental update returns all state 
+syn When user joins a room the state is included in the next sync 
+syn A change to displayname should not result in a full state sync 
+syn A change to displayname should appear in incremental /sync
+syn When user joins a room the state is included in a gapped sync 
+syn When user joins and leaves a room in the same batch, the full state is still included in the next sync 
+syn Current state appears in timeline in private history
+syn Current state appears in timeline in private history with many messages before
+syn Current state appears in timeline in private history with many messages after 
+syn Rooms a user is invited to appear in an initial sync
+syn Rooms a user is invited to appear in an incremental sync
+syn Newly joined room is included in an incremental sync after invite
+syn Sync can be polled for updates
+syn Sync is woken up for leaves
+syn Left rooms appear in the leave section of sync 
+syn Newly left rooms appear in the leave section of incremental sync
+syn We should see our own leave event, even if history_visibility is restricted (SYN-662)
+syn We should see our own leave event when rejecting an invite, even if history_visibility is restricted (riot-web/3462)
+syn Newly left rooms appear in the leave section of gapped sync
+syn Previously left rooms don't appear in the leave section of sync
+syn Left rooms appear in the leave section of full state sync
+syn Archived rooms only contain history from before the user left 
+syn Banned rooms appear in the leave section of sync 
+syn Newly banned rooms appear in the leave section of incremental sync
+syn Newly banned rooms appear in the leave section of incremental sync
+syn Typing events appear in initial sync
+syn Typing events appear in incremental sync
+syn Typing events appear in gapped sync
+syn Read receipts appear in initial v2 /sync 
+syn New read receipts appear in incremental v2 /sync 
+syn Can pass a JSON filter as a query parameter 
+syn Can request federation format via the filter 
+syn Read markers appear in incremental v2 /sync 
+syn Read markers appear in initial v2 /sync 
+syn Read markers can be updated 
+syn Lazy loading parameters in the filter are strictly boolean
+syn The only membership state included in an initial sync is for all the senders in the timeline 
+syn The only membership state included in an incremental sync is for senders in the timeline 
+syn The only membership state included in a gapped incremental sync is for senders in the timeline 
+syn Gapped incremental syncs include all state changes 
+syn Old leaves are present in gapped incremental syncs 
+syn Leaves are present in non-gapped incremental syncs 
+syn Old members are included in gappy incr LL sync if they start speaking 
+syn Members from the gap are included in gappy incr LL sync 
+syn We don't send redundant membership state across incremental syncs by default 
+syn We do send redundant membership state across incremental syncs if asked 
+syn Unnamed room comes with a name summary 
+syn Named room comes with just joined member count summary 
+syn Room summary only has 5 heroes 
+syn Room summary counts change when membership changes 
+rmv User can create and send/receive messages in a room with version 1 
+rmv User can create and send/receive messages in a room with version 1 (2 subtests)
+rmv local user can join room with version 1
+rmv User can invite local user to room with version 1
+rmv remote user can join room with version 1
+rmv User can invite remote user to room with version 1 
+rmv Remote user can backfill in a room with version 1
+rmv Can reject invites over federation for rooms with version 1 
+rmv Can receive redactions from regular users over federation in room version 1 
+rmv User can create and send/receive messages in a room with version 2 
+rmv User can create and send/receive messages in a room with version 2 (2 subtests)
+rmv local user can join room with version 2
+rmv User can invite local user to room with version 2
+rmv remote user can join room with version 2
+rmv User can invite remote user to room with version 2 
+rmv Remote user can backfill in a room with version 2
+rmv Can reject invites over federation for rooms with version 2 
+rmv Can receive redactions from regular users over federation in room version 2 
+rmv User can create and send/receive messages in a room with version 3 
+rmv User can create and send/receive messages in a room with version 3 (2 subtests)
+rmv local user can join room with version 3 
+rmv User can invite local user to room with version 3 
+rmv remote user can join room with version 3 
+rmv User can invite remote user to room with version 3 
+rmv Remote user can backfill in a room with version 3 
+rmv Can reject invites over federation for rooms with version 3 
+rmv Can receive redactions from regular users over federation in room version 3 
+rmv User can create and send/receive messages in a room with version 4 
+rmv User can create and send/receive messages in a room with version 4 (2 subtests)
+rmv local user can join room with version 4 
+rmv User can invite local user to room with version 4 
+rmv remote user can join room with version 4 
+rmv User can invite remote user to room with version 4 
+rmv Remote user can backfill in a room with version 4 
+rmv Can reject invites over federation for rooms with version 4 
+rmv Can receive redactions from regular users over federation in room version 4 
+rmv User can create and send/receive messages in a room with version 5 
+rmv User can create and send/receive messages in a room with version 5 (2 subtests)
+rmv local user can join room with version 5 
+rmv User can invite local user to room with version 5 
+rmv remote user can join room with version 5 
+rmv User can invite remote user to room with version 5 
+rmv Remote user can backfill in a room with version 5 
+rmv Can reject invites over federation for rooms with version 5 
+rmv Can receive redactions from regular users over federation in room version 5 
+pre Presence changes are reported to local room members 
+f,pre Presence changes are also reported to remote room members 
+pre Presence changes to UNAVAILABLE are reported to local room members 
+f,pre Presence changes to UNAVAILABLE are reported to remote room members 
+v1s Newly created users see their own presence in /initialSync (SYT-34) 
+dvk Can upload device keys 
+dvk Should reject keys claiming to belong to a different user
+dvk Can query device keys using POST 
+dvk Can query specific device keys using POST 
+dvk query for user with no keys returns empty key dict 
+dvk Can claim one time key using POST 
+f,dvk Can query remote device keys using POST 
+f,dvk Can claim remote one time key using POST 
+dvk Local device key changes appear in v2 /sync 
+dvk Local new device changes appear in v2 /sync 
+dvk Local delete device changes appear in v2 /sync 
+dvk Local update device changes appear in v2 /sync 
+dvk Can query remote device keys using POST after notification 
+f,dev Device deletion propagates over federation 
+f,dev If remote user leaves room, changes device and rejoins we see update in sync 
+f,dev If remote user leaves room we no longer receive device updates 
+dvk Local device key changes appear in /keys/changes 
+dvk New users appear in /keys/changes 
+f,dvk If remote user leaves room, changes device and rejoins we see update in /keys/changes 
+dvk Get left notifs in sync and /keys/changes when other user leaves 
+dvk Get left notifs for other users in sync and /keys/changes when user leaves 
+f,dvk If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes 
+dvk Can create backup version 
+dvk Can update backup version 
+dvk Responds correctly when backup is empty 
+dvk Can backup keys 
+dvk Can update keys with better versions 
+dvk Will not update keys with worse versions 
+dvk Will not back up to an old backup version 
+dvk Can delete backup 
+dvk Deleted & recreated backups are empty 
+dvk Can create more than 10 backup versions 
+dvk Can upload self-signing keys 
+dvk Fails to upload self-signing keys with no auth 
+dvk Fails to upload self-signing key without master key 
+dvk Changing master key notifies local users 
+dvk Changing user-signing key notifies local users 
+f,dvk can fetch self-signing keys over federation 
+f,dvk uploading self-signing key notifies over federation 
+f,dvk uploading signed devices gets propagated over federation 
+tag Can add tag
+tag Can remove tag
+tag Can list tags for a room
+v1s Tags appear in the v1 /events stream 
+v1s Tags appear in the v1 /initalSync 
+v1s Tags appear in the v1 room initial sync 
+tag Tags appear in an initial v2 /sync
+tag Newly updated tags appear in an incremental v2 /sync
+tag Deleted tags appear in an incremental v2 /sync
+tag local user has tags copied to the new room 
+f,tag remote user has tags copied to the new room 
+sch Can search for an event by body 
+sch Can get context around search results 
+sch Can back-paginate search results 
+sch Search works across an upgraded room and its predecessor 
+sch Search results with rank ordering do not include redacted events 
+sch Search results with recent ordering do not include redacted events 
+acc Can add account data
+acc Can add account data to room
+acc Can get account data without syncing 
+acc Can get room account data without syncing 
+v1s Latest account data comes down in /initialSync 
+v1s Latest account data comes down in room initialSync 
+v1s Account data appears in v1 /events stream 
+v1s Room account data appears in v1 /events stream 
+acc Latest account data appears in v2 /sync 
+acc New account data appears in incremental v2 /sync
+oid Can generate a openid access_token that can be exchanged for information about a user 
+oid Invalid openid access tokens are rejected 
+oid Requests to userinfo without access tokens are rejected 
+std Can send a message directly to a device using PUT /sendToDevice 
+std Can recv a device message using /sync 
+std Can recv device messages until they are acknowledged 
+std Device messages with the same txn_id are deduplicated 
+std Device messages wake up /sync 
+std Can recv device messages over federation 
+std Device messages over federation wake up /sync 
+std Can send messages with a wildcard device id 
+std Can send messages with a wildcard device id to two devices 
+std Wildcard device messages wake up /sync 
+std Wildcard device messages over federation wake up /sync 
+adm /whois 
+nsp /purge_history 
+nsp /purge_history by ts 
+nsp Can backfill purged history 
+nsp Shutdown room 
+ign Ignore user in existing room 
+ign Ignore invite in full sync 
+ign Ignore invite in incremental sync 
+fky Checking local federation server
+fky Federation key API allows unsigned requests for keys
+fky Federation key API can act as a notary server via a GET request 
+fky Federation key API can act as a notary server via a POST request 
+fky Key notary server should return an expired key if it can't find any others 
+fky Key notary server must not overwrite a valid key with a spurious result from the origin server 
+fqu Non-numeric ports in server names are rejected 
+fqu Outbound federation can query profile data
+fqu Inbound federation can query profile data
+fqu Outbound federation can query room alias directory
+fqu Inbound federation can query room alias directory
+fsj Outbound federation can query v1 /send_join 
+fsj Outbound federation can query v2 /send_join
+fmj Outbound federation passes make_join failures through to the client 
+fsj Inbound federation can receive v1 /send_join 
+fsj Inbound federation can receive v2 /send_join
+fmj Inbound /v1/make_join rejects remote attempts to join local users to rooms
+fsj Inbound /v1/send_join rejects incorrectly-signed joins 
+fsj Inbound /v1/send_join rejects joins from other servers 
+fau Inbound federation rejects remote attempts to kick local users to rooms
+frv Inbound federation rejects attempts to join v1 rooms from servers without v1 support 
+frv Inbound federation rejects attempts to join v2 rooms from servers lacking version support 
+frv Inbound federation rejects attempts to join v2 rooms from servers only supporting v1 
+frv Inbound federation accepts attempts to join v2 rooms from servers with support
+frv Outbound federation correctly handles unsupported room versions 
+frv A pair of servers can establish a join in a v2 room 
+fsj Outbound federation rejects send_join responses with no m.room.create event 
+frv Outbound federation rejects m.room.create events with an unknown room version 
+fsj Event with an invalid signature in the send_join response should not cause room join to fail 
+fed Outbound federation can send events
+fed Inbound federation can receive events 
+fed Inbound federation can receive redacted events 
+fed Ephemeral messages received from servers are correctly expired 
+fed Events whose auth_events are in the wrong room do not mess up the room state 
+fed Inbound federation can return events 
+fed Inbound federation redacts events from erased users 
+fme Outbound federation can request missing events 
+fme Inbound federation can return missing events for world_readable visibility 
+fme Inbound federation can return missing events for shared visibility 
+fme Inbound federation can return missing events for invite visibility 
+fme Inbound federation can return missing events for joined visibility 
+fme outliers whose auth_events are in a different room are correctly rejected 
+fbk Outbound federation can backfill events 
+fbk Inbound federation can backfill events 
+fbk Backfill checks the events requested belong to the room 
+fbk Backfilled events whose prev_events are in a different room do not allow cross-room back-pagination 
+fiv Outbound federation can send invites via v1 API 
+fiv Outbound federation can send invites via v2 API 
+fiv Inbound federation can receive invites via v1 API 
+fiv Inbound federation can receive invites via v2 API 
+fiv Inbound federation can receive invite and reject when remote replies with a 403 
+fiv Inbound federation can receive invite and reject when remote replies with a 500 
+fiv Inbound federation can receive invite and reject when remote is unreachable 
+fiv Inbound federation rejects invites which are not signed by the sender 
+fiv Inbound federation can receive invite rejections 
+fiv Inbound federation rejects incorrectly-signed invite rejections 
+fsl Inbound /v1/send_leave rejects leaves from other servers 
+fst Inbound federation can get state for a room 
+fst Inbound federation of state requires event_id as a mandatory paramater
+fst Inbound federation can get state_ids for a room 
+fst Inbound federation of state_ids requires event_id as a mandatory paramater
+fst  Federation rejects inbound events where the prev_events cannot be found 
+fst Room state at a rejected message event is the same as its predecessor 
+fst Room state at a rejected state event is the same as its predecessor 
+fst Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state 
+fst Federation handles empty auth_events in state_ids sanely 
+fst Getting state checks the events requested belong to the room 
+fst Getting state IDs checks the events requested belong to the room 
+fst Should not be able to take over the room by pretending there is no PL event 
+fpb Inbound federation can get public room list 
+fed Outbound federation sends receipts 
+fed Inbound federation rejects receipts from wrong remote 
+fed Inbound federation ignores redactions from invalid servers room > v3 
+fed An event which redacts an event in a different room should be ignored 
+fed An event which redacts itself should be ignored 
+fed A pair of events which redact each other should be ignored 
+fdk Local device key changes get to remote servers 
+fdk Server correctly handles incoming m.device_list_update 
+fdk Server correctly resyncs when client query keys and there is no remote cache 
+fdk Server correctly resyncs when server leaves and rejoins a room 
+fdk Local device key changes get to remote servers with correct prev_id 
+fdk Device list doesn't change if remote server is down 
+fdk If a device list update goes missing, the server resyncs on the next one 
+fst Name/topic keys are correct 
+fau Remote servers cannot set power levels in rooms without existing powerlevels 
+fau Remote servers should reject attempts by non-creators to set the power levels 
+fau Inbound federation rejects typing notifications from wrong remote 
+fed Forward extremities remain so even after the next events are populated as outliers 
+fau Banned servers cannot send events 
+fau Banned servers cannot /make_join 
+fau Banned servers cannot /send_join 
+fau Banned servers cannot /make_leave 
+fau Banned servers cannot /send_leave 
+fau Banned servers cannot /invite 
+fau Banned servers cannot get room state 
+fau Banned servers cannot get room state ids 
+fau Banned servers cannot backfill 
+fau Banned servers cannot /event_auth 
+fau Banned servers cannot get missing events 
+fau Server correctly handles transactions that break edu limits 
+fau Inbound federation correctly soft fails events 
+fau Inbound federation accepts a second soft-failed event 
+fau Inbound federation correctly handles soft failed events as extremities 
+med Can upload with Unicode file name 
+med Can download with Unicode file name locally 
+f,med Can download with Unicode file name over federation 
+med Alternative server names do not cause a routing loop 
+med Can download specifying a different Unicode file name 
+med Can upload without a file name
+med Can download without a file name locally
+f,med Can download without a file name over federation 
+med Can upload with ASCII file name
+med Can download file 'ascii' 
+med Can download file 'name with spaces' 
+med Can download file 'name;with;semicolons' 
+med Can download specifying a different ASCII file name 
+med Can send image in room message
+med Can fetch images in room 
+med POSTed media can be thumbnailed 
+f,med Remote media can be thumbnailed 
+med Test URL preview 
+med Can read configuration endpoint 
+nsp Can quarantine media in rooms 
+udr User appears in user directory 
+udr User in private room doesn't appear in user directory 
+udr User joining then leaving public room appears and dissappears from directory 
+udr Users appear/disappear from directory when join_rules are changed 
+udr Users appear/disappear from directory when history_visibility are changed 
+udr Users stay in directory when join_rules are changed but history_visibility is world_readable 
+f,udr User in remote room doesn't appear in user directory after server left room 
+udr User directory correctly update on display name change 
+udr User in shared private room does appear in user directory 
+udr User in shared private room does appear in user directory until leave 
+udr User in dir while user still shares private rooms 
+nsp Create group 
+nsp Add group rooms 
+nsp Remove group rooms 
+nsp Get local group profile 
+nsp Get local group users 
+nsp Add/remove local group rooms 
+nsp Get local group summary 
+nsp Get remote group profile 
+nsp Get remote group users 
+nsp Add/remove remote group rooms 
+nsp Get remote group summary 
+nsp Add local group users 
+nsp Remove self from local group 
+nsp Remove other from local group 
+nsp Add remote group users 
+nsp Remove self from remote group 
+nsp Listing invited users of a remote group when not a member returns a 403 
+nsp Add group category 
+nsp Remove group category 
+nsp Get group categories 
+nsp Add group role 
+nsp Remove group role 
+nsp Get group roles 
+nsp Add room to group summary 
+nsp Adding room to group summary keeps room_id when fetching rooms in group 
+nsp Adding multiple rooms to group summary have correct order 
+nsp Remove room from group summary 
+nsp Add room to group summary with category 
+nsp Remove room from group summary with category 
+nsp Add user to group summary 
+nsp Adding multiple users to group summary have correct order 
+nsp Remove user from group summary 
+nsp Add user to group summary with role 
+nsp Remove user from group summary with role 
+nsp Local group invites come down sync 
+nsp Group creator sees group in sync 
+nsp Group creator sees group in initial sync 
+nsp Get/set local group publicity 
+nsp Bulk get group publicity 
+nsp Joinability comes down summary 
+nsp Set group joinable and join it 
+nsp Group is not joinable by default 
+nsp Group is joinable over federation 
+nsp Room is transitioned on local and remote groups upon room upgrade 
+3pd Can bind 3PID via home server 
+3pd Can bind and unbind 3PID via homeserver 
+3pd Can unbind 3PID via homeserver when bound out of band 
+3pd 3PIDs are unbound after account deactivation 
+3pd Can bind and unbind 3PID via /unbind by specifying the identity server 
+3pd Can bind and unbind 3PID via /unbind without specifying the identity server 
+app AS can create a user 
+app AS can create a user with an underscore 
+app AS can create a user with inhibit_login 
+app AS cannot create users outside its own namespace
+app Regular users cannot register within the AS namespace
+app AS can make room aliases 
+app Regular users cannot create room aliases within the AS namespace
+app AS-ghosted users can use rooms via AS 
+app AS-ghosted users can use rooms themselves 
+app Ghost user must register before joining room 
+app AS can set avatar for ghosted users 
+app AS can set displayname for ghosted users 
+app AS can't set displayname for random users
+app Inviting an AS-hosted user asks the AS server 
+app Accesing an AS-hosted room alias asks the AS server 
+app Events in rooms with AS-hosted room aliases are sent to AS server 
+app AS user (not ghost) can join room without registering 
+app AS user (not ghost) can join room without registering, with user_id query param
+app HS provides query metadata 
+app HS can provide query metadata on a single protocol 
+app HS will proxy request for 3PU mapping 
+app HS will proxy request for 3PL mapping 
+app AS can publish rooms in their own list 
+app AS and main public room lists are separate 
+app AS can deactivate a user 
+psh Test that a message is pushed 
+psh Invites are pushed 
+psh Rooms with names are correctly named in pushed 
+psh Rooms with canonical alias are correctly named in pushed 
+psh Rooms with many users are correctly pushed 
+psh Don't get pushed for rooms you've muted 
+psh Rejected events are not pushed 
+psh Can add global push rule for room 
+psh Can add global push rule for sender 
+psh Can add global push rule for content 
+psh Can add global push rule for override 
+psh Can add global push rule for underride 
+psh Can add global push rule for content 
+psh New rules appear before old rules by default 
+psh Can add global push rule before an existing rule 
+psh Can add global push rule after an existing rule 
+psh Can delete a push rule 
+psh Can disable a push rule 
+psh Adding the same push rule twice is idempotent 
+psh Messages that notify from another user increment unread notification count 
+psh Messages that highlight from another user increment unread highlight count 
+psh Can change the actions of default rules 
+psh Changing the actions of an unknown default rule fails with 404
+psh Can change the actions of a user specified rule 
+psh Changing the actions of an unknown rule fails with 404
+psh Can fetch a user's pushers 
+psh Push rules come down in an initial /sync
+psh Adding a push rule wakes up an incremental /sync 
+psh Disabling a push rule wakes up an incremental /sync 
+psh Enabling a push rule wakes up an incremental /sync 
+psh Setting actions for a push rule wakes up an incremental /sync 
+psh Can enable/disable default rules 
+psh Enabling an unknown default rule fails with 404
+psh Test that rejected pushers are removed. 
+psh Notifications can be viewed with GET /notifications 
+psh Trying to add push rule with no scope fails with 400 
+psh Trying to add push rule with invalid scope fails with 400 
+psh Trying to add push rule with missing template fails with 400 
+psh Trying to add push rule with missing rule_id fails with 400 
+psh Trying to add push rule with empty rule_id fails with 400 
+psh Trying to add push rule with invalid template fails with 400 
+psh Trying to add push rule with rule_id with slashes fails with 400 
+psh Trying to add push rule with override rule without conditions fails with 400 
+psh Trying to add push rule with underride rule without conditions fails with 400 
+psh Trying to add push rule with condition without kind fails with 400 
+psh Trying to add push rule with content rule without pattern fails with 400 
+psh Trying to add push rule with no actions fails with 400 
+psh Trying to add push rule with invalid action fails with 400 
+psh Trying to add push rule with invalid attr fails with 400 
+psh Trying to add push rule with invalid value for enabled fails with 400 
+psh Trying to get push rules with no trailing slash fails with 400 
+psh Trying to get push rules with scope without trailing slash fails with 400 
+psh Trying to get push rules with template without tailing slash fails with 400 
+psh Trying to get push rules with unknown scope fails with 400 
+psh Trying to get push rules with unknown template fails with 400 
+psh Trying to get push rules with unknown attribute fails with 400 
+psh Trying to get push rules with unknown rule_id fails with 404
+v1s GET /initialSync with non-numeric 'limit' 
+v1s GET /events with non-numeric 'limit' 
+v1s GET /events with negative 'limit' 
+v1s GET /events with non-numeric 'timeout' 
+ath Event size limits 
+syn Check creating invalid filters returns 4xx 
+f,pre New federated private chats get full presence information (SYN-115) 
+pre Left room members do not cause problems for presence 
+crm Rooms can be created with an initial invite list (SYN-205) 
+typ Typing notifications don't leak 
+ban Non-present room members cannot ban others 
+psh Getting push rules doesn't corrupt the cache SYN-390 
+inv Test that we can be reinvited to a room we created 
+syn Multiple calls to /sync should not cause 500 errors 
+gst Guest user can call /events on another world_readable room (SYN-606) 
+gst Real user can call /events on another world_readable room (SYN-606) 
+gst Events come down the correct room
+pub Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list

are-we-synapse-yet.py

@@ -0,0 +1,252 @@
+#!/usr/bin/env python3
+
+from __future__ import division
+import argparse
+import re
+import sys
+
+# Usage: $ ./are-we-synapse-yet.py [-v] results.tap
+# This script scans a results.tap file from Dendrite's CI process and spits out
+# a rating of how close we are to Synapse parity, based purely on SyTests.
+# The main complexity is grouping tests sensibly into features like 'Registration'
+# and 'Federation'. Then it just checks the ones which are passing and calculates
+# percentages for each group. Produces results like:
+# 
+# Client-Server APIs: 29% (196/666 tests)
+# -------------------
+#   Registration             :  62% (20/32 tests)
+#   Login                    :   7% (1/15 tests)
+#   V1 CS APIs               :  10% (3/30 tests)
+#   ...
+#
+# or in verbose mode:
+#
+# Client-Server APIs: 29% (196/666 tests)
+# -------------------
+#  Registration             :  62% (20/32 tests)
+#    ✓ GET /register yields a set of flows
+#    ✓ POST /register can create a user
+#    ✓ POST /register downcases capitals in usernames
+#    ...
+# 
+# You can also tack `-v` on to see exactly which tests each category falls under.
+
+test_mappings = {
+    "nsp": "Non-Spec API",
+    "f": "Federation", # flag to mark test involves federation
+
+    "federation_apis": {
+        "fky": "Key API",
+        "fsj": "send_join API",
+        "fmj": "make_join API",
+        "fsl": "send_leave API",
+        "fiv": "Invite API",
+        "fqu": "Query API",
+        "frv": "room versions",
+        "fau": "Auth",
+        "fbk": "Backfill API",
+        "fme": "get_missing_events API",
+        "fst": "State APIs",
+        "fpb": "Public Room API",
+        "fdk": "Device Key APIs",
+        "fed": "Federation API",
+    },
+
+    "client_apis": {
+        "reg": "Registration",
+        "log": "Login",
+        "lox": "Logout",
+        "v1s": "V1 CS APIs",
+        "csa": "Misc CS APIs",
+        "pro": "Profile",
+        "dev": "Devices",
+        "dvk": "Device Keys",
+        "pre": "Presence",
+        "crm": "Create Room",
+        "syn": "Sync API",
+        "rmv": "Room Versions",
+        "rst": "Room State APIs",
+        "pub": "Public Room APIs",
+        "mem": "Room Membership",
+        "ali": "Room Aliases",
+        "jon": "Joining Rooms",
+        "lev": "Leaving Rooms",
+        "inv": "Inviting users to Rooms",
+        "ban": "Banning users",
+        "snd": "Sending events",
+        "get": "Getting events for Rooms",
+        "rct": "Receipts",
+        "red": "Read markers",
+        "med": "Media APIs",
+        "cap": "Capabilities API",
+        "typ": "Typing API",
+        "psh": "Push APIs",
+        "acc": "Account APIs",
+        "eph": "Ephemeral Events",
+        "plv": "Power Levels",
+        "xxx": "Redaction",
+        "3pd": "Third-Party ID APIs",
+        "gst": "Guest APIs",
+        "ath": "Room Auth",
+        "fgt": "Forget APIs",
+        "ctx": "Context APIs",
+        "upg": "Room Upgrade APIs",
+        "tag": "Tagging APIs",
+        "sch": "Search APIs",
+        "oid": "OpenID API",
+        "std": "Send-to-Device APIs",
+        "adm": "Server Admin API",
+        "ign": "Ignore Users",
+        "udr": "User Directory APIs",
+        "app": "Application Services API",
+    },
+}
+
+# optional 'not ' with test number then anything but '#'
+re_testname = re.compile(r"^(not )?ok [0-9]+ ([^#]+)")
+
+# Parses lines like the following:
+#
+# SUCCESS:     ok 3 POST /register downcases capitals in usernames
+# FAIL:        not ok 54 (expected fail) POST /createRoom creates a room with the given version
+# SKIP:        ok 821 Multiple calls to /sync should not cause 500 errors # skip lack of can_post_room_receipts
+# EXPECT FAIL: not ok 822 (expected fail) Guest user can call /events on another world_readable room (SYN-606) # TODO expected fail
+#
+# Only SUCCESS lines are treated as success, the rest are not implemented.
+#
+# Returns a dict like:
+# { name: "...", ok: True }
+def parse_test_line(line):
+    if not line.startswith("ok ") and not line.startswith("not ok "):
+        return
+    re_match = re_testname.match(line)
+    test_name = re_match.groups()[1].replace("(expected fail) ", "").strip()
+    test_pass = False
+    if line.startswith("ok ") and not "# skip " in line:
+        test_pass = True
+    return {
+        "name": test_name,
+        "ok": test_pass,
+    }
+
+# Prints the stats for a complete section.
+#   header_name => "Client-Server APIs"
+#   gid_to_tests => { gid: { <name>: True|False }}
+#   gid_to_name  => { gid: "Group Name" }
+#   verbose => True|False
+# Produces:
+# Client-Server APIs: 29% (196/666 tests)
+# -------------------
+#   Registration             :  62% (20/32 tests)
+#   Login                    :   7% (1/15 tests)
+#   V1 CS APIs               :  10% (3/30 tests)
+#   ...
+# or in verbose mode:
+# Client-Server APIs: 29% (196/666 tests)
+# -------------------
+#  Registration             :  62% (20/32 tests)
+#    ✓ GET /register yields a set of flows
+#    ✓ POST /register can create a user
+#    ✓ POST /register downcases capitals in usernames
+#    ...
+def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
+    subsections = [] # Registration: 100% (13/13 tests)
+    subsection_test_names = {} # 'subsection name': ["✓ Test 1", "✓ Test 2", "× Test 3"]
+    total_passing = 0
+    total_tests = 0
+    for gid, tests in gid_to_tests.items():
+        group_total = len(tests)
+        group_passing = 0
+        test_names_and_marks = []
+        for name, passing in tests.items():
+            if passing:
+                group_passing += 1
+            test_names_and_marks.append(f"{'✓' if passing else '×'} {name}")
+            
+        total_tests += group_total
+        total_passing += group_passing
+        pct = "{0:.0f}%".format(group_passing/group_total * 100)
+        line = "%s: %s (%d/%d tests)" % (gid_to_name[gid].ljust(25, ' '), pct.rjust(4, ' '), group_passing, group_total)
+        subsections.append(line)
+        subsection_test_names[line] = test_names_and_marks
+    
+    pct = "{0:.0f}%".format(total_passing/total_tests * 100)
+    print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests))
+    print("-" * (len(header_name)+1))
+    for line in subsections:
+        print("  %s" % (line,))
+        if verbose:
+            for test_name_and_pass_mark in subsection_test_names[line]:
+                print("    %s" % (test_name_and_pass_mark,))
+            print("")
+    print("")
+
+def main(results_tap_path, verbose):
+    # Load up test mappings
+    test_name_to_group_id = {}
+    fed_tests = set()
+    client_tests = set()
+    with open("./are-we-synapse-yet.list", "r") as f:
+        for line in f.readlines():
+            test_name = " ".join(line.split(" ")[1:]).strip()
+            groups = line.split(" ")[0].split(",")
+            for gid in groups:
+                if gid == "f" or gid in test_mappings["federation_apis"]:
+                    fed_tests.add(test_name)
+                else:
+                    client_tests.add(test_name)
+                if gid == "f":
+                    continue # we expect another group ID
+                test_name_to_group_id[test_name] = gid
+
+    # parse results.tap
+    summary = {
+        "client": {
+            # gid: {
+            #   test_name: OK
+            # }
+        },
+        "federation": {
+            # gid: {
+            #   test_name: OK
+            # }
+        },
+        "nonspec": {
+            "nsp": {}
+        },
+    }
+    with open(results_tap_path, "r") as f:
+        for line in f.readlines():
+            test_result = parse_test_line(line)
+            if not test_result:
+                continue
+            name = test_result["name"]
+            group_id = test_name_to_group_id.get(name)
+            if not group_id:
+                raise Exception("The test '%s' doesn't have a group" % (name,))
+            if group_id == "nsp":
+                summary["nonspec"]["nsp"][name] = test_result["ok"]
+            elif group_id in test_mappings["federation_apis"]:
+                group = summary["federation"].get(group_id, {})
+                group[name] = test_result["ok"]
+                summary["federation"][group_id] = group
+            elif group_id in test_mappings["client_apis"]:
+                group = summary["client"].get(group_id, {})
+                group[name] = test_result["ok"]
+                summary["client"][group_id] = group
+
+    print("Are We Synapse Yet?")
+    print("===================")
+    print("")
+    print_stats("Non-Spec APIs", summary["nonspec"], test_mappings, verbose)
+    print_stats("Client-Server APIs", summary["client"], test_mappings["client_apis"], verbose)
+    print_stats("Federation APIs", summary["federation"], test_mappings["federation_apis"], verbose)
+
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("tap_file", help="path to results.tap")
+    parser.add_argument("-v", action="store_true", help="show individual test names in output")
+    args = parser.parse_args()
+    main(args.tap_file, args.v)

clientapi/auth/storage/accounts/postgres/storage.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 	"golang.org/x/crypto/bcrypt"
 
@@ -46,7 +47,7 @@ type Database struct {
 func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sql.Open("postgres", dataSourceName); err != nil {
+	if db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
 		return nil, err
 	}
 	partitions := common.PartitionOffsetStatements{}

clientapi/auth/storage/accounts/sqlite3/storage.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 	"golang.org/x/crypto/bcrypt"
 
@@ -49,7 +50,7 @@ type Database struct {
 func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
 		return nil, err
 	}
 	partitions := common.PartitionOffsetStatements{}

clientapi/auth/storage/devices/postgres/storage.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -38,7 +39,7 @@ type Database struct {
 func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sql.Open("postgres", dataSourceName); err != nil {
+	if db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
 		return nil, err
 	}
 	d := devicesStatements{}

clientapi/auth/storage/devices/sqlite3/storage.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	_ "github.com/mattn/go-sqlite3"
@@ -40,7 +41,7 @@ type Database struct {
 func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
 		return nil, err
 	}
 	d := devicesStatements{}

clientapi/routing/joinroom.go

@@ -32,6 +32,7 @@ import (
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
+	"github.com/sirupsen/logrus"
 )
 
 // JoinRoomByIDOrAlias implements the "/join/{roomIDOrAlias}" API.
@@ -290,8 +291,16 @@ func (r joinRoomReq) joinRoomUsingServers(
 			// There was a problem talking to one of the servers.
 			util.GetLogger(r.req.Context()).WithError(lastErr).WithField("server", server).Warn("Failed to join room using server")
 			// Try the next server.
+			if r.req.Context().Err() != nil {
+				// The request context has expired so don't bother trying any
+				// more servers - they will immediately fail due to the expired
+				// context.
+				break
+			} else {
+				// The request context hasn't expired yet so try the next server.
 				continue
 			}
+		}
 		return *response
 	}
 
@@ -365,16 +374,22 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib
 		return nil, fmt.Errorf("r.federation.SendJoin: %w", err)
 	}
 
-	if err = respSendJoin.Check(r.req.Context(), r.keyRing, event); err != nil {
+	if err = r.checkSendJoinResponse(event, server, respMakeJoin, respSendJoin); err != nil {
-		return nil, fmt.Errorf("respSendJoin: %w", err)
+		return nil, err
 	}
 
+	util.GetLogger(r.req.Context()).WithFields(logrus.Fields{
+		"room_id":          roomID,
+		"num_auth_events":  len(respSendJoin.AuthEvents),
+		"num_state_events": len(respSendJoin.StateEvents),
+	}).Info("Room join signature and auth verification passed")
+
 	if err = r.producer.SendEventWithState(
 		r.req.Context(),
 		gomatrixserverlib.RespState(respSendJoin.RespState),
 		event.Headered(respMakeJoin.RoomVersion),
 	); err != nil {
-		return nil, fmt.Errorf("r.producer.SendEventWithState: %w", err)
+		util.GetLogger(r.req.Context()).WithError(err).Error("r.producer.SendEventWithState")
 	}
 
 	return &util.JSONResponse{
@@ -385,3 +400,49 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib
 		}{roomID},
 	}, nil
 }
+
+// checkSendJoinResponse checks that all of the signatures are correct
+// and that the join is allowed by the supplied state.
+func (r joinRoomReq) checkSendJoinResponse(
+	event gomatrixserverlib.Event,
+	server gomatrixserverlib.ServerName,
+	respMakeJoin gomatrixserverlib.RespMakeJoin,
+	respSendJoin gomatrixserverlib.RespSendJoin,
+) error {
+	// A list of events that we have retried, if they were not included in
+	// the auth events supplied in the send_join.
+	retries := map[string]bool{}
+
+retryCheck:
+	// TODO: Can we expand Check here to return a list of missing auth
+	// events rather than failing one at a time?
+	if err := respSendJoin.Check(r.req.Context(), r.keyRing, event); err != nil {
+		switch e := err.(type) {
+		case gomatrixserverlib.MissingAuthEventError:
+			// Check that we haven't already retried for this event, prevents
+			// us from ending up in endless loops
+			if !retries[e.AuthEventID] {
+				// Ask the server that we're talking to right now for the event
+				tx, txerr := r.federation.GetEvent(r.req.Context(), server, e.AuthEventID)
+				if txerr != nil {
+					return fmt.Errorf("r.federation.GetEvent: %w", txerr)
+				}
+				// For each event returned, add it to the auth events.
+				for _, pdu := range tx.PDUs {
+					ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, respMakeJoin.RoomVersion)
+					if everr != nil {
+						return fmt.Errorf("gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr)
+					}
+					respSendJoin.AuthEvents = append(respSendJoin.AuthEvents, ev)
+				}
+				// Mark the event as retried and then give the check another go.
+				retries[e.AuthEventID] = true
+				goto retryCheck
+			}
+			return fmt.Errorf("respSendJoin (after retries): %w", e)
+		default:
+			return fmt.Errorf("respSendJoin: %w", err)
+		}
+	}
+	return nil
+}

clientapi/routing/register.go

@@ -362,7 +362,7 @@ func UsernameMatchesMultipleExclusiveNamespaces(
 	// Check namespaces and see if more than one match
 	matchCount := 0
 	for _, appservice := range cfg.Derived.ApplicationServices {
-		if appservice.IsInterestedInUserID(userID) {
+		if appservice.OwnsNamespaceCoveringUserId(userID) {
 			if matchCount++; matchCount > 1 {
 				return true
 			}
@@ -1000,7 +1000,7 @@ func RegisterAvailable(
 	// Check if this username is reserved by an application service
 	userID := userutil.MakeUserID(username, cfg.Matrix.ServerName)
 	for _, appservice := range cfg.Derived.ApplicationServices {
-		if appservice.IsInterestedInUserID(userID) {
+		if appservice.OwnsNamespaceCoveringUserId(userID) {
 			return util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: jsonerror.UserInUse("Desired user ID is reserved by an application service."),

clientapi/routing/routing.go

@@ -149,6 +149,31 @@ func Setup(
 			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, queryAPI, federation, keyRing)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/rooms/{roomID}/state", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		vars, err := common.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
+		return OnIncomingStateRequest(req.Context(), queryAPI, vars["roomID"])
+	})).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		vars, err := common.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
+		return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], "")
+	})).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		vars, err := common.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
+		return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], vars["stateKey"])
+	})).Methods(http.MethodGet, http.MethodOptions)
+
 	r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}",
 		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
 			vars, err := common.URLDecodeMapValues(mux.Vars(req))
@@ -164,6 +189,7 @@ func Setup(
 			return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, queryAPI, producer, nil)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
+
 	r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}",
 		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
 			vars, err := common.URLDecodeMapValues(mux.Vars(req))

clientapi/routing/state.go

@@ -15,11 +15,12 @@
 package routing
 
 import (
+	"context"
 	"encoding/json"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -39,22 +40,29 @@ type stateEventInStateResp struct {
 // TODO: Check if the user is in the room. If not, check if the room's history
 // is publicly visible. Current behaviour is returning an empty array if the
 // user cannot see the room's history.
-func OnIncomingStateRequest(req *http.Request, db storage.Database, roomID string) util.JSONResponse {
+func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string) util.JSONResponse {
 	// TODO(#287): Auth request and handle the case where the user has left (where
 	// we should return the state at the poin they left)
+	stateReq := api.QueryLatestEventsAndStateRequest{
+		RoomID: roomID,
+	}
+	stateRes := api.QueryLatestEventsAndStateResponse{}
 
-	stateFilter := gomatrixserverlib.DefaultStateFilter()
+	if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil {
-	// TODO: stateFilter should not limit the number of state events (or only limits abusive number of events)
+		util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed")
-
-	stateEvents, err := db.GetStateEventsForRoom(req.Context(), roomID, &stateFilter)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("db.GetStateEventsForRoom failed")
 		return jsonerror.InternalServerError()
 	}
 
+	if len(stateRes.StateEvents) == 0 {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("cannot find state"),
+		}
+	}
+
 	resp := []stateEventInStateResp{}
 	// Fill the prev_content and replaces_state keys if necessary
-	for _, event := range stateEvents {
+	for _, event := range stateRes.StateEvents {
 		stateEvent := stateEventInStateResp{
 			ClientEvent: gomatrixserverlib.HeaderedToClientEvents(
 				[]gomatrixserverlib.HeaderedEvent{event}, gomatrixserverlib.FormatAll,
@@ -63,7 +71,7 @@ func OnIncomingStateRequest(req *http.Request, db storage.Database, roomID strin
 		var prevEventRef types.PrevEventRef
 		if len(event.Unsigned()) > 0 {
 			if err := json.Unmarshal(event.Unsigned(), &prevEventRef); err != nil {
-				util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal failed")
+				util.GetLogger(ctx).WithError(err).Error("json.Unmarshal failed")
 				return jsonerror.InternalServerError()
 			}
 			// Fills the previous state event ID if the state event replaces another
@@ -90,24 +98,32 @@ func OnIncomingStateRequest(req *http.Request, db storage.Database, roomID strin
 // /rooms/{roomID}/state/{type}/{statekey} request. It will look in current
 // state to see if there is an event with that type and state key, if there
 // is then (by default) we return the content, otherwise a 404.
-func OnIncomingStateTypeRequest(req *http.Request, db storage.Database, roomID string, evType, stateKey string) util.JSONResponse {
+func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string, evType, stateKey string) util.JSONResponse {
 	// TODO(#287): Auth request and handle the case where the user has left (where
 	// we should return the state at the poin they left)
-
+	util.GetLogger(ctx).WithFields(log.Fields{
-	logger := util.GetLogger(req.Context())
-	logger.WithFields(log.Fields{
 		"roomID":   roomID,
 		"evType":   evType,
 		"stateKey": stateKey,
 	}).Info("Fetching state")
 
-	event, err := db.GetStateEvent(req.Context(), roomID, evType, stateKey)
+	stateReq := api.QueryLatestEventsAndStateRequest{
-	if err != nil {
+		RoomID: roomID,
-		util.GetLogger(req.Context()).WithError(err).Error("db.GetStateEvent failed")
+		StateToFetch: []gomatrixserverlib.StateKeyTuple{
+			gomatrixserverlib.StateKeyTuple{
+				EventType: evType,
+				StateKey:  stateKey,
+			},
+		},
+	}
+	stateRes := api.QueryLatestEventsAndStateResponse{}
+
+	if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil {
+		util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed")
 		return jsonerror.InternalServerError()
 	}
 
-	if event == nil {
+	if len(stateRes.StateEvents) == 0 {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,
 			JSON: jsonerror.NotFound("cannot find state"),
@@ -115,7 +131,7 @@ func OnIncomingStateTypeRequest(req *http.Request, db storage.Database, roomID s
 	}
 
 	stateEvent := stateEventInStateResp{
-		ClientEvent: gomatrixserverlib.HeaderedToClientEvent(*event, gomatrixserverlib.FormatAll),
+		ClientEvent: gomatrixserverlib.HeaderedToClientEvent(stateRes.StateEvents[0], gomatrixserverlib.FormatAll),
 	}
 
 	return util.JSONResponse{

cmd/dendrite-client-api-server/main.go

@@ -33,7 +33,7 @@ func main() {
 	deviceDB := base.CreateDeviceDB()
 	keyDB := base.CreateKeyDB()
 	federation := base.CreateFederationClient()
-	keyRing := keydb.CreateKeyRing(federation.Client, keyDB)
+	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
 
 	asQuery := base.CreateHTTPAppServiceAPIs()
 	alias, input, query := base.CreateHTTPRoomserverAPIs()

cmd/dendrite-demo-libp2p/main.go

@@ -0,0 +1,203 @@
+// Copyright 2017 Vector Creations Ltd
+// Copyright 2018 New Vector Ltd
+// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"crypto/ed25519"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"time"
+
+	gostream "github.com/libp2p/go-libp2p-gostream"
+	p2phttp "github.com/libp2p/go-libp2p-http"
+	p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery"
+	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/clientapi"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/common/keydb"
+	"github.com/matrix-org/dendrite/common/transactions"
+	"github.com/matrix-org/dendrite/eduserver"
+	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/federationsender"
+	"github.com/matrix-org/dendrite/mediaapi"
+	"github.com/matrix-org/dendrite/publicroomsapi"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/syncapi"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/eduserver/cache"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/sirupsen/logrus"
+)
+
+func createKeyDB(
+	base *P2PDendrite,
+) keydb.Database {
+	db, err := keydb.NewDatabase(
+		string(base.Base.Cfg.Database.ServerKey),
+		base.Base.Cfg.Matrix.ServerName,
+		base.Base.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey),
+		base.Base.Cfg.Matrix.KeyID,
+	)
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to connect to keys db")
+	}
+	mdns := mDNSListener{
+		host:  base.LibP2P,
+		keydb: db,
+	}
+	serv, err := p2pdisc.NewMdnsService(
+		base.LibP2PContext,
+		base.LibP2P,
+		time.Second*10,
+		"_matrix-dendrite-p2p._tcp",
+	)
+	if err != nil {
+		panic(err)
+	}
+	serv.RegisterNotifee(&mdns)
+	return db
+}
+
+func createFederationClient(
+	base *P2PDendrite,
+) *gomatrixserverlib.FederationClient {
+	fmt.Println("Running in libp2p federation mode")
+	fmt.Println("Warning: Federation with non-libp2p homeservers will not work in this mode yet!")
+	tr := &http.Transport{}
+	tr.RegisterProtocol(
+		"matrix",
+		p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")),
+	)
+	return gomatrixserverlib.NewFederationClientWithTransport(
+		base.Base.Cfg.Matrix.ServerName, base.Base.Cfg.Matrix.KeyID, base.Base.Cfg.Matrix.PrivateKey, tr,
+	)
+}
+
+func main() {
+	instanceName := flag.String("name", "dendrite-p2p", "the name of this P2P demo instance")
+	instancePort := flag.Int("port", 8080, "the port that the client API will listen on")
+	flag.Parse()
+
+	filename := fmt.Sprintf("%s-private.key", *instanceName)
+	_, err := os.Stat(filename)
+	var privKey ed25519.PrivateKey
+	if os.IsNotExist(err) {
+		_, privKey, _ = ed25519.GenerateKey(nil)
+		if err = ioutil.WriteFile(filename, privKey, 0600); err != nil {
+			fmt.Printf("Couldn't write private key to file '%s': %s\n", filename, err)
+		}
+	} else {
+		privKey, err = ioutil.ReadFile(filename)
+		if err != nil {
+			fmt.Printf("Couldn't read private key from file '%s': %s\n", filename, err)
+			_, privKey, _ = ed25519.GenerateKey(nil)
+		}
+	}
+
+	cfg := config.Dendrite{}
+	cfg.Matrix.ServerName = "p2p"
+	cfg.Matrix.PrivateKey = privKey
+	cfg.Matrix.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName))
+	cfg.Kafka.UseNaffka = true
+	cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput"
+	cfg.Kafka.Topics.OutputClientData = "clientapiOutput"
+	cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput"
+	cfg.Kafka.Topics.UserUpdates = "userUpdates"
+	cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
+	cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName))
+	cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
+	cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
+	cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
+	cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName))
+	cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
+	cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
+	cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s-publicroomsa.db", *instanceName))
+	cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
+	if err = cfg.Derive(); err != nil {
+		panic(err)
+	}
+
+	base := NewP2PDendrite(&cfg, "Monolith")
+	defer base.Base.Close() // nolint: errcheck
+
+	accountDB := base.Base.CreateAccountsDB()
+	deviceDB := base.Base.CreateDeviceDB()
+	keyDB := createKeyDB(base)
+	federation := createFederationClient(base)
+	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
+
+	alias, input, query := roomserver.SetupRoomServerComponent(&base.Base)
+	eduInputAPI := eduserver.SetupEDUServerComponent(&base.Base, cache.New())
+	asQuery := appservice.SetupAppServiceAPIComponent(
+		&base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(),
+	)
+	fedSenderAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query)
+
+	clientapi.SetupClientAPIComponent(
+		&base.Base, deviceDB, accountDB,
+		federation, &keyRing, alias, input, query,
+		eduInputAPI, asQuery, transactions.New(), fedSenderAPI,
+	)
+	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
+	federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer)
+	mediaapi.SetupMediaAPIComponent(&base.Base, deviceDB)
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub)
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to connect to public rooms db")
+	}
+	publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, query, federation, nil) // Check this later
+	syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, query, federation, &cfg)
+
+	httpHandler := common.WrapHandlerInCORS(base.Base.APIMux)
+
+	// Set up the API endpoints we handle. /metrics is for prometheus, and is
+	// not wrapped by CORS, while everything else is
+	http.Handle("/metrics", promhttp.Handler())
+	http.Handle("/", httpHandler)
+
+	// Expose the matrix APIs directly rather than putting them under a /api path.
+	go func() {
+		httpBindAddr := fmt.Sprintf(":%d", *instancePort)
+		logrus.Info("Listening on ", httpBindAddr)
+		logrus.Fatal(http.ListenAndServe(httpBindAddr, nil))
+	}()
+	// Expose the matrix APIs also via libp2p
+	if base.LibP2P != nil {
+		go func() {
+			logrus.Info("Listening on libp2p host ID ", base.LibP2P.ID())
+			listener, err := gostream.Listen(base.LibP2P, "/matrix")
+			if err != nil {
+				panic(err)
+			}
+			defer func() {
+				logrus.Fatal(listener.Close())
+			}()
+			logrus.Fatal(http.Serve(listener, nil))
+		}()
+	}
+
+	// We want to block forever to let the HTTP and HTTPS handler serve the APIs
+	select {}
+}

cmd/dendrite-demo-libp2p/mdnslistener.go

@@ -0,0 +1,63 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"math"
+
+	"github.com/libp2p/go-libp2p-core/host"
+	"github.com/libp2p/go-libp2p-core/peer"
+	"github.com/matrix-org/dendrite/common/keydb"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type mDNSListener struct {
+	keydb keydb.Database
+	host  host.Host
+}
+
+func (n *mDNSListener) HandlePeerFound(p peer.AddrInfo) {
+	if err := n.host.Connect(context.Background(), p); err != nil {
+		fmt.Println("Error adding peer", p.ID.String(), "via mDNS:", err)
+	}
+	if pubkey, err := p.ID.ExtractPublicKey(); err == nil {
+		raw, _ := pubkey.Raw()
+		if err := n.keydb.StoreKeys(
+			context.Background(),
+			map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{
+				{
+					ServerName: gomatrixserverlib.ServerName(p.ID.String()),
+					KeyID:      "ed25519:p2pdemo",
+				}: {
+					VerifyKey: gomatrixserverlib.VerifyKey{
+						Key: gomatrixserverlib.Base64String(raw),
+					},
+					ValidUntilTS: math.MaxUint64 >> 1,
+					ExpiredTS:    gomatrixserverlib.PublicKeyNotExpired,
+				},
+			},
+		); err != nil {
+			fmt.Println("Failed to store keys:", err)
+		}
+	}
+	fmt.Println("Discovered", len(n.host.Peerstore().Peers())-1, "other libp2p peer(s):")
+	for _, peer := range n.host.Peerstore().Peers() {
+		if peer != n.host.ID() {
+			fmt.Println("-", peer)
+		}
+	}
+}

cmd/dendrite-demo-libp2p/p2pdendrite.go

@@ -0,0 +1,126 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"errors"
+
+	pstore "github.com/libp2p/go-libp2p-core/peerstore"
+	record "github.com/libp2p/go-libp2p-record"
+	"github.com/matrix-org/dendrite/common/basecomponent"
+
+	"github.com/libp2p/go-libp2p"
+	circuit "github.com/libp2p/go-libp2p-circuit"
+	crypto "github.com/libp2p/go-libp2p-core/crypto"
+	routing "github.com/libp2p/go-libp2p-core/routing"
+
+	host "github.com/libp2p/go-libp2p-core/host"
+	dht "github.com/libp2p/go-libp2p-kad-dht"
+	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	"github.com/matrix-org/dendrite/common/config"
+)
+
+// P2PDendrite is a Peer-to-Peer variant of BaseDendrite.
+type P2PDendrite struct {
+	Base basecomponent.BaseDendrite
+
+	// Store our libp2p object so that we can make outgoing connections from it
+	// later
+	LibP2P        host.Host
+	LibP2PContext context.Context
+	LibP2PCancel  context.CancelFunc
+	LibP2PDHT     *dht.IpfsDHT
+	LibP2PPubsub  *pubsub.PubSub
+}
+
+// NewP2PDendrite creates a new instance to be used by a component.
+// The componentName is used for logging purposes, and should be a friendly name
+// of the component running, e.g. SyncAPI.
+func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite {
+	baseDendrite := basecomponent.NewBaseDendrite(cfg, componentName)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	privKey, err := crypto.UnmarshalEd25519PrivateKey(cfg.Matrix.PrivateKey[:])
+	if err != nil {
+		panic(err)
+	}
+
+	//defaultIP6ListenAddr, _ := multiaddr.NewMultiaddr("/ip6/::/tcp/0")
+	var libp2pdht *dht.IpfsDHT
+	libp2p, err := libp2p.New(ctx,
+		libp2p.Identity(privKey),
+		libp2p.DefaultListenAddrs,
+		//libp2p.ListenAddrs(defaultIP6ListenAddr),
+		libp2p.DefaultTransports,
+		libp2p.Routing(func(h host.Host) (r routing.PeerRouting, err error) {
+			libp2pdht, err = dht.New(ctx, h)
+			if err != nil {
+				return nil, err
+			}
+			libp2pdht.Validator = libP2PValidator{}
+			r = libp2pdht
+			return
+		}),
+		libp2p.EnableAutoRelay(),
+		libp2p.EnableRelay(circuit.OptHop),
+	)
+	if err != nil {
+		panic(err)
+	}
+
+	libp2ppubsub, err := pubsub.NewFloodSub(context.Background(), libp2p, []pubsub.Option{
+		pubsub.WithMessageSigning(true),
+	}...)
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Println("Our public key:", privKey.GetPublic())
+	fmt.Println("Our node ID:", libp2p.ID())
+	fmt.Println("Our addresses:", libp2p.Addrs())
+
+	cfg.Matrix.ServerName = gomatrixserverlib.ServerName(libp2p.ID().String())
+
+	return &P2PDendrite{
+		Base:          *baseDendrite,
+		LibP2P:        libp2p,
+		LibP2PContext: ctx,
+		LibP2PCancel:  cancel,
+		LibP2PDHT:     libp2pdht,
+		LibP2PPubsub:  libp2ppubsub,
+	}
+}
+
+type libP2PValidator struct {
+	KeyBook pstore.KeyBook
+}
+
+func (v libP2PValidator) Validate(key string, value []byte) error {
+	ns, _, err := record.SplitKey(key)
+	if err != nil || ns != "matrix" {
+		return errors.New("not Matrix path")
+	}
+	return nil
+}
+
+func (v libP2PValidator) Select(k string, vals [][]byte) (int, error) {
+	return 0, nil
+}

cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go

@@ -0,0 +1,164 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgreswithdht
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/matrix-org/dendrite/publicroomsapi/storage/postgres"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	dht "github.com/libp2p/go-libp2p-kad-dht"
+)
+
+const DHTInterval = time.Second * 10
+
+// PublicRoomsServerDatabase represents a public rooms server database.
+type PublicRoomsServerDatabase struct {
+	dht *dht.IpfsDHT
+	postgres.PublicRoomsServerDatabase
+	ourRoomsContext  context.Context                         // our current value in the DHT
+	ourRoomsCancel   context.CancelFunc                      // cancel when we want to expire our value
+	foundRooms       map[string]gomatrixserverlib.PublicRoom // additional rooms we have learned about from the DHT
+	foundRoomsMutex  sync.RWMutex                            // protects foundRooms
+	maintenanceTimer *time.Timer                             //
+	roomsAdvertised  atomic.Value                            // stores int
+	roomsDiscovered  atomic.Value                            // stores int
+}
+
+// NewPublicRoomsServerDatabase creates a new public rooms server database.
+func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT) (*PublicRoomsServerDatabase, error) {
+	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName)
+	if err != nil {
+		return nil, err
+	}
+	provider := PublicRoomsServerDatabase{
+		dht:                       dht,
+		PublicRoomsServerDatabase: *pg,
+	}
+	go provider.ResetDHTMaintenance()
+	provider.roomsAdvertised.Store(0)
+	provider.roomsDiscovered.Store(0)
+	return &provider, nil
+}
+
+func (d *PublicRoomsServerDatabase) GetRoomVisibility(ctx context.Context, roomID string) (bool, error) {
+	return d.PublicRoomsServerDatabase.GetRoomVisibility(ctx, roomID)
+}
+
+func (d *PublicRoomsServerDatabase) SetRoomVisibility(ctx context.Context, visible bool, roomID string) error {
+	d.ResetDHTMaintenance()
+	return d.PublicRoomsServerDatabase.SetRoomVisibility(ctx, visible, roomID)
+}
+
+func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) {
+	count, err := d.PublicRoomsServerDatabase.CountPublicRooms(ctx)
+	if err != nil {
+		return 0, err
+	}
+	d.foundRoomsMutex.RLock()
+	defer d.foundRoomsMutex.RUnlock()
+	return count + int64(len(d.foundRooms)), nil
+}
+
+func (d *PublicRoomsServerDatabase) GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) {
+	realfilter := filter
+	if realfilter == "__local__" {
+		realfilter = ""
+	}
+	rooms, err := d.PublicRoomsServerDatabase.GetPublicRooms(ctx, offset, limit, realfilter)
+	if err != nil {
+		return []gomatrixserverlib.PublicRoom{}, err
+	}
+	if filter != "__local__" {
+		d.foundRoomsMutex.RLock()
+		defer d.foundRoomsMutex.RUnlock()
+		for _, room := range d.foundRooms {
+			rooms = append(rooms, room)
+		}
+	}
+	return rooms, nil
+}
+
+func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error {
+	return d.PublicRoomsServerDatabase.UpdateRoomFromEvents(ctx, eventsToAdd, eventsToRemove)
+}
+
+func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error {
+	return d.PublicRoomsServerDatabase.UpdateRoomFromEvent(ctx, event)
+}
+
+func (d *PublicRoomsServerDatabase) ResetDHTMaintenance() {
+	if d.maintenanceTimer != nil && !d.maintenanceTimer.Stop() {
+		<-d.maintenanceTimer.C
+	}
+	d.Interval()
+}
+
+func (d *PublicRoomsServerDatabase) Interval() {
+	if err := d.AdvertiseRoomsIntoDHT(); err != nil {
+		//	fmt.Println("Failed to advertise room in DHT:", err)
+	}
+	if err := d.FindRoomsInDHT(); err != nil {
+		//	fmt.Println("Failed to find rooms in DHT:", err)
+	}
+	fmt.Println("Found", d.roomsDiscovered.Load(), "room(s), advertised", d.roomsAdvertised.Load(), "room(s)")
+	d.maintenanceTimer = time.AfterFunc(DHTInterval, d.Interval)
+}
+
+func (d *PublicRoomsServerDatabase) AdvertiseRoomsIntoDHT() error {
+	dbCtx, dbCancel := context.WithTimeout(context.Background(), 3*time.Second)
+	_ = dbCancel
+	ourRooms, err := d.GetPublicRooms(dbCtx, 0, 1024, "__local__")
+	if err != nil {
+		return err
+	}
+	if j, err := json.Marshal(ourRooms); err == nil {
+		d.roomsAdvertised.Store(len(ourRooms))
+		d.ourRoomsContext, d.ourRoomsCancel = context.WithCancel(context.Background())
+		if err := d.dht.PutValue(d.ourRoomsContext, "/matrix/publicRooms", j); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (d *PublicRoomsServerDatabase) FindRoomsInDHT() error {
+	d.foundRoomsMutex.Lock()
+	searchCtx, searchCancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer searchCancel()
+	defer d.foundRoomsMutex.Unlock()
+	results, err := d.dht.GetValues(searchCtx, "/matrix/publicRooms", 1024)
+	if err != nil {
+		return err
+	}
+	d.foundRooms = make(map[string]gomatrixserverlib.PublicRoom)
+	for _, result := range results {
+		var received []gomatrixserverlib.PublicRoom
+		if err := json.Unmarshal(result.Val, &received); err != nil {
+			return err
+		}
+		for _, room := range received {
+			d.foundRooms[room.RoomID] = room
+		}
+	}
+	d.roomsDiscovered.Store(len(d.foundRooms))
+	return nil
+}

cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go

@@ -0,0 +1,179 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postgreswithpubsub
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/matrix-org/dendrite/publicroomsapi/storage/postgres"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	pubsub "github.com/libp2p/go-libp2p-pubsub"
+)
+
+const MaintenanceInterval = time.Second * 10
+
+type discoveredRoom struct {
+	time time.Time
+	room gomatrixserverlib.PublicRoom
+}
+
+// PublicRoomsServerDatabase represents a public rooms server database.
+type PublicRoomsServerDatabase struct {
+	postgres.PublicRoomsServerDatabase                           //
+	pubsub                             *pubsub.PubSub            //
+	subscription                       *pubsub.Subscription      //
+	foundRooms                         map[string]discoveredRoom // additional rooms we have learned about from the DHT
+	foundRoomsMutex                    sync.RWMutex              // protects foundRooms
+	maintenanceTimer                   *time.Timer               //
+	roomsAdvertised                    atomic.Value              // stores int
+}
+
+// NewPublicRoomsServerDatabase creates a new public rooms server database.
+func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub) (*PublicRoomsServerDatabase, error) {
+	pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName)
+	if err != nil {
+		return nil, err
+	}
+	provider := PublicRoomsServerDatabase{
+		pubsub:                    pubsub,
+		PublicRoomsServerDatabase: *pg,
+		foundRooms:                make(map[string]discoveredRoom),
+	}
+	if topic, err := pubsub.Join("/matrix/publicRooms"); err != nil {
+		return nil, err
+	} else if sub, err := topic.Subscribe(); err == nil {
+		provider.subscription = sub
+		go provider.MaintenanceTimer()
+		go provider.FindRooms()
+		provider.roomsAdvertised.Store(0)
+		return &provider, nil
+	} else {
+		return nil, err
+	}
+}
+
+func (d *PublicRoomsServerDatabase) GetRoomVisibility(ctx context.Context, roomID string) (bool, error) {
+	return d.PublicRoomsServerDatabase.GetRoomVisibility(ctx, roomID)
+}
+
+func (d *PublicRoomsServerDatabase) SetRoomVisibility(ctx context.Context, visible bool, roomID string) error {
+	d.MaintenanceTimer()
+	return d.PublicRoomsServerDatabase.SetRoomVisibility(ctx, visible, roomID)
+}
+
+func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) {
+	d.foundRoomsMutex.RLock()
+	defer d.foundRoomsMutex.RUnlock()
+	return int64(len(d.foundRooms)), nil
+}
+
+func (d *PublicRoomsServerDatabase) GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) {
+	var rooms []gomatrixserverlib.PublicRoom
+	if filter == "__local__" {
+		if r, err := d.PublicRoomsServerDatabase.GetPublicRooms(ctx, offset, limit, ""); err == nil {
+			rooms = append(rooms, r...)
+		} else {
+			return []gomatrixserverlib.PublicRoom{}, err
+		}
+	} else {
+		d.foundRoomsMutex.RLock()
+		defer d.foundRoomsMutex.RUnlock()
+		for _, room := range d.foundRooms {
+			rooms = append(rooms, room.room)
+		}
+	}
+	return rooms, nil
+}
+
+func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error {
+	return d.PublicRoomsServerDatabase.UpdateRoomFromEvents(ctx, eventsToAdd, eventsToRemove)
+}
+
+func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error {
+	return d.PublicRoomsServerDatabase.UpdateRoomFromEvent(ctx, event)
+}
+
+func (d *PublicRoomsServerDatabase) MaintenanceTimer() {
+	if d.maintenanceTimer != nil && !d.maintenanceTimer.Stop() {
+		<-d.maintenanceTimer.C
+	}
+	d.Interval()
+}
+
+func (d *PublicRoomsServerDatabase) Interval() {
+	d.foundRoomsMutex.Lock()
+	for k, v := range d.foundRooms {
+		if time.Since(v.time) > time.Minute {
+			delete(d.foundRooms, k)
+		}
+	}
+	d.foundRoomsMutex.Unlock()
+	if err := d.AdvertiseRooms(); err != nil {
+		fmt.Println("Failed to advertise room in DHT:", err)
+	}
+	d.foundRoomsMutex.RLock()
+	defer d.foundRoomsMutex.RUnlock()
+	fmt.Println("Found", len(d.foundRooms), "room(s), advertised", d.roomsAdvertised.Load(), "room(s)")
+	d.maintenanceTimer = time.AfterFunc(MaintenanceInterval, d.Interval)
+}
+
+func (d *PublicRoomsServerDatabase) AdvertiseRooms() error {
+	dbCtx, dbCancel := context.WithTimeout(context.Background(), 3*time.Second)
+	_ = dbCancel
+	ourRooms, err := d.GetPublicRooms(dbCtx, 0, 1024, "__local__")
+	if err != nil {
+		return err
+	}
+	advertised := 0
+	for _, room := range ourRooms {
+		if j, err := json.Marshal(room); err == nil {
+			if topic, err := d.pubsub.Join("/matrix/publicRooms"); err != nil {
+				fmt.Println("Failed to subscribe to topic:", err)
+			} else if err := topic.Publish(context.TODO(), j); err != nil {
+				fmt.Println("Failed to publish public room:", err)
+			} else {
+				advertised++
+			}
+		}
+	}
+
+	d.roomsAdvertised.Store(advertised)
+	return nil
+}
+
+func (d *PublicRoomsServerDatabase) FindRooms() {
+	for {
+		msg, err := d.subscription.Next(context.Background())
+		if err != nil {
+			continue
+		}
+		received := discoveredRoom{
+			time: time.Now(),
+		}
+		if err := json.Unmarshal(msg.Data, &received.room); err != nil {
+			fmt.Println("Unmarshal error:", err)
+			continue
+		}
+		d.foundRoomsMutex.Lock()
+		d.foundRooms[received.room.RoomID] = received
+		d.foundRoomsMutex.Unlock()
+	}
+}

cmd/dendrite-demo-libp2p/storage/storage.go

@@ -0,0 +1,61 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"net/url"
+
+	dht "github.com/libp2p/go-libp2p-kad-dht"
+	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage/postgreswithdht"
+	"github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub"
+	"github.com/matrix-org/dendrite/publicroomsapi/storage"
+	"github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3"
+)
+
+const schemePostgres = "postgres"
+const schemeFile = "file"
+
+// NewPublicRoomsServerDatabase opens a database connection.
+func NewPublicRoomsServerDatabaseWithDHT(dataSourceName string, dht *dht.IpfsDHT) (storage.Database, error) {
+	uri, err := url.Parse(dataSourceName)
+	if err != nil {
+		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht)
+	}
+	switch uri.Scheme {
+	case schemePostgres:
+		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht)
+	case schemeFile:
+		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName)
+	default:
+		return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht)
+	}
+}
+
+// NewPublicRoomsServerDatabase opens a database connection.
+func NewPublicRoomsServerDatabaseWithPubSub(dataSourceName string, pubsub *pubsub.PubSub) (storage.Database, error) {
+	uri, err := url.Parse(dataSourceName)
+	if err != nil {
+		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub)
+	}
+	switch uri.Scheme {
+	case schemePostgres:
+		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub)
+	case schemeFile:
+		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName)
+	default:
+		return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub)
+	}
+}

cmd/dendrite-federation-api-server/main.go

@@ -33,7 +33,7 @@ func main() {
 	keyDB := base.CreateKeyDB()
 	federation := base.CreateFederationClient()
 	federationSender := base.CreateHTTPFederationSenderAPIs()
-	keyRing := keydb.CreateKeyRing(federation.Client, keyDB)
+	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
 
 	alias, input, query := base.CreateHTTPRoomserverAPIs()
 	asQuery := base.CreateHTTPAppServiceAPIs()

cmd/dendrite-monolith-server/main.go

@@ -31,10 +31,11 @@ import (
 	"github.com/matrix-org/dendrite/federationsender"
 	"github.com/matrix-org/dendrite/mediaapi"
 	"github.com/matrix-org/dendrite/publicroomsapi"
+	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/syncapi"
-
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+
 	"github.com/sirupsen/logrus"
 )
 
@@ -54,7 +55,7 @@ func main() {
 	deviceDB := base.CreateDeviceDB()
 	keyDB := base.CreateKeyDB()
 	federation := base.CreateFederationClient()
-	keyRing := keydb.CreateKeyRing(federation.Client, keyDB)
+	keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives)
 
 	alias, input, query := roomserver.SetupRoomServerComponent(base)
 	eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New())
@@ -71,28 +72,44 @@ func main() {
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
 	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer)
 	mediaapi.SetupMediaAPIComponent(base, deviceDB)
-	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, federation, nil)
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI))
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to connect to public rooms db")
+	}
+	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, federation, nil)
 	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg)
 
 	httpHandler := common.WrapHandlerInCORS(base.APIMux)
 
 	// Set up the API endpoints we handle. /metrics is for prometheus, and is
 	// not wrapped by CORS, while everything else is
-	http.Handle("/metrics", promhttp.Handler())
+	if cfg.Metrics.Enabled {
+		http.Handle("/metrics", common.WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth))
+	}
 	http.Handle("/", httpHandler)
 
 	// Expose the matrix APIs directly rather than putting them under a /api path.
 	go func() {
-		logrus.Info("Listening on ", *httpBindAddr)
+		serv := http.Server{
-		logrus.Fatal(http.ListenAndServe(*httpBindAddr, nil))
+			Addr:         *httpBindAddr,
+			WriteTimeout: basecomponent.HTTPServerTimeout,
+		}
+
+		logrus.Info("Listening on ", serv.Addr)
+		logrus.Fatal(serv.ListenAndServe())
 	}()
 	// Handle HTTPS if certificate and key are provided
-	go func() {
 	if *certFile != "" && *keyFile != "" {
-			logrus.Info("Listening on ", *httpsBindAddr)
+		go func() {
-			logrus.Fatal(http.ListenAndServeTLS(*httpsBindAddr, *certFile, *keyFile, nil))
+			serv := http.Server{
+				Addr:         *httpsBindAddr,
+				WriteTimeout: basecomponent.HTTPServerTimeout,
 			}
+
+			logrus.Info("Listening on ", serv.Addr)
+			logrus.Fatal(serv.ListenAndServeTLS(*certFile, *keyFile))
 		}()
+	}
 
 	// We want to block forever to let the HTTP and HTTPS handler serve the APIs
 	select {}

cmd/dendrite-public-rooms-api-server/main.go

@@ -17,6 +17,8 @@ package main
 import (
 	"github.com/matrix-org/dendrite/common/basecomponent"
 	"github.com/matrix-org/dendrite/publicroomsapi"
+	"github.com/matrix-org/dendrite/publicroomsapi/storage"
+	"github.com/sirupsen/logrus"
 )
 
 func main() {
@@ -27,8 +29,11 @@ func main() {
 	deviceDB := base.CreateDeviceDB()
 
 	_, _, query := base.CreateHTTPRoomserverAPIs()
-
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI))
-	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, nil, nil)
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to connect to public rooms db")
+	}
+	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, nil, nil)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI))
 

cmd/dendritejs/main.go

@@ -34,6 +34,7 @@ import (
 	"github.com/matrix-org/dendrite/federationsender"
 	"github.com/matrix-org/dendrite/mediaapi"
 	"github.com/matrix-org/dendrite/publicroomsapi"
+	"github.com/matrix-org/dendrite/publicroomsapi/storage"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/syncapi"
 	"github.com/matrix-org/go-http-js-libp2p/go_http_js_libp2p"
@@ -137,7 +138,11 @@ func main() {
 	eduProducer := producers.NewEDUServerProducer(eduInputAPI)
 	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer)
 	mediaapi.SetupMediaAPIComponent(base, deviceDB)
-	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, federation, p2pPublicRoomProvider)
+	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI))
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to connect to public rooms db")
+	}
+	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, federation, p2pPublicRoomProvider)
 	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg)
 
 	httpHandler := common.WrapHandlerInCORS(base.APIMux)

cmd/roomserver-integration-tests/main.go

@@ -28,6 +28,7 @@ import (
 
 	"net/http"
 
+	"github.com/matrix-org/dendrite/common/caching"
 	"github.com/matrix-org/dendrite/common/test"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -253,6 +254,11 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R
 		panic(err)
 	}
 
+	cache, err := caching.NewImmutableInMemoryLRUCache()
+	if err != nil {
+		panic(err)
+	}
+
 	doInput := func() {
 		fmt.Printf("Roomserver is ready to receive input, sending %d events\n", len(input))
 		if err = writeToRoomServer(input, cfg.RoomServerURL()); err != nil {
@@ -270,7 +276,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R
 	cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)}
 
 	gotOutput, err := runAndReadFromTopic(cmd, cfg.RoomServerURL()+"/metrics", doInput, outputTopic, len(wantOutput), func() {
-		queryAPI, _ := api.NewRoomserverQueryAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP})
+		queryAPI, _ := api.NewRoomserverQueryAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache)
 		checkQueries(queryAPI)
 	})
 	if err != nil {

common/basecomponent/base.go

@@ -23,7 +23,9 @@ import (
 
 	"golang.org/x/crypto/ed25519"
 
+	"github.com/matrix-org/dendrite/common/caching"
 	"github.com/matrix-org/dendrite/common/keydb"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/naffka"
 
@@ -55,10 +57,14 @@ type BaseDendrite struct {
 	APIMux         *mux.Router
 	httpClient     *http.Client
 	Cfg            *config.Dendrite
+	ImmutableCache caching.ImmutableCache
 	KafkaConsumer  sarama.Consumer
 	KafkaProducer  sarama.SyncProducer
 }
 
+const HTTPServerTimeout = time.Minute * 5
+const HTTPClientTimeout = time.Second * 30
+
 // NewBaseDendrite creates a new instance to be used by a component.
 // The componentName is used for logging purposes, and should be a friendly name
 // of the compontent running, e.g. "SyncAPI"
@@ -79,14 +85,18 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string) *BaseDendrite {
 		kafkaConsumer, kafkaProducer = setupKafka(cfg)
 	}
 
-	const defaultHTTPTimeout = 30 * time.Second
+	cache, err := caching.NewImmutableInMemoryLRUCache()
+	if err != nil {
+		logrus.WithError(err).Warnf("Failed to create cache")
+	}
 
 	return &BaseDendrite{
 		componentName:  componentName,
 		tracerCloser:   closer,
 		Cfg:            cfg,
+		ImmutableCache: cache,
 		APIMux:         mux.NewRouter().UseEncodedPath(),
-		httpClient:    &http.Client{Timeout: defaultHTTPTimeout},
+		httpClient:     &http.Client{Timeout: HTTPClientTimeout},
 		KafkaConsumer:  kafkaConsumer,
 		KafkaProducer:  kafkaProducer,
 	}
@@ -114,7 +124,6 @@ func (b *BaseDendrite) CreateHTTPRoomserverAPIs() (
 	roomserverAPI.RoomserverInputAPI,
 	roomserverAPI.RoomserverQueryAPI,
 ) {
-
 	alias, err := roomserverAPI.NewRoomserverAliasAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient)
 	if err != nil {
 		logrus.WithError(err).Panic("NewRoomserverAliasAPIHTTP failed")
@@ -123,7 +132,7 @@ func (b *BaseDendrite) CreateHTTPRoomserverAPIs() (
 	if err != nil {
 		logrus.WithError(err).Panic("NewRoomserverInputAPIHTTP failed", b.httpClient)
 	}
-	query, err := roomserverAPI.NewRoomserverQueryAPIHTTP(b.Cfg.RoomServerURL(), nil)
+	query, err := roomserverAPI.NewRoomserverQueryAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache)
 	if err != nil {
 		logrus.WithError(err).Panic("NewRoomserverQueryAPIHTTP failed", b.httpClient)
 	}
@@ -133,7 +142,7 @@ func (b *BaseDendrite) CreateHTTPRoomserverAPIs() (
 // CreateHTTPEDUServerAPIs returns eduInputAPI for hitting the EDU
 // server over HTTP
 func (b *BaseDendrite) CreateHTTPEDUServerAPIs() eduServerAPI.EDUServerInputAPI {
-	e, err := eduServerAPI.NewEDUServerInputAPIHTTP(b.Cfg.EDUServerURL(), nil)
+	e, err := eduServerAPI.NewEDUServerInputAPIHTTP(b.Cfg.EDUServerURL(), b.httpClient)
 	if err != nil {
 		logrus.WithError(err).Panic("NewEDUServerInputAPIHTTP failed", b.httpClient)
 	}
@@ -143,7 +152,7 @@ func (b *BaseDendrite) CreateHTTPEDUServerAPIs() eduServerAPI.EDUServerInputAPI
 // CreateHTTPFederationSenderAPIs returns FederationSenderQueryAPI for hitting
 // the federation sender over HTTP
 func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderQueryAPI {
-	f, err := federationSenderAPI.NewFederationSenderQueryAPIHTTP(b.Cfg.FederationSenderURL(), nil)
+	f, err := federationSenderAPI.NewFederationSenderQueryAPIHTTP(b.Cfg.FederationSenderURL(), b.httpClient)
 	if err != nil {
 		logrus.WithError(err).Panic("NewFederationSenderQueryAPIHTTP failed", b.httpClient)
 	}
@@ -208,16 +217,20 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) {
 		addr = listenaddr
 	}
 
-	common.SetupHTTPAPI(http.DefaultServeMux, common.WrapHandlerInCORS(b.APIMux))
+	serv := http.Server{
-	logrus.Infof("Starting %s server on %s", b.componentName, addr)
+		Addr:         addr,
+		WriteTimeout: HTTPServerTimeout,
+	}
 
-	err := http.ListenAndServe(addr, nil)
+	common.SetupHTTPAPI(http.DefaultServeMux, common.WrapHandlerInCORS(b.APIMux), b.Cfg)
+	logrus.Infof("Starting %s server on %s", b.componentName, serv.Addr)
 
+	err := serv.ListenAndServe()
 	if err != nil {
 		logrus.WithError(err).Fatal("failed to serve http")
 	}
 
-	logrus.Infof("Stopped %s server on %s", b.componentName, addr)
+	logrus.Infof("Stopped %s server on %s", b.componentName, serv.Addr)
 }
 
 // setupKafka creates kafka consumer/producer pair from the config.
@@ -243,7 +256,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) {
 
 	uri, err := url.Parse(string(cfg.Database.Naffka))
 	if err != nil || uri.Scheme == "file" {
-		db, err = sql.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka))
+		db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka))
 		if err != nil {
 			logrus.WithError(err).Panic("Failed to open naffka database")
 		}
@@ -253,7 +266,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) {
 			logrus.WithError(err).Panic("Failed to setup naffka database")
 		}
 	} else {
-		db, err = sql.Open("postgres", string(cfg.Database.Naffka))
+		db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka))
 		if err != nil {
 			logrus.WithError(err).Panic("Failed to open naffka database")
 		}

common/caching/immutablecache.go

@@ -0,0 +1,12 @@
+package caching
+
+import "github.com/matrix-org/gomatrixserverlib"
+
+const (
+	RoomVersionMaxCacheEntries = 128
+)
+
+type ImmutableCache interface {
+	GetRoomVersion(roomId string) (gomatrixserverlib.RoomVersion, bool)
+	StoreRoomVersion(roomId string, roomVersion gomatrixserverlib.RoomVersion)
+}

common/caching/immutableinmemorylru.go

@@ -0,0 +1,43 @@
+package caching
+
+import (
+	"fmt"
+
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+type ImmutableInMemoryLRUCache struct {
+	roomVersions *lru.Cache
+}
+
+func NewImmutableInMemoryLRUCache() (*ImmutableInMemoryLRUCache, error) {
+	roomVersionCache, rvErr := lru.New(RoomVersionMaxCacheEntries)
+	if rvErr != nil {
+		return nil, rvErr
+	}
+	return &ImmutableInMemoryLRUCache{
+		roomVersions: roomVersionCache,
+	}, nil
+}
+
+func checkForInvalidMutation(cache *lru.Cache, key string, value interface{}) {
+	if peek, ok := cache.Peek(key); ok && peek != value {
+		panic(fmt.Sprintf("invalid use of immutable cache tries to mutate existing value of %q", key))
+	}
+}
+
+func (c *ImmutableInMemoryLRUCache) GetRoomVersion(roomID string) (gomatrixserverlib.RoomVersion, bool) {
+	val, found := c.roomVersions.Get(roomID)
+	if found && val != nil {
+		if roomVersion, ok := val.(gomatrixserverlib.RoomVersion); ok {
+			return roomVersion, true
+		}
+	}
+	return "", false
+}
+
+func (c *ImmutableInMemoryLRUCache) StoreRoomVersion(roomID string, roomVersion gomatrixserverlib.RoomVersion) {
+	checkForInvalidMutation(c.roomVersions, roomID, roomVersion)
+	c.roomVersions.Add(roomID, roomVersion)
+}

common/config/appservice.go

@@ -98,6 +98,22 @@ func (a *ApplicationService) IsInterestedInUserID(
 	return false
 }
 
+// OwnsNamespaceCoveringUserId returns a bool on whether an application service's
+// namespace is exclusive and includes the given user ID
+func (a *ApplicationService) OwnsNamespaceCoveringUserId(
+	userID string,
+) bool {
+	if namespaceSlice, ok := a.NamespaceMap["users"]; ok {
+		for _, namespace := range namespaceSlice {
+			if namespace.Exclusive && namespace.RegexpObject.MatchString(userID) {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
 // IsInterestedInRoomAlias returns a bool on whether an application service's
 // namespace includes the given room alias
 func (a *ApplicationService) IsInterestedInRoomAlias(

common/config/config.go

@@ -99,6 +99,9 @@ type Dendrite struct {
 		// If set disables new users from registering (except via shared
 		// secrets)
 		RegistrationDisabled bool `yaml:"registration_disabled"`
+		// Perspective keyservers, to use as a backup when direct key fetch
+		// requests don't succeed
+		KeyPerspectives KeyPerspectives `yaml:"key_perspectives"`
 	} `yaml:"matrix"`
 
 	// The configuration specific to the media repostitory.
@@ -119,6 +122,19 @@ type Dendrite struct {
 		ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"`
 	} `yaml:"media"`
 
+	// The configuration to use for Prometheus metrics
+	Metrics struct {
+		// Whether or not the metrics are enabled
+		Enabled bool `yaml:"enabled"`
+		// Use BasicAuth for Authorization
+		BasicAuth struct {
+			// Authorization via Static Username & Password
+			// Hardcoded Username and Password
+			Username string `yaml:"username"`
+			Password string `yaml:"password"`
+		} `yaml:"basic_auth"`
+	} `yaml:"metrics"`
+
 	// The configuration for talking to kafka.
 	Kafka struct {
 		// A list of kafka addresses to connect to.
@@ -272,6 +288,21 @@ type Dendrite struct {
 	} `yaml:"-"`
 }
 
+// KeyPerspectives are used to configure perspective key servers for
+// retrieving server keys.
+type KeyPerspectives []struct {
+	// The server name of the perspective key server
+	ServerName gomatrixserverlib.ServerName `yaml:"server_name"`
+	// Server keys for the perspective user, used to verify the
+	// keys have been signed by the perspective server
+	Keys []struct {
+		// The key ID, e.g. ed25519:auto
+		KeyID gomatrixserverlib.KeyID `yaml:"key_id"`
+		// The public key in base64 unpadded format
+		PublicKey string `yaml:"public_key"`
+	} `yaml:"keys"`
+}
+
 // A Path on the filesystem.
 type Path string
 

common/eventcontent.go

@@ -48,6 +48,7 @@ func InitialPowerLevelsContent(roomCreator string) (c gomatrixserverlib.PowerLev
 		"m.room.history_visibility": 100,
 		"m.room.canonical_alias":    50,
 		"m.room.avatar":             50,
+		"m.room.aliases":            0, // anyone can publish aliases by default. Has to be 0 else state_default is used.
 	}
 	c.Users = map[string]int64{roomCreator: 100}
 	return c

common/events.go

@@ -73,6 +73,10 @@ func AddPrevEventsToEvent(
 		return fmt.Errorf("gomatrixserverlib.StateNeededForEventBuilder: %w", err)
 	}
 
+	if len(eventsNeeded.Tuples()) == 0 {
+		return errors.New("expecting state tuples for event builder, got none")
+	}
+
 	// Ask the roomserver for information about this room
 	queryReq := api.QueryLatestEventsAndStateRequest{
 		RoomID:       builder.RoomID,

common/httpapi.go

@@ -1,11 +1,17 @@
 package common
 
 import (
+	"io"
 	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"os"
+	"strings"
 	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/common/config"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	opentracing "github.com/opentracing/opentracing-go"
@@ -13,8 +19,15 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/sirupsen/logrus"
 )
 
+// BasicAuth is used for authorization on /metrics handlers
+type BasicAuth struct {
+	Username string `yaml:"username"`
+	Password string `yaml:"password"`
+}
+
 // MakeAuthAPI turns a util.JSONRequestHandler function into an http.Handler which authenticates the request.
 func MakeAuthAPI(
 	metricsName string, data auth.Data,
@@ -38,12 +51,60 @@ func MakeAuthAPI(
 // MakeExternalAPI turns a util.JSONRequestHandler function into an http.Handler.
 // This is used for APIs that are called from the internet.
 func MakeExternalAPI(metricsName string, f func(*http.Request) util.JSONResponse) http.Handler {
+	// TODO: We shouldn't be directly reading env vars here, inject it in instead.
+	// Refactor this when we split out config structs.
+	verbose := false
+	if os.Getenv("DENDRITE_TRACE_HTTP") == "1" {
+		verbose = true
+	}
 	h := util.MakeJSONAPI(util.NewJSONRequestHandler(f))
 	withSpan := func(w http.ResponseWriter, req *http.Request) {
+		nextWriter := w
+		if verbose {
+			logger := logrus.NewEntry(logrus.StandardLogger())
+			// Log outgoing response
+			rec := httptest.NewRecorder()
+			nextWriter = rec
+			defer func() {
+				resp := rec.Result()
+				dump, err := httputil.DumpResponse(resp, true)
+				if err != nil {
+					logger.Debugf("Failed to dump outgoing response: %s", err)
+				} else {
+					strSlice := strings.Split(string(dump), "\n")
+					for _, s := range strSlice {
+						logger.Debug(s)
+					}
+				}
+				// copy the response to the client
+				for hdr, vals := range resp.Header {
+					for _, val := range vals {
+						w.Header().Add(hdr, val)
+					}
+				}
+				w.WriteHeader(resp.StatusCode)
+				// discard errors as this is for debugging
+				_, _ = io.Copy(w, resp.Body)
+				_ = resp.Body.Close()
+			}()
+
+			// Log incoming request
+			dump, err := httputil.DumpRequest(req, true)
+			if err != nil {
+				logger.Debugf("Failed to dump incoming request: %s", err)
+			} else {
+				strSlice := strings.Split(string(dump), "\n")
+				for _, s := range strSlice {
+					logger.Debug(s)
+				}
+			}
+		}
+
 		span := opentracing.StartSpan(metricsName)
 		defer span.Finish()
 		req = req.WithContext(opentracing.ContextWithSpan(req.Context(), span))
-		h.ServeHTTP(w, req)
+		h.ServeHTTP(nextWriter, req)
+
 	}
 
 	return http.HandlerFunc(withSpan)
@@ -123,11 +184,34 @@ func MakeFedAPI(
 
 // SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics
 // listener.
-func SetupHTTPAPI(servMux *http.ServeMux, apiMux http.Handler) {
+func SetupHTTPAPI(servMux *http.ServeMux, apiMux http.Handler, cfg *config.Dendrite) {
-	servMux.Handle("/metrics", promhttp.Handler())
+	if cfg.Metrics.Enabled {
+		servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth))
+	}
 	servMux.Handle("/api/", http.StripPrefix("/api", apiMux))
 }
 
+// WrapHandlerInBasicAuth adds basic auth to a handler. Only used for /metrics
+func WrapHandlerInBasicAuth(h http.Handler, b BasicAuth) http.HandlerFunc {
+	if b.Username == "" || b.Password == "" {
+		logrus.Warn("Metrics are exposed without protection. Make sure you set up protection at proxy level.")
+	}
+	return func(w http.ResponseWriter, r *http.Request) {
+		// Serve without authorization if either Username or Password is unset
+		if b.Username == "" || b.Password == "" {
+			h.ServeHTTP(w, r)
+			return
+		}
+		user, pass, ok := r.BasicAuth()
+
+		if !ok || user != b.Username || pass != b.Password {
+			http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
+			return
+		}
+		h.ServeHTTP(w, r)
+	}
+}
+
 // WrapHandlerInCORS adds CORS headers to all responses, including all error
 // responses.
 // Handles OPTIONS requests directly.

common/httpapi_test.go

@@ -0,0 +1,95 @@
+package common
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestWrapHandlerInBasicAuth(t *testing.T) {
+	type args struct {
+		h http.Handler
+		b BasicAuth
+	}
+
+	dummyHandler := http.HandlerFunc(func(h http.ResponseWriter, r *http.Request) {
+		h.WriteHeader(http.StatusOK)
+	})
+
+	tests := []struct {
+		name    string
+		args    args
+		want    int
+		reqAuth bool
+	}{
+		{
+			name:    "no user or password setup",
+			args:    args{h: dummyHandler},
+			want:    http.StatusOK,
+			reqAuth: false,
+		},
+		{
+			name: "only user set",
+			args: args{
+				h: dummyHandler,
+				b: BasicAuth{Username: "test"}, // no basic auth
+			},
+			want:    http.StatusOK,
+			reqAuth: false,
+		},
+		{
+			name: "only pass set",
+			args: args{
+				h: dummyHandler,
+				b: BasicAuth{Password: "test"}, // no basic auth
+			},
+			want:    http.StatusOK,
+			reqAuth: false,
+		},
+		{
+			name: "credentials correct",
+			args: args{
+				h: dummyHandler,
+				b: BasicAuth{Username: "test", Password: "test"}, // basic auth enabled
+			},
+			want:    http.StatusOK,
+			reqAuth: true,
+		},
+		{
+			name: "credentials wrong",
+			args: args{
+				h: dummyHandler,
+				b: BasicAuth{Username: "test1", Password: "test"}, // basic auth enabled
+			},
+			want:    http.StatusForbidden,
+			reqAuth: true,
+		},
+		{
+			name: "no basic auth in request",
+			args: args{
+				h: dummyHandler,
+				b: BasicAuth{Username: "test", Password: "test"}, // basic auth enabled
+			},
+			want:    http.StatusForbidden,
+			reqAuth: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			baHandler := WrapHandlerInBasicAuth(tt.args.h, tt.args.b)
+
+			req := httptest.NewRequest("GET", "http://localhost/metrics", nil)
+			if tt.reqAuth {
+				req.SetBasicAuth("test", "test")
+			}
+
+			w := httptest.NewRecorder()
+			baHandler(w, req)
+			resp := w.Result()
+
+			if resp.StatusCode != tt.want {
+				t.Errorf("Expected status code %d, got %d", resp.StatusCode, tt.want)
+			}
+		})
+	}
+}

common/keydb/keyring.go

@@ -14,19 +14,61 @@
 
 package keydb
 
-import "github.com/matrix-org/gomatrixserverlib"
+import (
+	"encoding/base64"
+
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/crypto/ed25519"
+)
 
 // CreateKeyRing creates and configures a KeyRing object.
 //
 // It creates the necessary key fetchers and collects them into a KeyRing
 // backed by the given KeyDatabase.
 func CreateKeyRing(client gomatrixserverlib.Client,
-	keyDB gomatrixserverlib.KeyDatabase) gomatrixserverlib.KeyRing {
+	keyDB gomatrixserverlib.KeyDatabase,
-	return gomatrixserverlib.KeyRing{
+	cfg config.KeyPerspectives) gomatrixserverlib.KeyRing {
+
+	fetchers := gomatrixserverlib.KeyRing{
 		KeyFetchers: []gomatrixserverlib.KeyFetcher{
-			// TODO: Use perspective key fetchers for production.
+			&gomatrixserverlib.DirectKeyFetcher{
-			&gomatrixserverlib.DirectKeyFetcher{Client: client},
+				Client: client,
+			},
 		},
 		KeyDatabase: keyDB,
 	}
+
+	logrus.Info("Enabled direct key fetcher")
+
+	var b64e = base64.StdEncoding.WithPadding(base64.NoPadding)
+	for _, ps := range cfg {
+		perspective := &gomatrixserverlib.PerspectiveKeyFetcher{
+			PerspectiveServerName: ps.ServerName,
+			PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{},
+			Client:                client,
+		}
+
+		for _, key := range ps.Keys {
+			rawkey, err := b64e.DecodeString(key.PublicKey)
+			if err != nil {
+				logrus.WithError(err).WithFields(logrus.Fields{
+					"server_name": ps.ServerName,
+					"public_key":  key.PublicKey,
+				}).Warn("Couldn't parse perspective key")
+				continue
+			}
+			perspective.PerspectiveServerKeys[key.KeyID] = rawkey
+		}
+
+		fetchers.KeyFetchers = append(fetchers.KeyFetchers, perspective)
+
+		logrus.WithFields(logrus.Fields{
+			"server_name":     ps.ServerName,
+			"num_public_keys": len(ps.Keys),
+		}).Info("Enabled perspective key fetcher")
+	}
+
+	return fetchers
 }

common/keydb/postgres/keydb.go

@@ -17,11 +17,11 @@ package postgres
 
 import (
 	"context"
-	"database/sql"
 	"math"
 
 	"golang.org/x/crypto/ed25519"
 
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -41,7 +41,7 @@ func NewDatabase(
 	serverKey ed25519.PublicKey,
 	serverKeyID gomatrixserverlib.KeyID,
 ) (*Database, error) {
-	db, err := sql.Open("postgres", dataSourceName)
+	db, err := sqlutil.Open("postgres", dataSourceName)
 	if err != nil {
 		return nil, err
 	}

common/keydb/sqlite3/keydb.go

@@ -17,12 +17,12 @@ package sqlite3
 
 import (
 	"context"
-	"database/sql"
 	"math"
 
 	"golang.org/x/crypto/ed25519"
 
 	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	_ "github.com/mattn/go-sqlite3"
@@ -44,7 +44,7 @@ func NewDatabase(
 	serverKey ed25519.PublicKey,
 	serverKeyID gomatrixserverlib.KeyID,
 ) (*Database, error) {
-	db, err := sql.Open(common.SQLiteDriverName(), dataSourceName)
+	db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName)
 	if err != nil {
 		return nil, err
 	}

dendrite-config.yaml

@@ -17,6 +17,14 @@ matrix:
     trusted_third_party_id_servers:
       - vector.im
       - matrix.org
+    # Perspective key servers which are used when direct key requests fail
+    #key_perspectives:
+    #  - server_name: matrix.org
+    #    keys:
+    #      - key_id: ed25519:auto
+    #        public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw
+    #      - key_id: ed25519:a_RXGa
+    #        public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ
 
 # The media repository config
 media:
@@ -53,6 +61,15 @@ media:
         height: 600
         method: scale
 
+# Metrics config for Prometheus
+metrics:
+    # Whether or not metrics are enabled
+    enabled: false
+    # Use basic auth to protect the metrics. Uncomment to the complete block to enable.
+    #basic_auth:
+    #  username: prometheusUser
+    #  password: y0ursecr3tPa$$w0rd
+
 # The config for the TURN server
 turn:
     # Whether or not guests can request TURN credentials

federationapi/routing/backfill.go

@@ -16,6 +16,7 @@ package routing
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"strconv"
 	"time"
@@ -73,7 +74,10 @@ func Backfill(
 	}
 	if req.Limit, err = strconv.Atoi(limit); err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(fmt.Sprintf("limit %q is invalid format", limit)),
+		}
 	}
 
 	// Query the roomserver.
@@ -93,7 +97,7 @@ func Backfill(
 	}
 
 	var eventJSONs []json.RawMessage
-	for _, e := range evs {
+	for _, e := range gomatrixserverlib.ReverseTopologicalOrdering(evs) {
 		eventJSONs = append(eventJSONs, e.JSON())
 	}
 

federationapi/routing/invite.go

@@ -66,6 +66,7 @@ func Invite(
 		ServerName:             event.Origin(),
 		Message:                redacted.JSON(),
 		AtTS:                   event.OriginServerTS(),
+		StrictValidityChecking: true,
 	}}
 	verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests)
 	if err != nil {

federationapi/routing/join.go

@@ -199,6 +199,7 @@ func SendJoin(
 		ServerName:             event.Origin(),
 		Message:                redacted.JSON(),
 		AtTS:                   event.OriginServerTS(),
+		StrictValidityChecking: true,
 	}}
 	verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests)
 	if err != nil {

federationapi/routing/leave.go

@@ -148,6 +148,7 @@ func SendLeave(
 		ServerName:             event.Origin(),
 		Message:                redacted.JSON(),
 		AtTS:                   event.OriginServerTS(),
+		StrictValidityChecking: true,
 	}}
 	verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests)
 	if err != nil {

federationapi/routing/profile.go

@@ -15,6 +15,7 @@
 package routing
 
 import (
+	"fmt"
 	"net/http"
 
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
@@ -46,12 +47,17 @@ func GetProfile(
 	_, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
 		util.GetLogger(httpReq.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.MissingArgument(fmt.Sprintf("Format of user ID %q is invalid", userID)),
+		}
 	}
 
 	if domain != cfg.Matrix.ServerName {
-		util.GetLogger(httpReq.Context()).WithError(err).Error("domain != cfg.Matrix.ServerName failed")
+		return util.JSONResponse{
-		return jsonerror.InternalServerError()
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(fmt.Sprintf("Domain %q does not match this server", domain)),
+		}
 	}
 
 	profile, err := appserviceAPI.RetrieveUserProfile(httpReq.Context(), userID, asAPI, accountDB)

federationapi/routing/send.go

@@ -71,15 +71,29 @@ func Send(
 	util.GetLogger(httpReq.Context()).Infof("Received transaction %q containing %d PDUs, %d EDUs", txnID, len(t.PDUs), len(t.EDUs))
 
 	resp, err := t.processTransaction()
-	if err != nil {
+	switch err.(type) {
-		util.GetLogger(httpReq.Context()).WithError(err).Error("t.processTransaction failed")
+	// No error? Great! Send back a 200.
-		return jsonerror.InternalServerError()
+	case nil:
-	}
-
 		return util.JSONResponse{
 			Code: http.StatusOK,
 			JSON: resp,
 		}
+	// Handle known error cases as we will return a 400 error for these.
+	case roomNotFoundError:
+	case unmarshalError:
+	case verifySigError:
+	// Handle unknown error cases. Sending 500 errors back should be a last
+	// resort as this can make other homeservers back off sending federation
+	// events.
+	default:
+		util.GetLogger(httpReq.Context()).WithError(err).Error("t.processTransaction failed")
+		return jsonerror.InternalServerError()
+	}
+	// Return a 400 error for bad requests as fallen through from above.
+	return util.JSONResponse{
+		Code: http.StatusBadRequest,
+		JSON: jsonerror.BadJSON(err.Error()),
+	}
 }
 
 type txnReq struct {
@@ -93,6 +107,8 @@ type txnReq struct {
 }
 
 func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) {
+	results := make(map[string]gomatrixserverlib.PDUResult)
+
 	var pdus []gomatrixserverlib.HeaderedEvent
 	for _, pdu := range t.PDUs {
 		var header struct {
@@ -100,28 +116,27 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) {
 		}
 		if err := json.Unmarshal(pdu, &header); err != nil {
 			util.GetLogger(t.context).WithError(err).Warn("Transaction: Failed to extract room ID from event")
-			return nil, err
+			return nil, unmarshalError{err}
 		}
 		verReq := api.QueryRoomVersionForRoomRequest{RoomID: header.RoomID}
 		verRes := api.QueryRoomVersionForRoomResponse{}
 		if err := t.query.QueryRoomVersionForRoom(t.context, &verReq, &verRes); err != nil {
 			util.GetLogger(t.context).WithError(err).Warn("Transaction: Failed to query room version for room", verReq.RoomID)
-			return nil, err
+			return nil, roomNotFoundError{verReq.RoomID}
 		}
 		event, err := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, verRes.RoomVersion)
 		if err != nil {
 			util.GetLogger(t.context).WithError(err).Warnf("Transaction: Failed to parse event JSON of event %q", event.EventID())
-			return nil, err
+			return nil, unmarshalError{err}
 		}
 		if err := gomatrixserverlib.VerifyAllEventSignatures(t.context, []gomatrixserverlib.Event{event}, t.keys); err != nil {
 			util.GetLogger(t.context).WithError(err).Warnf("Transaction: Couldn't validate signature of event %q", event.EventID())
-			return nil, err
+			return nil, verifySigError{event.EventID(), err}
 		}
 		pdus = append(pdus, event.Headered(verRes.RoomVersion))
 	}
 
 	// Process the events.
-	results := map[string]gomatrixserverlib.PDUResult{}
 	for _, e := range pdus {
 		err := t.processEvent(e.Unwrap())
 		if err != nil {
@@ -141,7 +156,7 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) {
 			// If we bail and stop processing then we risk wedging incoming
 			// transactions from that server forever.
 			switch err.(type) {
-			case unknownRoomError:
+			case roomNotFoundError:
 			case *gomatrixserverlib.NotAllowed:
 			default:
 				// Any other error should be the result of a temporary error in
@@ -162,11 +177,22 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) {
 	return &gomatrixserverlib.RespSend{PDUs: results}, nil
 }
 
-type unknownRoomError struct {
+type roomNotFoundError struct {
 	roomID string
 }
+type unmarshalError struct {
+	err error
+}
+type verifySigError struct {
+	eventID string
+	err     error
+}
 
-func (e unknownRoomError) Error() string { return fmt.Sprintf("unknown room %q", e.roomID) }
+func (e roomNotFoundError) Error() string { return fmt.Sprintf("room %q not found", e.roomID) }
+func (e unmarshalError) Error() string    { return fmt.Sprintf("unable to parse event: %s", e.err) }
+func (e verifySigError) Error() string {
+	return fmt.Sprintf("unable to verify signature of event %q: %s", e.eventID, e.err)
+}
 
 func (t *txnReq) processEDUs(edus []gomatrixserverlib.EDU) {
 	for _, e := range edus {
@@ -213,7 +239,7 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event) error {
 		// that this server is unaware of.
 		// However generally speaking we should reject events for rooms we
 		// aren't a member of.
-		return unknownRoomError{e.RoomID()}
+		return roomNotFoundError{e.RoomID()}
 	}
 
 	if !stateResp.PrevEventsExist {

federationapi/routing/threepid.go

@@ -271,6 +271,10 @@ func buildMembershipEvent(
 		return nil, err
 	}
 
+	if len(eventsNeeded.Tuples()) == 0 {
+		return nil, errors.New("expecting state tuples for event builder, got none")
+	}
+
 	// Ask the roomserver for information about this room
 	queryReq := roomserverAPI.QueryLatestEventsAndStateRequest{
 		RoomID:       builder.RoomID,

federationsender/storage/postgres/storage.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/federationsender/types"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
 // Database stores information needed by the federation sender
@@ -35,7 +36,7 @@ type Database struct {
 func NewDatabase(dataSourceName string) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sql.Open("postgres", dataSourceName); err != nil {
+	if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {

federationsender/storage/sqlite3/storage.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/federationsender/types"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 )
 
 // Database stores information needed by the federation sender
@@ -37,7 +38,7 @@ type Database struct {
 func NewDatabase(dataSourceName string) (*Database, error) {
 	var result Database
 	var err error
-	if result.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
 		return nil, err
 	}
 	if err = result.prepare(); err != nil {

go.mod

@@ -3,19 +3,29 @@ module github.com/matrix-org/dendrite
 require (
 	github.com/gorilla/mux v1.7.3
 	github.com/hashicorp/golang-lru v0.5.4
+	github.com/kr/pretty v0.2.0 // indirect
 	github.com/lib/pq v1.2.0
+	github.com/libp2p/go-libp2p v0.6.0
+	github.com/libp2p/go-libp2p-circuit v0.1.4
 	github.com/libp2p/go-libp2p-core v0.5.0
+	github.com/libp2p/go-libp2p-gostream v0.2.1
+	github.com/libp2p/go-libp2p-http v0.1.5
+	github.com/libp2p/go-libp2p-kad-dht v0.5.0
+	github.com/libp2p/go-libp2p-pubsub v0.2.5
+	github.com/libp2p/go-libp2p-record v0.1.2
 	github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20200409140603-8b9a51fe9b89
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20200421090225-4ea81b29f5f7
 	github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1
 	github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7
-	github.com/mattn/go-sqlite3 v2.0.2+incompatible
+	github.com/mattn/go-sqlite3 v2.0.3+incompatible
 	github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5
+	github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6
 	github.com/opentracing/opentracing-go v1.1.0
-	github.com/pkg/errors v0.8.1
+	github.com/pierrec/lz4 v2.5.0+incompatible // indirect
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.4.1
 	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
 	github.com/sirupsen/logrus v1.4.2
@@ -24,11 +34,11 @@ require (
 	github.com/uber/jaeger-client-go v2.22.1+incompatible
 	github.com/uber/jaeger-lib v2.2.0+incompatible
 	go.uber.org/atomic v1.6.0
-	golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d
+	golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d
 	golang.org/x/tools v0.0.0-20200402223321-bcf690261a44 // indirect
 	gopkg.in/Shopify/sarama.v1 v1.20.1
 	gopkg.in/h2non/bimg.v1 v1.0.18
-	gopkg.in/yaml.v2 v2.2.5
+	gopkg.in/yaml.v2 v2.2.8
 )
 
 go 1.13

go.sum

@@ -1,22 +1,32 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
+github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE=
 github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
+github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
 github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
+github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
+github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
 github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw=
 github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
+github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
 github.com/btcsuite/btcutil v1.0.1 h1:GKOz8BnRjYrb/JTKgaOk+zh26NWNdSNvdvv0xoAZMSA=
 github.com/btcsuite/btcutil v1.0.1/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts=
@@ -31,13 +41,28 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC5IlbaIF5Q7JNieBoACT7iW0YTxQHR0in0=
+github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4=
+github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
+github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
+github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
+github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
+github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
+github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
@@ -48,40 +73,109 @@ github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3B
 github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI=
+github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
+github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
+github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo=
+github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc=
+github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
+github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
+github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
+github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
 github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU=
 github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
+github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
+github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
+github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
+github.com/ipfs/go-datastore v0.3.1 h1:SS1t869a6cctoSYmZXUk8eL6AzVXgASmKIWFNQkQ1jU=
+github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
+github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
+github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
+github.com/ipfs/go-datastore v0.4.4 h1:rjvQ9+muFaJ+QZ7dN5B1MSDNQ0JVZKkkES/rMZmA8X8=
+github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
+github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk=
+github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
+github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8=
+github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s=
+github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk=
+github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE=
+github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc=
+github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8=
+github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s=
+github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
+github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50=
+github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
+github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
+github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
+github.com/ipfs/go-log v1.0.2 h1:s19ZwJxH8rPWzypjcDpqPLIyV7BnbLqvpli3iZoqYK0=
+github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk=
+github.com/ipfs/go-log/v2 v2.0.2 h1:xguurydRdfKMJjKyxNXNU8lYP0VZH1NUwJRwUorjuEw=
+github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
+github.com/ipfs/go-todocounter v0.0.2 h1:9UBngSQhylg2UDcxSAtpkT+rEWFr26hDPXVStE8LFyc=
+github.com/ipfs/go-todocounter v0.0.2/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4=
+github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc=
+github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
+github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA=
+github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
+github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4uRSZMmzKNLoXvTu1sfx+1kv/DojUlPrSZGs=
+github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc=
 github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
+github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2 h1:vhC1OXXiT9R2pczegwz6moDvuRpggaroAXhPIseh57A=
+github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod h1:8GXXJV31xl8whumTzdZsTt3RnUIiPqzkyf7mxToRCMs=
+github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
 github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10=
 github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
@@ -92,31 +186,171 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ=
+github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/libp2p/go-addr-util v0.0.1 h1:TpTQm9cXVRVSKsYbgQ7GKc3KbbHVTnbostgGaDEP+88=
+github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ=
+github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ=
 github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs=
 github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM=
+github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD21CIEy5eYd1Hlp0juHY0=
+github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc=
+github.com/libp2p/go-eventbus v0.1.0 h1:mlawomSAjjkk97QnYiEmHsLu7E136+2oCWSHRUvMfzQ=
+github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4=
+github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8=
+github.com/libp2p/go-flow-metrics v0.0.2/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs=
+github.com/libp2p/go-flow-metrics v0.0.3 h1:8tAs/hSdNvUiLgtlSy3mxwxWP4I9y/jlkPFT7epKdeM=
 github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs=
+github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs=
+github.com/libp2p/go-libp2p v0.5.0 h1:/nnb5mc2TK6TwknECsWIkfCwMTHv0AXbvzxlnVivfeg=
+github.com/libp2p/go-libp2p v0.5.0/go.mod h1:Os7a5Z3B+ErF4v7zgIJ7nBHNu2LYt8ZMLkTQUB3G/wA=
+github.com/libp2p/go-libp2p v0.6.0 h1:EFArryT9N7AVA70LCcOh8zxsW+FeDnxwcpWQx9k7+GM=
+github.com/libp2p/go-libp2p v0.6.0/go.mod h1:mfKWI7Soz3ABX+XEBR61lGbg+ewyMtJHVt043oWeqwg=
+github.com/libp2p/go-libp2p-autonat v0.1.1 h1:WLBZcIRsjZlWdAZj9CiBSvU2wQXoUOiS1Zk1tM7DTJI=
+github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE=
+github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro=
+github.com/libp2p/go-libp2p-blankhost v0.1.4 h1:I96SWjR4rK9irDHcHq3XHN6hawCRTPUADzkJacgZLvk=
+github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU=
+github.com/libp2p/go-libp2p-circuit v0.1.4 h1:Phzbmrg3BkVzbqd4ZZ149JxCuUWu2wZcXf/Kr6hZJj8=
+github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU=
+github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco=
+github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I=
+github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI=
+github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0=
+github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g=
+github.com/libp2p/go-libp2p-core v0.2.5/go.mod h1:6+5zJmKhsf7yHn1RbmYDu08qDUpIUxGdqHuEZckmZOA=
+github.com/libp2p/go-libp2p-core v0.3.0 h1:F7PqduvrztDtFsAa/bcheQ3azmNo+Nq7m8hQY5GiUW8=
+github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw=
+github.com/libp2p/go-libp2p-core v0.3.1/go.mod h1:thvWy0hvaSBhnVBaW37BvzgVV68OUhgJJLAa6almrII=
+github.com/libp2p/go-libp2p-core v0.4.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0=
 github.com/libp2p/go-libp2p-core v0.5.0 h1:FBQ1fpq2Fo/ClyjojVJ5AKXlKhvNc/B6U0O+7AN1ffE=
 github.com/libp2p/go-libp2p-core v0.5.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0=
+github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ=
+github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI=
+github.com/libp2p/go-libp2p-discovery v0.2.0 h1:1p3YSOq7VsgaL+xVHPi8XAmtGyas6D2J6rWBEfz/aiY=
+github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg=
+github.com/libp2p/go-libp2p-gostream v0.2.1 h1:JjA9roGokaR2BgWmaI/3HQu1/+jSbVVDLatQGnVdGjI=
+github.com/libp2p/go-libp2p-gostream v0.2.1/go.mod h1:1Mjp3LDmkqICe5tH9yLVNCqFaRTy6OwBvuJV6j1b9Nk=
+github.com/libp2p/go-libp2p-http v0.1.5 h1:FfLnzjlEzV4/6UCXCpPXRYZNoGCfogqCFjd7eF0Jbm8=
+github.com/libp2p/go-libp2p-http v0.1.5/go.mod h1:2YfPjsQxUlBGFQl2u461unkQ7ukwiSs7NX2eSslOJiU=
+github.com/libp2p/go-libp2p-kad-dht v0.5.0 h1:kDMtCftpQOL2s84/dZmw5z4NmBe6ByeDLKpcn6TcyxU=
+github.com/libp2p/go-libp2p-kad-dht v0.5.0/go.mod h1:42YDfiKXzIgaIexiEQ3rKZbVPVPziLOyHpXbOCVd814=
+github.com/libp2p/go-libp2p-kbucket v0.2.3 h1:XtNfN4WUy0cfeJoJgWCf1lor4Pp3kBkFJ9vQ+Zs+VUM=
+github.com/libp2p/go-libp2p-kbucket v0.2.3/go.mod h1:opWrBZSWnBYPc315q497huxY3sz1t488X6OiXUEYWKA=
+github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8=
+github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90=
+github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo=
+github.com/libp2p/go-libp2p-mplex v0.2.1 h1:E1xaJBQnbSiTHGI1gaBKmKhu1TUKkErKJnE8iGvirYI=
+github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE=
+github.com/libp2p/go-libp2p-mplex v0.2.2 h1:+Ld7YDAfVERQ0E+qqjE7o6fHwKuM0SqTzYiwN1lVVSA=
+github.com/libp2p/go-libp2p-mplex v0.2.2/go.mod h1:74S9eum0tVQdAfFiKxAyKzNdSuLqw5oadDq7+L/FELo=
+github.com/libp2p/go-libp2p-nat v0.0.5 h1:/mH8pXFVKleflDL1YwqMg27W9GD8kjEx7NY0P6eGc98=
+github.com/libp2p/go-libp2p-nat v0.0.5/go.mod h1:1qubaE5bTZMJE+E/uu2URroMbzdubFz1ChgiN79yKPE=
+github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLKcKF72EAMQ=
+github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU=
+github.com/libp2p/go-libp2p-peer v0.2.0 h1:EQ8kMjaCUwt/Y5uLgjT8iY2qg0mGUT0N1zUjer50DsY=
+github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY=
+github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY=
+github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI=
+github.com/libp2p/go-libp2p-peerstore v0.1.4 h1:d23fvq5oYMJ/lkkbO4oTwBp/JP+I/1m5gZJobNXCE/k=
+github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs=
+github.com/libp2p/go-libp2p-peerstore v0.2.0 h1:XcgJhI8WyUOCbHyRLNEX5542YNj8hnLSJ2G1InRjDhk=
+github.com/libp2p/go-libp2p-peerstore v0.2.0/go.mod h1:N2l3eVIeAitSg3Pi2ipSrJYnqhVnMNQZo9nkSCuAbnQ=
+github.com/libp2p/go-libp2p-pnet v0.2.0 h1:J6htxttBipJujEjz1y0a5+eYoiPcFHhSYHH6na5f0/k=
+github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA=
+github.com/libp2p/go-libp2p-pubsub v0.2.5 h1:tPKbkjAUI0xLGN3KKTKKy9TQEviVfrP++zJgH5Muke4=
+github.com/libp2p/go-libp2p-pubsub v0.2.5/go.mod h1:9Q2RRq8ofXkoewORcyVlgUFDKLKw7BuYSlJVWRcVk3Y=
+github.com/libp2p/go-libp2p-record v0.1.2 h1:M50VKzWnmUrk/M5/Dz99qO9Xh4vs8ijsK+7HkJvRP+0=
+github.com/libp2p/go-libp2p-record v0.1.2/go.mod h1:pal0eNcT5nqZaTV7UGhqeGqxFgGdsU/9W//C8dqjQDk=
+github.com/libp2p/go-libp2p-routing v0.1.0 h1:hFnj3WR3E2tOcKaGpyzfP4gvFZ3t8JkQmbapN0Ct+oU=
+github.com/libp2p/go-libp2p-routing v0.1.0/go.mod h1:zfLhI1RI8RLEzmEaaPwzonRvXeeSHddONWkcTcB54nE=
+github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8=
+github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g=
+github.com/libp2p/go-libp2p-secio v0.2.1 h1:eNWbJTdyPA7NxhP7J3c5lT97DC5d+u+IldkgCYFTPVA=
+github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8=
+github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4=
+github.com/libp2p/go-libp2p-swarm v0.2.2 h1:T4hUpgEs2r371PweU3DuH7EOmBIdTBCwWs+FLcgx3bQ=
+github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU=
+github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
+github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
+github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
+github.com/libp2p/go-libp2p-testing v0.1.0/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0=
+github.com/libp2p/go-libp2p-testing v0.1.1 h1:U03z3HnGI7Ni8Xx6ONVZvUFOAzWYmolWf5W5jAOPNmU=
+github.com/libp2p/go-libp2p-testing v0.1.1/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0=
+github.com/libp2p/go-libp2p-transport-upgrader v0.1.1 h1:PZMS9lhjK9VytzMCW3tWHAXtKXmlURSc3ZdvwEcKCzw=
+github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA=
+github.com/libp2p/go-libp2p-transport-upgrader v0.2.0 h1:5EhPgQhXZNyfL22ERZTUoVp9UVVbNowWNVtELQaKCHk=
+github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns=
+github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8=
+github.com/libp2p/go-libp2p-yamux v0.2.1 h1:Q3XYNiKCC2vIxrvUJL+Jg1kiyeEaIDNKLjgEjo3VQdI=
+github.com/libp2p/go-libp2p-yamux v0.2.1/go.mod h1:1FBXiHDk1VyRM1C0aez2bCfHQ4vMZKkAQzZbkSQt5fI=
+github.com/libp2p/go-libp2p-yamux v0.2.2 h1:eGvbqWqWY9S5lrpe2gA0UCOLCdzCgYSAR3vo/xCsNQg=
+github.com/libp2p/go-libp2p-yamux v0.2.2/go.mod h1:lIohaR0pT6mOt0AZ0L2dFze9hds9Req3OfS+B+dv4qw=
+github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q=
+github.com/libp2p/go-maddr-filter v0.0.5 h1:CW3AgbMO6vUvT4kf87y4N+0P8KUl2aqLYhrGyDUbLSg=
+github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M=
+github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0=
+github.com/libp2p/go-mplex v0.1.0 h1:/nBTy5+1yRyY82YaO6HXQRnO5IAGsXTjEJaR3LdTPc0=
+github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU=
+github.com/libp2p/go-mplex v0.1.1 h1:huPH/GGRJzmsHR9IZJJsrSwIM5YE2gL4ssgl1YWb/ps=
+github.com/libp2p/go-mplex v0.1.1/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk=
+github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ=
+github.com/libp2p/go-msgio v0.0.4 h1:agEFehY3zWJFUHK6SEMR7UYmk2z6kC3oeCM7ybLhguA=
+github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ=
+github.com/libp2p/go-nat v0.0.4 h1:KbizNnq8YIf7+Hn7+VFL/xE0eDrkPru2zIO9NMwL8UQ=
+github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo=
+github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0=
+github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc=
+github.com/libp2p/go-openssl v0.0.4 h1:d27YZvLoTyMhIN4njrkr8zMDOM4lfpHIp6A+TK9fovg=
 github.com/libp2p/go-openssl v0.0.4 h1:d27YZvLoTyMhIN4njrkr8zMDOM4lfpHIp6A+TK9fovg=
 github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc=
+github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc=
+github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FWpFLw=
+github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA=
+github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4=
+github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs=
+github.com/libp2p/go-stream-muxer v0.0.1 h1:Ce6e2Pyu+b5MC1k3eeFtAax0pW4gc6MosYSLV05UeLw=
+github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14=
+github.com/libp2p/go-stream-muxer-multistream v0.2.0 h1:714bRJ4Zy9mdhyTLJ+ZKiROmAFwUHpeRidG+q7LTQOg=
+github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc=
+github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc=
+github.com/libp2p/go-tcp-transport v0.1.1 h1:yGlqURmqgNA2fvzjSgZNlHcsd/IulAnKM8Ncu+vlqnw=
+github.com/libp2p/go-tcp-transport v0.1.1/go.mod h1:3HzGvLbx6etZjnFlERyakbaYPdfjg2pWP97dFZworkY=
+github.com/libp2p/go-ws-transport v0.2.0 h1:MJCw2OrPA9+76YNRvdo1wMnSOxb9Bivj6sVFY1Xrj6w=
+github.com/libp2p/go-ws-transport v0.2.0/go.mod h1:9BHJz/4Q5A9ludYWKoGCFC5gUElzlHoKzu0yY9p/klM=
+github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
+github.com/libp2p/go-yamux v1.2.3 h1:xX8A36vpXb59frIzWFdEgptLMsOANMFq2K7fPRlunYI=
+github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
+github.com/libp2p/go-yamux v1.3.0 h1:FsYzT16Wq2XqUGJsBbOxoz9g+dFklvNi7jN6YFPfl7U=
+github.com/libp2p/go-yamux v1.3.0/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/matrix-org/dendrite v0.0.0-20200220135450-0352f250b857/go.mod h1:DZ35IoR+ViBNVPe9umdlOSnjvKl7wfyRmZg4QfWGvTo=
 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY=
+github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY=
+github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg=
 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg=
 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f h1:5TOte9uk/epk8L+Pbp6qwaV8YsKYXKjyECPHUhJTWQc=
 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk=
@@ -130,8 +364,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200409140603-8b9a51fe9b89 h1:YAlUJK/Ty2ZrP/DL41CiR0Cp3pteshnyIS420KVs220=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200421090225-4ea81b29f5f7 h1:4vE84tE3r7BitCt2HQvT231JrhMjDfjDVDqVoiVPv0w=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20200409140603-8b9a51fe9b89/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20200421090225-4ea81b29f5f7/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI=
 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk=
 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A=
 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8=
@@ -139,9 +373,15 @@ github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXL
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U=
 github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
+github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
@@ -151,24 +391,61 @@ github.com/miekg/dns v1.1.12 h1:WMhc1ik4LNkTg8U9l3hI1LvxKmIL+f1+WV/SZtCbDDA=
 github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
+github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
+github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
+github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
 github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
+github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
+github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc=
 github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
 github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
+github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
+github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
+github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
+github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
+github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo=
+github.com/multiformats/go-multiaddr v0.2.0 h1:lR52sFwcTCuQb6bTfnXF6zA2XfyYvyd+5a9qECv/J90=
+github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4=
 github.com/multiformats/go-multiaddr v0.2.1 h1:SgG/cw5vqyB5QQe5FPe2TqggU9WtrA9X4nZw7LlVqOI=
 github.com/multiformats/go-multiaddr v0.2.1/go.mod h1:s/Apk6IyxfvMjDafnhJgJ3/46z7tZ04iMk5wP4QMGGE=
+github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
+github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
+github.com/multiformats/go-multiaddr-dns v0.2.0 h1:YWJoIDwLePniH7OU5hBnDZV6SWuvJqJ0YtN6pLeH9zA=
+github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0=
+github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q=
+github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E=
+github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo=
+github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU=
+github.com/multiformats/go-multiaddr-net v0.1.0/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ=
+github.com/multiformats/go-multiaddr-net v0.1.1 h1:jFFKUuXTXv+3ARyHZi3XUqQO+YWMKgBdhEvuGRfnL6s=
+github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ=
+github.com/multiformats/go-multiaddr-net v0.1.2 h1:P7zcBH9FRETdPkDrylcXVjQLQ2t1JQtNItZULWNWgeg=
+github.com/multiformats/go-multiaddr-net v0.1.2/go.mod h1:QsWt3XK/3hwvNxZJp92iMQKME1qHfpYmyIjFVsSOY6Y=
 github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA=
 github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
+github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
+github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po=
+github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
+github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
+github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
 github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc=
 github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
+github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ=
+github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
+github.com/multiformats/go-multistream v0.1.1 h1:JlAdpIFhBhGRLxe9W6Om0w++Gd6KMWoFPZL/dEnm9nI=
+github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38=
+github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
 github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
 github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg=
 github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
@@ -177,18 +454,34 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
 github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY=
 github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
+github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 h1:evlcQnJY+v8XRRchV3hXzpHDl6GcEZeLXAhlH9Csdww=
+github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6/go.mod h1:E26fwEtRNigBfFfHDWsklmo0T7Ixbg0XXgck+Hq4O9k=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU=
+github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg=
+github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg=
 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg=
 github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pierrec/lz4 v2.5.0+incompatible h1:MbdIZ43A//duwOjQqK3nP+up+65yraNFyX3Vp6Rwues=
+github.com/pierrec/lz4 v2.5.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -213,17 +506,27 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ=
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
+github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME=
 github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY=
+github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0=
 github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU=
 github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -231,6 +534,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/tidwall/gjson v1.1.5 h1:QysILxBeUEY3GTLA0fQVgkQG1zme8NxGvhh2SSqWNwI=
 github.com/tidwall/gjson v1.1.5/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA=
 github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc=
@@ -252,25 +556,57 @@ github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo
 github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw=
 github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=
+github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc=
+github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
+github.com/whyrusleeping/go-logging v0.0.1 h1:fwpzlmT0kRC/Fmd0MdmGgJG/CXIZ6gFq46FQZjprUcc=
+github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE=
+github.com/whyrusleeping/mafmt v1.2.8 h1:TCghSl5kkwEE0j+sU/gudyhVMRlpBin8fMBBHg59EbA=
+github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA=
+github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 h1:Y1/FEOpaCpD21WxrmfeIYCFPuVPRCY2XZTWzTNHGw30=
+github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4=
+github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds=
+github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI=
+github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow=
+github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg=
 github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010=
 go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc=
 golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI=
 golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw=
+golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -282,8 +618,10 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU=
 golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -307,18 +645,28 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTu
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -337,11 +685,14 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE=
 gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -349,6 +700,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4=
 gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So=
@@ -358,6 +711,7 @@ gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI=
 gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o=
 gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8=
 gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
@@ -365,4 +719,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099 h1:XJP7lxbSxWLOMNdBE4B/STaqVy6L73o0knwj2vIlxnw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

internal/sqlutil/trace.go

@@ -0,0 +1,89 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlutil
+
+import (
+	"context"
+	"database/sql"
+	"database/sql/driver"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/ngrok/sqlmw"
+	"github.com/sirupsen/logrus"
+)
+
+var tracingEnabled = os.Getenv("DENDRITE_TRACE_SQL") == "1"
+
+type traceInterceptor struct {
+	sqlmw.NullInterceptor
+}
+
+func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.StmtQueryContext, query string, args []driver.NamedValue) (driver.Rows, error) {
+	startedAt := time.Now()
+	rows, err := stmt.QueryContext(ctx, args)
+
+	logrus.WithField("duration", time.Since(startedAt)).WithField(logrus.ErrorKey, err).Debug("executed sql query ", query, " args: ", args)
+
+	return rows, err
+}
+
+func (in *traceInterceptor) StmtExecContext(ctx context.Context, stmt driver.StmtExecContext, query string, args []driver.NamedValue) (driver.Result, error) {
+	startedAt := time.Now()
+	result, err := stmt.ExecContext(ctx, args)
+
+	logrus.WithField("duration", time.Since(startedAt)).WithField(logrus.ErrorKey, err).Debug("executed sql query ", query, " args: ", args)
+
+	return result, err
+}
+
+func (in *traceInterceptor) RowsNext(c context.Context, rows driver.Rows, dest []driver.Value) error {
+	err := rows.Next(dest)
+	if err == io.EOF {
+		// For all cases, we call Next() n+1 times, the first to populate the initial dest, then eventually
+		// it will io.EOF. If we log on each Next() call we log the last element twice, so don't.
+		return err
+	}
+	cols := rows.Columns()
+	logrus.Debug(strings.Join(cols, " | "))
+
+	b := strings.Builder{}
+	for i, val := range dest {
+		b.WriteString(fmt.Sprintf("%v", val))
+		if i+1 <= len(dest)-1 {
+			b.WriteString(" | ")
+		}
+	}
+	logrus.Debug(b.String())
+	return err
+}
+
+// Open opens a database specified by its database driver name and a driver-specific data source name,
+// usually consisting of at least a database name and connection information. Includes tracing driver
+// if DENDRITE_TRACE_SQL=1
+func Open(driverName, dsn string) (*sql.DB, error) {
+	if tracingEnabled {
+		// install the wrapped driver
+		driverName += "-trace"
+	}
+	return sql.Open(driverName, dsn)
+}
+
+func init() {
+	registerDrivers()
+}

internal/sqlutil/trace_driver.go

@@ -0,0 +1,35 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !wasm
+
+package sqlutil
+
+import (
+	"database/sql"
+
+	"github.com/lib/pq"
+	sqlite "github.com/mattn/go-sqlite3"
+	"github.com/ngrok/sqlmw"
+)
+
+func registerDrivers() {
+	if !tracingEnabled {
+		return
+	}
+	// install the wrapped drivers
+	sql.Register("postgres-trace", sqlmw.Driver(&pq.Driver{}, new(traceInterceptor)))
+	sql.Register("sqlite3-trace", sqlmw.Driver(&sqlite.SQLiteDriver{}, new(traceInterceptor)))
+
+}

internal/sqlutil/trace_driver_wasm.go

@@ -0,0 +1,33 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build wasm
+
+package sqlutil
+
+import (
+	"database/sql"
+
+	sqlitejs "github.com/matrix-org/go-sqlite3-js"
+	"github.com/ngrok/sqlmw"
+)
+
+func registerDrivers() {
+	if !tracingEnabled {
+		return
+	}
+	// install the wrapped drivers
+	sql.Register("sqlite3_js-trace", sqlmw.Driver(&sqlitejs.SqliteJsDriver{}, new(traceInterceptor)))
+
+}

mediaapi/storage/postgres/storage.go

@@ -21,6 +21,7 @@ import (
 
 	// Import the postgres database driver.
 	_ "github.com/lib/pq"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -35,7 +36,7 @@ type Database struct {
 func Open(dataSourceName string) (*Database, error) {
 	var d Database
 	var err error
-	if d.db, err = sql.Open("postgres", dataSourceName); err != nil {
+	if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
 		return nil, err
 	}
 	if err = d.statements.prepare(d.db); err != nil {

mediaapi/storage/sqlite3/storage.go

@@ -21,6 +21,7 @@ import (
 
 	// Import the postgres database driver.
 	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/mediaapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	_ "github.com/mattn/go-sqlite3"
@@ -36,7 +37,7 @@ type Database struct {
 func Open(dataSourceName string) (*Database, error) {
 	var d Database
 	var err error
-	if d.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if d.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
 		return nil, err
 	}
 	if err = d.statements.prepare(d.db); err != nil {

publicroomsapi/directory/directory.go

@@ -17,6 +17,9 @@ package directory
 import (
 	"net/http"
 
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/roomserver/api"
+
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/publicroomsapi/storage"
@@ -54,11 +57,51 @@ func GetVisibility(
 }
 
 // SetVisibility implements PUT /directory/list/room/{roomID}
-// TODO: Check if user has the power level to edit the room visibility
+// TODO: Allow admin users to edit the room visibility
 func SetVisibility(
-	req *http.Request, publicRoomsDatabase storage.Database,
+	req *http.Request, publicRoomsDatabase storage.Database, queryAPI api.RoomserverQueryAPI, dev *authtypes.Device,
 	roomID string,
 ) util.JSONResponse {
+	queryMembershipReq := api.QueryMembershipForUserRequest{
+		RoomID: roomID,
+		UserID: dev.UserID,
+	}
+	var queryMembershipRes api.QueryMembershipForUserResponse
+	err := queryAPI.QueryMembershipForUser(req.Context(), &queryMembershipReq, &queryMembershipRes)
+	if err != nil {
+		util.GetLogger(req.Context()).WithError(err).Error("could not query membership for user")
+		return jsonerror.InternalServerError()
+	}
+	// Check if user id is in room
+	if !queryMembershipRes.IsInRoom {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("user does not belong to room"),
+		}
+	}
+	queryEventsReq := api.QueryLatestEventsAndStateRequest{
+		RoomID: roomID,
+		StateToFetch: []gomatrixserverlib.StateKeyTuple{{
+			EventType: gomatrixserverlib.MRoomPowerLevels,
+			StateKey:  "",
+		}},
+	}
+	var queryEventsRes api.QueryLatestEventsAndStateResponse
+	err = queryAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes)
+	if err != nil || len(queryEventsRes.StateEvents) == 0 {
+		util.GetLogger(req.Context()).WithError(err).Error("could not query events from room")
+		return jsonerror.InternalServerError()
+	}
+
+	// NOTSPEC: Check if the user's power is greater than power required to change m.room.aliases event
+	power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event)
+	if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomAliases, true) {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("userID doesn't have power level to change visibility"),
+		}
+	}
+
 	var v roomVisibility
 	if reqErr := httputil.UnmarshalJSONRequest(req, &v); reqErr != nil {
 		return *reqErr

publicroomsapi/publicroomsapi.go

@@ -31,21 +31,17 @@ import (
 func SetupPublicRoomsAPIComponent(
 	base *basecomponent.BaseDendrite,
 	deviceDB devices.Database,
+	publicRoomsDB storage.Database,
 	rsQueryAPI roomserverAPI.RoomserverQueryAPI,
 	fedClient *gomatrixserverlib.FederationClient,
 	extRoomsProvider types.ExternalPublicRoomsProvider,
 ) {
-	publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI))
-	if err != nil {
-		logrus.WithError(err).Panicf("failed to connect to public rooms db")
-	}
-
 	rsConsumer := consumers.NewOutputRoomEventConsumer(
 		base.Cfg, base.KafkaConsumer, publicRoomsDB, rsQueryAPI,
 	)
-	if err = rsConsumer.Start(); err != nil {
+	if err := rsConsumer.Start(); err != nil {
 		logrus.WithError(err).Panic("failed to start public rooms server consumer")
 	}
 
-	routing.Setup(base.APIMux, deviceDB, publicRoomsDB, fedClient, extRoomsProvider)
+	routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsQueryAPI, fedClient, extRoomsProvider)
 }

publicroomsapi/routing/routing.go

@@ -17,6 +17,8 @@ package routing
 import (
 	"net/http"
 
+	"github.com/matrix-org/dendrite/roomserver/api"
+
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
@@ -37,7 +39,7 @@ const pathPrefixR0 = "/_matrix/client/r0"
 // applied:
 // nolint: gocyclo
 func Setup(
-	apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database,
+	apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, queryAPI api.RoomserverQueryAPI,
 	fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider,
 ) {
 	r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
@@ -64,7 +66,7 @@ func Setup(
 			if err != nil {
 				return util.ErrorResponse(err)
 			}
-			return directory.SetVisibility(req, publicRoomsDB, vars["roomID"])
+			return directory.SetVisibility(req, publicRoomsDB, queryAPI, device, vars["roomID"])
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	r0mux.Handle("/publicRooms",

publicroomsapi/storage/postgres/storage.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 
 	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -38,7 +39,7 @@ type attributeValue interface{}
 func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sql.Open("postgres", dataSourceName); err != nil {
+	if db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
 		return nil, err
 	}
 	storage := PublicRoomsServerDatabase{

publicroomsapi/storage/sqlite3/storage.go

@@ -23,6 +23,7 @@ import (
 	_ "github.com/mattn/go-sqlite3"
 
 	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -40,7 +41,7 @@ type attributeValue interface{}
 func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) {
 	var db *sql.DB
 	var err error
-	if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
+	if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil {
 		return nil, err
 	}
 	storage := PublicRoomsServerDatabase{

publicroomsapi/storage/storage.go

@@ -23,6 +23,9 @@ import (
 	"github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3"
 )
 
+const schemePostgres = "postgres"
+const schemeFile = "file"
+
 // NewPublicRoomsServerDatabase opens a database connection.
 func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) {
 	uri, err := url.Parse(dataSourceName)
@@ -30,9 +33,9 @@ func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) {
 		return postgres.NewPublicRoomsServerDatabase(dataSourceName)
 	}
 	switch uri.Scheme {
-	case "postgres":
+	case schemePostgres:
 		return postgres.NewPublicRoomsServerDatabase(dataSourceName)
-	case "file":
+	case schemeFile:
 		return sqlite3.NewPublicRoomsServerDatabase(dataSourceName)
 	default:
 		return postgres.NewPublicRoomsServerDatabase(dataSourceName)

roomserver/alias/alias.go

@@ -17,6 +17,7 @@ package alias
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"net/http"
 	"time"
 
@@ -218,6 +219,9 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent(
 	if err != nil {
 		return err
 	}
+	if len(eventsNeeded.Tuples()) == 0 {
+		return errors.New("expecting state tuples for event builder, got none")
+	}
 	req := roomserverAPI.QueryLatestEventsAndStateRequest{
 		RoomID:       roomID,
 		StateToFetch: eventsNeeded.Tuples(),

roomserver/api/query.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"net/http"
 
+	"github.com/matrix-org/dendrite/common/caching"
 	commonHTTP "github.com/matrix-org/dendrite/common/http"
 	"github.com/matrix-org/gomatrixserverlib"
 	opentracing "github.com/opentracing/opentracing-go"
@@ -411,16 +412,17 @@ const RoomserverQueryRoomVersionForRoomPath = "/api/roomserver/queryRoomVersionF
 
 // NewRoomserverQueryAPIHTTP creates a RoomserverQueryAPI implemented by talking to a HTTP POST API.
 // If httpClient is nil an error is returned
-func NewRoomserverQueryAPIHTTP(roomserverURL string, httpClient *http.Client) (RoomserverQueryAPI, error) {
+func NewRoomserverQueryAPIHTTP(roomserverURL string, httpClient *http.Client, cache caching.ImmutableCache) (RoomserverQueryAPI, error) {
 	if httpClient == nil {
 		return nil, errors.New("NewRoomserverQueryAPIHTTP: httpClient is <nil>")
 	}
-	return &httpRoomserverQueryAPI{roomserverURL, httpClient}, nil
+	return &httpRoomserverQueryAPI{roomserverURL, httpClient, cache}, nil
 }
 
 type httpRoomserverQueryAPI struct {
 	roomserverURL  string
 	httpClient     *http.Client
+	immutableCache caching.ImmutableCache
 }
 
 // QueryLatestEventsAndState implements RoomserverQueryAPI
@@ -585,9 +587,18 @@ func (h *httpRoomserverQueryAPI) QueryRoomVersionForRoom(
 	request *QueryRoomVersionForRoomRequest,
 	response *QueryRoomVersionForRoomResponse,
 ) error {
+	if roomVersion, ok := h.immutableCache.GetRoomVersion(request.RoomID); ok {
+		response.RoomVersion = roomVersion
+		return nil
+	}
+
 	span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRoomVersionForRoom")
 	defer span.Finish()
 
 	apiURL := h.roomserverURL + RoomserverQueryRoomVersionForRoomPath
-	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	err := commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+	if err == nil {
+		h.immutableCache.StoreRoomVersion(request.RoomID, response.RoomVersion)
+	}
+	return err
 }

roomserver/query/query.go

@@ -22,6 +22,7 @@ import (
 	"net/http"
 
 	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/caching"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/auth"
 	"github.com/matrix-org/dendrite/roomserver/state"
@@ -98,6 +99,7 @@ type RoomserverQueryAPIDatabase interface {
 // RoomserverQueryAPI is an implementation of api.RoomserverQueryAPI
 type RoomserverQueryAPI struct {
 	DB             RoomserverQueryAPIDatabase
+	ImmutableCache caching.ImmutableCache
 }
 
 // QueryLatestEventsAndState implements api.RoomserverQueryAPI
@@ -132,10 +134,18 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState(
 		return err
 	}
 
+	var stateEntries []types.StateEntry
+	if len(request.StateToFetch) == 0 {
+		// Look up all room state.
+		stateEntries, err = roomState.LoadStateAtSnapshot(
+			ctx, currentStateSnapshotNID,
+		)
+	} else {
 		// Look up the current state for the requested tuples.
-	stateEntries, err := roomState.LoadStateAtSnapshotForStringTuples(
+		stateEntries, err = roomState.LoadStateAtSnapshotForStringTuples(
 			ctx, currentStateSnapshotNID, request.StateToFetch,
 		)
+	}
 	if err != nil {
 		return err
 	}
@@ -888,11 +898,17 @@ func (r *RoomserverQueryAPI) QueryRoomVersionForRoom(
 	request *api.QueryRoomVersionForRoomRequest,
 	response *api.QueryRoomVersionForRoomResponse,
 ) error {
+	if roomVersion, ok := r.ImmutableCache.GetRoomVersion(request.RoomID); ok {
+		response.RoomVersion = roomVersion
+		return nil
+	}
+
 	roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID)
 	if err != nil {
 		return err
 	}
 	response.RoomVersion = roomVersion
+	r.ImmutableCache.StoreRoomVersion(request.RoomID, response.RoomVersion)
 	return nil
 }
 

roomserver/roomserver.go

@@ -48,7 +48,10 @@ func SetupRoomServerComponent(
 
 	inputAPI.SetupHTTP(http.DefaultServeMux)
 
-	queryAPI := query.RoomserverQueryAPI{DB: roomserverDB}
+	queryAPI := query.RoomserverQueryAPI{
+		DB:             roomserverDB,
+		ImmutableCache: base.ImmutableCache,
+	}
 
 	queryAPI.SetupHTTP(http.DefaultServeMux)
 

roomserver/storage/postgres/storage.go

@@ -20,7 +20,7 @@ import (
 	"database/sql"
 	"encoding/json"
 
-	roomserverVersion "github.com/matrix-org/dendrite/roomserver/version"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 
 	// Import the postgres database driver.
 	_ "github.com/lib/pq"
@@ -39,7 +39,7 @@ type Database struct {
 func Open(dataSourceName string) (*Database, error) {
 	var d Database
 	var err error
-	if d.db, err = sql.Open("postgres", dataSourceName); err != nil {
+	if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil {
 		return nil, err
 	}
 	if err = d.statements.prepare(d.db); err != nil {
@@ -146,7 +146,7 @@ func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) (
 	if event.Type() != gomatrixserverlib.MRoomCreate {
 		return gomatrixserverlib.RoomVersion(""), nil
 	}
-	roomVersion = roomserverVersion.DefaultRoomVersion()
+	roomVersion = gomatrixserverlib.RoomVersionV1
 	var createContent gomatrixserverlib.CreateContent
 	// The m.room.create event contains an optional "room_version" key in
 	// the event content, so we need to unmarshal that first.

roomserver/storage/sqlite3/storage.go

@@ -22,7 +22,7 @@ import (
 	"errors"
 	"net/url"
 
-	roomserverVersion "github.com/matrix-org/dendrite/roomserver/version"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/roomserver/api"
@@ -52,7 +52,7 @@ func Open(dataSourceName string) (*Database, error) {
 	} else {
 		return nil, errors.New("no filename or path in connect string")
 	}
-	if d.db, err = sql.Open(common.SQLiteDriverName(), cs); err != nil {
+	if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs); err != nil {
 		return nil, err
 	}
 	//d.db.Exec("PRAGMA journal_mode=WAL;")
@@ -175,7 +175,7 @@ func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) (
 	if event.Type() != gomatrixserverlib.MRoomCreate {
 		return gomatrixserverlib.RoomVersion(""), nil
 	}
-	roomVersion = roomserverVersion.DefaultRoomVersion()
+	roomVersion = gomatrixserverlib.RoomVersionV1
 	var createContent gomatrixserverlib.CreateContent
 	// The m.room.create event contains an optional "room_version" key in
 	// the event content, so we need to unmarshal that first.

roomserver/version/version.go

@@ -59,7 +59,7 @@ var roomVersions = map[gomatrixserverlib.RoomVersion]RoomVersionDescription{
 // DefaultRoomVersion contains the room version that will, by
 // default, be used to create new rooms on this server.
 func DefaultRoomVersion() gomatrixserverlib.RoomVersion {
-	return gomatrixserverlib.RoomVersionV2
+	return gomatrixserverlib.RoomVersionV4
 }
 
 // RoomVersions returns a map of all known room versions to this

syncapi/routing/messages.go

@@ -209,18 +209,12 @@ func (r *messagesReq) retrieveEvents() (
 		return []gomatrixserverlib.ClientEvent{}, r.from, r.to, nil
 	}
 
-	// Sort the events to ensure we send them in the right order. We currently
+	// Sort the events to ensure we send them in the right order.
-	// do that based on the event's timestamp.
+	events = gomatrixserverlib.HeaderedReverseTopologicalOrdering(events)
 	if r.backwardOrdering {
-		sort.SliceStable(events, func(i int, j int) bool {
+		// This reverses the array from old->new to new->old
-			// Backward ordering is antichronological (latest event to oldest
+		sort.SliceStable(events, func(i, j int) bool {
-			// one).
+			return true
-			return sortEvents(&(events[j]), &(events[i]))
-		})
-	} else {
-		sort.SliceStable(events, func(i int, j int) bool {
-			// Forward ordering is chronological (oldest event to latest one).
-			return sortEvents(&(events[i]), &(events[j]))
 		})
 	}
 
@@ -493,12 +487,3 @@ func setToDefault(
 
 	return
 }
-
-// sortEvents is a function to give to sort.SliceStable, and compares the
-// timestamp of two Matrix events.
-// Returns true if the first event happened before the second one, false
-// otherwise.
-func sortEvents(e1 *gomatrixserverlib.HeaderedEvent, e2 *gomatrixserverlib.HeaderedEvent) bool {
-	t := e1.OriginServerTS().Time()
-	return e2.OriginServerTS().Time().After(t)
-}

syncapi/routing/routing.go

@@ -56,30 +56,6 @@ func Setup(
 		return srp.OnIncomingSyncRequest(req, device)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/state", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-		vars, err := common.URLDecodeMapValues(mux.Vars(req))
-		if err != nil {
-			return util.ErrorResponse(err)
-		}
-		return OnIncomingStateRequest(req, syncDB, vars["roomID"])
-	})).Methods(http.MethodGet, http.MethodOptions)
-
-	r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-		vars, err := common.URLDecodeMapValues(mux.Vars(req))
-		if err != nil {
-			return util.ErrorResponse(err)
-		}
-		return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], "")
-	})).Methods(http.MethodGet, http.MethodOptions)
-
-	r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-		vars, err := common.URLDecodeMapValues(mux.Vars(req))
-		if err != nil {
-			return util.ErrorResponse(err)
-		}
-		return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], vars["stateKey"])
-	})).Methods(http.MethodGet, http.MethodOptions)
-
 	r0mux.Handle("/rooms/{roomID}/messages", common.MakeAuthAPI("room_messages", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
 		vars, err := common.URLDecodeMapValues(mux.Vars(req))
 		if err != nil {

syncapi/storage/postgres/backward_extremities_table.go

@@ -60,7 +60,7 @@ const insertBackwardExtremitySQL = "" +
 	" ON CONFLICT DO NOTHING"
 
 const selectBackwardExtremitiesForRoomSQL = "" +
-	"SELECT event_id FROM syncapi_backward_extremities WHERE room_id = $1"
+	"SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1"
 
 const deleteBackwardExtremitySQL = "" +
 	"DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2"

syncapi/storage/postgres/syncserver.go

@@ -25,6 +25,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 
 	// Import the postgres database driver.
@@ -62,7 +63,7 @@ type SyncServerDatasource struct {
 func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, error) {
 	var d SyncServerDatasource
 	var err error
-	if d.db, err = sql.Open("postgres", dbDataSourceName); err != nil {
+	if d.db, err = sqlutil.Open("postgres", dbDataSourceName); err != nil {
 		return nil, err
 	}
 	if err = d.PartitionOffsetStatements.Prepare(d.db, "syncapi"); err != nil {

syncapi/storage/sqlite3/backward_extremities_table.go

@@ -60,7 +60,7 @@ const insertBackwardExtremitySQL = "" +
 	" ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING"
 
 const selectBackwardExtremitiesForRoomSQL = "" +
-	"SELECT event_id FROM syncapi_backward_extremities WHERE room_id = $1"
+	"SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1"
 
 const deleteBackwardExtremitySQL = "" +
 	"DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2"

syncapi/storage/sqlite3/syncserver.go

@@ -27,6 +27,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/roomserver/api"
 
 	// Import the sqlite3 package
@@ -78,7 +79,7 @@ func NewSyncServerDatasource(dataSourceName string) (*SyncServerDatasource, erro
 	} else {
 		return nil, errors.New("no filename or path in connect string")
 	}
-	if d.db, err = sql.Open(common.SQLiteDriverName(), cs); err != nil {
+	if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs); err != nil {
 		return nil, err
 	}
 	if err = d.prepare(); err != nil {

sytest-blacklist

@@ -35,3 +35,7 @@ Inbound federation rejects invites which are not signed by the sender
 
 # Blacklisted because we don't support ignores yet
 Ignore invite in incremental sync
+
+# Blacklisted because this test calls /r0/events which we don't implement
+New room members see their own join event
+Existing members see new members' join events

sytest-whitelist

@@ -17,6 +17,11 @@ POST /register rejects registration of usernames with 'é'
 POST /register rejects registration of usernames with '\n'
 POST /register rejects registration of usernames with '''
 GET /login yields a set of flows
+POST /login can log in as a user
+POST /login returns the same device_id as that in the request
+POST /login can log in as a user with just the local part of the id
+POST /login as non-existing user is rejected
+POST /login wrong password is rejected
 GET /events initially
 GET /initialSync initially
 Version responds 200 OK with valid structure
@@ -55,8 +60,9 @@ Request to logout with invalid an access token is rejected
 Request to logout without an access token is rejected
 Room creation reports m.room.create to myself
 Room creation reports m.room.member to myself
-New room members see their own join event
+# Blacklisted because these tests call /r0/events which we don't implement
-Existing members see new members' join events
+# New room members see their own join event
+# Existing members see new members' join events
 setting 'm.room.power_levels' respects room powerlevel
 Unprivileged users can set m.room.topic if it only needs level 0
 Users cannot set ban powerlevel higher than their own
@@ -245,3 +251,5 @@ Remote user can backfill in a room with version 4
 Outbound federation can send invites via v2 API
 User can invite local user to room with version 3
 User can invite local user to room with version 4
+A pair of servers can establish a join in a v2 room
+Can logout all devices