From f8d2860765325c29d516f766002b46515b3e0a8c Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Tue, 6 Aug 2019 22:07:36 +0800 Subject: [PATCH 01/11] Replace membership and visibility values with constants (#774) Signed-off-by: Alex Chen --- clientapi/auth/storage/accounts/storage.go | 2 +- clientapi/routing/createroom.go | 14 ++++------ clientapi/routing/joinroom.go | 2 +- clientapi/routing/membership.go | 6 ++--- clientapi/routing/profile.go | 2 +- clientapi/routing/routing.go | 2 +- clientapi/threepid/invites.go | 2 +- cmd/create-room-events/main.go | 2 +- federationapi/routing/join.go | 2 +- federationapi/routing/leave.go | 4 +-- federationapi/routing/threepid.go | 2 +- federationsender/consumers/roomserver.go | 2 +- go.mod | 2 +- go.sum | 2 ++ publicroomsapi/directory/directory.go | 5 ++-- publicroomsapi/storage/storage.go | 2 +- roomserver/auth/auth.go | 2 +- roomserver/input/membership.go | 21 +++++---------- roomserver/query/query.go | 2 +- syncapi/storage/syncserver.go | 30 +++++++++------------- syncapi/sync/notifier.go | 8 +++--- 21 files changed, 51 insertions(+), 65 deletions(-) diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 5c8ffffeb..41d75daad 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -230,7 +230,7 @@ func (d *Database) newMembership( } // Only "join" membership events can be considered as new memberships - if membership == "join" { + if membership == gomatrixserverlib.Join { if err := d.saveMembership(ctx, txn, localpart, roomID, eventID); err != nil { return err } diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 8c5ee975c..4a76e1b06 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -55,10 +55,6 @@ const ( presetPublicChat = "public_chat" ) -const ( - joinRulePublic = "public" - joinRuleInvite = "invite" -) const ( historyVisibilityShared = "shared" // TODO: These should be implemented once history visibility is implemented @@ -201,7 +197,7 @@ func createRoom( } membershipContent := common.MemberContent{ - Membership: "join", + Membership: gomatrixserverlib.Join, DisplayName: profile.DisplayName, AvatarURL: profile.AvatarURL, } @@ -209,19 +205,19 @@ func createRoom( var joinRules, historyVisibility string switch r.Preset { case presetPrivateChat: - joinRules = joinRuleInvite + joinRules = gomatrixserverlib.Invite historyVisibility = historyVisibilityShared case presetTrustedPrivateChat: - joinRules = joinRuleInvite + joinRules = gomatrixserverlib.Invite historyVisibility = historyVisibilityShared // TODO If trusted_private_chat, all invitees are given the same power level as the room creator. case presetPublicChat: - joinRules = joinRulePublic + joinRules = gomatrixserverlib.Public historyVisibility = historyVisibilityShared default: // Default room rules, r.Preset was previously checked for valid values so // only a request with no preset should end up here. - joinRules = joinRuleInvite + joinRules = gomatrixserverlib.Invite historyVisibility = historyVisibilityShared } diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 9c02a93ca..432c982b4 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -70,7 +70,7 @@ func JoinRoomByIDOrAlias( return httputil.LogThenError(req, err) } - content["membership"] = "join" + content["membership"] = gomatrixserverlib.Join content["displayname"] = profile.DisplayName content["avatar_url"] = profile.AvatarURL diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 61898fecd..5e183fa0f 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -102,7 +102,7 @@ func SendMembership( var returnData interface{} = struct{}{} // The join membership requires the room id to be sent in the response - if membership == "join" { + if membership == gomatrixserverlib.Join { returnData = struct { RoomID string `json:"room_id"` }{roomID} @@ -141,7 +141,7 @@ func buildMembershipEvent( // "unban" or "kick" isn't a valid membership value, change it to "leave" if membership == "unban" || membership == "kick" { - membership = "leave" + membership = gomatrixserverlib.Leave } content := common.MemberContent{ @@ -192,7 +192,7 @@ func loadProfile( func getMembershipStateKey( body threepid.MembershipRequest, device *authtypes.Device, membership string, ) (stateKey string, reason string, err error) { - if membership == "ban" || membership == "unban" || membership == "kick" || membership == "invite" { + if membership == gomatrixserverlib.Ban || membership == "unban" || membership == "kick" || membership == gomatrixserverlib.Invite { // If we're in this case, the state key is contained in the request body, // possibly along with a reason (for "kick" and "ban") so we need to parse // it diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 034b9ac84..8d28b3660 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -264,7 +264,7 @@ func buildMembershipEvents( } content := common.MemberContent{ - Membership: "join", + Membership: gomatrixserverlib.Join, } content.DisplayName = newProfile.DisplayName diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index ab8f89731..c262db3d8 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -93,7 +93,7 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", - common.MakeAuthAPI("join", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI(gomatrixserverlib.Join, authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 2538577fd..251afb0d3 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -91,7 +91,7 @@ func CheckAndProcessInvite( producer *producers.RoomserverProducer, membership string, roomID string, evTime time.Time, ) (inviteStoredOnIDServer bool, err error) { - if membership != "invite" || (body.Address == "" && body.IDServer == "" && body.Medium == "") { + if membership != gomatrixserverlib.Invite || (body.Address == "" && body.IDServer == "" && body.Medium == "") { // If none of the 3PID-specific fields are supplied, it's a standard invite // so return nil for it to be processed as such return diff --git a/cmd/create-room-events/main.go b/cmd/create-room-events/main.go index 1d05b2a12..8475914f0 100644 --- a/cmd/create-room-events/main.go +++ b/cmd/create-room-events/main.go @@ -86,7 +86,7 @@ func main() { // Build a m.room.member event. b.Type = "m.room.member" b.StateKey = userID - b.SetContent(map[string]string{"membership": "join"}) // nolint: errcheck + b.SetContent(map[string]string{"membership": gomatrixserverlib.Join}) // nolint: errcheck b.AuthEvents = []gomatrixserverlib.EventReference{create} member := buildAndOutput() diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 0b60408f7..6f6574dd7 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -58,7 +58,7 @@ func MakeJoin( Type: "m.room.member", StateKey: &userID, } - err = builder.SetContent(map[string]interface{}{"membership": "join"}) + err = builder.SetContent(map[string]interface{}{"membership": gomatrixserverlib.Join}) if err != nil { return httputil.LogThenError(httpReq, err) } diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 3c57d39d1..a982b87f8 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -56,7 +56,7 @@ func MakeLeave( Type: "m.room.member", StateKey: &userID, } - err = builder.SetContent(map[string]interface{}{"membership": "leave"}) + err = builder.SetContent(map[string]interface{}{"membership": gomatrixserverlib.Leave}) if err != nil { return httputil.LogThenError(httpReq, err) } @@ -153,7 +153,7 @@ func SendLeave( mem, err := event.Membership() if err != nil { return httputil.LogThenError(httpReq, err) - } else if mem != "leave" { + } else if mem != gomatrixserverlib.Leave { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.BadJSON("The membership in the event content must be set to leave"), diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 05ca8892e..cff311cc4 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -202,7 +202,7 @@ func createInviteFrom3PIDInvite( content := common.MemberContent{ AvatarURL: profile.AvatarURL, DisplayName: profile.DisplayName, - Membership: "invite", + Membership: gomatrixserverlib.Invite, ThirdPartyInvite: &common.TPInvite{ Signed: inv.Signed, }, diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 45e48f166..3ba978b1d 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -233,7 +233,7 @@ func joinedHostsFromEvents(evs []gomatrixserverlib.Event) ([]types.JoinedHost, e if err != nil { return nil, err } - if membership != "join" { + if membership != gomatrixserverlib.Join { continue } _, serverName, err := gomatrixserverlib.SplitID('@', *ev.StateKey()) diff --git a/go.mod b/go.mod index 5d01012c9..8e14253ca 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6 + github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6 github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 github.com/matttproud/golang_protobuf_extensions v1.0.1 diff --git a/go.sum b/go.sum index 53151121b..0d59d1dd6 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2 h1:pY github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6 h1:B8n1H5Wb1B5jwLzTylBpY0kJCMRqrofT7PmOw4aJFJA= github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= +github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6 h1:xr69Hk6QM3RIN6JSvx3RpDowBGpHpDDqhqXCeySwYow= +github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 h1:p7WTwG+aXM86+yVrYAiCMW3ZHSmotVvuRbjtt3jC+4A= github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171013132526-8b1c8ab81986 h1:TiWl4hLvezAhRPM8tPcPDFTysZ7k4T/1J4GPp/iqlZo= diff --git a/publicroomsapi/directory/directory.go b/publicroomsapi/directory/directory.go index bb0153850..626a1c153 100644 --- a/publicroomsapi/directory/directory.go +++ b/publicroomsapi/directory/directory.go @@ -19,6 +19,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/publicroomsapi/storage" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -39,7 +40,7 @@ func GetVisibility( var v roomVisibility if isPublic { - v.Visibility = "public" + v.Visibility = gomatrixserverlib.Public } else { v.Visibility = "private" } @@ -61,7 +62,7 @@ func SetVisibility( return *reqErr } - isPublic := v.Visibility == "public" + isPublic := v.Visibility == gomatrixserverlib.Public if err := publicRoomsDatabase.SetRoomVisibility(req.Context(), isPublic, roomID); err != nil { return httputil.LogThenError(req, err) } diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go index eab27041b..aa9806945 100644 --- a/publicroomsapi/storage/storage.go +++ b/publicroomsapi/storage/storage.go @@ -185,7 +185,7 @@ func (d *PublicRoomsServerDatabase) updateNumJoinedUsers( return err } - if membership != "join" { + if membership != gomatrixserverlib.Join { return nil } diff --git a/roomserver/auth/auth.go b/roomserver/auth/auth.go index 2dce6f6dc..5ff1fadad 100644 --- a/roomserver/auth/auth.go +++ b/roomserver/auth/auth.go @@ -23,7 +23,7 @@ func IsServerAllowed( ) bool { for _, ev := range authEvents { membership, err := ev.Membership() - if err != nil || membership != "join" { + if err != nil || membership != gomatrixserverlib.Join { continue } diff --git a/roomserver/input/membership.go b/roomserver/input/membership.go index 0c3fbb80a..841c5fec6 100644 --- a/roomserver/input/membership.go +++ b/roomserver/input/membership.go @@ -23,13 +23,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// Membership values -// TODO: Factor these out somewhere sensible? -const join = "join" -const leave = "leave" -const invite = "invite" -const ban = "ban" - // updateMembership updates the current membership and the invites for each // user affected by a change in the current state of the room. // Returns a list of output events to write to the kafka log to inform the @@ -91,8 +84,8 @@ func updateMembership( ) ([]api.OutputEvent, error) { var err error // Default the membership to Leave if no event was added or removed. - oldMembership := leave - newMembership := leave + oldMembership := gomatrixserverlib.Leave + newMembership := gomatrixserverlib.Leave if remove != nil { oldMembership, err = remove.Membership() @@ -106,7 +99,7 @@ func updateMembership( return nil, err } } - if oldMembership == newMembership && newMembership != join { + if oldMembership == newMembership && newMembership != gomatrixserverlib.Join { // If the membership is the same then nothing changed and we can return // immediately, unless it's a Join update (e.g. profile update). return updates, nil @@ -118,11 +111,11 @@ func updateMembership( } switch newMembership { - case invite: + case gomatrixserverlib.Invite: return updateToInviteMembership(mu, add, updates) - case join: + case gomatrixserverlib.Join: return updateToJoinMembership(mu, add, updates) - case leave, ban: + case gomatrixserverlib.Leave, gomatrixserverlib.Ban: return updateToLeaveMembership(mu, add, newMembership, updates) default: panic(fmt.Errorf( @@ -183,7 +176,7 @@ func updateToJoinMembership( for _, eventID := range retired { orie := api.OutputRetireInviteEvent{ EventID: eventID, - Membership: join, + Membership: gomatrixserverlib.Join, RetiredByEventID: add.EventID(), TargetUserID: *add.StateKey(), } diff --git a/roomserver/query/query.go b/roomserver/query/query.go index b97d50b17..a62a1f706 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -359,7 +359,7 @@ func (r *RoomserverQueryAPI) getMembershipsBeforeEventNID( return nil, err } - if membership == "join" { + if membership == gomatrixserverlib.Join { events = append(events, event) } } diff --git a/syncapi/storage/syncserver.go b/syncapi/storage/syncserver.go index e914bddfe..ebec6c3e1 100644 --- a/syncapi/storage/syncserver.go +++ b/syncapi/storage/syncserver.go @@ -35,12 +35,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -const ( - membershipJoin = "join" - membershipLeave = "leave" - membershipBan = "ban" -) - type stateDelta struct { roomID string stateEvents []gomatrixserverlib.Event @@ -356,7 +350,7 @@ func (d *SyncServerDatasource) IncrementalSync( ) } else { joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership( - ctx, nil, device.UserID, membershipJoin, + ctx, nil, device.UserID, gomatrixserverlib.Join, ) } if err != nil { @@ -405,7 +399,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( res = types.NewResponse(toPos) // Extract room state and recent events for all rooms the user is joined to. - joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, membershipJoin) + joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) if err != nil { return } @@ -583,7 +577,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( res *types.Response, ) error { endPos := toPos - if delta.membershipPos > 0 && delta.membership == membershipLeave { + if delta.membershipPos > 0 && delta.membership == gomatrixserverlib.Leave { // make sure we don't leak recent events after the leave event. // TODO: History visibility makes this somewhat complex to handle correctly. For example: // TODO: This doesn't work for join -> leave in a single /sync request (see events prior to join). @@ -621,7 +615,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( } switch delta.membership { - case membershipJoin: + case gomatrixserverlib.Join: jr := types.NewJoinResponse() // Use the short form of batch token for prev_batch jr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) @@ -629,9 +623,9 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true jr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[delta.roomID] = *jr - case membershipLeave: + case gomatrixserverlib.Leave: fallthrough // transitions to leave are the same as ban - case membershipBan: + case gomatrixserverlib.Ban: // TODO: recentEvents may contain events that this user is not allowed to see because they are // no longer in the room. lr := types.NewLeaveResponse() @@ -768,7 +762,7 @@ func (d *SyncServerDatasource) getStateDeltas( // the 'state' part of the response though, so is transparent modulo bandwidth concerns as it is not added to // the timeline. if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { - if membership == membershipJoin { + if membership == gomatrixserverlib.Join { // send full room state down instead of a delta var s []streamEvent s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID) @@ -791,13 +785,13 @@ func (d *SyncServerDatasource) getStateDeltas( } // Add in currently joined rooms - joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, membershipJoin) + joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) if err != nil { return nil, nil, err } for _, joinedRoomID := range joinedRoomIDs { deltas = append(deltas, stateDelta{ - membership: membershipJoin, + membership: gomatrixserverlib.Join, stateEvents: streamEventsToEvents(device, state[joinedRoomID]), roomID: joinedRoomID, }) @@ -814,7 +808,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( ctx context.Context, device *authtypes.Device, txn *sql.Tx, fromPos, toPos int64, userID string, ) ([]stateDelta, []string, error) { - joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join") + joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) if err != nil { return nil, nil, err } @@ -829,7 +823,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( return nil, nil, stateErr } deltas = append(deltas, stateDelta{ - membership: "join", + membership: gomatrixserverlib.Join, stateEvents: streamEventsToEvents(device, s), roomID: joinedRoomID, }) @@ -848,7 +842,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( for roomID, stateStreamEvents := range state { for _, ev := range stateStreamEvents { if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { - if membership != "join" { // We've already added full state for all joined rooms above. + if membership != gomatrixserverlib.Join { // We've already added full state for all joined rooms above. deltas = append(deltas, stateDelta{ membership: membership, membershipPos: ev.streamPosition, diff --git a/syncapi/sync/notifier.go b/syncapi/sync/notifier.go index 14bc2efb6..15d6b070c 100644 --- a/syncapi/sync/notifier.go +++ b/syncapi/sync/notifier.go @@ -93,16 +93,16 @@ func (n *Notifier) OnNewEvent( } else { // Keep the joined user map up-to-date switch membership { - case "invite": + case gomatrixserverlib.Invite: usersToNotify = append(usersToNotify, targetUserID) - case "join": + case gomatrixserverlib.Join: // Manually append the new user's ID so they get notified // along all members in the room usersToNotify = append(usersToNotify, targetUserID) n.addJoinedUser(ev.RoomID(), targetUserID) - case "leave": + case gomatrixserverlib.Leave: fallthrough - case "ban": + case gomatrixserverlib.Ban: n.removeJoinedUser(ev.RoomID(), targetUserID) } } From 83f8e05032b82e2dd85d93e77643139bb3231ba2 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 6 Aug 2019 15:52:04 +0100 Subject: [PATCH 02/11] Add /event/ on non world readable room does not work (#777) --- testfile | 1 + 1 file changed, 1 insertion(+) diff --git a/testfile b/testfile index 1d97eb37e..81e47780f 100644 --- a/testfile +++ b/testfile @@ -165,3 +165,4 @@ Can list tags for a room Tags appear in an initial v2 /sync Newly updated tags appear in an incremental v2 /sync Deleted tags appear in an incremental v2 /sync +/event/ on non world readable room does not work From 66bf615360cfe6eac11e901ffe34f70f97330a22 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Tue, 6 Aug 2019 23:33:53 +0800 Subject: [PATCH 03/11] Fix transaction IDs in transaction cache have global scope (#772) --- clientapi/routing/sendevent.go | 4 +-- common/transactions/transactions.go | 21 +++++++++----- common/transactions/transactions_test.go | 35 +++++++++++++++++++++--- testfile | 1 + 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index e916e451e..9696b360e 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -50,7 +50,7 @@ func SendEvent( ) util.JSONResponse { if txnID != nil { // Try to fetch response from transactionsCache - if res, ok := txnCache.FetchTransaction(*txnID); ok { + if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok { return *res } } @@ -83,7 +83,7 @@ func SendEvent( } // Add response to transactionsCache if txnID != nil { - txnCache.AddTransaction(*txnID, &res) + txnCache.AddTransaction(device.AccessToken, *txnID, &res) } return res diff --git a/common/transactions/transactions.go b/common/transactions/transactions.go index febcb9a75..80b403a98 100644 --- a/common/transactions/transactions.go +++ b/common/transactions/transactions.go @@ -22,7 +22,14 @@ import ( // DefaultCleanupPeriod represents the default time duration after which cacheCleanService runs. const DefaultCleanupPeriod time.Duration = 30 * time.Minute -type txnsMap map[string]*util.JSONResponse +type txnsMap map[CacheKey]*util.JSONResponse + +// CacheKey is the type for the key in a transactions cache. +// This is needed because the spec requires transaction IDs to have a per-access token scope. +type CacheKey struct { + AccessToken string + TxnID string +} // Cache represents a temporary store for response entries. // Entries are evicted after a certain period, defined by cleanupPeriod. @@ -50,14 +57,14 @@ func NewWithCleanupPeriod(cleanupPeriod time.Duration) *Cache { return &t } -// FetchTransaction looks up an entry for txnID in Cache. +// FetchTransaction looks up an entry for the (accessToken, txnID) tuple in Cache. // Looks in both the txnMaps. // Returns (JSON response, true) if txnID is found, else the returned bool is false. -func (t *Cache) FetchTransaction(txnID string) (*util.JSONResponse, bool) { +func (t *Cache) FetchTransaction(accessToken, txnID string) (*util.JSONResponse, bool) { t.RLock() defer t.RUnlock() for _, txns := range t.txnsMaps { - res, ok := txns[txnID] + res, ok := txns[CacheKey{accessToken, txnID}] if ok { return res, true } @@ -65,13 +72,13 @@ func (t *Cache) FetchTransaction(txnID string) (*util.JSONResponse, bool) { return nil, false } -// AddTransaction adds an entry for txnID in Cache for later access. +// AddTransaction adds an entry for the (accessToken, txnID) tuple in Cache. // Adds to the front txnMap. -func (t *Cache) AddTransaction(txnID string, res *util.JSONResponse) { +func (t *Cache) AddTransaction(accessToken, txnID string, res *util.JSONResponse) { t.Lock() defer t.Unlock() - t.txnsMaps[0][txnID] = res + t.txnsMaps[0][CacheKey{accessToken, txnID}] = res } // cacheCleanService is responsible for cleaning up entries after cleanupPeriod. diff --git a/common/transactions/transactions_test.go b/common/transactions/transactions_test.go index 0cdb776cc..f565e4846 100644 --- a/common/transactions/transactions_test.go +++ b/common/transactions/transactions_test.go @@ -24,27 +24,54 @@ type fakeType struct { } var ( - fakeTxnID = "aRandomTxnID" - fakeResponse = &util.JSONResponse{Code: http.StatusOK, JSON: fakeType{ID: "0"}} + fakeAccessToken = "aRandomAccessToken" + fakeAccessToken2 = "anotherRandomAccessToken" + fakeTxnID = "aRandomTxnID" + fakeResponse = &util.JSONResponse{ + Code: http.StatusOK, JSON: fakeType{ID: "0"}, + } + fakeResponse2 = &util.JSONResponse{ + Code: http.StatusOK, JSON: fakeType{ID: "1"}, + } ) // TestCache creates a New Cache and tests AddTransaction & FetchTransaction func TestCache(t *testing.T) { fakeTxnCache := New() - fakeTxnCache.AddTransaction(fakeTxnID, fakeResponse) + fakeTxnCache.AddTransaction(fakeAccessToken, fakeTxnID, fakeResponse) // Add entries for noise. for i := 1; i <= 100; i++ { fakeTxnCache.AddTransaction( + fakeAccessToken, fakeTxnID+string(i), &util.JSONResponse{Code: http.StatusOK, JSON: fakeType{ID: string(i)}}, ) } - testResponse, ok := fakeTxnCache.FetchTransaction(fakeTxnID) + testResponse, ok := fakeTxnCache.FetchTransaction(fakeAccessToken, fakeTxnID) if !ok { t.Error("Failed to retrieve entry for txnID: ", fakeTxnID) } else if testResponse.JSON != fakeResponse.JSON { t.Error("Fetched response incorrect. Expected: ", fakeResponse.JSON, " got: ", testResponse.JSON) } } + +// TestCacheScope ensures transactions with the same transaction ID are not shared +// across multiple access tokens. +func TestCacheScope(t *testing.T) { + cache := New() + cache.AddTransaction(fakeAccessToken, fakeTxnID, fakeResponse) + cache.AddTransaction(fakeAccessToken2, fakeTxnID, fakeResponse2) + + if res, ok := cache.FetchTransaction(fakeAccessToken, fakeTxnID); !ok { + t.Errorf("failed to retrieve entry for (%s, %s)", fakeAccessToken, fakeTxnID) + } else if res.JSON != fakeResponse.JSON { + t.Errorf("Wrong cache entry for (%s, %s). Expected: %v; got: %v", fakeAccessToken, fakeTxnID, fakeResponse.JSON, res.JSON) + } + if res, ok := cache.FetchTransaction(fakeAccessToken2, fakeTxnID); !ok { + t.Errorf("failed to retrieve entry for (%s, %s)", fakeAccessToken, fakeTxnID) + } else if res.JSON != fakeResponse2.JSON { + t.Errorf("Wrong cache entry for (%s, %s). Expected: %v; got: %v", fakeAccessToken, fakeTxnID, fakeResponse2.JSON, res.JSON) + } +} diff --git a/testfile b/testfile index 81e47780f..5791938f7 100644 --- a/testfile +++ b/testfile @@ -159,6 +159,7 @@ Inbound federation rejects remote attempts to kick local users to rooms An event which redacts itself should be ignored A pair of events which redact each other should be ignored Full state sync includes joined rooms +A message sent after an initial sync appears in the timeline of an incremental sync. Can add tag Can remove tag Can list tags for a room From 324ca22b358d53339b2971ec85a02de7282604e6 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 7 Aug 2019 00:02:12 +0800 Subject: [PATCH 04/11] Implement profile retrieval over federation (#726) --- appservice/api/query.go | 4 +- clientapi/auth/authtypes/profile.go | 2 +- clientapi/routing/profile.go | 107 +++++++++++++++++++++++----- clientapi/routing/routing.go | 6 +- common/types.go | 5 ++ testfile | 1 + 6 files changed, 100 insertions(+), 25 deletions(-) diff --git a/appservice/api/query.go b/appservice/api/query.go index 8ce3b4e04..9542df565 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -20,13 +20,13 @@ package api import ( "context" "database/sql" - "errors" "net/http" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/common" commonHTTP "github.com/matrix-org/dendrite/common/http" opentracing "github.com/opentracing/opentracing-go" ) @@ -164,7 +164,7 @@ func RetrieveUserProfile( // If no user exists, return if !userResp.UserIDExists { - return nil, errors.New("no known profile for given user ID") + return nil, common.ErrProfileNoExists } // Try to query the user from the local database again diff --git a/clientapi/auth/authtypes/profile.go b/clientapi/auth/authtypes/profile.go index 6cf508f4f..0bc49658b 100644 --- a/clientapi/auth/authtypes/profile.go +++ b/clientapi/auth/authtypes/profile.go @@ -14,7 +14,7 @@ package authtypes -// Profile represents the profile for a Matrix account on this home server. +// Profile represents the profile for a Matrix account. type Profile struct { Localpart string DisplayName string diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 8d28b3660..e8ea6cf13 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -30,43 +30,61 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrix" "github.com/matrix-org/util" ) // GetProfile implements GET /profile/{userID} func GetProfile( - req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI, + req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite, + userID string, + asAPI appserviceAPI.AppServiceQueryAPI, + federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { - profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { + if err == common.ErrProfileNoExists { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), + } + } + return httputil.LogThenError(req, err) } - res := common.ProfileResponse{ - AvatarURL: profile.AvatarURL, - DisplayName: profile.DisplayName, - } return util.JSONResponse{ Code: http.StatusOK, - JSON: res, + JSON: common.ProfileResponse{ + AvatarURL: profile.AvatarURL, + DisplayName: profile.DisplayName, + }, } } // GetAvatarURL implements GET /profile/{userID}/avatar_url func GetAvatarURL( - req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI, + req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite, + userID string, asAPI appserviceAPI.AppServiceQueryAPI, + federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { - profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { + if err == common.ErrProfileNoExists { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), + } + } + return httputil.LogThenError(req, err) } - res := common.AvatarURL{ - AvatarURL: profile.AvatarURL, - } return util.JSONResponse{ Code: http.StatusOK, - JSON: res, + JSON: common.AvatarURL{ + AvatarURL: profile.AvatarURL, + }, } } @@ -152,18 +170,27 @@ func SetAvatarURL( // GetDisplayName implements GET /profile/{userID}/displayname func GetDisplayName( - req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI, + req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite, + userID string, asAPI appserviceAPI.AppServiceQueryAPI, + federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { - profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { + if err == common.ErrProfileNoExists { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), + } + } + return httputil.LogThenError(req, err) } - res := common.DisplayName{ - DisplayName: profile.DisplayName, - } + return util.JSONResponse{ Code: http.StatusOK, - JSON: res, + JSON: common.DisplayName{ + DisplayName: profile.DisplayName, + }, } } @@ -247,6 +274,48 @@ func SetDisplayName( } } +// getProfile gets the full profile of a user by querying the database or a +// remote homeserver. +// Returns an error when something goes wrong or specifically +// common.ErrProfileNoExists when the profile doesn't exist. +func getProfile( + ctx context.Context, accountDB *accounts.Database, cfg *config.Dendrite, + userID string, + asAPI appserviceAPI.AppServiceQueryAPI, + federation *gomatrixserverlib.FederationClient, +) (*authtypes.Profile, error) { + localpart, domain, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + return nil, err + } + + if domain != cfg.Matrix.ServerName { + profile, fedErr := federation.LookupProfile(ctx, domain, userID, "") + if fedErr != nil { + if x, ok := fedErr.(gomatrix.HTTPError); ok { + if x.Code == http.StatusNotFound { + return nil, common.ErrProfileNoExists + } + } + + return nil, fedErr + } + + return &authtypes.Profile{ + Localpart: localpart, + DisplayName: profile.DisplayName, + AvatarURL: profile.AvatarURL, + }, nil + } + + profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB) + if err != nil { + return nil, err + } + + return profile, nil +} + func buildMembershipEvents( ctx context.Context, memberships []authtypes.Membership, diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index c262db3d8..825dd97aa 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -283,7 +283,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetProfile(req, accountDB, vars["userID"], asAPI) + return GetProfile(req, accountDB, &cfg, vars["userID"], asAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -293,7 +293,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetAvatarURL(req, accountDB, vars["userID"], asAPI) + return GetAvatarURL(req, accountDB, &cfg, vars["userID"], asAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -315,7 +315,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetDisplayName(req, accountDB, vars["userID"], asAPI) + return GetDisplayName(req, accountDB, &cfg, vars["userID"], asAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) diff --git a/common/types.go b/common/types.go index 6888d3806..91765be00 100644 --- a/common/types.go +++ b/common/types.go @@ -15,9 +15,14 @@ package common import ( + "errors" "strconv" ) +// ErrProfileNoExists is returned when trying to lookup a user's profile that +// doesn't exist locally. +var ErrProfileNoExists = errors.New("no known profile for given user ID") + // AccountData represents account data sent from the client API server to the // sync API server type AccountData struct { diff --git a/testfile b/testfile index 5791938f7..74c9d9e4f 100644 --- a/testfile +++ b/testfile @@ -167,3 +167,4 @@ Tags appear in an initial v2 /sync Newly updated tags appear in an incremental v2 /sync Deleted tags appear in an incremental v2 /sync /event/ on non world readable room does not work +Outbound federation can query profile data From 8c721b555ed51b15010feada9f9f7ed8854ff42f Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 6 Aug 2019 20:26:15 +0100 Subject: [PATCH 05/11] Scope the buildkite build badge only to tests running on master (#779) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8eadaf431..4e628c0ff 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg)](https://buildkite.com/matrix-dot-org/dendrite) [![CircleCI](https://circleci.com/gh/matrix-org/dendrite.svg?style=svg)](https://circleci.com/gh/matrix-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) +# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![CircleCI](https://circleci.com/gh/matrix-org/dendrite.svg?style=svg)](https://circleci.com/gh/matrix-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) Dendrite will be a matrix homeserver written in go. From 94ea325c93d03362795ce85c49008797d37d36dd Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 7 Aug 2019 11:00:58 +0800 Subject: [PATCH 06/11] Fix permission and 404 response for alias deletion - #654 (#706) --- clientapi/routing/directory.go | 25 +++++++++++++++- roomserver/alias/alias.go | 37 ++++++++++++++++++++++-- roomserver/alias/alias_test.go | 8 ++++- roomserver/api/alias.go | 35 ++++++++++++++++++++++ roomserver/storage/room_aliases_table.go | 33 ++++++++++++++++----- roomserver/storage/storage.go | 11 +++++-- 6 files changed, 135 insertions(+), 14 deletions(-) diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index ab85e86a9..0d91d0426 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -164,13 +164,36 @@ func SetLocalAlias( } // RemoveLocalAlias implements DELETE /directory/room/{roomAlias} -// TODO: Check if the user has the power level to remove an alias func RemoveLocalAlias( req *http.Request, device *authtypes.Device, alias string, aliasAPI roomserverAPI.RoomserverAliasAPI, ) util.JSONResponse { + + creatorQueryReq := roomserverAPI.GetCreatorIDForAliasRequest{ + Alias: alias, + } + var creatorQueryRes roomserverAPI.GetCreatorIDForAliasResponse + if err := aliasAPI.GetCreatorIDForAlias(req.Context(), &creatorQueryReq, &creatorQueryRes); err != nil { + return httputil.LogThenError(req, err) + } + + if creatorQueryRes.UserID == "" { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Alias does not exist"), + } + } + + if creatorQueryRes.UserID != device.UserID { + // TODO: Still allow deletion if user is admin + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("You do not have permission to delete this alias"), + } + } + queryReq := roomserverAPI.RemoveRoomAliasRequest{ Alias: alias, UserID: device.UserID, diff --git a/roomserver/alias/alias.go b/roomserver/alias/alias.go index f699e3362..aeaf5ae94 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/alias/alias.go @@ -33,13 +33,16 @@ import ( type RoomserverAliasAPIDatabase interface { // Save a given room alias with the room ID it refers to. // Returns an error if there was a problem talking to the database. - SetRoomAlias(ctx context.Context, alias string, roomID string) error + SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error // Look up the room ID a given alias refers to. // Returns an error if there was a problem talking to the database. GetRoomIDForAlias(ctx context.Context, alias string) (string, error) // Look up all aliases referring to a given room ID. // Returns an error if there was a problem talking to the database. GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) + // Get the user ID of the creator of an alias. + // Returns an error if there was a problem talking to the database. + GetCreatorIDForAlias(ctx context.Context, alias string) (string, error) // Remove a given room alias. // Returns an error if there was a problem talking to the database. RemoveRoomAlias(ctx context.Context, alias string) error @@ -73,7 +76,7 @@ func (r *RoomserverAliasAPI) SetRoomAlias( response.AliasExists = false // Save the new alias - if err := r.DB.SetRoomAlias(ctx, request.Alias, request.RoomID); err != nil { + if err := r.DB.SetRoomAlias(ctx, request.Alias, request.RoomID, request.UserID); err != nil { return err } @@ -133,6 +136,22 @@ func (r *RoomserverAliasAPI) GetAliasesForRoomID( return nil } +// GetCreatorIDForAlias implements alias.RoomserverAliasAPI +func (r *RoomserverAliasAPI) GetCreatorIDForAlias( + ctx context.Context, + request *roomserverAPI.GetCreatorIDForAliasRequest, + response *roomserverAPI.GetCreatorIDForAliasResponse, +) error { + // Look up the aliases in the database for the given RoomID + creatorID, err := r.DB.GetCreatorIDForAlias(ctx, request.Alias) + if err != nil { + return err + } + + response.UserID = creatorID + return nil +} + // RemoveRoomAlias implements alias.RoomserverAliasAPI func (r *RoomserverAliasAPI) RemoveRoomAlias( ctx context.Context, @@ -277,6 +296,20 @@ func (r *RoomserverAliasAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + servMux.Handle( + roomserverAPI.RoomserverGetCreatorIDForAliasPath, + common.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { + var request roomserverAPI.GetCreatorIDForAliasRequest + var response roomserverAPI.GetCreatorIDForAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetCreatorIDForAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) servMux.Handle( roomserverAPI.RoomserverGetAliasesForRoomIDPath, common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { diff --git a/roomserver/alias/alias_test.go b/roomserver/alias/alias_test.go index 4b9ca022d..6ddb63a73 100644 --- a/roomserver/alias/alias_test.go +++ b/roomserver/alias/alias_test.go @@ -30,7 +30,7 @@ type MockRoomserverAliasAPIDatabase struct { } // These methods can be essentially noop -func (db MockRoomserverAliasAPIDatabase) SetRoomAlias(ctx context.Context, alias string, roomID string) error { +func (db MockRoomserverAliasAPIDatabase) SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error { return nil } @@ -43,6 +43,12 @@ func (db MockRoomserverAliasAPIDatabase) RemoveRoomAlias(ctx context.Context, al return nil } +func (db *MockRoomserverAliasAPIDatabase) GetCreatorIDForAlias( + ctx context.Context, alias string, +) (string, error) { + return "", nil +} + // This method needs to change depending on test case func (db *MockRoomserverAliasAPIDatabase) GetRoomIDForAlias( ctx context.Context, diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go index 576710713..cb78f726a 100644 --- a/roomserver/api/alias.go +++ b/roomserver/api/alias.go @@ -62,6 +62,18 @@ type GetAliasesForRoomIDResponse struct { Aliases []string `json:"aliases"` } +// GetCreatorIDForAliasRequest is a request to GetCreatorIDForAlias +type GetCreatorIDForAliasRequest struct { + // The alias we want to find the creator of + Alias string `json:"alias"` +} + +// GetCreatorIDForAliasResponse is a response to GetCreatorIDForAlias +type GetCreatorIDForAliasResponse struct { + // The user ID of the alias creator + UserID string `json:"user_id"` +} + // RemoveRoomAliasRequest is a request to RemoveRoomAlias type RemoveRoomAliasRequest struct { // ID of the user removing the alias @@ -96,6 +108,13 @@ type RoomserverAliasAPI interface { response *GetAliasesForRoomIDResponse, ) error + // Get the user ID of the creator of an alias + GetCreatorIDForAlias( + ctx context.Context, + req *GetCreatorIDForAliasRequest, + response *GetCreatorIDForAliasResponse, + ) error + // Remove a room alias RemoveRoomAlias( ctx context.Context, @@ -113,6 +132,9 @@ const RoomserverGetRoomIDForAliasPath = "/api/roomserver/GetRoomIDForAlias" // RoomserverGetAliasesForRoomIDPath is the HTTP path for the GetAliasesForRoomID API. const RoomserverGetAliasesForRoomIDPath = "/api/roomserver/GetAliasesForRoomID" +// RoomserverGetCreatorIDForAliasPath is the HTTP path for the GetCreatorIDForAlias API. +const RoomserverGetCreatorIDForAliasPath = "/api/roomserver/GetCreatorIDForAlias" + // RoomserverRemoveRoomAliasPath is the HTTP path for the RemoveRoomAlias API. const RoomserverRemoveRoomAliasPath = "/api/roomserver/removeRoomAlias" @@ -169,6 +191,19 @@ func (h *httpRoomserverAliasAPI) GetAliasesForRoomID( return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } +// GetCreatorIDForAlias implements RoomserverAliasAPI +func (h *httpRoomserverAliasAPI) GetCreatorIDForAlias( + ctx context.Context, + request *GetCreatorIDForAliasRequest, + response *GetCreatorIDForAliasResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "GetCreatorIDForAlias") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverGetCreatorIDForAliasPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + // RemoveRoomAlias implements RoomserverAliasAPI func (h *httpRoomserverAliasAPI) RemoveRoomAlias( ctx context.Context, diff --git a/roomserver/storage/room_aliases_table.go b/roomserver/storage/room_aliases_table.go index f640c37fe..3ed20e8e3 100644 --- a/roomserver/storage/room_aliases_table.go +++ b/roomserver/storage/room_aliases_table.go @@ -25,14 +25,16 @@ CREATE TABLE IF NOT EXISTS roomserver_room_aliases ( -- Alias of the room alias TEXT NOT NULL PRIMARY KEY, -- Room ID the alias refers to - room_id TEXT NOT NULL + room_id TEXT NOT NULL, + -- User ID of the creator of this alias + creator_id TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS roomserver_room_id_idx ON roomserver_room_aliases(room_id); ` const insertRoomAliasSQL = "" + - "INSERT INTO roomserver_room_aliases (alias, room_id) VALUES ($1, $2)" + "INSERT INTO roomserver_room_aliases (alias, room_id, creator_id) VALUES ($1, $2, $3)" const selectRoomIDFromAliasSQL = "" + "SELECT room_id FROM roomserver_room_aliases WHERE alias = $1" @@ -40,14 +42,18 @@ const selectRoomIDFromAliasSQL = "" + const selectAliasesFromRoomIDSQL = "" + "SELECT alias FROM roomserver_room_aliases WHERE room_id = $1" +const selectCreatorIDFromAliasSQL = "" + + "SELECT creator_id FROM roomserver_room_aliases WHERE alias = $1" + const deleteRoomAliasSQL = "" + "DELETE FROM roomserver_room_aliases WHERE alias = $1" type roomAliasesStatements struct { - insertRoomAliasStmt *sql.Stmt - selectRoomIDFromAliasStmt *sql.Stmt - selectAliasesFromRoomIDStmt *sql.Stmt - deleteRoomAliasStmt *sql.Stmt + insertRoomAliasStmt *sql.Stmt + selectRoomIDFromAliasStmt *sql.Stmt + selectAliasesFromRoomIDStmt *sql.Stmt + selectCreatorIDFromAliasStmt *sql.Stmt + deleteRoomAliasStmt *sql.Stmt } func (s *roomAliasesStatements) prepare(db *sql.DB) (err error) { @@ -59,14 +65,15 @@ func (s *roomAliasesStatements) prepare(db *sql.DB) (err error) { {&s.insertRoomAliasStmt, insertRoomAliasSQL}, {&s.selectRoomIDFromAliasStmt, selectRoomIDFromAliasSQL}, {&s.selectAliasesFromRoomIDStmt, selectAliasesFromRoomIDSQL}, + {&s.selectCreatorIDFromAliasStmt, selectCreatorIDFromAliasSQL}, {&s.deleteRoomAliasStmt, deleteRoomAliasSQL}, }.prepare(db) } func (s *roomAliasesStatements) insertRoomAlias( - ctx context.Context, alias string, roomID string, + ctx context.Context, alias string, roomID string, creatorUserID string, ) (err error) { - _, err = s.insertRoomAliasStmt.ExecContext(ctx, alias, roomID) + _, err = s.insertRoomAliasStmt.ExecContext(ctx, alias, roomID, creatorUserID) return } @@ -101,6 +108,16 @@ func (s *roomAliasesStatements) selectAliasesFromRoomID( return } +func (s *roomAliasesStatements) selectCreatorIDFromAlias( + ctx context.Context, alias string, +) (creatorID string, err error) { + err = s.selectCreatorIDFromAliasStmt.QueryRowContext(ctx, alias).Scan(&creatorID) + if err == sql.ErrNoRows { + return "", nil + } + return +} + func (s *roomAliasesStatements) deleteRoomAlias( ctx context.Context, alias string, ) (err error) { diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index f6c2fccd4..71c13b7ca 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -441,8 +441,8 @@ func (d *Database) GetInvitesForUser( } // SetRoomAlias implements alias.RoomserverAliasAPIDB -func (d *Database) SetRoomAlias(ctx context.Context, alias string, roomID string) error { - return d.statements.insertRoomAlias(ctx, alias, roomID) +func (d *Database) SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error { + return d.statements.insertRoomAlias(ctx, alias, roomID, creatorUserID) } // GetRoomIDForAlias implements alias.RoomserverAliasAPIDB @@ -455,6 +455,13 @@ func (d *Database) GetAliasesForRoomID(ctx context.Context, roomID string) ([]st return d.statements.selectAliasesFromRoomID(ctx, roomID) } +// GetCreatorIDForAlias implements alias.RoomserverAliasAPIDB +func (d *Database) GetCreatorIDForAlias( + ctx context.Context, alias string, +) (string, error) { + return d.statements.selectCreatorIDFromAlias(ctx, alias) +} + // RemoveRoomAlias implements alias.RoomserverAliasAPIDB func (d *Database) RemoveRoomAlias(ctx context.Context, alias string) error { return d.statements.deleteRoomAlias(ctx, alias) From 76e4ebaf78a698eb775ccb358fb5c253084747b9 Mon Sep 17 00:00:00 2001 From: Thibaut CHARLES Date: Wed, 7 Aug 2019 12:12:09 +0200 Subject: [PATCH 07/11] State events filtering database api (#438) This PR adds a gomatrixserverlib.Filter parameter to functions handling the syncapi_current_room_state table. It does not implement any filtering logic inside the syncapi IncrementalSync/CompleteSync functions, just the APIs for future use. Default filters are provided as placeholders in IncrementalSync/CompleteSync, so behaviour should be unchanged (except the default 20 event limit) SQL table will be changed. You can upgrade an existing database using: ``` ALTER TABLE syncapi_current_room_state ADD COLUMN IF NOT EXISTS sender text; UPDATE syncapi_current_room_state SET sender=(event_json::json->>'sender'); ALTER TABLE syncapi_current_room_state ALTER COLUMN sender SET NOT NULL; ALTER TABLE syncapi_current_room_state ADD COLUMN IF NOT EXISTS contains_url bool; UPDATE syncapi_current_room_state SET contains_url=(event_json::json->>'content')::json->>'url' IS NOT NULL; ALTER TABLE syncapi_current_room_state ALTER COLUMN contains_url SET NOT NULL; ``` Note: This depends on #436 (and includes all its commits). I'm not sure if Github will remove the duplicated commits once #436 is merged. --- syncapi/routing/state.go | 5 ++- syncapi/storage/current_room_state_table.go | 42 ++++++++++++++++++--- syncapi/storage/filtering.go | 36 ++++++++++++++++++ syncapi/storage/output_room_events_table.go | 41 ++++++++++++++++++-- syncapi/storage/syncserver.go | 31 ++++++++++----- 5 files changed, 134 insertions(+), 21 deletions(-) create mode 100644 syncapi/storage/filtering.go diff --git a/syncapi/routing/state.go b/syncapi/routing/state.go index 5571a0525..87a93d194 100644 --- a/syncapi/routing/state.go +++ b/syncapi/routing/state.go @@ -44,7 +44,10 @@ func OnIncomingStateRequest(req *http.Request, db *storage.SyncServerDatasource, // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) - stateEvents, err := db.GetStateEventsForRoom(req.Context(), roomID) + stateFilterPart := gomatrixserverlib.DefaultFilterPart() + // TODO: stateFilterPart should not limit the number of state events (or only limits abusive number of events) + + stateEvents, err := db.GetStateEventsForRoom(req.Context(), roomID, &stateFilterPart) if err != nil { return httputil.LogThenError(req, err) } diff --git a/syncapi/storage/current_room_state_table.go b/syncapi/storage/current_room_state_table.go index 852bfd760..88e7a76c3 100644 --- a/syncapi/storage/current_room_state_table.go +++ b/syncapi/storage/current_room_state_table.go @@ -17,6 +17,7 @@ package storage import ( "context" "database/sql" + "encoding/json" "github.com/lib/pq" "github.com/matrix-org/dendrite/common" @@ -32,6 +33,10 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( event_id TEXT NOT NULL, -- The state event type e.g 'm.room.member' type TEXT NOT NULL, + -- The 'sender' property of the event. + sender TEXT NOT NULL, + -- true if the event content contains a url key + contains_url BOOL NOT NULL, -- The state_key value for this state event e.g '' state_key TEXT NOT NULL, -- The JSON for the event. Stored as TEXT because this should be valid UTF-8. @@ -46,16 +51,16 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( CONSTRAINT syncapi_room_state_unique UNIQUE (room_id, type, state_key) ); -- for event deletion -CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_state(event_id); +CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_state(event_id, room_id, type, sender, contains_url); -- for querying membership states of users CREATE INDEX IF NOT EXISTS syncapi_membership_idx ON syncapi_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave'; ` const upsertRoomStateSQL = "" + - "INSERT INTO syncapi_current_room_state (room_id, event_id, type, state_key, event_json, membership, added_at)" + - " VALUES ($1, $2, $3, $4, $5, $6, $7)" + + "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, event_json, membership, added_at)" + + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + " ON CONFLICT ON CONSTRAINT syncapi_room_state_unique" + - " DO UPDATE SET event_id = $2, event_json = $5, membership = $6, added_at = $7" + " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, event_json = $7, membership = $8, added_at = $9" const deleteRoomStateByEventIDSQL = "" + "DELETE FROM syncapi_current_room_state WHERE event_id = $1" @@ -64,7 +69,13 @@ const selectRoomIDsWithMembershipSQL = "" + "SELECT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2" const selectCurrentStateSQL = "" + - "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1" + "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1" + + " AND ( $2::text[] IS NULL OR sender = ANY($2) )" + + " AND ( $3::text[] IS NULL OR NOT(sender = ANY($3)) )" + + " AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" + + " AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" + + " AND ( $6::bool IS NULL OR contains_url = $6 )" + + " LIMIT $7" const selectJoinedUsersSQL = "" + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" @@ -166,9 +177,17 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( // CurrentState returns all the current state events for the given room. func (s *currentRoomStateStatements) selectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, + stateFilterPart *gomatrixserverlib.FilterPart, ) ([]gomatrixserverlib.Event, error) { stmt := common.TxStmt(txn, s.selectCurrentStateStmt) - rows, err := stmt.QueryContext(ctx, roomID) + rows, err := stmt.QueryContext(ctx, roomID, + pq.StringArray(stateFilterPart.Senders), + pq.StringArray(stateFilterPart.NotSenders), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), + stateFilterPart.ContainsURL, + stateFilterPart.Limit, + ) if err != nil { return nil, err } @@ -189,12 +208,23 @@ func (s *currentRoomStateStatements) upsertRoomState( ctx context.Context, txn *sql.Tx, event gomatrixserverlib.Event, membership *string, addedAt int64, ) error { + // Parse content as JSON and search for an "url" key + containsURL := false + var content map[string]interface{} + if json.Unmarshal(event.Content(), &content) != nil { + // Set containsURL to true if url is present + _, containsURL = content["url"] + } + + // upsert state event stmt := common.TxStmt(txn, s.upsertRoomStateStmt) _, err := stmt.ExecContext( ctx, event.RoomID(), event.EventID(), event.Type(), + event.Sender(), + containsURL, *event.StateKey(), event.JSON(), membership, diff --git a/syncapi/storage/filtering.go b/syncapi/storage/filtering.go new file mode 100644 index 000000000..27b0b888a --- /dev/null +++ b/syncapi/storage/filtering.go @@ -0,0 +1,36 @@ +// Copyright 2017 Thibaut CHARLES +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "strings" +) + +// filterConvertWildcardToSQL converts wildcards as defined in +// https://matrix.org/docs/spec/client_server/r0.3.0.html#post-matrix-client-r0-user-userid-filter +// to SQL wildcards that can be used with LIKE() +func filterConvertTypeWildcardToSQL(values []string) []string { + if values == nil { + // Return nil instead of []string{} so IS NULL can work correctly when + // the return value is passed into SQL queries + return nil + } + + ret := make([]string, len(values)) + for i := range values { + ret[i] = strings.Replace(values[i], "*", "%", -1) + } + return ret +} diff --git a/syncapi/storage/output_room_events_table.go b/syncapi/storage/output_room_events_table.go index 34632aedf..8fbeb18c9 100644 --- a/syncapi/storage/output_room_events_table.go +++ b/syncapi/storage/output_room_events_table.go @@ -17,6 +17,7 @@ package storage import ( "context" "database/sql" + "encoding/json" "sort" "github.com/matrix-org/dendrite/roomserver/api" @@ -43,6 +44,12 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( room_id TEXT NOT NULL, -- The JSON for the event. Stored as TEXT because this should be valid UTF-8. event_json TEXT NOT NULL, + -- The event type e.g 'm.room.member'. + type TEXT NOT NULL, + -- The 'sender' property of the event. + sender TEXT NOT NULL, + -- true if the event content contains a url key. + contains_url BOOL NOT NULL, -- A list of event IDs which represent a delta of added/removed room state. This can be NULL -- if there is no delta. add_state_ids TEXT[], @@ -56,8 +63,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_output_room_ev const insertEventSQL = "" + "INSERT INTO syncapi_output_room_events (" + - " room_id, event_id, event_json, add_state_ids, remove_state_ids, device_id, transaction_id" + - ") VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id" + "room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, device_id, transaction_id" + + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id" const selectEventsSQL = "" + "SELECT id, event_json FROM syncapi_output_room_events WHERE event_id = ANY($1)" @@ -75,7 +82,13 @@ const selectStateInRangeSQL = "" + "SELECT id, event_json, add_state_ids, remove_state_ids" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2) AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + - " ORDER BY id ASC" + " AND ( $3::text[] IS NULL OR sender = ANY($3) )" + + " AND ( $4::text[] IS NULL OR NOT(sender = ANY($4)) )" + + " AND ( $5::text[] IS NULL OR type LIKE ANY($5) )" + + " AND ( $6::text[] IS NULL OR NOT(type LIKE ANY($6)) )" + + " AND ( $7::bool IS NULL OR contains_url = $7 )" + + " ORDER BY id ASC" + + " LIMIT $8" type outputRoomEventsStatements struct { insertEventStmt *sql.Stmt @@ -113,10 +126,19 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) { // two positions, only the most recent state is returned. func (s *outputRoomEventsStatements) selectStateInRange( ctx context.Context, txn *sql.Tx, oldPos, newPos int64, + stateFilterPart *gomatrixserverlib.FilterPart, ) (map[string]map[string]bool, map[string]streamEvent, error) { stmt := common.TxStmt(txn, s.selectStateInRangeStmt) - rows, err := stmt.QueryContext(ctx, oldPos, newPos) + rows, err := stmt.QueryContext( + ctx, oldPos, newPos, + pq.StringArray(stateFilterPart.Senders), + pq.StringArray(stateFilterPart.NotSenders), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), + stateFilterPart.ContainsURL, + stateFilterPart.Limit, + ) if err != nil { return nil, nil, err } @@ -205,12 +227,23 @@ func (s *outputRoomEventsStatements) insertEvent( txnID = &transactionID.TransactionID } + // Parse content as JSON and search for an "url" key + containsURL := false + var content map[string]interface{} + if json.Unmarshal(event.Content(), &content) != nil { + // Set containsURL to true if url is present + _, containsURL = content["url"] + } + stmt := common.TxStmt(txn, s.insertEventStmt) err = stmt.QueryRowContext( ctx, event.RoomID(), event.EventID(), event.JSON(), + event.Type(), + event.Sender(), + containsURL, pq.StringArray(addState), pq.StringArray(removeState), deviceID, diff --git a/syncapi/storage/syncserver.go b/syncapi/storage/syncserver.go index ebec6c3e1..c57a90256 100644 --- a/syncapi/storage/syncserver.go +++ b/syncapi/storage/syncserver.go @@ -185,10 +185,10 @@ func (d *SyncServerDatasource) GetStateEvent( // Returns an empty slice if no state events could be found for this room. // Returns an error if there was an issue with the retrieval. func (d *SyncServerDatasource) GetStateEventsForRoom( - ctx context.Context, roomID string, + ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.FilterPart, ) (stateEvents []gomatrixserverlib.Event, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { - stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID) + stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilterPart) return err }) return @@ -245,6 +245,8 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse( var succeeded bool defer common.EndTransaction(txn, &succeeded) + stateFilterPart := gomatrixserverlib.DefaultFilterPart() // TODO: use filter provided in request + // Work out which rooms to return in the response. This is done by getting not only the currently // joined rooms, but also which rooms have membership transitions for this user between the 2 PDU stream positions. // This works out what the 'state' key should be for each room as well as which membership block @@ -252,9 +254,13 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse( var deltas []stateDelta var joinedRoomIDs []string if !wantFullState { - deltas, joinedRoomIDs, err = d.getStateDeltas(ctx, &device, txn, fromPos, toPos, device.UserID) + deltas, joinedRoomIDs, err = d.getStateDeltas( + ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilterPart, + ) } else { - deltas, joinedRoomIDs, err = d.getStateDeltasForFullStateSync(ctx, &device, txn, fromPos, toPos, device.UserID) + deltas, joinedRoomIDs, err = d.getStateDeltasForFullStateSync( + ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilterPart, + ) } if err != nil { return nil, err @@ -404,10 +410,12 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( return } + stateFilterPart := gomatrixserverlib.DefaultFilterPart() // TODO: use filter provided in request + // Build up a /sync response. Add joined rooms. for _, roomID := range joinedRoomIDs { var stateEvents []gomatrixserverlib.Event - stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID) + stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, &stateFilterPart) if err != nil { return } @@ -733,6 +741,7 @@ func (d *SyncServerDatasource) fetchMissingStateEvents( func (d *SyncServerDatasource) getStateDeltas( ctx context.Context, device *authtypes.Device, txn *sql.Tx, fromPos, toPos int64, userID string, + stateFilterPart *gomatrixserverlib.FilterPart, ) ([]stateDelta, []string, error) { // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 // - Get membership list changes for this user in this sync response @@ -745,7 +754,7 @@ func (d *SyncServerDatasource) getStateDeltas( var deltas []stateDelta // get all the state events ever between these two positions - stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos) + stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos, stateFilterPart) if err != nil { return nil, nil, err } @@ -765,7 +774,7 @@ func (d *SyncServerDatasource) getStateDeltas( if membership == gomatrixserverlib.Join { // send full room state down instead of a delta var s []streamEvent - s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID) + s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID, stateFilterPart) if err != nil { return nil, nil, err } @@ -807,6 +816,7 @@ func (d *SyncServerDatasource) getStateDeltas( func (d *SyncServerDatasource) getStateDeltasForFullStateSync( ctx context.Context, device *authtypes.Device, txn *sql.Tx, fromPos, toPos int64, userID string, + stateFilterPart *gomatrixserverlib.FilterPart, ) ([]stateDelta, []string, error) { joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) if err != nil { @@ -818,7 +828,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( // Add full states for all joined rooms for _, joinedRoomID := range joinedRoomIDs { - s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID) + s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID, stateFilterPart) if stateErr != nil { return nil, nil, stateErr } @@ -830,7 +840,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( } // Get all the state events ever between these two positions - stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos) + stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos, stateFilterPart) if err != nil { return nil, nil, err } @@ -861,8 +871,9 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( func (d *SyncServerDatasource) currentStateStreamEventsForRoom( ctx context.Context, txn *sql.Tx, roomID string, + stateFilterPart *gomatrixserverlib.FilterPart, ) ([]streamEvent, error) { - allState, err := d.roomstate.selectCurrentState(ctx, txn, roomID) + allState, err := d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilterPart) if err != nil { return nil, err } From 76040bfa87e4ed9bb8c97deb0d596942825799a7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 7 Aug 2019 11:46:36 +0100 Subject: [PATCH 08/11] Add CI information to CONTRIBUTING.md (#778) Add information about how the continuous integration is set up in Dendrite and how to run the tests locally so that people don't need to wait around for things to churn. --- CONTRIBUTING.md | 33 ++++++++++++++++++++++++ docs/images/details-button-location.jpg | Bin 0 -> 25460 bytes 2 files changed, 33 insertions(+) create mode 100644 docs/images/details-button-location.jpg diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 22ad0586f..dc962fee7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,6 +20,39 @@ should pick up any unit test and run it). There are also [scripts](scripts) for [linting](scripts/find-lint.sh) and doing a [build/test/lint run](scripts/build-test-lint.sh). +## Continuous Integration + +When a Pull Request is submitted, continuous integration jobs are run +automatically to ensure the code builds and is relatively well-written. Checks +are run on [Buildkite](https://buildkite.com/matrix-dot-org/dendrite/) and +[CircleCI](https://circleci.com/gh/matrix-org/dendrite/). + +If a job fails, click the "details" button and you should be taken to the job's +logs. + +![Click the details button on the failing build step](docs/images/details-button-location.jpg) + +Scroll down to the failing step and you should see some log output. Scan +the logs until you find what it's complaining about, fix it, submit a new +commit, then rinse and repeat until CI passes. + +### Running CI Tests Locally + +To save waiting for CI to finish after every commit, it is ideal to run the +checks locally before pushing, fixing errors first. This also saves other +people time as only so many PRs can be tested at a given time. + +To execute what Buildkite tests, simply run `./scripts/build-test-lint.sh`. +This script will build the code, lint it, and run `go test ./...` with race +condition checking enabled. If something needs to be changed, fix it and then +run the script again until it no longer complains. Be warned that the linting +can take a significant amount of CPU and RAM. + +CircleCI simply runs [Sytest](https://github.com/matrix-org/sytest) with a test +whitelist. See +[docs/sytest.md](https://github.com/matrix-org/dendrite/blob/master/docs/sytest.md#using-a-sytest-docker-image) +for instructions on setting it up to run locally. + ## Picking Things To Do diff --git a/docs/images/details-button-location.jpg b/docs/images/details-button-location.jpg new file mode 100644 index 0000000000000000000000000000000000000000..53129a6e1bfbd3c991c3483894ee05c5e4bac608 GIT binary patch literal 25460 zcmeFYbx<8!w=deg6nA%rg}b}EyW7Iu2_Z;u2oN-QAh-pGgb*Z1kN^P!!8IX-U`YsU z65z4FecpG@d3A5yy7m5eRj=+`y}C!wp*_}bc6ZG=*6qjJEr38%RYMg3?;1DA4FI=0 z%t;zbN_P5&da4@QDt8P3U=H(g@(zR$0D!koP=KMDB7?bwB?EF3zy`1Y9DoQAcW?^y z|AK{nCw?b8x<+x9(`}fV%(z0E+W>zvDkN!{4;)KeX`Qbbzs;(w&St05IUV{ukQuztB!@ z0p54lZ+s`u@=pwR6adf!-X(zt?d65$F%{$q3~+S}3UK0ece;!5-;EXs1PlQ+KoQ^p z(16$72YpAG0)lt@Kp+5c1ss8(yQ34p3%K9y|Euo5jr_kMg3w`ixd8wrU%&7GcUQL{ z20?xS5pD*RyRaFIf_#0@3>=|o$3XWWv?PODP>`P_AD@rA584^+8h}OzI-ITSC0v737!!yS4U`~rMk-0$4noqD-Dp?w0;&J4jm z&gcLJ2L`18v_p`42%163*W26ICy+rQC@8?)F*wNG*N4Z*%^?7-;N|X#W)R}#XE5>c z^9=~PQ`Y{+3IoUAfd<}1>hvF0?gISV2(NE|E8l;(3gmMP=LvM+yEDM&g?4rDa`JUX zqj>+$#oxI7`v?I4iN`<}A;EujfjEi)0N=R1{lN+V$ZP<(&b+<-U4DCeU2%t#H2@g! z`A>b{rvM;+e%GJ*pM5N40DvC{0A0iX+2@!K0NsfIK(Xu;91!v^Kfv8Ux&to&0bl?e zKmbSp^KKkr-;E?Z0RN6j2oM7#04YEQPymzw6+jKp0CWI7zyL4;%m53(3a|kj01Ds) zxbCpT3-AH_cc>BqgaHvi3=jt-07*a!kO5=?IY9mne@cKdpmK*qbwC5q1hfGiKo`&h z4DQfr1Q-J*fEi#8SOAuQHDCkS0(OAI9WtE&XTaqSqi%QD^t{8aH{b*K0{(Y+4ZK5b z2oMT{0pUOda36>S9stom3=j)E1mb}NAQ4Ccl7Unp4R{2k1DU{MAPdL_a)CS`A9w;3 z0!2VEPy&6fh0U0JFe*U>;ZiJ^)L=M_?IP0oH(ZU<23!wt*dB z5BLOp2EG6Xz*pcKa0HwHr@$HT1NaI20xp0{;0pK+Tmv`2?cH4v0>VHzhy*b}ED#&S z1@S<9kPsvSNkCGN9HaoLKx&W{qz4&5CXg9q1=&Cj5Cw99JRmQ~4+?-npfD&3ih~lM z6etbKg7TmOs06Bjs-QZk32K2lpdP3X8iK~431|jdfR>;&Xbakd4xkf=23(>1HiFGy3)lvBfL&k@_!4{tz6J-tA#fNR1>b@b;1oCw&VqB`0=Ni%1XsW{ za0A>1cfn8KK6n5gf=A#9cn1Ce&%sOZ3j71!KmZ5?0)rqSm=J6TE(9M!1R;TtK`0

wg?xjYK)yrHAs3J<$Tj2^3W36*7*H%IE|dUD3?+q9K&hd0PzES7lnsi4azlBc z0#IS77*rA}1C@g+LRFyZP%Wq~)BtJ>HHBJ0t)X^MM<^QV2K9vcK>eXX&`{_-Xe2Zm z8VikwCP7o7>Cnf}9Ox5h5wsLq4y}aNKHyLFe{iX%n^o$xxu_(zOX=82rL{H35$ls!4hGq zuyj}!EEiS)D~3IVJ%iQ2>S4{WR#+#j2lfiq4|@X}g-yVwVeerdU>{*?ur1gwY#;U& zb`1LtJBR%RyM_aB7#stR4abKQ!^z-Oa5^{>oE46O^S}k*!fs7<>{w1D}U4!B^p%@Ll*m{1AQu{|Uc@|3Lr<7y<)y1IdFFK#C$Ik#a~Sq&iXuX@E3ES|aU`PDnSTH_{&&f{Z{$A>)wA$VbR5WInP8 z`4m}&tV1><+mYSKSI9x+C~^WhgIqu^BiE5T$bIBD4NEr>5mzL8G#vtnShy!`4}?~vk0>svl_Dj zvlX)o^A+YG<|yVA=6lQ~%r(p%%zexw%G}mL`@ymMNAMmIIaxmKRn4Rwz~^);tXEirSYuf4 zu;#HoVr^jUVSU9q!8*tKjSXTWv2n18uqm+VuvxG9OP#jeI~#BRgx!G4WBj6H!pi@k`whP{J*fPI2}j{O@4 z#6jZV;t=Cd;xOQ_;c(*!;)vtO;;7(g;TYhU;n?6f<9Ois;e_Bs;ylDj#>v3R!zsq8 zz^TP~j?;Qy~Z2Co5GvNTfy7L`+|3Z_Y3bDABK;OPlQj2&w$U4&xx{LlDD_~-b42%rR5 z1VjXs1Plc11bhUd1TqB51lj~f1eOGj1nvZW1YrbG1PKJ`1i1ty1eF901nmSb35E#9 z31$gC5^NIe6C4x#BDf}m6XFn(5KB^)B0AesL}^4hM8!mvM2$opM6Za3iKd7ah*pVqi4KW=5d9{G5@Qn+6H^m2 z6LS#@5lay(6KfM26I&BI6MGQ{5#J|{BTggEAub`VB5oq?B<>>~A$~`^NW4M(nfRFa z7x66#k_4ZGoP>ddgM^<%oJ4^{gT#QulEjh3gCvk7g5)7d8c7aGF-a9k6G<1zYm!lt z8ImQEEs`%JXC!}-f}~iaM5NTD%%t3;!lW{!Dx|ukW~6qcZlr#s;iNI7DWut?MWmIa zjijBVuSrKqXGlMiZjm04ekZ*mgOcHpk&w}nv61nSiIK^ZX^{k}>QYf-1iYcloo>TNt3{gx{EKqDv>{FalTv0+PaVW_s=_yf^f|OE}DwMjE z=9CVUo|Hk94=58UA5#`mR#G-mc2f>ePEgKMu2X)dJf*y%f>Pm9kx?;FaZ(9U$xx|M z=~G!!IZ=62g;K>(rBLNil~UDEwNmv`jZjTfeWco<`bKq5bxVy&O-xNo%}&ivElI6R ztxIiA?MUrK9ZVfXolKogT|!+$-Adg{JwiQ0y-dAJeMEgh1JYpAkkZi8plF0>WN6fA z3}~!q&@_HD_h{m1(rKR1RM0fibkPjbOwxRy*`zt3`AKs_i$P07OGC>}D?lqnt3sq{F>`;a!B_6cnTZ4+%b?I7(G?IP_q?IG$-?hieZo`{}?o}FHhUYcHw-jLp!-jzOp z{yu#oeHMKQeGPpZeINZ>`Z@Y_`Y-f9=x-P>7>F6@7*Gtt3~~&b48{z03?2-@4ABg! z40#OY42=xk3_}d>7(O!WG8{Af#Ry}>XQX0eW#ng+VpL@`V6mVq#_DXOd=8V=`p2WpZZJZcF*7|gC$k8%0<$)=IkOY9FY`U-1m?%gCCs(V9nAgAlgx|EJIu$- zf3d(>2v}%XI9P;O zW1ZuG;};5o!bee~I8eeU1(XiT66J~tL`9?0Pz9)JR2!-vHHG?!`h+?|-Ed-aQgE_x z3UJDDYI2%!I&=DSMslWb=5tnZwsQ7yPH--9?s1-R-f&@Yk#n(d32@1BX>plzp}7LM zqPWty3b?Ac+PM0;rnr{5K6CxJo7}=6hG39aQ@#lHKlgd-TQ_a)PGsrW|v&wV8^NSb8OUO&d z%grmntHx{0>%i;Fd!ILjH=nnPw~cpz_Z{yl?-$-*d{90@J~}>bK1n_`J`+AiK0m$( ze5rf|d^LO>d_#P*eCvFNe1GvH_(}Mg`1$x{__g>g_+9yf_+$Aq`Ahj5`Cssl@qggo zj~QmdkIGfCksCjt`_bTej_|5yd``hd@X__LM4I{ z5ff1rF&1$W2@r`E$q*?KX%y)d85j8|vM+Kj3KJz3WfbKXl@rw!wHEahy(gM1nlD-{ z+9^6LIxo5{dMbJ=h9^cV#w{i#rXgl7<|-B<7B7|~_DrlzY*1`gY*Xx5>{=X0oLZby zTtZx3+)Ug>JXkzVJX^d%yj6Tqd{%r@{8;>20!M;cf=fbDLPNq_!c`(vB0(ZoqEezm z;*G?d#J0qlBp`_|NiWGODI=*PX)Wm~86lY>Stwa2*(3Q@a!GPu@|P4sid2eKN?1x+ z%1FvdDnKe$>akS0REyMr)U4E|)QQxMG@dk_G>^25w2rj3w3qaK=~U?==?3YS(i74v z(qE-lG&HJkVVRp%d*Rg$*Rek$-2sh z$|lO@%ht$t%Z|w|$?nTu$RXv(<=ExKlJ$yCl%Kej})(!@RaD4_>|<843r#{ z0+b#q&PODStB+bH`eM=572mn*j^4=K+p?uGxh-mN}idfVqmfxw)5lw0X99jrmLSY4aWPa|=uhS_^&)Weal)Pm3sv zY>OI;mlo3&I~KnzF)e8=1uRu8EiAn(qb+kRYb{?{&RXtSURq&W(OU^wsaaWB`C2`+ z%C~B;>bIJ=+PC^`jc?6tEoQA{ZEqc9on&2X-C{j#{n7fH4QNAZgR+saF|a|~+_Ooy zsj%s?nY7uo`C*G;OJmD#t88mw>unorn`c{Z+iyE>yKno)j=+x9PQp&t&e1N^F4gX- zU5DMc-MZa(dxSl;J)ga@y}7-&eXM<+eS>|!{eu00{j~#;1G|HigT4dW;hsaf!!w5- zhj$L!4!<0+9T^-&95ozm9RnSc97`PA9LF429Z#L$PE<~OPRdRePCiZ#ot`)~ISn~2 zIel{moynZJoaLQOoIRYQoO7M)o%@{^oDZCD(8OpIS{7}Dc0)&^v(dHaKJ*;=3;NoH z$c5cS#>LRZ)g{s;%caJp&t=Z#i_5hukt>I*jH{8Wo9hGDY}Y#1*RBh$2d=knByOB; za&9JW9&XWYd2S7EgKmp%-`pYYK-;8 zfgZ^oWgeX#6CPV0zdUg~nLNckbv>Ou?|Ei=R(rnkob&wRdE-Um#pxyQW$NYS73=lH ztJ!PVYsKr-8{tjkE#$4?ZRZ{Ao$6ig-Qzv&z2|-9L+HcqBjaP_e+{RjM){Eq`*0n`D40qOyE0U-g80xAPu2D}gW5^x(x7RVi_6lfXf zAD9$a7T6W|E^sgKDu^(MBS#fBAzwT6v{ZG~Ni|Gr0jkMo}5JaS?RXjHU9v}JTabV~HI=-%js=x;Hw7}^+-7~L3`n5dZi zn3kBgG21bJ#S+DG#wx{H#RkQu#a6|>j$MpBd5C<-@KEBR!9$OS4<8mi?0ER@;irc; zapZCQaq4jnarfe~;~L|JkCP*e2C3q#o zCzK|1C(I@sBtjCY6NM9X5?vFc6AKdC5+@V)60eiUllYS~k{pvFl5&%plg5&^lCF}8 zlev>slkJkjk{>5GBo8NVBwwTuq;RGvrP!o|q-3Pjr3|I4rJSeYr*fn!rdp*2r>3XY zrVgd9rk zuAFY09+sY!-k3g`zLkEJL7c&pp_bv05s{Ib@jPQZV>jbElRQ%(Q!5jl8I@U>*^xP& z`QG0kJq$NG;wAICo~d;H?@{Nv*+j4Y-s=`6D>|E$!k>a2mRm8_rH_}QpzCRD2Ok3TF_gtSa4Q|Q^;PZSZG@qUYJw(yl|rMQxQ-^QzTYoSma%l zR8(2iU$jzmUQAfbU948@RQ#a0pt!Smw)n6FS;ABzQ({pPRFYZJP%>JwQ*vEOSt?wr zU+P(!SXxo~x^$)Vyo|7nyG*^zsVu6jsI04OuI%_J=2O3%u1 zoW5ML+_XHPJiWZWe6)PK{JMg&LZrf=!n-25qN-xBV!h(8XQa;rp6NVue-{6&{8`_# zdMdIIGmEoT{R#imH057OGCGajH?(D%B3v52_2R zyQ=4_PinAhIBJw@>}w)x3TnD)=4wuAv1>VMm1`YpBWnw5yK3iaPwH^$P<1MGj&%>} zit2jm7V6IG@#;D2)#{z=qw7oRU)C?x|7aj+;AzloaBXqLs8&uvNd+yEUb?wsoX+w+(2cYm;iT zXbWk}X=`npY5UfW)z03o((c$E-Co+>+rHfXtAnINutTrIyCb!uwqvwouM^Tq-zn2+ z)fwKI*V)lI*Lm87*TvnX(dF6|-}S6(uxqpHx|_ONqT8%HxI4SMwR@)fs0X_T)uYyf z?s?c#-qYW+-t+qfvH- zyf>sbx3|6beedZjyjMK0v|f3lg$IoW z0|p-twhYb;9uMITaSv$?c?=~F)eemfeSQOf!}3PyjpLh`H&5RTyxDwnGfX=yJ!~}` zF2X8Una=cZ0>-sj~ZS~ucx1Yx0<1FLKN%ou2`+yUnyPbU)fp( zSD99oSJA5ptF^1+t6$f!*Lc=+*L>F=ueGf$tevlut&6W)tVgUDufJa3+yFP2Hk3Ee z8;KjW8xtGfHgPxkHuX0HHgh&RHye3(bJ<2(4B<-7WK&+qBqTfcw! ze))s?hujaxAMroxeoXy1{Ym^&^rz*|$e-muhkt%K$3EvfH#`qHFF5Zz-}(jn#r{j{ zm+!BvUtPbJe_dZNUMOFu7t15ukK$xy&Asy@*C$j|8L{pVZV!i5B%QygYk#^kN%(FKLvmK{%l{vuTj@J z*8$gg*DtR(ZXh@8H(EEoH#s*wH*2@RE$gkut@mx#ZP)GU?d?BxC3jT-(7#Ioo-F}@ z@qGZ$eg^;rg#f_jdRIqaeY<+Ma*u$)5pWm+0Y@N_2n;L&EG$e+ED}6?90GC@3JP)( zGBPS!R(dLGW*Raw25tssHg*&zijtm(pNE5=l>^1`w-FGDMBXjA5MyBxbKFgta{O<% z+W~+O1Bim;K|v+}LI^?$!Q0`xqJ+B=2I$?2`QMlDPlF;La3l=GxZ~;*03Z|!!ePj} z4TiwJD|moF@9G^05k$m{0`HMb3Wg*GMh=g&vPqc*{bR_416#>i6w&3CJ3?ZPg8{J* z+qX|y*_2eAf^#d{hUV`qQvL1f&fov-?C$utNjTz;CqM}P?G6IFTOj>cO8>6;Mg=}48}>Tlu2S1E@Ua}RMp{*X8) zT`X$-9Qc*1$<+e3=Mu~F-}-b~PoBq%Q$9`oU#M#0h&fH|FPK^Sj1|n(Hg}eA{BL{z zKPLY3m-zoA7mE$XwuJUAlP`MaO0___5IoUjef8b#;S}7we-NYd4>Vqg{T#m@xe2@` zxhatA>7S0OXgkQ6lAQiu2$cxB(7OIN7+DuxIp6Hu0unbVQQBo|cKdSgbqj>>+W!IU z{6DhII@f|x{!zuZfJ#s#Uzp0x8=}vvnrOcG$YqITMiusq3<3{rZOTdXf75C_FfjiI ziXH!q0#8YOhyRC!Lc%{0Yfk473|>#jozDNM_ej7$Hn;^6%03zCbCr7j{&r0n=5oCD zTqlO#huqUY%+NSf-O5@bykBmT7OxX?epb$%=CycyuZ#{t-`aG}QhmQguD$r|78u^g z+HnyLcIRE~wEhqxM6ugnegF0S&XQ8j`;(H*7gB{TFZ>yQc8pI7WT(NH$C$@9WWw1# zSf=8Y#W;opG$s32SM|O&l8Oq6cS1TyXP3G@1-!q$CnijGsB13Dcc~XI6&>sDhI^hY zDY++1ZZ~~3-cn5PkBJSuA#G{QiIaS?|;k0IMw9pNtHEln%sIqg1#6zgInX|&_zN&z1sW-bP z{qc;^E%5!K{|~Nyh3D#Z?C6svjJ&GagUwFwul7v}jF@`b^*-On+yt;i5Xx?F93u8ByAuT@2N+JpDKP{PkRqpA9~O|*qWot(Mkmu zp2o%+;)!O*#Dg8uxM=kIE^3y%%2n^}I$U4=uO*g?nnQ{yDLIe0%tm4a5DN$Rg zxECyo=)^)jTP~s*f4NX`p|f>{Iy+hhRa1nJK3m{HgT1b59eZ4xomnwvulj_@{Yg`KUag& z92tksvH=y|kk{%U%-1wj%D&3_!^T_oGnZGU8`B+CZJ*x_!OOc?+f1u-@~z)Q*jFi_ zRt8`kr%)O<4g(RDyIY>?Y#orYv`_L9|J9F5_r0pkCT|9WN_>0``CZFpT*tadBErqA7C5b+H z`-l`)%P1kivX`Ytq;bYBa#H)8_T|pVoA4gxN-rm=%jVYN{k#>`@UVbp=bx-1h=V|# zY6!uH6{U0=>cFdcH|<<+4t9H2!0GJ!>1lJjrLlXeAxqCG`p2tay}ByRU>(vrE9K*f zE;+&(-e5=yP468^X`T1R23}vJ^zhCilh=`WyQ7tO+dZz4FtSxKqmaD_aNe3#k!U)3lGhTLB zCj1Dnt+y7>E=4~*#9>8szcq_+*X2twhV(Y2GZz~BM`JcrwT(kot~Qe<-WxzF} zGf>|Xtk6Y9pVmE?Ogwm<|Kg^#Z`1pTeyn$WpK@$_`|M2kZIfWKuAM&51!mSKF_eo_ zmF7>=q7#Rv*pQF}eW|{0u^KcD?=S^TjcdtYJV<92U+XQ-&E>C2hZxTp`|*eL;}O60 z7wLD>vfOQ~%XufyX_x}L_$64O+1nc-t-+rEh{@eb+ccF&gjJg?RVv>_`Y;0K z8kJc?J>HvlxLk?-ggY{&9n`V1?BV?A>cfV)QL?EnkzeV71jU|^WjCAl!&<)du0OQD z+o{r*1BB9fIB?+GD8-cgv>U>htSZ-`ro2gIE{`>9)wK{=Qs6fC{Bj_xKxg-BaqrdP*_Ep`VG89)81I^^H^c3=FnkX774 zs-i`3&hQ&u(L+w%^RSOWl|4v0N)CuKxf#LVV(4ZtI`|e&yb5_UOH7{t)K5Nu5 zZb?R>zT)maG>5T+4=cNRP5BOys8fEv?$k{3*L+2JZnamF5E=mi^HKye~B3 zrAA-5K8?bP`pYXA9oOM6Qt+x&biGk)l?3uM)LVXj_wf@ZKGb;ss-ZR}aq zsv%r+_ozGP)un>QiSw@W<<@@EXD(5`B_?L|VE${;gYS$!bMf%#cuAtZrpBmmmuX@9 z%(=dmmJaHfyE8%^hbd@HDGy2>?xucvI8o${M1?nh);v#D`mP{33hHi6V_;zT;xe}( zkykDkZh8?_dYqfIi;YeuZhZdEDp|8FzN+iP+E5IA+ScCI)}=qY(b_!Y=ng zg>LwFTNaikcRw`7=n^a|FrCGv`~AcG_pgW;4 zJ`H^KT=Gg{kaywt*F}tq=4h~IG-f?uxd}3pv~9I|zEM5$Qq5np%eO+=dyg+jG54Ls zpe=tDKAv@g$4oU&QocLC%atXq5k@3Oh(k`6pn~appvq#@Q0i6duGLc0d%DaYCxW>> zKV;KCFRB}bvpc7o?>rD8ZJt3a){x>>vHAeHJY%LiU8;V?wOPRUz@CuyGu@-_5tiBS z?X=xaQxDL6hq~x3L#gpFG-#y;!XWa);P`785 z4N15bjF_9$9M$&}j7oNY8aSi=a6B0ekXcQSn}K7L+jdqhR#s&P$!Vdh9)6IWRCmkv zj;FC9is`!Qlsl0@R8ieNt=xw%W;~#2FYxCq+;dT!is{n?6&fqF+quyK>DDR^vcE&S zUe^W~fq!`jB6Ega0*3DvwHl4|MFir`@I z&I+877MLuf8j)1Qe6jlM1nsk@l65sb7eKOXt}FGb_nBOYPtdzEo0Vp5o=e{sI1@z9 z46;KD&q~kWT3hN{eyaC9;g%6fp7*9Wrp0iY=2l3)1sLXLX$BbUihrLL(!RFWgV4>~jXkP)Yzq2BjgS$JY?l1ATUIgik7F(Eq{&=D!13M^1e-1w2{6GV%q5cfX zA-+`5H+>HsAaXzE@3UwkI})n3j7?n6#d8u^{Fygj9NU;fAPAmE#0NEx-vaB`4L3_S z=fN|)ro8WY{ql)|&{DK$PD`!UK?6M|bqV&(T^l8ND^`n+N3U^y!Zh5tn-~?$S1fbW z(rJ?Fq`3ncS2O?c5PC1*TKi}fLV2b0t9pL;!ns*BI4l^PiDLyGrmVC!Q)UNhyS;m3 zX%p#RaPz9<%wI~Lv;({=(HadaW%#z#wpmG$GNnGf)@-;nY-l=Vw(!x5q_{>3L&Fpg zs(&hGoOADZc#swCPdJ*@rIxnhfca3_DJ#k zE&oq5XZ|-s?-?;xpw4nE@$Atv1jRI27^gIWFPt>S(vz5EhPLL)+iKr6ar(NoRp1Ru z(ImVbq%gw^O0KjYL3d^T>16B}Q54O|jvEWs6_vwpGM8lmceCt>btvnZ@88@4P8EN4 z*h0pRF7CH|MX#OkDH2k-D0AV>5Z{faA2g(b#W)zE&z2j5jVlR4o~m*?8}_((@~ClBle60XgYj8LqTjK<(c9 zQf?2H{dLs)yBXrKZ_V>fG)_|ij>KQXd2fN(%lY~U`uQf4>->X@;XRX%JC)HlSQ&V^-P{0)IVZhQlJr>bPqsnz;!%f3 ztHGA_ZVgYkzQDde&xIYF9ewNAtIA>Y{D?K`7gj2V)!wcXJnvIb|6*9_eOsh4Lwayc zS^c-uQuo|Rt@IvMqNOOhAL%9?k9d+FU$ok2HGQAZWBw{zU*P%7$mLU@Ps)x*21i~{ z{oS+CjMB6OH?qfSeQoLy?viDoGZ$Kcc^Ky!c3Qv-&PPrUB&NZ*HFBCjr&VCoh50O=g?3Rx13p3u> z{c7D(5(b7&vZB9i+_5EgPX_k3+COaf-NY#BorropMWwRb@n%VSh7zo^cQ{tZCu6okgkrp8o zO0iflnO9@N#-yLzRGw8?*GW?h$G_AgeEd-1L!I^e*Y7JfpUh_+*?Fy2=)7pB;Q7dt zGrpo#;Onlrm1!{IHjV^b)0x^rYC1-i%_gn8Ha$*{x=7sKvFggT3AL%7KI(|crHMPE zV0&0=Set0sc~3If6j|c^!qComSbP>wbRmf2(8g%bG4RHo!By0HZO(ULc@mk<{J{3wCxj0d16g-8r=4x z@q0HdwfNiuy9;q26_8VeR#(1Y+h&u}mPq)qfvHu*u{>%r)0^hJryrbR0=a%ZPi%hB z5Zo%TZ|Ot#_`rzLk_Pv9mULa;9O_J4wMTbAF}`hQAJR}$#dtxhGQ^CLvbSiHH9Fzk zmN8s?sv(PnK_5=UUTc1OncZo)nB23$9pV|wsZNFG~nJwZ=JwXZ60?TRTBXng^0{Tstu46!zN-hQCdUaN@GlBXb`)M)FFc>Vv)6OnzwQK{tG;+O1!okh zR?FxbHhg8zV===d6zyNQKGR#kt@)_>lO=k?l9F&UH?AfL_S_z&M0p-Kuw97zfiroz zx#{Q4jMs&?L14nva{QL**VzeE!|!ttHE;lsD{&XGb9WeDd{P)Jw5oXW{%CsI*!aoJ zqrQAg#p$r$rxG+&cuks>&pL_(n-7wh*ZN3}WM_2Vj@{HdyHbPUGqRMULc6w+gwNwY z+ToRJN604z-6IKP92Tb0^k0|o`!swDc+zg1j)u+J)fsbn|IlyftTUA=d)n=-Qy0iA zLaTzddq9vYUd2Mt2wkUyO20!teV+e2q4mS!y0QbqY9E2%l%Vmd!!Szib!C+%dv9?% z#r6sII!Y7H+0e+#H{5H=6$&)?6QtW?5>X)j*+4AXF@wt~8UOD0W0ApAFkCgHWwmTZKEHsZw z&BvOJzkXzF`kBC{nqFOsz^j^|Okd*NoK z7LGW*CW6S)a473(U-o6-Kr!%CUVO@=F!z~hg|`@N#N_q8&~))hygE!xI)6;V6r)_| zFsnHWtT?by_}#UcdPq61mcas(O=aozdW^VFckuot#j5 zpC7MkrgB&?vMPD9KmNmL-t?`T)|Q@oC5-P!w#fyI!n3i^lW?&Q+~DS zEOl7XmhV)K{;ni^8@d#$b`W!ADx%A{zP)lUn0%-37U&Wy%VC%=G(B0tnd$1}uCa`+ zid;cLTEEKg<@`Qu{`ifomzui4n(bMdeJxI%U10{X6&aLAdsifgbh$JA$_Nz0xjwWlhT ztV~!<6!B{MFy7k>G#v%s-@? z%*^tU)bLE@X}(M+lV=GhouUurn|Pkrr}%E#qacIGXWx5to})`l%aF9KJX1rDz`q0G z|3>6x6FHBLP|CB=kZhZP;d6@ypAEy$k;Ne~2Td~G3+Ze!Lman&$4O`yIEY**zY(r? z^$Gdx;L+tg24X!K@E?#i#-zNKCuyP$)SY?uZ19_>`y&_4UvHUI;bj|94G6 zx2$n}?k%A4qK!7z3V!d&KTNgu&%^x%-B$e@1l0xc*6lyLackUXfq}@zPts zXW}NKeWq3AF~s;x6zU1VG?!Xj9ow?l^M4zW_=TIKib;SXLCnV_q^`Ny`MJ!*I^8~j z0E*?AlzipZ!?BFZrHa?_1f0`HVm$K!1rurGhS;iv z*y+I=mjLM+{oQFv8`DugL10u**2I2Q_UomwdZXi$MA8iGdH3UyRpT8;fj{SebxJnR z@?ClCyYZ31J{~&VBkpYN5GZzb-8HVg&Tsm>sq*UZA1?_idrplsYX8anEgS#ctN;KtEVfv9xT?AInItm*iiF?0R} zNF=cD3PIm=iiSuQHw~2p-g)BhyS5be9AhOyDI&CvAT~~`@!Be6o19qI&^6s3OBQA* z#$55q;*TE-tC^H&>!cK!Hl}%t6K&pd$IL_)tX#`@N>g*wUuXWw1@g5tv@M^byYMNY z&8tI;z2|kJg*a9(G0Ttxwu}R>8l8{|qbzzb7%?;^j*0eS3mt=IvNN+UFGkii6Y0T-bLAyOpBfBh0vBf<=6v^N_TXf61=n<>w^~9PJD$j~ zhoH|5QBY^smxt@V2GMTa7KGLo!f15wH4N}fhsGQq2${~8 z-_nFQb!SbE7j|!Cw@>!G3wq5Ks4w$E{Y5kvRm?gN57W8bfvs_7t<8s(WK>=&w^|>U ziRR=wmSQ)*Z^;flb<-pw35d9*jpw6NWtFL}M_G$fHde~DQ{LIz*ac8N?ee!{ z8R0mc?9daY(R=51c( zMaAKfcz}4nzdmLHJ7oh*nv^6bQLL_o*FMr?6ClB78LpmS2ir9}$$RH^24PdvjzDdt zZM8W31y%4-7AhKZqHwI|I7@#jL3nBoiq%_5t4ONwNGu(>P0dA`P#P9s>s5OVqw#ODAVrHc`aexhNA25OW(QjRA`-987w}Tx9N8S7suc zJ!il(PIHpmNk?T;?P%xd$Uad7^EOQaNfB-$2eiwPQz@C7^dgx~HvVF0Y>C3jMY?b%yRp zoK}+w$>8qG1~%U1aje?lz96^`+jF~_-MXd2x&x?Toz0ZxMC{#PSXe`-x>)iQf%%Z6 zbPzWDOyo1+wcf72!tUjBvo+0AX9cTZMV|te<-Y*7*+V7pI zDCw=sNnqq_LB+nUek{78Dldem#WQv9mpjmkVV++^Li1-Iwg}!T+gL-EA`vDI0hu&N zuZ}#%59sE)1X1}7P?|g~YC#)nQTG7ZDF}H!TU%y9rek(`{*OeF+>dT(%6F16@a^r0 z@4jBDJ*Q6s?N6Dk&D<0~x8Z z&b#(6L)*-y#j4|~5ZZwlenxhH@=yTArr|`nqj8aYo+DSk#Xt_h%#DywZ{B|&ENt+H zWLs&dr1`QM*{rl5)UvSe2Vx?)uGM`9E7Bgk84@0vSEvmLkx%|9ZJhlVP?smx@<#+( z?IB`)gnNMBt^}@0(KWTAn440XGpb`$CIOQoBB+GqTVLWtS*&6V0==Q_N^ZbI2cluk9VO&iltR zgnP$)Yu+mRsKlP*+f*Xxsu=&S_N8+wQHB=m+3PM! z8oTuua6BPs;m`86C;$6}z1HgsgWVFRkbks~f7y-T!{y|hni``auMUM=7u9f49ra$D z`T_o0T<#&>g?E-ImVFA;C01DBbI8Ad^pjfGpyL?EK(}GC0|QP9tF?j8-#`XG%ra%) z&cFR-@7>H^9qD-+td@=o{|g{~xEDIqWZi2zx)B(ZsmO!SvMC7yU+ldyoU_hpM-^6; zjkdHl+)=Qj&VrB@aRLf0Mj;q~rH77@wk@r<@KXphlJ5-uDbDCj8vBe!Fw4|N&U^LT zb(<#^FL|zJ86|t`!1-i$n0Nl?+{$1u|8L`khRADl=vMhpllg|Nck3d{54H=M{-bH; z7vlZjf&X2P|1aUtywmnR&fDG3tF@J%v)nM+{@Z9>h){#n<2`=Ww##Q_yOl6criv=% z_gl#kaai89B&$({obOYLvT695UXN$|}>bG@36oBN}d<*18Z6|84+V|pdATo1eG1q~wW1Ef% zr8yHedSRyfTW0J8@AM4~>#*KK2ndafRyy|As-NE@m`#34_tpz{`jQ3T&gU;}?~Ag7 z^nGQWuxHE(>@Z;He)SjdzTL?sE@3D%;9GFBBjGai@tW9mvi7;qQK;X|Yp6!4giXTt z_Z3+Af>~r!PCtQ&!V%pK=7BNqx9e+reeO(BJR2)HzO^QNT724a6l}NbUEDFNElhzo z(g8Spy@&N~>n_|V*Gk)1N~y8JLOKoBdZOd%DyW3MeACzTu7xVgG#qv8-T-|xR22GJ z>100>)vWgjVwP;{vbV`HI4M*i?9O2m?g-1)SwC%@0)0#Gxvu#mE!yXe8-peQ@#2U`afx?kK@nn_3z%p>wuB~5Xe`S?Uy1eUQ>$)CkC!914g_$ z4eq*Qeh>zR5|ickf-IZ*5|IRrCoj6oR3p6#1{*3C&3G)N+n;zW0^1W^s=2F?T-r&SaKv zXJ)D7K&EMTYEv4;dw>HC=?|jxEekRR=|dUj&Vf8Y)es z8E++o`SeFt_AM+xTd$9crP$pbWpl&Ljw)c}yj^e43*H-B(J^pU&_fvdwm2ypQo&$k*kMB#Tt)D8MR?wK?2&)Geb!yX@v2=_u)6EWXbArRt%bff z8Pd^3RBiRhu04Eb4y~yC!UcZ%-7G3dfBFrZ_#MP5+ zeSN61(ZUYO;9cHv{h;k*X*ylgvJF#efgq`E>YLBMd~ntIJ)B`{R&3$&jK)UZX*0Pi z9vw4j5hXO{14$Oc9bI^P^=Dh_=T6s;KSZphR(inlK4#82GB=fC0$9u_F3!f10YdWV z33E!%B~b=M7LzV|g%a*b&5MeTrfWJ=aPII*9XK22HC$>wDd-yP)Nvwk#9l7D#a=hw zLxB0bpD1W}h$M|dCTXiGGC4?)EeE0`tkt^rt!>vVvwVfDf>psZi5MZye`Ld0_y{*C zc?XG6JCAoxYfb&+yI1>omRAr-3qOdC=2tMc_Aq}b&CkKxRvm6aXXcJ{SH_IKsU3)+ zb#|h9{Hj&1I#(F_RXfOiY5s8Gn;p0s%^z2+3D(_L_?nu*ggmuHLMBBDT&VAQwT#(G zJ@3846wqkWe-AyoOFQC)pzTxx^jqB~g0RmJDIzg59ad&I5%qz#Wwt~{gA#ZB(Z@rp z>!)mR!=lsysuZP{0pV|GaEyi)pOxCe{aVLTJJ)0Wb7umBf0_nu;5nLK98~x}&`L!1 z@DEC_1}18|cBiR~hb0`-M}3LRa0oD!f}ui@O|8p>N*s>Ov0I94dO)PNf zu#?~73!ilrn5-6cR|S1jaAhBq>&=ozx8)v`zs+9=4ob}bWf?*#YF{vQyYttgeDA(h z_1%2Q;}*xsli0DhuXjJC;R}=QIf@qBrsQErnQf7u6ZWcpx#JW*&}9xBJq|k$w)(1L zI;7yCv%<2h?V7(wyG=IeaPJu#O?l`d9yL+Ux1KmAy`Q?O4x=kX%rjSP^7)5-l1lid zqs-i@UxX93cB$-KNyrh>O_1b!hnTBy6o&a^5JMuMi*zq=$0q-v=cbfLU5k07owiC~ zgdnxKl}aNWWDx_Lft>7#;!DWF(4TjHpl*k1Ol^ORl9+}oO^zgQFNieivn$^Vx+Vgv zM+&k(61VC}hfOsV*cDTRjB?#|kk6c=xJn~)**RP~m}d)`r>53T4scQ*$)4TBa2KWb z0n~5>fh@y`#uq(dd7bVy(ZWAN{iKZf!5LmIP?o3);wej6@>3+N3-BI`%AWpJc7IKR z2hqW(8#DhA+PPLtE{0LSZX2rbYk%}|VBow1p(857;NeU=XWi7*X!?b<%O%90w)Lsw$d&UmHB)lFo9fZ&6-tMl z_48BV@p9{yr#q{l9vgwI=|+24h_D?l#XBl0f|k}|Q;%utZK8ZDtiQXJ)yPPwy>Tq7 z3C^&cI=k6iA6Z@yq^!#QiWX+?Z-fWfS7pCxoxp zM}g(ON~3T;Mn=cAnuPYCS6gh48m(LyEd5zvF-)41pLlL>Z@r}?Rq`IdA`}ebefKmV zB5!wcsKG_`S;XRa?Xeke{iRLh2gRd7jNkQD{Cp4f&dn+ zIf4fAkKaM6Yr7mC*Sc4nWh>D4oL_Z-xb^DVjzy4{cbMgB;We%;C_e~u)PkF>E7|3y z^w5)fVkN=0m-!YUEDZQMx~=nBb*|EKIrG3^#46FolRl-!t}Nv}EFb_OYtS2aR>Ru- z5wW+>c`-VNQOR6b^~#_%`k=CT9V^4kad;%jKSs*_Qh=`)ELQhG`#y%7U6~s=iR!jt z78aWrR2S+|deJ)Wr@>vQ3#7cTtpxL3H`s{$N$u?ZHMKMHSU2Lsi`OKhu(S*jd*_b8 zZ_3Q2bNE;Gv4zB+JQ3Ljs^=&wlX^T6zXPVfxQQ{tft1E!=S@Xpd$3A-12vUU#z&J! z6PA{Ee&(s zo9ED8zfByjKPi>>ZP49jZUOa?BR% zo^qQ6!{=@Ls)F~Tuu#DD&?9p@E6;6f*9WS8dd<-CS65zqmF!vUMiw4XLex!!qy?fY zL(7SEe2)H*9Qku~un4l`8)_;=QipW}V1lxmV;^8?wCjoBYwlKebX@`1g2D`VRFq5H zY6ojiFXy|jim*b7NWUu*$EYU5`fb$I?Uw~Vgz;S8`xJ$(G(@lsog`&Tgz7k=kq5F_ zxn#opjTt-DWP+#=0`Kc+@%x9JEAX}v(|#7~6iSK*)B7T-@V{gQ#H=XDM3g5>Tm7gWZ@-T5g%}CO@W2V1QycryAs^0SG}$|Mn-cx0(KU)*}lIip?+DR)anBM=^=( zYz>?NKwgSxmdWm2IQRNt+0XFj>el8aGw-tCJ^R0P`EJcI=q!_2w*lYGm)CxLA3Hk@ zyfZGFX?j%wmDmy0ryowy0dm$OKk*d4#<5mcK?%wh*Z0`IY)kB}%${^Lct`wRv}=%+ zO2cmx$o=?Te$UW63OY=;YqlN&##OvbX%Xnr5p|sA-T!($qC}PZr9|psb+h-g zRy}47W2i5_wu&I53H+a!Xoc-RyZ3&${$nabjK=(5YlPK{yR#WOnb1l+V=B@lvxkx0 z&=UVI=3e^bSkhB;l@Gt!t<>eA<1Np0h#Uv<@)E4r{#%UyFIE1gBSMXrD_)B_RYR$r u4RhA~Y6DLGj}T!2H;KAfreuh+f{j?gpGcBTo&QbN|94OS)err<{C@y{Rpo;K literal 0 HcmV?d00001 From 5716cd60b5b343e62f1ccdfbd764d4803515d4c7 Mon Sep 17 00:00:00 2001 From: Thibaut CHARLES Date: Thu, 8 Aug 2019 07:10:42 +0200 Subject: [PATCH 09/11] Add filtering database API to syncapi account data table (#513) --- syncapi/storage/account_data_table.go | 15 ++++++++++++--- syncapi/storage/syncserver.go | 3 ++- syncapi/sync/requestpool.go | 6 ++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/syncapi/storage/account_data_table.go b/syncapi/storage/account_data_table.go index 9b73ce7d6..7b4803e3d 100644 --- a/syncapi/storage/account_data_table.go +++ b/syncapi/storage/account_data_table.go @@ -18,7 +18,9 @@ import ( "context" "database/sql" + "github.com/lib/pq" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" ) const accountDataSchema = ` @@ -41,7 +43,7 @@ CREATE TABLE IF NOT EXISTS syncapi_account_data_type ( CONSTRAINT syncapi_account_data_unique UNIQUE (user_id, room_id, type) ); -CREATE UNIQUE INDEX IF NOT EXISTS syncapi_account_data_id_idx ON syncapi_account_data_type(id); +CREATE UNIQUE INDEX IF NOT EXISTS syncapi_account_data_id_idx ON syncapi_account_data_type(id, type); ` const insertAccountDataSQL = "" + @@ -53,7 +55,9 @@ const insertAccountDataSQL = "" + const selectAccountDataInRangeSQL = "" + "SELECT room_id, type FROM syncapi_account_data_type" + " WHERE user_id = $1 AND id > $2 AND id <= $3" + - " ORDER BY id ASC" + " AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" + + " AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" + + " ORDER BY id ASC LIMIT $6" const selectMaxAccountDataIDSQL = "" + "SELECT MAX(id) FROM syncapi_account_data_type" @@ -93,6 +97,7 @@ func (s *accountDataStatements) selectAccountDataInRange( ctx context.Context, userID string, oldPos, newPos int64, + accountDataFilterPart *gomatrixserverlib.FilterPart, ) (data map[string][]string, err error) { data = make(map[string][]string) @@ -103,7 +108,11 @@ func (s *accountDataStatements) selectAccountDataInRange( oldPos-- } - rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, oldPos, newPos) + rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, oldPos, newPos, + pq.StringArray(filterConvertTypeWildcardToSQL(accountDataFilterPart.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(accountDataFilterPart.NotTypes)), + accountDataFilterPart.Limit, + ) if err != nil { return } diff --git a/syncapi/storage/syncserver.go b/syncapi/storage/syncserver.go index c57a90256..fb883702c 100644 --- a/syncapi/storage/syncserver.go +++ b/syncapi/storage/syncserver.go @@ -495,8 +495,9 @@ var txReadOnlySnapshot = sql.TxOptions{ // If there was an issue with the retrieval, returns an error func (d *SyncServerDatasource) GetAccountDataInRange( ctx context.Context, userID string, oldPos, newPos int64, + accountDataFilterPart *gomatrixserverlib.FilterPart, ) (map[string][]string, error) { - return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos) + return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos, accountDataFilterPart) } // UpsertAccountData keeps track of new or updated account data, by saving the type diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index d773a4606..6b95f4698 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -141,12 +141,14 @@ func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.SyncP return } - res, err = rp.appendAccountData(res, req.device.UserID, req, latestPos.PDUPosition) + accountDataFilter := gomatrixserverlib.DefaultFilterPart() // TODO: use filter provided in req instead + res, err = rp.appendAccountData(res, req.device.UserID, req, latestPos.PDUPosition, &accountDataFilter) return } func (rp *RequestPool) appendAccountData( data *types.Response, userID string, req syncRequest, currentPos int64, + accountDataFilter *gomatrixserverlib.FilterPart, ) (*types.Response, error) { // TODO: Account data doesn't have a sync position of its own, meaning that // account data might be sent multiple time to the client if multiple account @@ -180,7 +182,7 @@ func (rp *RequestPool) appendAccountData( } // Sync is not initial, get all account data since the latest sync - dataTypes, err := rp.db.GetAccountDataInRange(req.ctx, userID, req.since.PDUPosition, currentPos) + dataTypes, err := rp.db.GetAccountDataInRange(req.ctx, userID, req.since.PDUPosition, currentPos, accountDataFilter) if err != nil { return nil, err } From aa0d22bf50da7e3d4637ee5d46a1151342eda8d3 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Fri, 9 Aug 2019 17:45:54 +0800 Subject: [PATCH 10/11] Implement client single event retrieval (#693) --- clientapi/routing/getevent.go | 127 ++++++++++++++++++++++++++++++++++ clientapi/routing/routing.go | 9 +++ testfile | 2 + 3 files changed, 138 insertions(+) create mode 100644 clientapi/routing/getevent.go diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go new file mode 100644 index 000000000..7071d16f0 --- /dev/null +++ b/clientapi/routing/getevent.go @@ -0,0 +1,127 @@ +// Copyright 2019 Alex Chen +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +type getEventRequest struct { + req *http.Request + device *authtypes.Device + roomID string + eventID string + cfg config.Dendrite + federation *gomatrixserverlib.FederationClient + keyRing gomatrixserverlib.KeyRing + requestedEvent gomatrixserverlib.Event +} + +// GetEvent implements GET /_matrix/client/r0/rooms/{roomId}/event/{eventId} +// https://matrix.org/docs/spec/client_server/r0.4.0.html#get-matrix-client-r0-rooms-roomid-event-eventid +func GetEvent( + req *http.Request, + device *authtypes.Device, + roomID string, + eventID string, + cfg config.Dendrite, + queryAPI api.RoomserverQueryAPI, + federation *gomatrixserverlib.FederationClient, + keyRing gomatrixserverlib.KeyRing, +) util.JSONResponse { + eventsReq := api.QueryEventsByIDRequest{ + EventIDs: []string{eventID}, + } + var eventsResp api.QueryEventsByIDResponse + err := queryAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp) + if err != nil { + return httputil.LogThenError(req, err) + } + + if len(eventsResp.Events) == 0 { + // Event not found locally + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), + } + } + + requestedEvent := eventsResp.Events[0] + + r := getEventRequest{ + req: req, + device: device, + roomID: roomID, + eventID: eventID, + cfg: cfg, + federation: federation, + keyRing: keyRing, + requestedEvent: requestedEvent, + } + + stateReq := api.QueryStateAfterEventsRequest{ + RoomID: r.requestedEvent.RoomID(), + PrevEventIDs: r.requestedEvent.PrevEventIDs(), + StateToFetch: []gomatrixserverlib.StateKeyTuple{{ + EventType: gomatrixserverlib.MRoomMember, + StateKey: device.UserID, + }}, + } + var stateResp api.QueryStateAfterEventsResponse + if err := queryAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil { + return httputil.LogThenError(req, err) + } + + if !stateResp.RoomExists { + util.GetLogger(req.Context()).Errorf("Expected to find room for event %s but failed", r.requestedEvent.EventID()) + return jsonerror.InternalServerError() + } + + if !stateResp.PrevEventsExist { + // Missing some events locally; stateResp.StateEvents unavailable. + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), + } + } + + for _, stateEvent := range stateResp.StateEvents { + if stateEvent.StateKeyEquals(r.device.UserID) { + membership, err := stateEvent.Membership() + if err != nil { + return httputil.LogThenError(req, err) + } + if membership == gomatrixserverlib.Join { + return util.JSONResponse{ + Code: http.StatusOK, + JSON: gomatrixserverlib.ToClientEvent(r.requestedEvent, gomatrixserverlib.FormatAll), + } + } + } + } + + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 825dd97aa..d36ed6957 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -132,6 +132,15 @@ func Setup( nil, cfg, queryAPI, producer, transactionsCache) }), ).Methods(http.MethodPut, http.MethodOptions) + r0mux.Handle("/rooms/{roomID}/event/{eventID}", + common.MakeAuthAPI("rooms_get_event", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, queryAPI, federation, keyRing) + }), + ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}", common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(req)) diff --git a/testfile b/testfile index 74c9d9e4f..d04ab731c 100644 --- a/testfile +++ b/testfile @@ -168,3 +168,5 @@ Newly updated tags appear in an incremental v2 /sync Deleted tags appear in an incremental v2 /sync /event/ on non world readable room does not work Outbound federation can query profile data +/event/ on joined room works +/event/ does not allow access to events before the user joined From 386cc975f08b44cd8f83e6d2fbcb08ee55fdb868 Mon Sep 17 00:00:00 2001 From: Victor Cuadrado Juan Date: Fri, 9 Aug 2019 12:30:38 +0200 Subject: [PATCH 11/11] Add typingserver service to docker-compose (#692) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Víctor Cuadrado Juan --- docker/README.md | 2 +- docker/dendrite-docker.yml | 1 + docker/docker-compose.yml | 10 ++++++++++ docker/services/typing-server.sh | 5 +++++ 4 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 docker/services/typing-server.sh diff --git a/docker/README.md b/docker/README.md index 7d18ce605..ff88c0818 100644 --- a/docker/README.md +++ b/docker/README.md @@ -58,7 +58,7 @@ docker-compose up kafka zookeeper postgres and the following dendrite components ``` -docker-compose up client_api media_api sync_api room_server public_rooms_api +docker-compose up client_api media_api sync_api room_server public_rooms_api typing_server docker-compose up client_api_proxy ``` diff --git a/docker/dendrite-docker.yml b/docker/dendrite-docker.yml index c2e7682eb..abb8c3307 100644 --- a/docker/dendrite-docker.yml +++ b/docker/dendrite-docker.yml @@ -114,6 +114,7 @@ listen: media_api: "media_api:7774" public_rooms_api: "public_rooms_api:7775" federation_sender: "federation_sender:7776" + typing_server: "typing_server:7777" # The configuration for tracing the dendrite components. tracing: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 763e5b0f0..9cf67457c 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -95,6 +95,16 @@ services: networks: - internal + typing_server: + container_name: dendrite_typing_server + hostname: typing_server + entrypoint: ["bash", "./docker/services/typing-server.sh"] + build: ./ + volumes: + - ..:/build + networks: + - internal + federation_api_proxy: container_name: dendrite_federation_api_proxy hostname: federation_api_proxy diff --git a/docker/services/typing-server.sh b/docker/services/typing-server.sh new file mode 100644 index 000000000..16ee0fa62 --- /dev/null +++ b/docker/services/typing-server.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +bash ./docker/build.sh + +./bin/dendrite-typing-server --config=dendrite.yaml