From b729a10366f9cb6f8b34db58c7bc1b9b69e67b5f Mon Sep 17 00:00:00 2001 From: Thibaut CHARLES Date: Wed, 24 Jul 2019 18:08:51 +0200 Subject: [PATCH 01/15] Store & retrieve filters as structs rather than []byte (#436) Manipulate filters as gomatrix.Filter structures, instead of their []byte JSON representation. This lays ground work for using filters in dendrite for /sync requests. --- .../auth/storage/accounts/filter_table.go | 38 ++++++++++++++----- clientapi/auth/storage/accounts/storage.go | 8 ++-- clientapi/routing/filter.go | 20 +++------- 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/clientapi/auth/storage/accounts/filter_table.go b/clientapi/auth/storage/accounts/filter_table.go index 81bae4545..2b07ef17e 100644 --- a/clientapi/auth/storage/accounts/filter_table.go +++ b/clientapi/auth/storage/accounts/filter_table.go @@ -17,6 +17,7 @@ package accounts import ( "context" "database/sql" + "encoding/json" "github.com/matrix-org/gomatrixserverlib" ) @@ -71,25 +72,44 @@ func (s *filterStatements) prepare(db *sql.DB) (err error) { func (s *filterStatements) selectFilter( ctx context.Context, localpart string, filterID string, -) (filter []byte, err error) { - err = s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filter) - return +) (*gomatrixserverlib.Filter, error) { + // Retrieve filter from database (stored as canonical JSON) + var filterData []byte + err := s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filterData) + if err != nil { + return nil, err + } + + // Unmarshal JSON into Filter struct + var filter gomatrixserverlib.Filter + if err = json.Unmarshal(filterData, &filter); err != nil { + return nil, err + } + return &filter, nil } func (s *filterStatements) insertFilter( - ctx context.Context, filter []byte, localpart string, + ctx context.Context, filter *gomatrixserverlib.Filter, localpart string, ) (filterID string, err error) { var existingFilterID string - // This can result in a race condition when two clients try to insert the - // same filter and localpart at the same time, however this is not a - // problem as both calls will result in the same filterID - filterJSON, err := gomatrixserverlib.CanonicalJSON(filter) + // Serialise json + filterJSON, err := json.Marshal(filter) + if err != nil { + return "", err + } + // Remove whitespaces and sort JSON data + // needed to prevent from inserting the same filter multiple times + filterJSON, err = gomatrixserverlib.CanonicalJSON(filterJSON) if err != nil { return "", err } - // Check if filter already exists in the database + // Check if filter already exists in the database using its localpart and content + // + // This can result in a race condition when two clients try to insert the + // same filter and localpart at the same time, however this is not a + // problem as both calls will result in the same filterID err = s.selectFilterIDByContentStmt.QueryRowContext(ctx, localpart, filterJSON).Scan(&existingFilterID) if err != nil && err != sql.ErrNoRows { diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 27c0a176a..5c8ffffeb 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -344,11 +344,11 @@ func (d *Database) GetThreePIDsForLocalpart( } // GetFilter looks up the filter associated with a given local user and filter ID. -// Returns a filter represented as a byte slice. Otherwise returns an error if -// no such filter exists or if there was an error talking to the database. +// Returns a filter structure. Otherwise returns an error if no such filter exists +// or if there was an error talking to the database. func (d *Database) GetFilter( ctx context.Context, localpart string, filterID string, -) ([]byte, error) { +) (*gomatrixserverlib.Filter, error) { return d.filter.selectFilter(ctx, localpart, filterID) } @@ -356,7 +356,7 @@ func (d *Database) GetFilter( // Returns the filterID as a string. Otherwise returns an error if something // goes wrong. func (d *Database) PutFilter( - ctx context.Context, localpart string, filter []byte, + ctx context.Context, localpart string, filter *gomatrixserverlib.Filter, ) (string, error) { return d.filter.insertFilter(ctx, filter, localpart) } diff --git a/clientapi/routing/filter.go b/clientapi/routing/filter.go index 291a165b7..eec501ff7 100644 --- a/clientapi/routing/filter.go +++ b/clientapi/routing/filter.go @@ -17,13 +17,10 @@ package routing import ( "net/http" - "encoding/json" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -43,7 +40,7 @@ func GetFilter( return httputil.LogThenError(req, err) } - res, err := accountDB.GetFilter(req.Context(), localpart, filterID) + filter, err := accountDB.GetFilter(req.Context(), localpart, filterID) if err != nil { //TODO better error handling. This error message is *probably* right, // but if there are obscure db errors, this will also be returned, @@ -53,11 +50,6 @@ func GetFilter( JSON: jsonerror.NotFound("No such filter"), } } - filter := gomatrix.Filter{} - err = json.Unmarshal(res, &filter) - if err != nil { - return httputil.LogThenError(req, err) - } return util.JSONResponse{ Code: http.StatusOK, @@ -85,21 +77,21 @@ func PutFilter( return httputil.LogThenError(req, err) } - var filter gomatrix.Filter + var filter gomatrixserverlib.Filter if reqErr := httputil.UnmarshalJSONRequest(req, &filter); reqErr != nil { return *reqErr } - filterArray, err := json.Marshal(filter) - if err != nil { + // Validate generates a user-friendly error + if err = filter.Validate(); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("Filter is malformed"), + JSON: jsonerror.BadJSON("Invalid filter: " + err.Error()), } } - filterID, err := accountDB.PutFilter(req.Context(), localpart, filterArray) + filterID, err := accountDB.PutFilter(req.Context(), localpart, &filter) if err != nil { return httputil.LogThenError(req, err) } From 604685c5035d836d7069be92963191d6d9f49f84 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Thu, 25 Jul 2019 00:15:36 +0800 Subject: [PATCH 02/15] Implement room creation content (#754) Fixes #660. Signed-off-by: Alex Chen minecnly@gmail.com --- clientapi/routing/createroom.go | 36 +++++++++++++++++++++++++++++++-- common/eventcontent.go | 12 +++++++++-- testfile | 3 +++ 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 220ba6ae8..8c5ee975c 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -15,6 +15,7 @@ package routing import ( + "encoding/json" "fmt" "net/http" "strings" @@ -97,6 +98,27 @@ func (r createRoomRequest) Validate() *util.JSONResponse { } } + // Validate creation_content fields defined in the spec by marshalling the + // creation_content map into bytes and then unmarshalling the bytes into + // common.CreateContent. + + creationContentBytes, err := json.Marshal(r.CreationContent) + if err != nil { + return &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("malformed creation_content"), + } + } + + var CreationContent common.CreateContent + err = json.Unmarshal(creationContentBytes, &CreationContent) + if err != nil { + return &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("malformed creation_content"), + } + } + return nil } @@ -154,7 +176,17 @@ func createRoom( JSON: jsonerror.InvalidArgumentValue(err.Error()), } } - // TODO: visibility/presets/raw initial state/creation content + + // Clobber keys: creator, room_version + + if r.CreationContent == nil { + r.CreationContent = make(map[string]interface{}, 2) + } + + r.CreationContent["creator"] = userID + r.CreationContent["room_version"] = "1" // TODO: We set this to 1 before we support Room versioning + + // TODO: visibility/presets/raw initial state // TODO: Create room alias association // Make sure this doesn't fall into an application service's namespace though! @@ -214,7 +246,7 @@ func createRoom( // harder to reason about, hence sticking to a strict static ordering. // TODO: Synapse has txn/token ID on each event. Do we need to do this here? eventsToMake := []fledglingEvent{ - {"m.room.create", "", common.CreateContent{Creator: userID}}, + {"m.room.create", "", r.CreationContent}, {"m.room.member", userID, membershipContent}, {"m.room.power_levels", "", common.InitialPowerLevelsContent(userID)}, // TODO: m.room.canonical_alias diff --git a/common/eventcontent.go b/common/eventcontent.go index 971c4f0a7..c45724fcd 100644 --- a/common/eventcontent.go +++ b/common/eventcontent.go @@ -16,8 +16,16 @@ package common // CreateContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-create type CreateContent struct { - Creator string `json:"creator"` - Federate *bool `json:"m.federate,omitempty"` + Creator string `json:"creator"` + Federate *bool `json:"m.federate,omitempty"` + RoomVersion string `json:"room_version,omitempty"` + Predecessor PreviousRoom `json:"predecessor,omitempty"` +} + +// PreviousRoom is the "Previous Room" structure defined at https://matrix.org/docs/spec/client_server/r0.5.0#m-room-create +type PreviousRoom struct { + RoomID string `json:"room_id"` + EventID string `json:"event_id"` } // MemberContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-member diff --git a/testfile b/testfile index 1b2ad9e69..4003638f1 100644 --- a/testfile +++ b/testfile @@ -151,3 +151,6 @@ Inbound federation of state requires event_id as a mandatory paramater Inbound federation of state_ids requires event_id as a mandatory paramater POST /register returns the same device_id as that in the request POST /login returns the same device_id as that in the request +POST /createRoom with creation content +User can create and send/receive messages in a room with version 1 +POST /createRoom ignores attempts to set the room version via creation_content From 45d24d3fb5b84b89bcd8eec7058896b3b4a6f2e3 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 24 Jul 2019 18:41:39 +0100 Subject: [PATCH 03/15] Remove the buildkite pipeline (#763) New repo: https://github.com/matrix-org/pipelines/ --- .buildkite/pipeline.yaml | 49 ---------------------------------------- 1 file changed, 49 deletions(-) delete mode 100644 .buildkite/pipeline.yaml diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml deleted file mode 100644 index 9d755a244..000000000 --- a/.buildkite/pipeline.yaml +++ /dev/null @@ -1,49 +0,0 @@ -steps: - - command: - # https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint - - "GOGC=20 ./scripts/find-lint.sh" - label: "\U0001F9F9 Lint / :go: 1.12" - agents: - # Use a larger instance as linting takes a looot of memory - queue: "medium" - plugins: - - docker#v3.0.1: - image: "golang:1.12" - - - wait - - - command: - - "go build ./cmd/..." - label: "\U0001F528 Build / :go: 1.11" - plugins: - - docker#v3.0.1: - image: "golang:1.11" - retry: - automatic: - - exit_status: 128 - limit: 3 - - - command: - - "go build ./cmd/..." - label: "\U0001F528 Build / :go: 1.12" - plugins: - - docker#v3.0.1: - image: "golang:1.12" - retry: - automatic: - - exit_status: 128 - limit: 3 - - - command: - - "go test ./..." - label: "\U0001F9EA Unit tests / :go: 1.11" - plugins: - - docker#v3.0.1: - image: "golang:1.11" - - - command: - - "go test ./..." - label: "\U0001F9EA Unit tests / :go: 1.12" - plugins: - - docker#v3.0.1: - image: "golang:1.12" From e66933b108a4656d01bfa6285c782bdbfb249756 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Fri, 26 Jul 2019 00:00:22 +0800 Subject: [PATCH 04/15] Fix data races reported by go test -race ./... (#748) --- syncapi/sync/notifier.go | 1 + syncapi/sync/notifier_test.go | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/syncapi/sync/notifier.go b/syncapi/sync/notifier.go index 30ac3a2e5..14bc2efb6 100644 --- a/syncapi/sync/notifier.go +++ b/syncapi/sync/notifier.go @@ -185,6 +185,7 @@ func (n *Notifier) wakeupUsers(userIDs []string, newPos types.SyncPosition) { // fetchUserStream retrieves a stream unique to the given user. If makeIfNotExists is true, // a stream will be made for this user if one doesn't exist and it will be returned. This // function does not wait for data to be available on the stream. +// NB: Callers should have locked the mutex before calling this function. func (n *Notifier) fetchUserStream(userID string, makeIfNotExists bool) *UserStream { stream, ok := n.userStreams[userID] if !ok && makeIfNotExists { diff --git a/syncapi/sync/notifier_test.go b/syncapi/sync/notifier_test.go index 904315e9f..808e07cc7 100644 --- a/syncapi/sync/notifier_test.go +++ b/syncapi/sync/notifier_test.go @@ -143,7 +143,7 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { wg.Done() }() - stream := n.fetchUserStream(bob, true) + stream := lockedFetchUserStream(n, bob) waitForBlocking(stream, 1) n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter) @@ -171,7 +171,7 @@ func TestNewInviteEventForUser(t *testing.T) { wg.Done() }() - stream := n.fetchUserStream(bob, true) + stream := lockedFetchUserStream(n, bob) waitForBlocking(stream, 1) n.OnNewEvent(&aliceInviteBobEvent, "", nil, syncPositionAfter) @@ -199,7 +199,7 @@ func TestEDUWakeup(t *testing.T) { wg.Done() }() - stream := n.fetchUserStream(bob, true) + stream := lockedFetchUserStream(n, bob) waitForBlocking(stream, 1) n.OnNewEvent(&aliceInviteBobEvent, "", nil, syncPositionNewEDU) @@ -230,7 +230,7 @@ func TestMultipleRequestWakeup(t *testing.T) { go poll() go poll() - stream := n.fetchUserStream(bob, true) + stream := lockedFetchUserStream(n, bob) waitForBlocking(stream, 3) n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter) @@ -266,14 +266,14 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { } leaveWG.Done() }() - bobStream := n.fetchUserStream(bob, true) + bobStream := lockedFetchUserStream(n, bob) waitForBlocking(bobStream, 1) n.OnNewEvent(&bobLeaveEvent, "", nil, syncPositionAfter) leaveWG.Wait() // send an event into the room. Make sure alice gets it. Bob should not. var aliceWG sync.WaitGroup - aliceStream := n.fetchUserStream(alice, true) + aliceStream := lockedFetchUserStream(n, alice) aliceWG.Add(1) go func() { pos, err := waitForEvents(n, newTestSyncRequest(alice, syncPositionAfter)) @@ -328,6 +328,15 @@ func waitForBlocking(s *UserStream, numBlocking uint) { } } +// lockedFetchUserStream invokes Notifier.fetchUserStream, respecting Notifier.streamLock. +// A new stream is made if it doesn't exist already. +func lockedFetchUserStream(n *Notifier, userID string) *UserStream { + n.streamLock.Lock() + defer n.streamLock.Unlock() + + return n.fetchUserStream(userID, true) +} + func newTestSyncRequest(userID string, since types.SyncPosition) syncRequest { return syncRequest{ device: authtypes.Device{UserID: userID}, From 3e6d0a6246ed1bc10760c214fce762e6880bc5be Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Mon, 29 Jul 2019 15:18:21 +0800 Subject: [PATCH 05/15] Add newly passing tests from matrix-org/sytest 56de891 (#769) --- testfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testfile b/testfile index 4003638f1..2f9229e87 100644 --- a/testfile +++ b/testfile @@ -154,3 +154,5 @@ POST /login returns the same device_id as that in the request POST /createRoom with creation content User can create and send/receive messages in a room with version 1 POST /createRoom ignores attempts to set the room version via creation_content +Inbound federation rejects remote attempts to join local users to rooms +Inbound federation rejects remote attempts to kick local users to rooms From 40e44c5f3b445b2e5bb7dc98d991a9177d0f2abb Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 31 Jul 2019 20:45:45 +0800 Subject: [PATCH 06/15] Add newly passing tests from matrix-org/sytest (#771) Signed-off-by: Alex Chen --- testfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testfile b/testfile index 2f9229e87..c23dda662 100644 --- a/testfile +++ b/testfile @@ -156,3 +156,5 @@ User can create and send/receive messages in a room with version 1 POST /createRoom ignores attempts to set the room version via creation_content Inbound federation rejects remote attempts to join local users to rooms Inbound federation rejects remote attempts to kick local users to rooms +An event which redacts itself should be ignored +A pair of events which redact each other should be ignored From 3e1abe9ad3464a6c47dfcf003bb4bf6422c820af Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 31 Jul 2019 21:20:11 +0800 Subject: [PATCH 07/15] Fix /sync may contain duplicate EDUs and EDUs for left rooms (#752) In 29841be (#718), EDUs are added to /sync responses for rooms listed in joinedRoomIDs returned by addPDUDeltaToResponse. However this list may contain rooms other than those currently joined. Some variable renamings are done to make golangci-lint pass. Signed-off-by: Alex Chen minecnly@gmail.com --- syncapi/storage/syncserver.go | 44 +++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/syncapi/storage/syncserver.go b/syncapi/storage/syncserver.go index b4d7ccbd2..20fa8a4e0 100644 --- a/syncapi/storage/syncserver.go +++ b/syncapi/storage/syncserver.go @@ -35,6 +35,12 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) +const ( + membershipJoin = "join" + membershipLeave = "leave" + membershipBan = "ban" +) + type stateDelta struct { roomID string stateEvents []gomatrixserverlib.Event @@ -248,14 +254,12 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse( // joined rooms, but also which rooms have membership transitions for this user between the 2 PDU stream positions. // This works out what the 'state' key should be for each room as well as which membership block // to put the room into. - deltas, err := d.getStateDeltas(ctx, &device, txn, fromPos, toPos, device.UserID) + deltas, joinedRoomIDs, err := d.getStateDeltas(ctx, &device, txn, fromPos, toPos, device.UserID) if err != nil { return nil, err } - joinedRoomIDs := make([]string, 0, len(deltas)) for _, delta := range deltas { - joinedRoomIDs = append(joinedRoomIDs, delta.roomID) err = d.addRoomDeltaToResponse(ctx, &device, txn, fromPos, toPos, delta, numRecentEventsPerRoom, res) if err != nil { return nil, err @@ -344,7 +348,7 @@ func (d *SyncServerDatasource) IncrementalSync( ) } else { joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership( - ctx, nil, device.UserID, "join", + ctx, nil, device.UserID, membershipJoin, ) } if err != nil { @@ -393,7 +397,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( res = types.NewResponse(toPos) // Extract room state and recent events for all rooms the user is joined to. - joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join") + joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, membershipJoin) if err != nil { return } @@ -571,7 +575,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( res *types.Response, ) error { endPos := toPos - if delta.membershipPos > 0 && delta.membership == "leave" { + if delta.membershipPos > 0 && delta.membership == membershipLeave { // make sure we don't leak recent events after the leave event. // TODO: History visibility makes this somewhat complex to handle correctly. For example: // TODO: This doesn't work for join -> leave in a single /sync request (see events prior to join). @@ -595,7 +599,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( } switch delta.membership { - case "join": + case membershipJoin: jr := types.NewJoinResponse() if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 { // Use the short form of batch token for prev_batch @@ -608,9 +612,9 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true jr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[delta.roomID] = *jr - case "leave": + case membershipLeave: fallthrough // transitions to leave are the same as ban - case "ban": + case membershipBan: // TODO: recentEvents may contain events that this user is not allowed to see because they are // no longer in the room. lr := types.NewLeaveResponse() @@ -716,10 +720,14 @@ func (d *SyncServerDatasource) fetchMissingStateEvents( return events, nil } +// getStateDeltas returns the state deltas between fromPos and toPos, +// exclusive of oldPos, inclusive of newPos, for the rooms in which +// the user has new membership events. +// A list of joined room IDs is also returned in case the caller needs it. func (d *SyncServerDatasource) getStateDeltas( ctx context.Context, device *authtypes.Device, txn *sql.Tx, fromPos, toPos int64, userID string, -) ([]stateDelta, error) { +) ([]stateDelta, []string, error) { // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 // - Get membership list changes for this user in this sync response // - For each room which has membership list changes: @@ -733,11 +741,11 @@ func (d *SyncServerDatasource) getStateDeltas( // get all the state events ever between these two positions stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos) if err != nil { - return nil, err + return nil, nil, err } state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap) if err != nil { - return nil, err + return nil, nil, err } for roomID, stateStreamEvents := range state { @@ -748,12 +756,12 @@ func (d *SyncServerDatasource) getStateDeltas( // the 'state' part of the response though, so is transparent modulo bandwidth concerns as it is not added to // the timeline. if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { - if membership == "join" { + if membership == membershipJoin { // send full room state down instead of a delta var allState []gomatrixserverlib.Event allState, err = d.roomstate.selectCurrentState(ctx, txn, roomID) if err != nil { - return nil, err + return nil, nil, err } s := make([]streamEvent, len(allState)) for i := 0; i < len(s); i++ { @@ -775,19 +783,19 @@ func (d *SyncServerDatasource) getStateDeltas( } // Add in currently joined rooms - joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join") + joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, membershipJoin) if err != nil { - return nil, err + return nil, nil, err } for _, joinedRoomID := range joinedRoomIDs { deltas = append(deltas, stateDelta{ - membership: "join", + membership: membershipJoin, stateEvents: streamEventsToEvents(device, state[joinedRoomID]), roomID: joinedRoomID, }) } - return deltas, nil + return deltas, joinedRoomIDs, nil } // streamEventsToEvents converts streamEvent to Event. If device is non-nil and From 92db6cd0eaecbd2c27f83488a176ef76ecf16f36 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 31 Jul 2019 21:36:21 +0800 Subject: [PATCH 08/15] Fix index in invites_table.go (#770) This PR fixes a possible typo in an index created in invites_table.go. Signed-off-by: Alex Chen minecnly@gmail.com --- syncapi/storage/invites_table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncapi/storage/invites_table.go b/syncapi/storage/invites_table.go index 88c98f7e3..9f52087f6 100644 --- a/syncapi/storage/invites_table.go +++ b/syncapi/storage/invites_table.go @@ -23,7 +23,7 @@ CREATE INDEX IF NOT EXISTS syncapi_invites_target_user_id_idx -- For deleting old invites CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx - ON syncapi_invite_events(target_user_id, id); + ON syncapi_invite_events (event_id); ` const insertInviteEventSQL = "" + From 3578d77d259c852a16dc430725d348e2c62c4ff5 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Thu, 1 Aug 2019 12:36:13 +0800 Subject: [PATCH 10/15] Implement "full_state" query parameter for /sync (#751) Closes #637. --- syncapi/storage/syncserver.go | 133 +++++++++++++++++++++++++++------- syncapi/sync/requestpool.go | 12 ++- testfile | 1 + 3 files changed, 117 insertions(+), 29 deletions(-) diff --git a/syncapi/storage/syncserver.go b/syncapi/storage/syncserver.go index 20fa8a4e0..e914bddfe 100644 --- a/syncapi/storage/syncserver.go +++ b/syncapi/storage/syncserver.go @@ -241,6 +241,7 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse( device authtypes.Device, fromPos, toPos int64, numRecentEventsPerRoom int, + wantFullState bool, res *types.Response, ) ([]string, error) { txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot) @@ -254,7 +255,13 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse( // joined rooms, but also which rooms have membership transitions for this user between the 2 PDU stream positions. // This works out what the 'state' key should be for each room as well as which membership block // to put the room into. - deltas, joinedRoomIDs, err := d.getStateDeltas(ctx, &device, txn, fromPos, toPos, device.UserID) + var deltas []stateDelta + var joinedRoomIDs []string + if !wantFullState { + deltas, joinedRoomIDs, err = d.getStateDeltas(ctx, &device, txn, fromPos, toPos, device.UserID) + } else { + deltas, joinedRoomIDs, err = d.getStateDeltasForFullStateSync(ctx, &device, txn, fromPos, toPos, device.UserID) + } if err != nil { return nil, err } @@ -336,15 +343,16 @@ func (d *SyncServerDatasource) IncrementalSync( device authtypes.Device, fromPos, toPos types.SyncPosition, numRecentEventsPerRoom int, + wantFullState bool, ) (*types.Response, error) { nextBatchPos := fromPos.WithUpdates(toPos) res := types.NewResponse(nextBatchPos) var joinedRoomIDs []string var err error - if fromPos.PDUPosition != toPos.PDUPosition { + if fromPos.PDUPosition != toPos.PDUPosition || wantFullState { joinedRoomIDs, err = d.addPDUDeltaToResponse( - ctx, device, fromPos.PDUPosition, toPos.PDUPosition, numRecentEventsPerRoom, res, + ctx, device, fromPos.PDUPosition, toPos.PDUPosition, numRecentEventsPerRoom, wantFullState, res, ) } else { joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership( @@ -593,21 +601,30 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( recentEvents := streamEventsToEvents(device, recentStreamEvents) delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) // roll back - // Don't bother appending empty room entries - if len(recentEvents) == 0 && len(delta.stateEvents) == 0 { - return nil + var prevPDUPos int64 + + if len(recentEvents) == 0 { + if len(delta.stateEvents) == 0 { + // Don't bother appending empty room entries + return nil + } + + // If full_state=true and since is already up to date, then we'll have + // state events but no recent events. + prevPDUPos = toPos - 1 + } else { + prevPDUPos = recentStreamEvents[0].streamPosition - 1 + } + + if prevPDUPos <= 0 { + prevPDUPos = 1 } switch delta.membership { case membershipJoin: jr := types.NewJoinResponse() - if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 { - // Use the short form of batch token for prev_batch - jr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) - } else { - // Use the short form of batch token for prev_batch - jr.Timeline.PrevBatch = "1" - } + // Use the short form of batch token for prev_batch + jr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true jr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) @@ -618,13 +635,8 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( // TODO: recentEvents may contain events that this user is not allowed to see because they are // no longer in the room. lr := types.NewLeaveResponse() - if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 { - // Use the short form of batch token for prev_batch - lr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) - } else { - // Use the short form of batch token for prev_batch - lr.Timeline.PrevBatch = "1" - } + // Use the short form of batch token for prev_batch + lr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true lr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) @@ -758,15 +770,11 @@ func (d *SyncServerDatasource) getStateDeltas( if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { if membership == membershipJoin { // send full room state down instead of a delta - var allState []gomatrixserverlib.Event - allState, err = d.roomstate.selectCurrentState(ctx, txn, roomID) + var s []streamEvent + s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID) if err != nil { return nil, nil, err } - s := make([]streamEvent, len(allState)) - for i := 0; i < len(s); i++ { - s[i] = streamEvent{Event: allState[i], streamPosition: 0} - } state[roomID] = s continue // we'll add this room in when we do joined rooms } @@ -798,6 +806,79 @@ func (d *SyncServerDatasource) getStateDeltas( return deltas, joinedRoomIDs, nil } +// getStateDeltasForFullStateSync is a variant of getStateDeltas used for /sync +// requests with full_state=true. +// Fetches full state for all joined rooms and uses selectStateInRange to get +// updates for other rooms. +func (d *SyncServerDatasource) getStateDeltasForFullStateSync( + ctx context.Context, device *authtypes.Device, txn *sql.Tx, + fromPos, toPos int64, userID string, +) ([]stateDelta, []string, error) { + joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join") + if err != nil { + return nil, nil, err + } + + // Use a reasonable initial capacity + deltas := make([]stateDelta, 0, len(joinedRoomIDs)) + + // Add full states for all joined rooms + for _, joinedRoomID := range joinedRoomIDs { + s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID) + if stateErr != nil { + return nil, nil, stateErr + } + deltas = append(deltas, stateDelta{ + membership: "join", + stateEvents: streamEventsToEvents(device, s), + roomID: joinedRoomID, + }) + } + + // Get all the state events ever between these two positions + stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos) + if err != nil { + return nil, nil, err + } + state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap) + if err != nil { + return nil, nil, err + } + + for roomID, stateStreamEvents := range state { + for _, ev := range stateStreamEvents { + if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { + if membership != "join" { // We've already added full state for all joined rooms above. + deltas = append(deltas, stateDelta{ + membership: membership, + membershipPos: ev.streamPosition, + stateEvents: streamEventsToEvents(device, stateStreamEvents), + roomID: roomID, + }) + } + + break + } + } + } + + return deltas, joinedRoomIDs, nil +} + +func (d *SyncServerDatasource) currentStateStreamEventsForRoom( + ctx context.Context, txn *sql.Tx, roomID string, +) ([]streamEvent, error) { + allState, err := d.roomstate.selectCurrentState(ctx, txn, roomID) + if err != nil { + return nil, err + } + s := make([]streamEvent, len(allState)) + for i := 0; i < len(s); i++ { + s[i] = streamEvent{Event: allState[i], streamPosition: 0} + } + return s, nil +} + // streamEventsToEvents converts streamEvent to Event. If device is non-nil and // matches the streamevent.transactionID device then the transaction ID gets // added to the unsigned section of the output event. diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index a6ec6bd92..d773a4606 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -65,8 +65,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype currPos := rp.notifier.CurrentPosition() - // If this is an initial sync or timeout=0 we return immediately - if syncReq.since == nil || syncReq.timeout == 0 { + if shouldReturnImmediately(syncReq) { syncData, err = rp.currentSyncForUser(*syncReq, currPos) if err != nil { return httputil.LogThenError(req, err) @@ -135,7 +134,7 @@ func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.SyncP if req.since == nil { res, err = rp.db.CompleteSync(req.ctx, req.device.UserID, req.limit) } else { - res, err = rp.db.IncrementalSync(req.ctx, req.device, *req.since, latestPos, req.limit) + res, err = rp.db.IncrementalSync(req.ctx, req.device, *req.since, latestPos, req.limit, req.wantFullState) } if err != nil { @@ -216,3 +215,10 @@ func (rp *RequestPool) appendAccountData( return data, nil } + +// shouldReturnImmediately returns whether the /sync request is an initial sync, +// or timeout=0, or full_state=true, in any of the cases the request should +// return immediately. +func shouldReturnImmediately(syncReq *syncRequest) bool { + return syncReq.since == nil || syncReq.timeout == 0 || syncReq.wantFullState +} diff --git a/testfile b/testfile index c23dda662..4c5163e5e 100644 --- a/testfile +++ b/testfile @@ -158,3 +158,4 @@ Inbound federation rejects remote attempts to join local users to rooms Inbound federation rejects remote attempts to kick local users to rooms An event which redacts itself should be ignored A pair of events which redact each other should be ignored +Full state sync includes joined rooms From d283676b9ac5137e745dcc61587929a296359265 Mon Sep 17 00:00:00 2001 From: Sumukha Pk Date: Fri, 2 Aug 2019 16:47:51 +0530 Subject: [PATCH 11/15] Implements room tagging. (#694) --- clientapi/routing/room_tagging.go | 234 ++++++++++++++++++++++++++++++ clientapi/routing/routing.go | 30 ++++ testfile | 6 + 3 files changed, 270 insertions(+) create mode 100644 clientapi/routing/room_tagging.go diff --git a/clientapi/routing/room_tagging.go b/clientapi/routing/room_tagging.go new file mode 100644 index 000000000..6e7324cd8 --- /dev/null +++ b/clientapi/routing/room_tagging.go @@ -0,0 +1,234 @@ +// Copyright 2019 Sumukha PK +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "encoding/json" + "net/http" + + "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/gomatrix" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +// newTag creates and returns a new gomatrix.TagContent +func newTag() gomatrix.TagContent { + return gomatrix.TagContent{ + Tags: make(map[string]gomatrix.TagProperties), + } +} + +// GetTags implements GET /_matrix/client/r0/user/{userID}/rooms/{roomID}/tags +func GetTags( + req *http.Request, + accountDB *accounts.Database, + device *authtypes.Device, + userID string, + roomID string, + syncProducer *producers.SyncAPIProducer, +) util.JSONResponse { + + if device.UserID != userID { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("Cannot retrieve another user's tags"), + } + } + + _, data, err := obtainSavedTags(req, userID, roomID, accountDB) + if err != nil { + return httputil.LogThenError(req, err) + } + + if len(data) == 0 { + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: data[0].Content, + } +} + +// PutTag implements PUT /_matrix/client/r0/user/{userID}/rooms/{roomID}/tags/{tag} +// Put functionality works by getting existing data from the DB (if any), adding +// the tag to the "map" and saving the new "map" to the DB +func PutTag( + req *http.Request, + accountDB *accounts.Database, + device *authtypes.Device, + userID string, + roomID string, + tag string, + syncProducer *producers.SyncAPIProducer, +) util.JSONResponse { + + if device.UserID != userID { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("Cannot modify another user's tags"), + } + } + + var properties gomatrix.TagProperties + if reqErr := httputil.UnmarshalJSONRequest(req, &properties); reqErr != nil { + return *reqErr + } + + localpart, data, err := obtainSavedTags(req, userID, roomID, accountDB) + if err != nil { + return httputil.LogThenError(req, err) + } + + var tagContent gomatrix.TagContent + if len(data) > 0 { + if err = json.Unmarshal(data[0].Content, &tagContent); err != nil { + return httputil.LogThenError(req, err) + } + } else { + tagContent = newTag() + } + tagContent.Tags[tag] = properties + if err = saveTagData(req, localpart, roomID, accountDB, tagContent); err != nil { + return httputil.LogThenError(req, err) + } + + // Send data to syncProducer in order to inform clients of changes + // Run in a goroutine in order to prevent blocking the tag request response + go func() { + if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil { + logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi") + } + }() + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} + +// DeleteTag implements DELETE /_matrix/client/r0/user/{userID}/rooms/{roomID}/tags/{tag} +// Delete functionality works by obtaining the saved tags, removing the intended tag from +// the "map" and then saving the new "map" in the DB +func DeleteTag( + req *http.Request, + accountDB *accounts.Database, + device *authtypes.Device, + userID string, + roomID string, + tag string, + syncProducer *producers.SyncAPIProducer, +) util.JSONResponse { + + if device.UserID != userID { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("Cannot modify another user's tags"), + } + } + + localpart, data, err := obtainSavedTags(req, userID, roomID, accountDB) + if err != nil { + return httputil.LogThenError(req, err) + } + + // If there are no tags in the database, exit + if len(data) == 0 { + // Spec only defines 200 responses for this endpoint so we don't return anything else. + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } + } + + var tagContent gomatrix.TagContent + err = json.Unmarshal(data[0].Content, &tagContent) + if err != nil { + return httputil.LogThenError(req, err) + } + + // Check whether the tag to be deleted exists + if _, ok := tagContent.Tags[tag]; ok { + delete(tagContent.Tags, tag) + } else { + // Spec only defines 200 responses for this endpoint so we don't return anything else. + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } + } + if err = saveTagData(req, localpart, roomID, accountDB, tagContent); err != nil { + return httputil.LogThenError(req, err) + } + + // Send data to syncProducer in order to inform clients of changes + // Run in a goroutine in order to prevent blocking the tag request response + go func() { + if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil { + logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi") + } + }() + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} + +// obtainSavedTags gets all tags scoped to a userID and roomID +// from the database +func obtainSavedTags( + req *http.Request, + userID string, + roomID string, + accountDB *accounts.Database, +) (string, []gomatrixserverlib.ClientEvent, error) { + localpart, _, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + return "", nil, err + } + + data, err := accountDB.GetAccountDataByType( + req.Context(), localpart, roomID, "m.tag", + ) + + return localpart, data, err +} + +// saveTagData saves the provided tag data into the database +func saveTagData( + req *http.Request, + localpart string, + roomID string, + accountDB *accounts.Database, + Tag gomatrix.TagContent, +) error { + newTagData, err := json.Marshal(Tag) + if err != nil { + return err + } + + return accountDB.SaveAccountData(req.Context(), localpart, roomID, "m.tag", string(newTagData)) +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 8135e49af..ab8f89731 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -483,4 +483,34 @@ func Setup( }} }), ).Methods(http.MethodGet, http.MethodOptions) + + r0mux.Handle("/user/{userId}/rooms/{roomId}/tags", + common.MakeAuthAPI("get_tags", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetTags(req, accountDB, device, vars["userId"], vars["roomId"], syncProducer) + }), + ).Methods(http.MethodGet, http.MethodOptions) + + r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}", + common.MakeAuthAPI("put_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return PutTag(req, accountDB, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer) + }), + ).Methods(http.MethodPut, http.MethodOptions) + + r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}", + common.MakeAuthAPI("delete_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return DeleteTag(req, accountDB, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer) + }), + ).Methods(http.MethodDelete, http.MethodOptions) } diff --git a/testfile b/testfile index 4c5163e5e..1d97eb37e 100644 --- a/testfile +++ b/testfile @@ -159,3 +159,9 @@ Inbound federation rejects remote attempts to kick local users to rooms An event which redacts itself should be ignored A pair of events which redact each other should be ignored Full state sync includes joined rooms +Can add tag +Can remove tag +Can list tags for a room +Tags appear in an initial v2 /sync +Newly updated tags appear in an incremental v2 /sync +Deleted tags appear in an incremental v2 /sync From f8d2860765325c29d516f766002b46515b3e0a8c Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Tue, 6 Aug 2019 22:07:36 +0800 Subject: [PATCH 12/15] Replace membership and visibility values with constants (#774) Signed-off-by: Alex Chen --- clientapi/auth/storage/accounts/storage.go | 2 +- clientapi/routing/createroom.go | 14 ++++------ clientapi/routing/joinroom.go | 2 +- clientapi/routing/membership.go | 6 ++--- clientapi/routing/profile.go | 2 +- clientapi/routing/routing.go | 2 +- clientapi/threepid/invites.go | 2 +- cmd/create-room-events/main.go | 2 +- federationapi/routing/join.go | 2 +- federationapi/routing/leave.go | 4 +-- federationapi/routing/threepid.go | 2 +- federationsender/consumers/roomserver.go | 2 +- go.mod | 2 +- go.sum | 2 ++ publicroomsapi/directory/directory.go | 5 ++-- publicroomsapi/storage/storage.go | 2 +- roomserver/auth/auth.go | 2 +- roomserver/input/membership.go | 21 +++++---------- roomserver/query/query.go | 2 +- syncapi/storage/syncserver.go | 30 +++++++++------------- syncapi/sync/notifier.go | 8 +++--- 21 files changed, 51 insertions(+), 65 deletions(-) diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 5c8ffffeb..41d75daad 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -230,7 +230,7 @@ func (d *Database) newMembership( } // Only "join" membership events can be considered as new memberships - if membership == "join" { + if membership == gomatrixserverlib.Join { if err := d.saveMembership(ctx, txn, localpart, roomID, eventID); err != nil { return err } diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 8c5ee975c..4a76e1b06 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -55,10 +55,6 @@ const ( presetPublicChat = "public_chat" ) -const ( - joinRulePublic = "public" - joinRuleInvite = "invite" -) const ( historyVisibilityShared = "shared" // TODO: These should be implemented once history visibility is implemented @@ -201,7 +197,7 @@ func createRoom( } membershipContent := common.MemberContent{ - Membership: "join", + Membership: gomatrixserverlib.Join, DisplayName: profile.DisplayName, AvatarURL: profile.AvatarURL, } @@ -209,19 +205,19 @@ func createRoom( var joinRules, historyVisibility string switch r.Preset { case presetPrivateChat: - joinRules = joinRuleInvite + joinRules = gomatrixserverlib.Invite historyVisibility = historyVisibilityShared case presetTrustedPrivateChat: - joinRules = joinRuleInvite + joinRules = gomatrixserverlib.Invite historyVisibility = historyVisibilityShared // TODO If trusted_private_chat, all invitees are given the same power level as the room creator. case presetPublicChat: - joinRules = joinRulePublic + joinRules = gomatrixserverlib.Public historyVisibility = historyVisibilityShared default: // Default room rules, r.Preset was previously checked for valid values so // only a request with no preset should end up here. - joinRules = joinRuleInvite + joinRules = gomatrixserverlib.Invite historyVisibility = historyVisibilityShared } diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 9c02a93ca..432c982b4 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -70,7 +70,7 @@ func JoinRoomByIDOrAlias( return httputil.LogThenError(req, err) } - content["membership"] = "join" + content["membership"] = gomatrixserverlib.Join content["displayname"] = profile.DisplayName content["avatar_url"] = profile.AvatarURL diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 61898fecd..5e183fa0f 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -102,7 +102,7 @@ func SendMembership( var returnData interface{} = struct{}{} // The join membership requires the room id to be sent in the response - if membership == "join" { + if membership == gomatrixserverlib.Join { returnData = struct { RoomID string `json:"room_id"` }{roomID} @@ -141,7 +141,7 @@ func buildMembershipEvent( // "unban" or "kick" isn't a valid membership value, change it to "leave" if membership == "unban" || membership == "kick" { - membership = "leave" + membership = gomatrixserverlib.Leave } content := common.MemberContent{ @@ -192,7 +192,7 @@ func loadProfile( func getMembershipStateKey( body threepid.MembershipRequest, device *authtypes.Device, membership string, ) (stateKey string, reason string, err error) { - if membership == "ban" || membership == "unban" || membership == "kick" || membership == "invite" { + if membership == gomatrixserverlib.Ban || membership == "unban" || membership == "kick" || membership == gomatrixserverlib.Invite { // If we're in this case, the state key is contained in the request body, // possibly along with a reason (for "kick" and "ban") so we need to parse // it diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 034b9ac84..8d28b3660 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -264,7 +264,7 @@ func buildMembershipEvents( } content := common.MemberContent{ - Membership: "join", + Membership: gomatrixserverlib.Join, } content.DisplayName = newProfile.DisplayName diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index ab8f89731..c262db3d8 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -93,7 +93,7 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", - common.MakeAuthAPI("join", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI(gomatrixserverlib.Join, authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 2538577fd..251afb0d3 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -91,7 +91,7 @@ func CheckAndProcessInvite( producer *producers.RoomserverProducer, membership string, roomID string, evTime time.Time, ) (inviteStoredOnIDServer bool, err error) { - if membership != "invite" || (body.Address == "" && body.IDServer == "" && body.Medium == "") { + if membership != gomatrixserverlib.Invite || (body.Address == "" && body.IDServer == "" && body.Medium == "") { // If none of the 3PID-specific fields are supplied, it's a standard invite // so return nil for it to be processed as such return diff --git a/cmd/create-room-events/main.go b/cmd/create-room-events/main.go index 1d05b2a12..8475914f0 100644 --- a/cmd/create-room-events/main.go +++ b/cmd/create-room-events/main.go @@ -86,7 +86,7 @@ func main() { // Build a m.room.member event. b.Type = "m.room.member" b.StateKey = userID - b.SetContent(map[string]string{"membership": "join"}) // nolint: errcheck + b.SetContent(map[string]string{"membership": gomatrixserverlib.Join}) // nolint: errcheck b.AuthEvents = []gomatrixserverlib.EventReference{create} member := buildAndOutput() diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 0b60408f7..6f6574dd7 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -58,7 +58,7 @@ func MakeJoin( Type: "m.room.member", StateKey: &userID, } - err = builder.SetContent(map[string]interface{}{"membership": "join"}) + err = builder.SetContent(map[string]interface{}{"membership": gomatrixserverlib.Join}) if err != nil { return httputil.LogThenError(httpReq, err) } diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 3c57d39d1..a982b87f8 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -56,7 +56,7 @@ func MakeLeave( Type: "m.room.member", StateKey: &userID, } - err = builder.SetContent(map[string]interface{}{"membership": "leave"}) + err = builder.SetContent(map[string]interface{}{"membership": gomatrixserverlib.Leave}) if err != nil { return httputil.LogThenError(httpReq, err) } @@ -153,7 +153,7 @@ func SendLeave( mem, err := event.Membership() if err != nil { return httputil.LogThenError(httpReq, err) - } else if mem != "leave" { + } else if mem != gomatrixserverlib.Leave { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.BadJSON("The membership in the event content must be set to leave"), diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 05ca8892e..cff311cc4 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -202,7 +202,7 @@ func createInviteFrom3PIDInvite( content := common.MemberContent{ AvatarURL: profile.AvatarURL, DisplayName: profile.DisplayName, - Membership: "invite", + Membership: gomatrixserverlib.Invite, ThirdPartyInvite: &common.TPInvite{ Signed: inv.Signed, }, diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 45e48f166..3ba978b1d 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -233,7 +233,7 @@ func joinedHostsFromEvents(evs []gomatrixserverlib.Event) ([]types.JoinedHost, e if err != nil { return nil, err } - if membership != "join" { + if membership != gomatrixserverlib.Join { continue } _, serverName, err := gomatrixserverlib.SplitID('@', *ev.StateKey()) diff --git a/go.mod b/go.mod index 5d01012c9..8e14253ca 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6 + github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6 github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 github.com/matttproud/golang_protobuf_extensions v1.0.1 diff --git a/go.sum b/go.sum index 53151121b..0d59d1dd6 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2 h1:pY github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6 h1:B8n1H5Wb1B5jwLzTylBpY0kJCMRqrofT7PmOw4aJFJA= github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= +github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6 h1:xr69Hk6QM3RIN6JSvx3RpDowBGpHpDDqhqXCeySwYow= +github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 h1:p7WTwG+aXM86+yVrYAiCMW3ZHSmotVvuRbjtt3jC+4A= github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171013132526-8b1c8ab81986 h1:TiWl4hLvezAhRPM8tPcPDFTysZ7k4T/1J4GPp/iqlZo= diff --git a/publicroomsapi/directory/directory.go b/publicroomsapi/directory/directory.go index bb0153850..626a1c153 100644 --- a/publicroomsapi/directory/directory.go +++ b/publicroomsapi/directory/directory.go @@ -19,6 +19,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/publicroomsapi/storage" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -39,7 +40,7 @@ func GetVisibility( var v roomVisibility if isPublic { - v.Visibility = "public" + v.Visibility = gomatrixserverlib.Public } else { v.Visibility = "private" } @@ -61,7 +62,7 @@ func SetVisibility( return *reqErr } - isPublic := v.Visibility == "public" + isPublic := v.Visibility == gomatrixserverlib.Public if err := publicRoomsDatabase.SetRoomVisibility(req.Context(), isPublic, roomID); err != nil { return httputil.LogThenError(req, err) } diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go index eab27041b..aa9806945 100644 --- a/publicroomsapi/storage/storage.go +++ b/publicroomsapi/storage/storage.go @@ -185,7 +185,7 @@ func (d *PublicRoomsServerDatabase) updateNumJoinedUsers( return err } - if membership != "join" { + if membership != gomatrixserverlib.Join { return nil } diff --git a/roomserver/auth/auth.go b/roomserver/auth/auth.go index 2dce6f6dc..5ff1fadad 100644 --- a/roomserver/auth/auth.go +++ b/roomserver/auth/auth.go @@ -23,7 +23,7 @@ func IsServerAllowed( ) bool { for _, ev := range authEvents { membership, err := ev.Membership() - if err != nil || membership != "join" { + if err != nil || membership != gomatrixserverlib.Join { continue } diff --git a/roomserver/input/membership.go b/roomserver/input/membership.go index 0c3fbb80a..841c5fec6 100644 --- a/roomserver/input/membership.go +++ b/roomserver/input/membership.go @@ -23,13 +23,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// Membership values -// TODO: Factor these out somewhere sensible? -const join = "join" -const leave = "leave" -const invite = "invite" -const ban = "ban" - // updateMembership updates the current membership and the invites for each // user affected by a change in the current state of the room. // Returns a list of output events to write to the kafka log to inform the @@ -91,8 +84,8 @@ func updateMembership( ) ([]api.OutputEvent, error) { var err error // Default the membership to Leave if no event was added or removed. - oldMembership := leave - newMembership := leave + oldMembership := gomatrixserverlib.Leave + newMembership := gomatrixserverlib.Leave if remove != nil { oldMembership, err = remove.Membership() @@ -106,7 +99,7 @@ func updateMembership( return nil, err } } - if oldMembership == newMembership && newMembership != join { + if oldMembership == newMembership && newMembership != gomatrixserverlib.Join { // If the membership is the same then nothing changed and we can return // immediately, unless it's a Join update (e.g. profile update). return updates, nil @@ -118,11 +111,11 @@ func updateMembership( } switch newMembership { - case invite: + case gomatrixserverlib.Invite: return updateToInviteMembership(mu, add, updates) - case join: + case gomatrixserverlib.Join: return updateToJoinMembership(mu, add, updates) - case leave, ban: + case gomatrixserverlib.Leave, gomatrixserverlib.Ban: return updateToLeaveMembership(mu, add, newMembership, updates) default: panic(fmt.Errorf( @@ -183,7 +176,7 @@ func updateToJoinMembership( for _, eventID := range retired { orie := api.OutputRetireInviteEvent{ EventID: eventID, - Membership: join, + Membership: gomatrixserverlib.Join, RetiredByEventID: add.EventID(), TargetUserID: *add.StateKey(), } diff --git a/roomserver/query/query.go b/roomserver/query/query.go index b97d50b17..a62a1f706 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -359,7 +359,7 @@ func (r *RoomserverQueryAPI) getMembershipsBeforeEventNID( return nil, err } - if membership == "join" { + if membership == gomatrixserverlib.Join { events = append(events, event) } } diff --git a/syncapi/storage/syncserver.go b/syncapi/storage/syncserver.go index e914bddfe..ebec6c3e1 100644 --- a/syncapi/storage/syncserver.go +++ b/syncapi/storage/syncserver.go @@ -35,12 +35,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -const ( - membershipJoin = "join" - membershipLeave = "leave" - membershipBan = "ban" -) - type stateDelta struct { roomID string stateEvents []gomatrixserverlib.Event @@ -356,7 +350,7 @@ func (d *SyncServerDatasource) IncrementalSync( ) } else { joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership( - ctx, nil, device.UserID, membershipJoin, + ctx, nil, device.UserID, gomatrixserverlib.Join, ) } if err != nil { @@ -405,7 +399,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( res = types.NewResponse(toPos) // Extract room state and recent events for all rooms the user is joined to. - joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, membershipJoin) + joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) if err != nil { return } @@ -583,7 +577,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( res *types.Response, ) error { endPos := toPos - if delta.membershipPos > 0 && delta.membership == membershipLeave { + if delta.membershipPos > 0 && delta.membership == gomatrixserverlib.Leave { // make sure we don't leak recent events after the leave event. // TODO: History visibility makes this somewhat complex to handle correctly. For example: // TODO: This doesn't work for join -> leave in a single /sync request (see events prior to join). @@ -621,7 +615,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( } switch delta.membership { - case membershipJoin: + case gomatrixserverlib.Join: jr := types.NewJoinResponse() // Use the short form of batch token for prev_batch jr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) @@ -629,9 +623,9 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true jr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[delta.roomID] = *jr - case membershipLeave: + case gomatrixserverlib.Leave: fallthrough // transitions to leave are the same as ban - case membershipBan: + case gomatrixserverlib.Ban: // TODO: recentEvents may contain events that this user is not allowed to see because they are // no longer in the room. lr := types.NewLeaveResponse() @@ -768,7 +762,7 @@ func (d *SyncServerDatasource) getStateDeltas( // the 'state' part of the response though, so is transparent modulo bandwidth concerns as it is not added to // the timeline. if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { - if membership == membershipJoin { + if membership == gomatrixserverlib.Join { // send full room state down instead of a delta var s []streamEvent s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID) @@ -791,13 +785,13 @@ func (d *SyncServerDatasource) getStateDeltas( } // Add in currently joined rooms - joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, membershipJoin) + joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) if err != nil { return nil, nil, err } for _, joinedRoomID := range joinedRoomIDs { deltas = append(deltas, stateDelta{ - membership: membershipJoin, + membership: gomatrixserverlib.Join, stateEvents: streamEventsToEvents(device, state[joinedRoomID]), roomID: joinedRoomID, }) @@ -814,7 +808,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( ctx context.Context, device *authtypes.Device, txn *sql.Tx, fromPos, toPos int64, userID string, ) ([]stateDelta, []string, error) { - joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join") + joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) if err != nil { return nil, nil, err } @@ -829,7 +823,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( return nil, nil, stateErr } deltas = append(deltas, stateDelta{ - membership: "join", + membership: gomatrixserverlib.Join, stateEvents: streamEventsToEvents(device, s), roomID: joinedRoomID, }) @@ -848,7 +842,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( for roomID, stateStreamEvents := range state { for _, ev := range stateStreamEvents { if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { - if membership != "join" { // We've already added full state for all joined rooms above. + if membership != gomatrixserverlib.Join { // We've already added full state for all joined rooms above. deltas = append(deltas, stateDelta{ membership: membership, membershipPos: ev.streamPosition, diff --git a/syncapi/sync/notifier.go b/syncapi/sync/notifier.go index 14bc2efb6..15d6b070c 100644 --- a/syncapi/sync/notifier.go +++ b/syncapi/sync/notifier.go @@ -93,16 +93,16 @@ func (n *Notifier) OnNewEvent( } else { // Keep the joined user map up-to-date switch membership { - case "invite": + case gomatrixserverlib.Invite: usersToNotify = append(usersToNotify, targetUserID) - case "join": + case gomatrixserverlib.Join: // Manually append the new user's ID so they get notified // along all members in the room usersToNotify = append(usersToNotify, targetUserID) n.addJoinedUser(ev.RoomID(), targetUserID) - case "leave": + case gomatrixserverlib.Leave: fallthrough - case "ban": + case gomatrixserverlib.Ban: n.removeJoinedUser(ev.RoomID(), targetUserID) } } From 83f8e05032b82e2dd85d93e77643139bb3231ba2 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 6 Aug 2019 15:52:04 +0100 Subject: [PATCH 13/15] Add /event/ on non world readable room does not work (#777) --- testfile | 1 + 1 file changed, 1 insertion(+) diff --git a/testfile b/testfile index 1d97eb37e..81e47780f 100644 --- a/testfile +++ b/testfile @@ -165,3 +165,4 @@ Can list tags for a room Tags appear in an initial v2 /sync Newly updated tags appear in an incremental v2 /sync Deleted tags appear in an incremental v2 /sync +/event/ on non world readable room does not work From 66bf615360cfe6eac11e901ffe34f70f97330a22 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Tue, 6 Aug 2019 23:33:53 +0800 Subject: [PATCH 14/15] Fix transaction IDs in transaction cache have global scope (#772) --- clientapi/routing/sendevent.go | 4 +-- common/transactions/transactions.go | 21 +++++++++----- common/transactions/transactions_test.go | 35 +++++++++++++++++++++--- testfile | 1 + 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index e916e451e..9696b360e 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -50,7 +50,7 @@ func SendEvent( ) util.JSONResponse { if txnID != nil { // Try to fetch response from transactionsCache - if res, ok := txnCache.FetchTransaction(*txnID); ok { + if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok { return *res } } @@ -83,7 +83,7 @@ func SendEvent( } // Add response to transactionsCache if txnID != nil { - txnCache.AddTransaction(*txnID, &res) + txnCache.AddTransaction(device.AccessToken, *txnID, &res) } return res diff --git a/common/transactions/transactions.go b/common/transactions/transactions.go index febcb9a75..80b403a98 100644 --- a/common/transactions/transactions.go +++ b/common/transactions/transactions.go @@ -22,7 +22,14 @@ import ( // DefaultCleanupPeriod represents the default time duration after which cacheCleanService runs. const DefaultCleanupPeriod time.Duration = 30 * time.Minute -type txnsMap map[string]*util.JSONResponse +type txnsMap map[CacheKey]*util.JSONResponse + +// CacheKey is the type for the key in a transactions cache. +// This is needed because the spec requires transaction IDs to have a per-access token scope. +type CacheKey struct { + AccessToken string + TxnID string +} // Cache represents a temporary store for response entries. // Entries are evicted after a certain period, defined by cleanupPeriod. @@ -50,14 +57,14 @@ func NewWithCleanupPeriod(cleanupPeriod time.Duration) *Cache { return &t } -// FetchTransaction looks up an entry for txnID in Cache. +// FetchTransaction looks up an entry for the (accessToken, txnID) tuple in Cache. // Looks in both the txnMaps. // Returns (JSON response, true) if txnID is found, else the returned bool is false. -func (t *Cache) FetchTransaction(txnID string) (*util.JSONResponse, bool) { +func (t *Cache) FetchTransaction(accessToken, txnID string) (*util.JSONResponse, bool) { t.RLock() defer t.RUnlock() for _, txns := range t.txnsMaps { - res, ok := txns[txnID] + res, ok := txns[CacheKey{accessToken, txnID}] if ok { return res, true } @@ -65,13 +72,13 @@ func (t *Cache) FetchTransaction(txnID string) (*util.JSONResponse, bool) { return nil, false } -// AddTransaction adds an entry for txnID in Cache for later access. +// AddTransaction adds an entry for the (accessToken, txnID) tuple in Cache. // Adds to the front txnMap. -func (t *Cache) AddTransaction(txnID string, res *util.JSONResponse) { +func (t *Cache) AddTransaction(accessToken, txnID string, res *util.JSONResponse) { t.Lock() defer t.Unlock() - t.txnsMaps[0][txnID] = res + t.txnsMaps[0][CacheKey{accessToken, txnID}] = res } // cacheCleanService is responsible for cleaning up entries after cleanupPeriod. diff --git a/common/transactions/transactions_test.go b/common/transactions/transactions_test.go index 0cdb776cc..f565e4846 100644 --- a/common/transactions/transactions_test.go +++ b/common/transactions/transactions_test.go @@ -24,27 +24,54 @@ type fakeType struct { } var ( - fakeTxnID = "aRandomTxnID" - fakeResponse = &util.JSONResponse{Code: http.StatusOK, JSON: fakeType{ID: "0"}} + fakeAccessToken = "aRandomAccessToken" + fakeAccessToken2 = "anotherRandomAccessToken" + fakeTxnID = "aRandomTxnID" + fakeResponse = &util.JSONResponse{ + Code: http.StatusOK, JSON: fakeType{ID: "0"}, + } + fakeResponse2 = &util.JSONResponse{ + Code: http.StatusOK, JSON: fakeType{ID: "1"}, + } ) // TestCache creates a New Cache and tests AddTransaction & FetchTransaction func TestCache(t *testing.T) { fakeTxnCache := New() - fakeTxnCache.AddTransaction(fakeTxnID, fakeResponse) + fakeTxnCache.AddTransaction(fakeAccessToken, fakeTxnID, fakeResponse) // Add entries for noise. for i := 1; i <= 100; i++ { fakeTxnCache.AddTransaction( + fakeAccessToken, fakeTxnID+string(i), &util.JSONResponse{Code: http.StatusOK, JSON: fakeType{ID: string(i)}}, ) } - testResponse, ok := fakeTxnCache.FetchTransaction(fakeTxnID) + testResponse, ok := fakeTxnCache.FetchTransaction(fakeAccessToken, fakeTxnID) if !ok { t.Error("Failed to retrieve entry for txnID: ", fakeTxnID) } else if testResponse.JSON != fakeResponse.JSON { t.Error("Fetched response incorrect. Expected: ", fakeResponse.JSON, " got: ", testResponse.JSON) } } + +// TestCacheScope ensures transactions with the same transaction ID are not shared +// across multiple access tokens. +func TestCacheScope(t *testing.T) { + cache := New() + cache.AddTransaction(fakeAccessToken, fakeTxnID, fakeResponse) + cache.AddTransaction(fakeAccessToken2, fakeTxnID, fakeResponse2) + + if res, ok := cache.FetchTransaction(fakeAccessToken, fakeTxnID); !ok { + t.Errorf("failed to retrieve entry for (%s, %s)", fakeAccessToken, fakeTxnID) + } else if res.JSON != fakeResponse.JSON { + t.Errorf("Wrong cache entry for (%s, %s). Expected: %v; got: %v", fakeAccessToken, fakeTxnID, fakeResponse.JSON, res.JSON) + } + if res, ok := cache.FetchTransaction(fakeAccessToken2, fakeTxnID); !ok { + t.Errorf("failed to retrieve entry for (%s, %s)", fakeAccessToken, fakeTxnID) + } else if res.JSON != fakeResponse2.JSON { + t.Errorf("Wrong cache entry for (%s, %s). Expected: %v; got: %v", fakeAccessToken, fakeTxnID, fakeResponse2.JSON, res.JSON) + } +} diff --git a/testfile b/testfile index 81e47780f..5791938f7 100644 --- a/testfile +++ b/testfile @@ -159,6 +159,7 @@ Inbound federation rejects remote attempts to kick local users to rooms An event which redacts itself should be ignored A pair of events which redact each other should be ignored Full state sync includes joined rooms +A message sent after an initial sync appears in the timeline of an incremental sync. Can add tag Can remove tag Can list tags for a room From 324ca22b358d53339b2971ec85a02de7282604e6 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 7 Aug 2019 00:02:12 +0800 Subject: [PATCH 15/15] Implement profile retrieval over federation (#726) --- appservice/api/query.go | 4 +- clientapi/auth/authtypes/profile.go | 2 +- clientapi/routing/profile.go | 107 +++++++++++++++++++++++----- clientapi/routing/routing.go | 6 +- common/types.go | 5 ++ testfile | 1 + 6 files changed, 100 insertions(+), 25 deletions(-) diff --git a/appservice/api/query.go b/appservice/api/query.go index 8ce3b4e04..9542df565 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -20,13 +20,13 @@ package api import ( "context" "database/sql" - "errors" "net/http" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/common" commonHTTP "github.com/matrix-org/dendrite/common/http" opentracing "github.com/opentracing/opentracing-go" ) @@ -164,7 +164,7 @@ func RetrieveUserProfile( // If no user exists, return if !userResp.UserIDExists { - return nil, errors.New("no known profile for given user ID") + return nil, common.ErrProfileNoExists } // Try to query the user from the local database again diff --git a/clientapi/auth/authtypes/profile.go b/clientapi/auth/authtypes/profile.go index 6cf508f4f..0bc49658b 100644 --- a/clientapi/auth/authtypes/profile.go +++ b/clientapi/auth/authtypes/profile.go @@ -14,7 +14,7 @@ package authtypes -// Profile represents the profile for a Matrix account on this home server. +// Profile represents the profile for a Matrix account. type Profile struct { Localpart string DisplayName string diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 8d28b3660..e8ea6cf13 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -30,43 +30,61 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrix" "github.com/matrix-org/util" ) // GetProfile implements GET /profile/{userID} func GetProfile( - req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI, + req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite, + userID string, + asAPI appserviceAPI.AppServiceQueryAPI, + federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { - profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { + if err == common.ErrProfileNoExists { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), + } + } + return httputil.LogThenError(req, err) } - res := common.ProfileResponse{ - AvatarURL: profile.AvatarURL, - DisplayName: profile.DisplayName, - } return util.JSONResponse{ Code: http.StatusOK, - JSON: res, + JSON: common.ProfileResponse{ + AvatarURL: profile.AvatarURL, + DisplayName: profile.DisplayName, + }, } } // GetAvatarURL implements GET /profile/{userID}/avatar_url func GetAvatarURL( - req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI, + req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite, + userID string, asAPI appserviceAPI.AppServiceQueryAPI, + federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { - profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { + if err == common.ErrProfileNoExists { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), + } + } + return httputil.LogThenError(req, err) } - res := common.AvatarURL{ - AvatarURL: profile.AvatarURL, - } return util.JSONResponse{ Code: http.StatusOK, - JSON: res, + JSON: common.AvatarURL{ + AvatarURL: profile.AvatarURL, + }, } } @@ -152,18 +170,27 @@ func SetAvatarURL( // GetDisplayName implements GET /profile/{userID}/displayname func GetDisplayName( - req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI, + req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite, + userID string, asAPI appserviceAPI.AppServiceQueryAPI, + federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { - profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { + if err == common.ErrProfileNoExists { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), + } + } + return httputil.LogThenError(req, err) } - res := common.DisplayName{ - DisplayName: profile.DisplayName, - } + return util.JSONResponse{ Code: http.StatusOK, - JSON: res, + JSON: common.DisplayName{ + DisplayName: profile.DisplayName, + }, } } @@ -247,6 +274,48 @@ func SetDisplayName( } } +// getProfile gets the full profile of a user by querying the database or a +// remote homeserver. +// Returns an error when something goes wrong or specifically +// common.ErrProfileNoExists when the profile doesn't exist. +func getProfile( + ctx context.Context, accountDB *accounts.Database, cfg *config.Dendrite, + userID string, + asAPI appserviceAPI.AppServiceQueryAPI, + federation *gomatrixserverlib.FederationClient, +) (*authtypes.Profile, error) { + localpart, domain, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + return nil, err + } + + if domain != cfg.Matrix.ServerName { + profile, fedErr := federation.LookupProfile(ctx, domain, userID, "") + if fedErr != nil { + if x, ok := fedErr.(gomatrix.HTTPError); ok { + if x.Code == http.StatusNotFound { + return nil, common.ErrProfileNoExists + } + } + + return nil, fedErr + } + + return &authtypes.Profile{ + Localpart: localpart, + DisplayName: profile.DisplayName, + AvatarURL: profile.AvatarURL, + }, nil + } + + profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB) + if err != nil { + return nil, err + } + + return profile, nil +} + func buildMembershipEvents( ctx context.Context, memberships []authtypes.Membership, diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index c262db3d8..825dd97aa 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -283,7 +283,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetProfile(req, accountDB, vars["userID"], asAPI) + return GetProfile(req, accountDB, &cfg, vars["userID"], asAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -293,7 +293,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetAvatarURL(req, accountDB, vars["userID"], asAPI) + return GetAvatarURL(req, accountDB, &cfg, vars["userID"], asAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -315,7 +315,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetDisplayName(req, accountDB, vars["userID"], asAPI) + return GetDisplayName(req, accountDB, &cfg, vars["userID"], asAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) diff --git a/common/types.go b/common/types.go index 6888d3806..91765be00 100644 --- a/common/types.go +++ b/common/types.go @@ -15,9 +15,14 @@ package common import ( + "errors" "strconv" ) +// ErrProfileNoExists is returned when trying to lookup a user's profile that +// doesn't exist locally. +var ErrProfileNoExists = errors.New("no known profile for given user ID") + // AccountData represents account data sent from the client API server to the // sync API server type AccountData struct { diff --git a/testfile b/testfile index 5791938f7..74c9d9e4f 100644 --- a/testfile +++ b/testfile @@ -167,3 +167,4 @@ Tags appear in an initial v2 /sync Newly updated tags appear in an incremental v2 /sync Deleted tags appear in an incremental v2 /sync /event/ on non world readable room does not work +Outbound federation can query profile data