From 92ceb46b4996330326aaf0cb99c3fdd523b9afba Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 2 Oct 2020 09:43:02 +0100 Subject: [PATCH 1/5] Update to matrix-org/gomatrixserverlib#227 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b4e16c0c4..b1119f608 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae + github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index 461b3d3c0..c657cc457 100644 --- a/go.sum +++ b/go.sum @@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae h1:9vxYG3+pXD3G74lZ8s67/afs2RkvyrJy6HEcqfHEVgw= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290 h1:ilT9QNIh2KXfvzIALtAe31IvLVZH7mVjVtOOTxdd0tY= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= From 28454d6fb712d69e299660bef098be0be45c9475 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 2 Oct 2020 11:38:35 +0100 Subject: [PATCH 2/5] Log origin in /send --- federationapi/routing/send.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index d4bbe31c6..e2ab9b334 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -81,7 +81,7 @@ func Send( t.TransactionID = txnID t.Destination = cfg.Matrix.ServerName - util.GetLogger(httpReq.Context()).Infof("Received transaction %q containing %d PDUs, %d EDUs", txnID, len(t.PDUs), len(t.EDUs)) + util.GetLogger(httpReq.Context()).Infof("Received transaction %q from %q containing %d PDUs, %d EDUs", txnID, request.Origin(), len(t.PDUs), len(t.EDUs)) resp, jsonErr := t.processTransaction(httpReq.Context()) if jsonErr != nil { From 1b29e5771feb335afe0e39e7f33f1d1896dbdb2f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 2 Oct 2020 11:49:13 +0100 Subject: [PATCH 3/5] Fix build.sh --- build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.sh b/build.sh index f790077e3..31e0519f5 100755 --- a/build.sh +++ b/build.sh @@ -7,7 +7,10 @@ if [ -d ".git" ] then export BUILD=`git rev-parse --short HEAD || ""` export BRANCH=`(git symbolic-ref --short HEAD | tr -d \/ ) || ""` - [[ $BRANCH == "master" ]] && export BRANCH="" + if [[ $BRANCH == "master" ]] + then + export BRANCH="" + fi export FLAGS="-X github.com/matrix-org/dendrite/internal.branch=$BRANCH -X github.com/matrix-org/dendrite/internal.build=$BUILD" else From fb9a8f215b0af25c02d62c6f44ed91adc3349a6e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 2 Oct 2020 12:50:58 +0100 Subject: [PATCH 4/5] Fix initial sync (#1465) * Fix complete sync check * Remove unnecessary 'since' copy * Fix failing test * Un-whitelist a couple of tests Co-authored-by: Kegan Dougal --- syncapi/storage/shared/syncserver.go | 17 +++++++++-------- syncapi/sync/requestpool.go | 13 ++++--------- sytest-whitelist | 2 -- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 05a8768e8..edb51b347 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -670,7 +670,7 @@ func (d *Database) RedactEvent(ctx context.Context, redactedEventID string, reda // nolint:nakedret func (d *Database) getResponseWithPDUsForCompleteSync( ctx context.Context, res *types.Response, - userID string, deviceID string, + userID string, device userapi.Device, numRecentEventsPerRoom int, ) ( toPos types.StreamingToken, @@ -712,7 +712,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync( for _, roomID := range joinedRoomIDs { var jr *types.JoinResponse jr, err = d.getJoinResponseForCompleteSync( - ctx, txn, roomID, r, &stateFilter, numRecentEventsPerRoom, + ctx, txn, roomID, r, &stateFilter, numRecentEventsPerRoom, device, ) if err != nil { return @@ -721,7 +721,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync( } // Add peeked rooms. - peeks, err := d.Peeks.SelectPeeksInRange(ctx, txn, userID, deviceID, r) + peeks, err := d.Peeks.SelectPeeksInRange(ctx, txn, userID, device.ID, r) if err != nil { return } @@ -729,7 +729,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync( if !peek.Deleted { var jr *types.JoinResponse jr, err = d.getJoinResponseForCompleteSync( - ctx, txn, peek.RoomID, r, &stateFilter, numRecentEventsPerRoom, + ctx, txn, peek.RoomID, r, &stateFilter, numRecentEventsPerRoom, device, ) if err != nil { return @@ -751,7 +751,7 @@ func (d *Database) getJoinResponseForCompleteSync( roomID string, r types.Range, stateFilter *gomatrixserverlib.StateFilter, - numRecentEventsPerRoom int, + numRecentEventsPerRoom int, device userapi.Device, ) (jr *types.JoinResponse, err error) { var stateEvents []gomatrixserverlib.HeaderedEvent stateEvents, err = d.CurrentRoomState.SelectCurrentState(ctx, txn, roomID, stateFilter) @@ -784,8 +784,9 @@ func (d *Database) getJoinResponseForCompleteSync( } // We don't include a device here as we don't need to send down - // transaction IDs for complete syncs - recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents) + // transaction IDs for complete syncs, but we do it anyway because Sytest demands it for: + // "Can sync a room with a message with a transaction id" - which does a complete sync to check. + recentEvents := d.StreamEventsToEvents(&device, recentStreamEvents) stateEvents = removeDuplicates(stateEvents, recentEvents) jr = types.NewJoinResponse() jr.Timeline.PrevBatch = prevBatchStr @@ -800,7 +801,7 @@ func (d *Database) CompleteSync( device userapi.Device, numRecentEventsPerRoom int, ) (*types.Response, error) { toPos, joinedRoomIDs, err := d.getResponseWithPDUsForCompleteSync( - ctx, res, device.UserID, device.ID, numRecentEventsPerRoom, + ctx, res, device.UserID, device, numRecentEventsPerRoom, ) if err != nil { return nil, fmt.Errorf("d.getResponseWithPDUsForCompleteSync: %w", err) diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index aaaf94917..8a79737aa 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -197,19 +197,14 @@ func (rp *RequestPool) OnIncomingKeyChangeRequest(req *http.Request, device *use func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.StreamingToken) (*types.Response, error) { res := types.NewResponse() - since := types.NewStreamToken(0, 0, nil) - if req.since != nil { - since = *req.since - } - // See if we have any new tasks to do for the send-to-device messaging. - events, updates, deletions, err := rp.db.SendToDeviceUpdatesForSync(req.ctx, req.device.UserID, req.device.ID, since) + events, updates, deletions, err := rp.db.SendToDeviceUpdatesForSync(req.ctx, req.device.UserID, req.device.ID, *req.since) if err != nil { return nil, fmt.Errorf("rp.db.SendToDeviceUpdatesForSync: %w", err) } // TODO: handle ignored users - if req.since == nil { + if req.since.PDUPosition() == 0 && req.since.EDUPosition() == 0 { res, err = rp.db.CompleteSync(req.ctx, res, req.device, req.limit) if err != nil { return res, fmt.Errorf("rp.db.CompleteSync: %w", err) @@ -226,7 +221,7 @@ func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.Strea if err != nil { return res, fmt.Errorf("rp.appendAccountData: %w", err) } - res, err = rp.appendDeviceLists(res, req.device.UserID, since, latestPos) + res, err = rp.appendDeviceLists(res, req.device.UserID, *req.since, latestPos) if err != nil { return res, fmt.Errorf("rp.appendDeviceLists: %w", err) } @@ -240,7 +235,7 @@ func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.Strea // Then add the updates into the sync response. if len(updates) > 0 || len(deletions) > 0 { // Handle the updates and deletions in the database. - err = rp.db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, since) + err = rp.db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, *req.since) if err != nil { return res, fmt.Errorf("rp.db.CleanSendToDeviceUpdates: %w", err) } diff --git a/sytest-whitelist b/sytest-whitelist index 29e9166ae..df71e275f 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -396,8 +396,6 @@ GET /rooms/:room_id/state fetches entire room state Setting room topic reports m.room.topic to myself setting 'm.room.name' respects room powerlevel Syncing a new room with a large timeline limit isn't limited -Left rooms appear in the leave section of sync -Banned rooms appear in the leave section of sync Getting state checks the events requested belong to the room Getting state IDs checks the events requested belong to the room Can invite users to invite-only rooms From 439bc235d7855d7efe7aed864eef7c6ee9eef88c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFck=20Bonniot?= Date: Fri, 2 Oct 2020 16:05:23 +0200 Subject: [PATCH 5/5] Add 'completed' field in Interactive Auth API (#1469) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This field must be present even after authentication failure, as tested by sytest. This is needed by #1455. Signed-off-by: Loïck Bonniot --- clientapi/auth/user_interactive.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/clientapi/auth/user_interactive.go b/clientapi/auth/user_interactive.go index c67eba150..b7414ebe9 100644 --- a/clientapi/auth/user_interactive.go +++ b/clientapi/auth/user_interactive.go @@ -103,7 +103,8 @@ type userInteractiveFlow struct { // the user already has a valid access token, but we want to double-check // that it isn't stolen by re-authenticating them. type UserInteractive struct { - Flows []userInteractiveFlow + Completed []string + Flows []userInteractiveFlow // Map of login type to implementation Types map[string]Type // Map of session ID to completed login types, will need to be extended in future @@ -117,6 +118,7 @@ func NewUserInteractive(getAccByPass GetAccountByPassword, cfg *config.ClientAPI } // TODO: Add SSO login return &UserInteractive{ + Completed: []string{}, Flows: []userInteractiveFlow{ { Stages: []string{typePassword.Name()}, @@ -140,6 +142,7 @@ func (u *UserInteractive) IsSingleStageFlow(authType string) bool { func (u *UserInteractive) AddCompletedStage(sessionID, authType string) { // TODO: Handle multi-stage flows + u.Completed = append(u.Completed, authType) delete(u.Sessions, sessionID) } @@ -148,11 +151,13 @@ func (u *UserInteractive) Challenge(sessionID string) *util.JSONResponse { return &util.JSONResponse{ Code: 401, JSON: struct { - Flows []userInteractiveFlow `json:"flows"` - Session string `json:"session"` + Completed []string `json:"completed"` + Flows []userInteractiveFlow `json:"flows"` + Session string `json:"session"` // TODO: Return any additional `params` Params map[string]interface{} `json:"params"` }{ + u.Completed, u.Flows, sessionID, make(map[string]interface{}),