Merge branch 'master' of https://github.com/matrix-org/dendrite into add-last-seen

build.sh

@@ -7,7 +7,10 @@ if [ -d ".git" ]
 then
     export BUILD=`git rev-parse --short HEAD || ""`
     export BRANCH=`(git symbolic-ref --short HEAD | tr -d \/ ) || ""`
-    [[ $BRANCH == "master" ]] && export BRANCH=""
+    if [[ $BRANCH == "master" ]]
+    then
+        export BRANCH=""
+    fi
 
     export FLAGS="-X github.com/matrix-org/dendrite/internal.branch=$BRANCH -X github.com/matrix-org/dendrite/internal.build=$BUILD"
 else

clientapi/auth/user_interactive.go

@@ -103,7 +103,8 @@ type userInteractiveFlow struct {
 // the user already has a valid access token, but we want to double-check
 // that it isn't stolen by re-authenticating them.
 type UserInteractive struct {
-	Flows []userInteractiveFlow
+	Completed []string
+	Flows     []userInteractiveFlow
 	// Map of login type to implementation
 	Types map[string]Type
 	// Map of session ID to completed login types, will need to be extended in future
@@ -117,6 +118,7 @@ func NewUserInteractive(getAccByPass GetAccountByPassword, cfg *config.ClientAPI
 	}
 	// TODO: Add SSO login
 	return &UserInteractive{
+		Completed: []string{},
 		Flows: []userInteractiveFlow{
 			{
 				Stages: []string{typePassword.Name()},
@@ -140,6 +142,7 @@ func (u *UserInteractive) IsSingleStageFlow(authType string) bool {
 
 func (u *UserInteractive) AddCompletedStage(sessionID, authType string) {
 	// TODO: Handle multi-stage flows
+	u.Completed = append(u.Completed, authType)
 	delete(u.Sessions, sessionID)
 }
 
@@ -148,11 +151,13 @@ func (u *UserInteractive) Challenge(sessionID string) *util.JSONResponse {
 	return &util.JSONResponse{
 		Code: 401,
 		JSON: struct {
-			Flows   []userInteractiveFlow `json:"flows"`
-			Session string                `json:"session"`
+			Completed []string              `json:"completed"`
+			Flows     []userInteractiveFlow `json:"flows"`
+			Session   string                `json:"session"`
 			// TODO: Return any additional `params`
 			Params map[string]interface{} `json:"params"`
 		}{
+			u.Completed,
 			u.Flows,
 			sessionID,
 			make(map[string]interface{}),

federationapi/routing/send.go

@@ -81,7 +81,7 @@ func Send(
 	t.TransactionID = txnID
 	t.Destination = cfg.Matrix.ServerName
 
-	util.GetLogger(httpReq.Context()).Infof("Received transaction %q containing %d PDUs, %d EDUs", txnID, len(t.PDUs), len(t.EDUs))
+	util.GetLogger(httpReq.Context()).Infof("Received transaction %q from %q containing %d PDUs, %d EDUs", txnID, request.Origin(), len(t.PDUs), len(t.EDUs))
 
 	resp, jsonErr := t.processTransaction(httpReq.Context())
 	if jsonErr != nil {

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3
 	github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290
 	github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.2

go.sum

@@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg=
 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae h1:9vxYG3+pXD3G74lZ8s67/afs2RkvyrJy6HEcqfHEVgw=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20201001100250-2bfdd2727cae/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290 h1:ilT9QNIh2KXfvzIALtAe31IvLVZH7mVjVtOOTxdd0tY=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20201002084023-8bcafefa3290/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4=
 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE=
 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=

syncapi/storage/shared/syncserver.go

@@ -670,7 +670,7 @@ func (d *Database) RedactEvent(ctx context.Context, redactedEventID string, reda
 // nolint:nakedret
 func (d *Database) getResponseWithPDUsForCompleteSync(
 	ctx context.Context, res *types.Response,
-	userID string, deviceID string,
+	userID string, device userapi.Device,
 	numRecentEventsPerRoom int,
 ) (
 	toPos types.StreamingToken,
@@ -712,7 +712,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync(
 	for _, roomID := range joinedRoomIDs {
 		var jr *types.JoinResponse
 		jr, err = d.getJoinResponseForCompleteSync(
-			ctx, txn, roomID, r, &stateFilter, numRecentEventsPerRoom,
+			ctx, txn, roomID, r, &stateFilter, numRecentEventsPerRoom, device,
 		)
 		if err != nil {
 			return
@@ -721,7 +721,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync(
 	}
 
 	// Add peeked rooms.
-	peeks, err := d.Peeks.SelectPeeksInRange(ctx, txn, userID, deviceID, r)
+	peeks, err := d.Peeks.SelectPeeksInRange(ctx, txn, userID, device.ID, r)
 	if err != nil {
 		return
 	}
@@ -729,7 +729,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync(
 		if !peek.Deleted {
 			var jr *types.JoinResponse
 			jr, err = d.getJoinResponseForCompleteSync(
-				ctx, txn, peek.RoomID, r, &stateFilter, numRecentEventsPerRoom,
+				ctx, txn, peek.RoomID, r, &stateFilter, numRecentEventsPerRoom, device,
 			)
 			if err != nil {
 				return
@@ -751,7 +751,7 @@ func (d *Database) getJoinResponseForCompleteSync(
 	roomID string,
 	r types.Range,
 	stateFilter *gomatrixserverlib.StateFilter,
-	numRecentEventsPerRoom int,
+	numRecentEventsPerRoom int, device userapi.Device,
 ) (jr *types.JoinResponse, err error) {
 	var stateEvents []gomatrixserverlib.HeaderedEvent
 	stateEvents, err = d.CurrentRoomState.SelectCurrentState(ctx, txn, roomID, stateFilter)
@@ -784,8 +784,9 @@ func (d *Database) getJoinResponseForCompleteSync(
 	}
 
 	// We don't include a device here as we don't need to send down
-	// transaction IDs for complete syncs
-	recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents)
+	// transaction IDs for complete syncs, but we do it anyway because Sytest demands it for:
+	// "Can sync a room with a message with a transaction id" - which does a complete sync to check.
+	recentEvents := d.StreamEventsToEvents(&device, recentStreamEvents)
 	stateEvents = removeDuplicates(stateEvents, recentEvents)
 	jr = types.NewJoinResponse()
 	jr.Timeline.PrevBatch = prevBatchStr
@@ -800,7 +801,7 @@ func (d *Database) CompleteSync(
 	device userapi.Device, numRecentEventsPerRoom int,
 ) (*types.Response, error) {
 	toPos, joinedRoomIDs, err := d.getResponseWithPDUsForCompleteSync(
-		ctx, res, device.UserID, device.ID, numRecentEventsPerRoom,
+		ctx, res, device.UserID, device, numRecentEventsPerRoom,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("d.getResponseWithPDUsForCompleteSync: %w", err)

syncapi/sync/requestpool.go

@@ -197,19 +197,14 @@ func (rp *RequestPool) OnIncomingKeyChangeRequest(req *http.Request, device *use
 func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.StreamingToken) (*types.Response, error) {
 	res := types.NewResponse()
 
-	since := types.NewStreamToken(0, 0, nil)
-	if req.since != nil {
-		since = *req.since
-	}
-
 	// See if we have any new tasks to do for the send-to-device messaging.
-	events, updates, deletions, err := rp.db.SendToDeviceUpdatesForSync(req.ctx, req.device.UserID, req.device.ID, since)
+	events, updates, deletions, err := rp.db.SendToDeviceUpdatesForSync(req.ctx, req.device.UserID, req.device.ID, *req.since)
 	if err != nil {
 		return nil, fmt.Errorf("rp.db.SendToDeviceUpdatesForSync: %w", err)
 	}
 
 	// TODO: handle ignored users
-	if req.since == nil {
+	if req.since.PDUPosition() == 0 && req.since.EDUPosition() == 0 {
 		res, err = rp.db.CompleteSync(req.ctx, res, req.device, req.limit)
 		if err != nil {
 			return res, fmt.Errorf("rp.db.CompleteSync: %w", err)
@@ -226,7 +221,7 @@ func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.Strea
 	if err != nil {
 		return res, fmt.Errorf("rp.appendAccountData: %w", err)
 	}
-	res, err = rp.appendDeviceLists(res, req.device.UserID, since, latestPos)
+	res, err = rp.appendDeviceLists(res, req.device.UserID, *req.since, latestPos)
 	if err != nil {
 		return res, fmt.Errorf("rp.appendDeviceLists: %w", err)
 	}
@@ -240,7 +235,7 @@ func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.Strea
 	// Then add the updates into the sync response.
 	if len(updates) > 0 || len(deletions) > 0 {
 		// Handle the updates and deletions in the database.
-		err = rp.db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, since)
+		err = rp.db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, *req.since)
 		if err != nil {
 			return res, fmt.Errorf("rp.db.CleanSendToDeviceUpdates: %w", err)
 		}

sytest-whitelist

@@ -396,8 +396,6 @@ GET /rooms/:room_id/state fetches entire room state
 Setting room topic reports m.room.topic to myself
 setting 'm.room.name' respects room powerlevel
 Syncing a new room with a large timeline limit isn't limited
-Left rooms appear in the leave section of sync
-Banned rooms appear in the leave section of sync
 Getting state checks the events requested belong to the room
 Getting state IDs checks the events requested belong to the room
 Can invite users to invite-only rooms