Merge pull request #61 from globekeeper/GK-merge-upstream-10.8

Merge upstream 0.10.8

.github/workflows/dendrite.yml

@@ -194,6 +194,42 @@ jobs:
         with:
           jobs: ${{ toJSON(needs) }}
 
+  # run database upgrade tests
+  upgrade_test:
+    name: Upgrade tests
+    timeout-minutes: 20
+    needs: initial-tests-done
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.18"
+          cache: true
+      - name: Build upgrade-tests
+        run: go build ./cmd/dendrite-upgrade-tests
+      - name: Test upgrade (PostgreSQL)
+        run: ./dendrite-upgrade-tests --head .
+
+  # run database upgrade tests, skipping over one version
+  upgrade_test_direct:
+    name: Upgrade tests from HEAD-2
+    timeout-minutes: 20
+    needs: initial-tests-done
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.18"
+          cache: true
+      - name: Build upgrade-tests
+        run: go build ./cmd/dendrite-upgrade-tests
+      - name: Test upgrade (PostgreSQL)
+        run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
+
   # run Sytest in different variations
   sytest:
     timeout-minutes: 20
@@ -344,3 +380,15 @@ jobs:
         uses: re-actors/alls-green@release/v1
         with:
           jobs: ${{ toJSON(needs) }}
+
+  update-docker-images:
+    name: Update Docker images
+    permissions:
+      packages: write
+      contents: read
+      security-events: write # To upload Trivy sarif files
+    if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main'
+    needs: [integration-tests-done]
+    uses: matrix-org/dendrite/.github/workflows/docker.yml@main
+    secrets:
+      DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}

Dockerfile

@@ -23,4 +23,4 @@ COPY --from=base /build/bin/* /usr/bin/
 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite
 
-ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
+ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]

clientapi/routing/password.go

@@ -61,6 +61,7 @@ func Password(
 		sessionID = util.RandomString(sessionIDLength)
 	}
 	var localpart string
+	var domain gomatrixserverlib.ServerName
 	switch r.Auth.Type {
 	case authtypes.LoginTypePassword:
 		// Check if the existing password is correct.
@@ -71,6 +72,13 @@ func Password(
 		if _, authErr := typePassword.Login(req.Context(), &r.Auth.PasswordRequest); authErr != nil {
 			return *authErr
 		}
+		// Get the local part.
+		var err error
+		localpart, domain, err = gomatrixserverlib.SplitID('@', device.UserID)
+		if err != nil {
+			util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
+			return jsonerror.InternalServerError()
+		}
 		sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypePassword)
 	case authtypes.LoginTypeEmail:
 		threePid := &authtypes.ThreePID{}
@@ -111,6 +119,8 @@ func Password(
 				},
 			}
 		}
+		localpart = res.Localpart
+		domain = res.ServerName
 		sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeEmail)
 	default:
 		flows := []authtypes.Flow{
@@ -141,13 +151,6 @@ func Password(
 		return *resErr
 	}
 
-	// Get the local part.
-	localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
-	if err != nil {
-		util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
-		return jsonerror.InternalServerError()
-	}
-
 	// Ask the user API to perform the password change.
 	passwordReq := &api.PerformPasswordUpdateRequest{
 		Localpart:  localpart,

clientapi/routing/register.go

@@ -833,7 +833,7 @@ func handleRegistrationFlow(
 	// A response with current registration flow and remaining available methods
 	// will be returned if a flow has not been successfully completed yet
 	return checkAndCompleteFlow(sessions.getCompletedStages(sessionID),
-		req, r, sessionID, cfg, userAPI)
+		req, r, sessionID, cfg, userAPI, threePid)
 }
 
 // handleApplicationServiceRegistration handles the registration of an
@@ -874,9 +874,8 @@ func handleApplicationServiceRegistration(
 	// Don't need to worry about appending to registration stages as
 	// application service registration is entirely separate.
 	return completeRegistration(
-		req.Context(), userAPI, r.Username, r.ServerName, "", appserviceID, req.RemoteAddr,
+		req.Context(), userAPI, r.Username, r.ServerName, "", appserviceID, req.RemoteAddr, req.UserAgent(), r.Auth.Session,
-		req.UserAgent(), r.Auth.Session, r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
+		r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeAppService, nil,
-		userapi.AccountTypeAppService, nil,
 	)
 }
 
@@ -890,13 +889,13 @@ func checkAndCompleteFlow(
 	sessionID string,
 	cfg *config.ClientAPI,
 	userAPI userapi.ClientUserAPI,
+	threePid *authtypes.ThreePID,
 ) util.JSONResponse {
 	if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
 		// This flow was completed, registration can continue
 		return completeRegistration(
-			req.Context(), userAPI, r.Username, r.ServerName, r.Password, "", req.RemoteAddr,
+			req.Context(), userAPI, r.Username, r.ServerName, r.Password, "", req.RemoteAddr, req.UserAgent(), sessionID,
-			req.UserAgent(), sessionID, r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
+			r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeUser, threePid,
-			userapi.AccountTypeUser, nil,
 		)
 	}
 	sessions.addParams(sessionID, r)
@@ -966,9 +965,10 @@ func completeRegistration(
 	// TODO-entry refuse register if threepid is already bound to account.
 	if threePid != nil {
 		err = userAPI.PerformSaveThreePIDAssociation(ctx, &userapi.PerformSaveThreePIDAssociationRequest{
-			Medium:    threePid.Medium,
+			Medium:     threePid.Medium,
-			ThreePID:  threePid.Address,
+			ThreePID:   threePid.Address,
-			Localpart: accRes.Account.Localpart,
+			Localpart:  accRes.Account.Localpart,
+			ServerName: accRes.Account.ServerName,
 		}, &struct{}{})
 		if err != nil {
 			return util.JSONResponse{

sytest-whitelist

@@ -758,9 +758,9 @@ Can filter rooms/{roomId}/members
 Current state appears in timeline in private history with many messages after
 AS can publish rooms in their own list
 AS and main public room lists are separate
+AS can deactivate a user
 /upgrade preserves direct room state
 local user has tags copied to the new room
 remote user has tags copied to the new room
 /upgrade moves remote aliases to the new room
 Local and remote users' homeservers remove a room from their public directory on upgrade
-AS can deactivate a user

userapi/storage/postgres/deltas/2022110411000000_server_names.go

@@ -35,6 +35,7 @@ var serverNamesDropPK = map[string]string{
 var serverNamesDropIndex = []string{
 	"userapi_pusher_localpart_idx",
 	"userapi_pusher_app_id_pushkey_localpart_idx",
+	"userapi_pusher_app_id_pushkey_idx",
 }
 
 // I know what you're thinking: you're wondering "why doesn't this use $1

userapi/storage/postgres/notifications_table.go

@@ -73,7 +73,7 @@ const selectNotificationSQL = "" +
 	") AND NOT read ORDER BY localpart, id LIMIT $5"
 
 const selectNotificationCountSQL = "" +
-	"SELECT COUNT(*) FROM userapi_notifications WHERE localpart = $1 AND server_name = $2 AND (" +
+	"SELECT COUNT(DISTINCT(room_id)) FROM userapi_notifications WHERE localpart = $1 AND server_name = $2 AND (" +
 	"(($3 & 1) <> 0 AND highlight) OR (($3 & 2) <> 0 AND NOT highlight)" +
 	") AND NOT read"
 

userapi/storage/postgres/pusher_table.go

@@ -51,13 +51,13 @@ CREATE TABLE IF NOT EXISTS userapi_pushers (
 CREATE INDEX IF NOT EXISTS userapi_pusher_localpart_idx ON userapi_pushers(localpart, server_name);
 
 -- Pushkey must be unique for a given user and app.
-CREATE UNIQUE INDEX IF NOT EXISTS userapi_pusher_app_id_pushkey_localpart_idx ON userapi_pushers(app_id, pushkey, localpart, server_name);
+CREATE UNIQUE INDEX IF NOT EXISTS userapi_pusher_app_id_pushkey_idx ON userapi_pushers(app_id, pushkey, server_name);
 `
 
 const insertPusherSQL = "" +
 	"INSERT INTO userapi_pushers (localpart, server_name, session_id, pushkey, pushkey_ts_ms, kind, app_id, app_display_name, device_display_name, profile_tag, lang, data)" +
 	"VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)" +
-	"ON CONFLICT (app_id, pushkey, localpart, server_name) DO UPDATE SET session_id = $3, pushkey_ts_ms = $5, kind = $6, app_display_name = $8, device_display_name = $9, profile_tag = $10, lang = $11, data = $12"
+	"ON CONFLICT (app_id, pushkey, server_name) DO UPDATE SET localpart = $1, session_id = $3, pushkey_ts_ms = $5, kind = $6, app_display_name = $8, device_display_name = $9, profile_tag = $10, lang = $11, data = $12"
 
 const selectPushersSQL = "" +
 	"SELECT session_id, pushkey, pushkey_ts_ms, kind, app_id, app_display_name, device_display_name, profile_tag, lang, data FROM userapi_pushers WHERE localpart = $1 AND server_name = $2"

userapi/storage/sqlite3/notifications_table.go

@@ -73,7 +73,7 @@ const selectNotificationSQL = "" +
 	") AND NOT read ORDER BY localpart, id LIMIT $5"
 
 const selectNotificationCountSQL = "" +
-	"SELECT COUNT(*) FROM userapi_notifications WHERE localpart = $1 AND server_name = $2 AND (" +
+	"SELECT COUNT(DISTINCT(room_id)) FROM userapi_notifications WHERE localpart = $1 AND server_name = $2 AND (" +
 	"(($3 & 1) <> 0 AND highlight) OR (($3 & 2) <> 0 AND NOT highlight)" +
 	") AND NOT read"
 

userapi/storage/storage_test.go

@@ -542,7 +542,7 @@ func Test_Notification(t *testing.T) {
 		// get notifications
 		count, err := db.GetNotificationCount(ctx, aliceLocalpart, aliceDomain, tables.AllNotifications)
 		assert.NoError(t, err, "unable to get notification count")
-		assert.Equal(t, int64(10), count)
+		assert.Equal(t, int64(2), count)
 		notifs, count, err := db.GetNotifications(ctx, aliceLocalpart, aliceDomain, 0, 15, tables.AllNotifications)
 		assert.NoError(t, err, "unable to get notifications")
 		assert.Equal(t, int64(10), count)