Merge branch 'matrix-org:main' into main

docs/installation/4_database.md

@@ -16,6 +16,9 @@ Dendrite can automatically populate the database with the relevant tables and in
 it is not capable of creating the databases themselves. You will need to create the databases
 manually.
 
+The databases **must** be created with UTF-8 encoding configured or you will likely run into problems
+with your Dendrite deployment.
+
 At this point, you can choose to either use a single database for all Dendrite components,
 or you can run each component with its own separate database:
 
@@ -65,7 +68,7 @@ sudo -u postgres createuser -P dendrite
 Create the database itself, using the `dendrite` role from above:
 
 ```bash
-sudo -u postgres createdb -O dendrite dendrite
+sudo -u postgres createdb -O dendrite -E UTF-8 dendrite
 ```
 
 ### Multiple database creation
@@ -85,7 +88,7 @@ The following eight components require a database. In this example they will be
 
 ```bash
 for i in appservice federationapi mediaapi mscs roomserver syncapi keyserver userapi; do
-    sudo -u postgres createdb -O dendrite dendrite_$i
+    sudo -u postgres createdb -O dendrite -E UTF-8 dendrite_$i
 done
 ```
 

federationapi/internal/perform.go

@@ -217,7 +217,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 		var remoteEvent *gomatrixserverlib.Event
 		remoteEvent, err = respSendJoin.Event.UntrustedEvent(respMakeJoin.RoomVersion)
 		if err == nil && isWellFormedMembershipEvent(
-			remoteEvent, roomID, userID, r.cfg.Matrix.ServerName,
+			remoteEvent, roomID, userID,
 		) {
 			event = remoteEvent
 		}
@@ -285,7 +285,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 
 // isWellFormedMembershipEvent returns true if the event looks like a legitimate
 // membership event.
-func isWellFormedMembershipEvent(event *gomatrixserverlib.Event, roomID, userID string, origin gomatrixserverlib.ServerName) bool {
+func isWellFormedMembershipEvent(event *gomatrixserverlib.Event, roomID, userID string) bool {
 	if membership, err := event.Membership(); err != nil {
 		return false
 	} else if membership != gomatrixserverlib.Join {
@@ -294,9 +294,6 @@ func isWellFormedMembershipEvent(event *gomatrixserverlib.Event, roomID, userID
 	if event.RoomID() != roomID {
 		return false
 	}
-	if event.Origin() != origin {
-		return false
-	}
 	if !event.StateKeyEquals(userID) {
 		return false
 	}

federationapi/routing/invite.go

@@ -148,8 +148,15 @@ func processInvite(
 			JSON: jsonerror.BadJSON("The event JSON could not be redacted"),
 		}
 	}
+	_, serverName, err := gomatrixserverlib.SplitID('@', event.Sender())
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("The event JSON contains an invalid sender"),
+		}
+	}
 	verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
-		ServerName:             event.Origin(),
+		ServerName:             serverName,
 		Message:                redacted,
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,

federationapi/routing/join.go

@@ -203,14 +203,6 @@ func SendJoin(
 		}
 	}
 
-	// Check that the event is from the server sending the request.
-	if event.Origin() != request.Origin() {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("The join must be sent by the server it originated on"),
-		}
-	}
-
 	// Check that a state key is provided.
 	if event.StateKey() == nil || event.StateKeyEquals("") {
 		return util.JSONResponse{
@@ -228,16 +220,16 @@ func SendJoin(
 	// Check that the sender belongs to the server that is sending us
 	// the request. By this point we've already asserted that the sender
 	// and the state key are equal so we don't need to check both.
-	var domain gomatrixserverlib.ServerName
-	if _, domain, err = gomatrixserverlib.SplitID('@', event.Sender()); err != nil {
+	var serverName gomatrixserverlib.ServerName
+	if _, serverName, err = gomatrixserverlib.SplitID('@', event.Sender()); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
 			JSON: jsonerror.Forbidden("The sender of the join is invalid"),
 		}
-	} else if domain != request.Origin() {
+	} else if serverName != request.Origin() {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("The sender of the join must belong to the origin server"),
+			JSON: jsonerror.Forbidden("The sender does not match the server that originated the request"),
 		}
 	}
 
@@ -292,7 +284,7 @@ func SendJoin(
 		}
 	}
 	verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
-		ServerName:             event.Origin(),
+		ServerName:             serverName,
 		Message:                redacted,
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,

federationapi/routing/leave.go

@@ -118,6 +118,7 @@ func MakeLeave(
 }
 
 // SendLeave implements the /send_leave API
+// nolint:gocyclo
 func SendLeave(
 	httpReq *http.Request,
 	request *gomatrixserverlib.FederationRequest,
@@ -167,14 +168,6 @@ func SendLeave(
 		}
 	}
 
-	// Check that the event is from the server sending the request.
-	if event.Origin() != request.Origin() {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("The leave must be sent by the server it originated on"),
-		}
-	}
-
 	if event.StateKey() == nil || event.StateKeyEquals("") {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -188,6 +181,22 @@ func SendLeave(
 		}
 	}
 
+	// Check that the sender belongs to the server that is sending us
+	// the request. By this point we've already asserted that the sender
+	// and the state key are equal so we don't need to check both.
+	var serverName gomatrixserverlib.ServerName
+	if _, serverName, err = gomatrixserverlib.SplitID('@', event.Sender()); err != nil {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("The sender of the join is invalid"),
+		}
+	} else if serverName != request.Origin() {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("The sender does not match the server that originated the request"),
+		}
+	}
+
 	// Check if the user has already left. If so, no-op!
 	queryReq := &api.QueryLatestEventsAndStateRequest{
 		RoomID: roomID,
@@ -240,7 +249,7 @@ func SendLeave(
 		}
 	}
 	verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{
-		ServerName:             event.Origin(),
+		ServerName:             serverName,
 		Message:                redacted,
 		AtTS:                   event.OriginServerTS(),
 		StrictValidityChecking: true,

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5
 	github.com/matrix-org/pinecone v0.0.0-20220923151905-0900fceecb89
 	github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
 	github.com/mattn/go-sqlite3 v1.14.15

go.sum

@@ -437,8 +437,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3 h1:u3FKZmXxfhv3XhD8RziBlt96QTt8eHFhg1upCloBh2g=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5 h1:cQMA9hip0WSp6cv7CUfButa9Jl/9E6kqWmQyOjx5A5s=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
 github.com/matrix-org/pinecone v0.0.0-20220923151905-0900fceecb89 h1:Ym50Fgn3GiYya4p29k3nJ5nYsalFGev3eIm3DeGNIq4=
 github.com/matrix-org/pinecone v0.0.0-20220923151905-0900fceecb89/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=

roomserver/internal/input/input_events.go

@@ -118,6 +118,10 @@ func (r *Inputer) processRoomEvent(
 	if roomInfo == nil && !isCreateEvent {
 		return fmt.Errorf("room %s does not exist for event %s", event.RoomID(), event.EventID())
 	}
+	_, senderDomain, err := gomatrixserverlib.SplitID('@', event.Sender())
+	if err != nil {
+		return fmt.Errorf("event has invalid sender %q", input.Event.Sender())
+	}
 
 	// If we already know about this outlier and it hasn't been rejected
 	// then we won't attempt to reprocess it. If it was rejected or has now
@@ -145,7 +149,8 @@ func (r *Inputer) processRoomEvent(
 	var missingAuth, missingPrev bool
 	serverRes := &fedapi.QueryJoinedHostServerNamesInRoomResponse{}
 	if !isCreateEvent {
-		missingAuthIDs, missingPrevIDs, err := r.DB.MissingAuthPrevEvents(ctx, event)
+		var missingAuthIDs, missingPrevIDs []string
+		missingAuthIDs, missingPrevIDs, err = r.DB.MissingAuthPrevEvents(ctx, event)
 		if err != nil {
 			return fmt.Errorf("updater.MissingAuthPrevEvents: %w", err)
 		}
@@ -158,7 +163,7 @@ func (r *Inputer) processRoomEvent(
 			RoomID:      event.RoomID(),
 			ExcludeSelf: true,
 		}
-		if err := r.FSAPI.QueryJoinedHostServerNamesInRoom(ctx, serverReq, serverRes); err != nil {
+		if err = r.FSAPI.QueryJoinedHostServerNamesInRoom(ctx, serverReq, serverRes); err != nil {
 			return fmt.Errorf("r.FSAPI.QueryJoinedHostServerNamesInRoom: %w", err)
 		}
 		// Sort all of the servers into a map so that we can randomise
@@ -173,9 +178,9 @@ func (r *Inputer) processRoomEvent(
 			serverRes.ServerNames = append(serverRes.ServerNames, input.Origin)
 			delete(servers, input.Origin)
 		}
-		if origin := event.Origin(); origin != input.Origin {
-			serverRes.ServerNames = append(serverRes.ServerNames, origin)
-			delete(servers, origin)
+		if senderDomain != input.Origin {
+			serverRes.ServerNames = append(serverRes.ServerNames, senderDomain)
+			delete(servers, senderDomain)
 		}
 		for server := range servers {
 			serverRes.ServerNames = append(serverRes.ServerNames, server)
@@ -188,7 +193,7 @@ func (r *Inputer) processRoomEvent(
 	isRejected := false
 	authEvents := gomatrixserverlib.NewAuthEvents(nil)
 	knownEvents := map[string]*types.Event{}
-	if err := r.fetchAuthEvents(ctx, logger, headered, &authEvents, knownEvents, serverRes.ServerNames); err != nil {
+	if err = r.fetchAuthEvents(ctx, logger, headered, &authEvents, knownEvents, serverRes.ServerNames); err != nil {
 		return fmt.Errorf("r.fetchAuthEvents: %w", err)
 	}
 
@@ -231,7 +236,6 @@ func (r *Inputer) processRoomEvent(
 	if input.Kind == api.KindNew {
 		// Check that the event passes authentication checks based on the
 		// current room state.
-		var err error
 		softfail, err = helpers.CheckForSoftFail(ctx, r.DB, headered, input.StateEventIDs)
 		if err != nil {
 			logger.WithError(err).Warn("Error authing soft-failed event")
@@ -265,7 +269,8 @@ func (r *Inputer) processRoomEvent(
 				hadEvents:  map[string]bool{},
 				haveEvents: map[string]*gomatrixserverlib.Event{},
 			}
-			if stateSnapshot, err := missingState.processEventWithMissingState(ctx, event, headered.RoomVersion); err != nil {
+			var stateSnapshot *parsedRespState
+			if stateSnapshot, err = missingState.processEventWithMissingState(ctx, event, headered.RoomVersion); err != nil {
 				// Something went wrong with retrieving the missing state, so we can't
 				// really do anything with the event other than reject it at this point.
 				isRejected = true
@@ -302,7 +307,6 @@ func (r *Inputer) processRoomEvent(
 	// burning CPU time.
 	historyVisibility := gomatrixserverlib.HistoryVisibilityShared // Default to shared.
 	if input.Kind != api.KindOutlier && rejectionErr == nil && !isRejected {
-		var err error
 		historyVisibility, rejectionErr, err = r.processStateBefore(ctx, input, missingPrev)
 		if err != nil {
 			return fmt.Errorf("r.processStateBefore: %w", err)

roomserver/internal/perform/perform_backfill.go

@@ -468,7 +468,9 @@ FindSuccessor:
 	// Store the server names in a temporary map to avoid duplicates.
 	serverSet := make(map[gomatrixserverlib.ServerName]bool)
 	for _, event := range memberEvents {
-		serverSet[event.Origin()] = true
+		if _, senderDomain, err := gomatrixserverlib.SplitID('@', event.Sender()); err == nil {
+			serverSet[senderDomain] = true
+		}
 	}
 	var servers []gomatrixserverlib.ServerName
 	for server := range serverSet {

roomserver/internal/perform/perform_invite.go

@@ -50,6 +50,10 @@ func (r *Inviter) PerformInvite(
 	if event.StateKey() == nil {
 		return nil, fmt.Errorf("invite must be a state event")
 	}
+	_, senderDomain, err := gomatrixserverlib.SplitID('@', event.Sender())
+	if err != nil {
+		return nil, fmt.Errorf("sender %q is invalid", event.Sender())
+	}
 
 	roomID := event.RoomID()
 	targetUserID := *event.StateKey()
@@ -67,7 +71,7 @@ func (r *Inviter) PerformInvite(
 		return nil, nil
 	}
 	isTargetLocal := domain == r.Cfg.Matrix.ServerName
-	isOriginLocal := event.Origin() == r.Cfg.Matrix.ServerName
+	isOriginLocal := senderDomain == r.Cfg.Matrix.ServerName
 	if !isOriginLocal && !isTargetLocal {
 		res.Error = &api.PerformError{
 			Code: api.PerformErrorBadRequest,
@@ -235,7 +239,7 @@ func (r *Inviter) PerformInvite(
 			{
 				Kind:         api.KindNew,
 				Event:        event,
-				Origin:       event.Origin(),
+				Origin:       senderDomain,
 				SendAsServer: req.SendAsServer,
 			},
 		},

roomserver/internal/perform/perform_leave.go

@@ -81,12 +81,11 @@ func (r *Leaver) performLeaveRoomByID(
 	// that.
 	isInvitePending, senderUser, eventID, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID)
 	if err == nil && isInvitePending {
-		var host gomatrixserverlib.ServerName
-		_, host, err = gomatrixserverlib.SplitID('@', senderUser)
-		if err != nil {
+		_, senderDomain, serr := gomatrixserverlib.SplitID('@', senderUser)
+		if serr != nil {
 			return nil, fmt.Errorf("sender %q is invalid", senderUser)
 		}
-		if host != r.Cfg.Matrix.ServerName {
+		if senderDomain != r.Cfg.Matrix.ServerName {
 			return r.performFederatedRejectInvite(ctx, req, res, senderUser, eventID)
 		}
 		// check that this is not a "server notice room"
@@ -172,6 +171,12 @@ func (r *Leaver) performLeaveRoomByID(
 		return nil, fmt.Errorf("eventutil.BuildEvent: %w", err)
 	}
 
+	// Get the sender domain.
+	_, senderDomain, serr := gomatrixserverlib.SplitID('@', event.Sender())
+	if serr != nil {
+		return nil, fmt.Errorf("sender %q is invalid", event.Sender())
+	}
+
 	// Give our leave event to the roomserver input stream. The
 	// roomserver will process the membership change and notify
 	// downstream automatically.
@@ -180,7 +185,7 @@ func (r *Leaver) performLeaveRoomByID(
 			{
 				Kind:         api.KindNew,
 				Event:        event.Headered(buildRes.RoomVersion),
-				Origin:       event.Origin(),
+				Origin:       senderDomain,
 				SendAsServer: string(r.Cfg.Matrix.ServerName),
 			},
 		},

roomserver/storage/shared/storage.go

@@ -896,7 +896,7 @@ func (d *Database) handleRedactions(
 	switch {
 	case redactUser >= pl.Redact:
 		// The power level of the redaction event’s sender is greater than or equal to the redact level.
-	case redactedEvent.Origin() == redactionEvent.Origin() && redactedEvent.Sender() == redactionEvent.Sender():
+	case redactedEvent.Sender() == redactionEvent.Sender():
 		// The domain of the redaction event’s sender matches that of the original event’s sender.
 	default:
 		return nil, "", nil

userapi/storage/shared/storage.go

@@ -777,7 +777,7 @@ func (d *Database) GetPushers(
 func (d *Database) RemovePusher(
 	ctx context.Context, appid, pushkey, localpart string,
 ) error {
-	return d.Writer.Do(nil, nil, func(txn *sql.Tx) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		err := d.Pushers.DeletePusher(ctx, txn, appid, pushkey, localpart)
 		if err == sql.ErrNoRows {
 			return nil
@@ -792,7 +792,7 @@ func (d *Database) RemovePusher(
 func (d *Database) RemovePushers(
 	ctx context.Context, appid, pushkey string,
 ) error {
-	return d.Writer.Do(nil, nil, func(txn *sql.Tx) error {
+	return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 		return d.Pushers.DeletePushers(ctx, txn, appid, pushkey)
 	})
 }

userapi/storage/sqlite3/pusher_table.go

@@ -96,7 +96,7 @@ func (s *pushersStatements) InsertPusher(
 	ctx context.Context, txn *sql.Tx, session_id int64,
 	pushkey string, pushkeyTS int64, kind api.PusherKind, appid, appdisplayname, devicedisplayname, profiletag, lang, data, localpart string,
 ) error {
-	_, err := s.insertPusherStmt.ExecContext(ctx, localpart, session_id, pushkey, pushkeyTS, kind, appid, appdisplayname, devicedisplayname, profiletag, lang, data)
+	_, err := sqlutil.TxStmt(txn, s.insertPusherStmt).ExecContext(ctx, localpart, session_id, pushkey, pushkeyTS, kind, appid, appdisplayname, devicedisplayname, profiletag, lang, data)
 	logrus.Debugf("Created pusher %d", session_id)
 	return err
 }
@@ -144,13 +144,13 @@ func (s *pushersStatements) SelectPushers(
 func (s *pushersStatements) DeletePusher(
 	ctx context.Context, txn *sql.Tx, appid, pushkey, localpart string,
 ) error {
-	_, err := s.deletePusherStmt.ExecContext(ctx, appid, pushkey, localpart)
+	_, err := sqlutil.TxStmt(txn, s.deletePusherStmt).ExecContext(ctx, appid, pushkey, localpart)
 	return err
 }
 
 func (s *pushersStatements) DeletePushers(
 	ctx context.Context, txn *sql.Tx, appid, pushkey string,
 ) error {
-	_, err := s.deletePushersByAppIdAndPushKeyStmt.ExecContext(ctx, appid, pushkey)
+	_, err := sqlutil.TxStmt(txn, s.deletePushersByAppIdAndPushKeyStmt).ExecContext(ctx, appid, pushkey)
 	return err
 }