Merge branch 'master' of https://github.com/matrix-org/dendrite into recaptcha

* Reduced cyclo complexity of handleRegistrationFlow.

Signed-off-by: Andrew Morgan (https://amorgan.xyz) <andrew@amorgan.xyz>

src/github.com/matrix-org/dendrite/clientapi/producers/roomserver.go

@@ -36,14 +36,16 @@ func NewRoomserverProducer(inputAPI api.RoomserverInputAPI) *RoomserverProducer
 // SendEvents writes the given events to the roomserver input log. The events are written with KindNew.
 func (c *RoomserverProducer) SendEvents(
 	ctx context.Context, events []gomatrixserverlib.Event, sendAsServer gomatrixserverlib.ServerName,
+	txnID *api.TransactionID,
 ) error {
 	ires := make([]api.InputRoomEvent, len(events))
 	for i, event := range events {
 		ires[i] = api.InputRoomEvent{
-			Kind:         api.KindNew,
-			Event:        event,
-			AuthEventIDs: event.AuthEventIDs(),
-			SendAsServer: string(sendAsServer),
+			Kind:          api.KindNew,
+			Event:         event,
+			AuthEventIDs:  event.AuthEventIDs(),
+			SendAsServer:  string(sendAsServer),
+			TransactionID: txnID,
 		}
 	}
 	return c.SendInputRoomEvents(ctx, ires)

src/github.com/matrix-org/dendrite/clientapi/routing/createroom.go

@@ -214,7 +214,7 @@ func createRoom(req *http.Request, device *authtypes.Device,
 	}
 
 	// send events to the room server
-	err = producer.SendEvents(req.Context(), builtEvents, cfg.Matrix.ServerName)
+	err = producer.SendEvents(req.Context(), builtEvents, cfg.Matrix.ServerName, nil)
 	if err != nil {
 		return httputil.LogThenError(req, err)
 	}

src/github.com/matrix-org/dendrite/clientapi/routing/joinroom.go

@@ -217,7 +217,7 @@ func (r joinRoomReq) joinRoomUsingServers(
 	var queryRes api.QueryLatestEventsAndStateResponse
 	event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.queryAPI, &queryRes)
 	if err == nil {
-		if err = r.producer.SendEvents(r.req.Context(), []gomatrixserverlib.Event{*event}, r.cfg.Matrix.ServerName); err != nil {
+		if err = r.producer.SendEvents(r.req.Context(), []gomatrixserverlib.Event{*event}, r.cfg.Matrix.ServerName, nil); err != nil {
 			return httputil.LogThenError(r.req, err)
 		}
 		return util.JSONResponse{

src/github.com/matrix-org/dendrite/clientapi/routing/membership.go

@@ -98,7 +98,7 @@ func SendMembership(
 	}
 
 	if err := producer.SendEvents(
-		req.Context(), []gomatrixserverlib.Event{*event}, cfg.Matrix.ServerName,
+		req.Context(), []gomatrixserverlib.Event{*event}, cfg.Matrix.ServerName, nil,
 	); err != nil {
 		return httputil.LogThenError(req, err)
 	}

src/github.com/matrix-org/dendrite/clientapi/routing/profile.go

@@ -138,7 +138,7 @@ func SetAvatarURL(
 		return httputil.LogThenError(req, err)
 	}
 
-	if err := rsProducer.SendEvents(req.Context(), events, cfg.Matrix.ServerName); err != nil {
+	if err := rsProducer.SendEvents(req.Context(), events, cfg.Matrix.ServerName, nil); err != nil {
 		return httputil.LogThenError(req, err)
 	}
 
@@ -230,7 +230,7 @@ func SetDisplayName(
 		return httputil.LogThenError(req, err)
 	}
 
-	if err := rsProducer.SendEvents(req.Context(), events, cfg.Matrix.ServerName); err != nil {
+	if err := rsProducer.SendEvents(req.Context(), events, cfg.Matrix.ServerName, nil); err != nil {
 		return httputil.LogThenError(req, err)
 	}
 

src/github.com/matrix-org/dendrite/clientapi/routing/register.go

@@ -53,7 +53,7 @@ const (
 var (
 	// TODO: Remove old sessions. Need to do so on a session-specific timeout.
 	sessions           = make(map[string][]authtypes.LoginType) // Sessions and completed flow stages
-	validUsernameRegex = regexp.MustCompile(`^[0-9a-zA-Z_\-./]+$`)
+	validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-./]+$`)
 )
 
 // registerRequest represents the submitted registration request.
@@ -173,6 +173,13 @@ func validateRecaptcha(
 	response string,
 	clientip string,
 ) *util.JSONResponse {
+	if !cfg.Matrix.RecaptchaEnabled {
+		return &util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.BadJSON("Captcha registration is disabled"),
+		}
+	}
+
 	if response == "" {
 		return &util.JSONResponse{
 			Code: 400,
@@ -256,6 +263,9 @@ func Register(
 		}
 	}
 
+	// Squash username to all lowercase letters
+	r.Username = strings.ToLower(r.Username)
+
 	if resErr = validateUserName(r.Username); resErr != nil {
 		return *resErr
 	}
@@ -292,18 +302,13 @@ func handleRegistrationFlow(
 	// TODO: Handle mapping registrationRequest parameters into session parameters
 
 	// TODO: email / msisdn auth types.
+
+	if cfg.Matrix.RegistrationDisabled && r.Auth.Type != authtypes.LoginTypeSharedSecret {
+		return util.MessageResponse(403, "Registration has been disabled")
+	}
+
 	switch r.Auth.Type {
 	case authtypes.LoginTypeRecaptcha:
-		if !cfg.Matrix.RecaptchaEnabled {
-			return util.MessageResponse(400, "Captcha registration is disabled")
-		}
-
-		logger := util.GetLogger(req.Context())
-		logger.WithFields(log.Fields{
-			"clientip": req.RemoteAddr,
-			"response": r.Auth.Response,
-		}).Info("Submitting recaptcha response")
-
 		// Check given captcha response
 		resErr := validateRecaptcha(cfg, r.Auth.Response, req.RemoteAddr)
 		if resErr != nil {
@@ -314,18 +319,12 @@ func handleRegistrationFlow(
 		sessions[sessionID] = append(sessions[sessionID], authtypes.LoginTypeRecaptcha)
 
 	case authtypes.LoginTypeSharedSecret:
-		if cfg.Matrix.RegistrationSharedSecret == "" {
-			return util.MessageResponse(400, "Shared secret registration is disabled")
-		}
-
-		valid, err := isValidMacLogin(r.Username, r.Password, r.Admin,
-			r.Auth.Mac, cfg.Matrix.RegistrationSharedSecret)
+		// Check shared secret against config
+		valid, err := isValidMacLogin(cfg, r.Username, r.Password, r.Admin, r.Auth.Mac)
 
 		if err != nil {
 			return httputil.LogThenError(req, err)
-		}
-
-		if !valid {
+		} else if !valid {
 			return util.MessageResponse(403, "HMAC incorrect")
 		}
 
@@ -367,16 +366,10 @@ func LegacyRegister(
 	cfg *config.Dendrite,
 ) util.JSONResponse {
 	var r legacyRegisterRequest
-	resErr := httputil.UnmarshalJSONRequest(req, &r)
+	resErr := parseAndValidateLegacyLogin(req, &r)
 	if resErr != nil {
 		return *resErr
 	}
-	if resErr = validateUserName(r.Username); resErr != nil {
-		return *resErr
-	}
-	if resErr = validatePassword(r.Password); resErr != nil {
-		return *resErr
-	}
 
 	logger := util.GetLogger(req.Context())
 	logger.WithFields(log.Fields{
@@ -384,12 +377,8 @@ func LegacyRegister(
 		"auth.type": r.Type,
 	}).Info("Processing registration request")
 
-	// All registration requests must specify what auth they are using to perform this request
-	if r.Type == "" {
-		return util.JSONResponse{
-			Code: 400,
-			JSON: jsonerror.BadJSON("invalid type"),
-		}
+	if cfg.Matrix.RegistrationDisabled && r.Type != authtypes.LoginTypeSharedSecret {
+		return util.MessageResponse(403, "Registration has been disabled")
 	}
 
 	switch r.Type {
@@ -398,7 +387,7 @@ func LegacyRegister(
 			return util.MessageResponse(400, "Shared secret registration is disabled")
 		}
 
-		valid, err := isValidMacLogin(r.Username, r.Password, r.Admin, r.Mac, cfg.Matrix.RegistrationSharedSecret)
+		valid, err := isValidMacLogin(cfg, r.Username, r.Password, r.Admin, r.Mac)
 		if err != nil {
 			return httputil.LogThenError(req, err)
 		}
@@ -419,6 +408,35 @@ func LegacyRegister(
 	}
 }
 
+// parseAndValidateLegacyLogin parses the request into r and checks that the
+// request is valid (e.g. valid user names, etc)
+func parseAndValidateLegacyLogin(req *http.Request, r *legacyRegisterRequest) *util.JSONResponse {
+	resErr := httputil.UnmarshalJSONRequest(req, &r)
+	if resErr != nil {
+		return resErr
+	}
+
+	// Squash username to all lowercase letters
+	r.Username = strings.ToLower(r.Username)
+
+	if resErr = validateUserName(r.Username); resErr != nil {
+		return resErr
+	}
+	if resErr = validatePassword(r.Password); resErr != nil {
+		return resErr
+	}
+
+	// All registration requests must specify what auth they are using to perform this request
+	if r.Type == "" {
+		return &util.JSONResponse{
+			Code: 400,
+			JSON: jsonerror.BadJSON("invalid type"),
+		}
+	}
+
+	return nil
+}
+
 func completeRegistration(
 	ctx context.Context,
 	accountDB *accounts.Database,
@@ -478,11 +496,18 @@ func completeRegistration(
 // Used for shared secret registration.
 // Checks if the username, password and isAdmin flag matches the given mac.
 func isValidMacLogin(
+	cfg *config.Dendrite,
 	username, password string,
 	isAdmin bool,
 	givenMac []byte,
-	sharedSecret string,
 ) (bool, error) {
+	sharedSecret := cfg.Matrix.RegistrationSharedSecret
+
+	// Check that shared secret registration isn't disabled.
+	if cfg.Matrix.RegistrationSharedSecret == "" {
+		return false, errors.New("Shared secret registration is disabled")
+	}
+
 	// Double check that username/password don't contain the HMAC delimiters. We should have
 	// already checked this.
 	if strings.Contains(username, "\x00") {
@@ -571,6 +596,9 @@ func RegisterAvailable(
 ) util.JSONResponse {
 	username := req.URL.Query().Get("username")
 
+	// Squash username to all lowercase letters
+	username = strings.ToLower(username)
+
 	if err := validateUserName(username); err != nil {
 		return *err
 	}

src/github.com/matrix-org/dendrite/clientapi/routing/sendevent.go

@@ -41,7 +41,7 @@ type sendEventResponse struct {
 func SendEvent(
 	req *http.Request,
 	device *authtypes.Device,
-	roomID, eventType string, _, stateKey *string,
+	roomID, eventType string, txnID, stateKey *string,
 	cfg config.Dendrite,
 	queryAPI api.RoomserverQueryAPI,
 	producer *producers.RoomserverProducer,
@@ -90,9 +90,17 @@ func SendEvent(
 		}
 	}
 
+	var txnAndDeviceID *api.TransactionID
+	if txnID != nil {
+		txnAndDeviceID = &api.TransactionID{
+			TransactionID: *txnID,
+			DeviceID:      device.ID,
+		}
+	}
+
 	// pass the new event to the roomserver
 	if err := producer.SendEvents(
-		req.Context(), []gomatrixserverlib.Event{*e}, cfg.Matrix.ServerName,
+		req.Context(), []gomatrixserverlib.Event{*e}, cfg.Matrix.ServerName, txnAndDeviceID,
 	); err != nil {
 		return httputil.LogThenError(req, err)
 	}

src/github.com/matrix-org/dendrite/clientapi/threepid/invites.go

@@ -355,5 +355,5 @@ func emit3PIDInviteEvent(
 		return err
 	}
 
-	return producer.SendEvents(ctx, []gomatrixserverlib.Event{*event}, cfg.Matrix.ServerName)
+	return producer.SendEvents(ctx, []gomatrixserverlib.Event{*event}, cfg.Matrix.ServerName, nil)
 }

src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go

@@ -387,7 +387,8 @@ func main() {
 			"adds_state_event_ids":["$1463671337126266wrSBX:matrix.org", "$1463671339126270PnVwC:matrix.org"],
 			"removes_state_event_ids":null,
 			"last_sent_event_id":"",
-			"send_as_server":""
+			"send_as_server":"",
+			"transaction_id": null
 		}}`,
 	}
 

src/github.com/matrix-org/dendrite/common/config/config.go

@@ -95,6 +95,9 @@ type Dendrite struct {
 		// HTTP API endpoint used to verify whether the captcha response
 		// was successful
 		RecaptchaSiteVerifyAPI string `yaml:"recaptcha_siteverify_api"`
+		// If set disables new users from registering (except via shared
+		// secrets)
+		RegistrationDisabled bool `yaml:"registration_disabled"`
 	} `yaml:"matrix"`
 
 	// The configuration specific to the media repostitory.

src/github.com/matrix-org/dendrite/federationapi/routing/join.go

@@ -169,7 +169,7 @@ func SendJoin(
 	// Send the events to the room server.
 	// We are responsible for notifying other servers that the user has joined
 	// the room, so set SendAsServer to cfg.Matrix.ServerName
-	err = producer.SendEvents(ctx, []gomatrixserverlib.Event{event}, cfg.Matrix.ServerName)
+	err = producer.SendEvents(ctx, []gomatrixserverlib.Event{event}, cfg.Matrix.ServerName, nil)
 	if err != nil {
 		return httputil.LogThenError(httpReq, err)
 	}

src/github.com/matrix-org/dendrite/federationapi/routing/send.go

@@ -170,7 +170,7 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event) error {
 	// TODO: Check that the event is allowed by its auth_events.
 
 	// pass the event to the roomserver
-	return t.producer.SendEvents(t.context, []gomatrixserverlib.Event{e}, api.DoNotSendToOtherServers)
+	return t.producer.SendEvents(t.context, []gomatrixserverlib.Event{e}, api.DoNotSendToOtherServers, nil)
 }
 
 func checkAllowedByState(e gomatrixserverlib.Event, stateEvents []gomatrixserverlib.Event) error {

src/github.com/matrix-org/dendrite/federationapi/routing/threepid.go

@@ -81,7 +81,7 @@ func CreateInvitesFrom3PIDInvites(
 	}
 
 	// Send all the events
-	if err := producer.SendEvents(req.Context(), evs, cfg.Matrix.ServerName); err != nil {
+	if err := producer.SendEvents(req.Context(), evs, cfg.Matrix.ServerName, nil); err != nil {
 		return httputil.LogThenError(req, err)
 	}
 
@@ -154,7 +154,7 @@ func ExchangeThirdPartyInvite(
 
 	// Send the event to the roomserver
 	if err = producer.SendEvents(
-		httpReq.Context(), []gomatrixserverlib.Event{signedEvent.Event}, cfg.Matrix.ServerName,
+		httpReq.Context(), []gomatrixserverlib.Event{signedEvent.Event}, cfg.Matrix.ServerName, nil,
 	); err != nil {
 		return httputil.LogThenError(httpReq, err)
 	}

src/github.com/matrix-org/dendrite/roomserver/api/input.go

@@ -68,6 +68,16 @@ type InputRoomEvent struct {
 	// The server name to use to push this event to other servers.
 	// Or empty if this event shouldn't be pushed to other servers.
 	SendAsServer string `json:"send_as_server"`
+	// The transaction ID of the send request if sent by a local user and one
+	// was specified
+	TransactionID *TransactionID `json:"transaction_id"`
+}
+
+// TransactionID contains the transaction ID sent by a client when sending an
+// event, along with the ID of that device.
+type TransactionID struct {
+	DeviceID      string `json:"device_id"`
+	TransactionID string `json:"id"`
 }
 
 // InputInviteEvent is a matrix invite event received over federation without

src/github.com/matrix-org/dendrite/roomserver/api/output.go

@@ -107,6 +107,9 @@ type OutputNewRoomEvent struct {
 	// We encode the server name that the event should be sent using here to
 	// future proof the API for virtual hosting.
 	SendAsServer string `json:"send_as_server"`
+	// The transaction ID of the send request if sent by a local user and one
+	// was specified
+	TransactionID *TransactionID `json:"transaction_id"`
 }
 
 // An OutputNewInviteEvent is written whenever an invite becomes active.

src/github.com/matrix-org/dendrite/roomserver/input/events.go

@@ -129,7 +129,7 @@ func processRoomEvent(
 	}
 
 	// Update the extremities of the event graph for the room
-	return updateLatestEvents(ctx, db, ow, roomNID, stateAtEvent, event, input.SendAsServer)
+	return updateLatestEvents(ctx, db, ow, roomNID, stateAtEvent, event, input.SendAsServer, input.TransactionID)
 }
 
 func processInviteEvent(

src/github.com/matrix-org/dendrite/roomserver/input/latest_events.go

@@ -50,6 +50,7 @@ func updateLatestEvents(
 	stateAtEvent types.StateAtEvent,
 	event gomatrixserverlib.Event,
 	sendAsServer string,
+	transactionID *api.TransactionID,
 ) (err error) {
 	updater, err := db.GetLatestEventsForUpdate(ctx, roomNID)
 	if err != nil {
@@ -61,6 +62,7 @@ func updateLatestEvents(
 	u := latestEventsUpdater{
 		ctx: ctx, db: db, updater: updater, ow: ow, roomNID: roomNID,
 		stateAtEvent: stateAtEvent, event: event, sendAsServer: sendAsServer,
+		transactionID: transactionID,
 	}
 	if err = u.doUpdateLatestEvents(); err != nil {
 		return err
@@ -75,13 +77,14 @@ func updateLatestEvents(
 // The state could be passed using function arguments, but it becomes impractical
 // when there are so many variables to pass around.
 type latestEventsUpdater struct {
-	ctx          context.Context
-	db           RoomEventDatabase
-	updater      types.RoomRecentEventsUpdater
-	ow           OutputRoomEventWriter
-	roomNID      types.RoomNID
-	stateAtEvent types.StateAtEvent
-	event        gomatrixserverlib.Event
+	ctx           context.Context
+	db            RoomEventDatabase
+	updater       types.RoomRecentEventsUpdater
+	ow            OutputRoomEventWriter
+	roomNID       types.RoomNID
+	stateAtEvent  types.StateAtEvent
+	event         gomatrixserverlib.Event
+	transactionID *api.TransactionID
 	// Which server to send this event as.
 	sendAsServer string
 	// The eventID of the event that was processed before this one.
@@ -241,6 +244,7 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error)
 		Event:           u.event,
 		LastSentEventID: u.lastEventIDSent,
 		LatestEventIDs:  latestEventIDs,
+		TransactionID:   u.transactionID,
 	}
 
 	var stateEventNIDs []types.EventNID