Fix a bug where we would accidentally skip over some events

.forgejo/workflows/docker.yml

@@ -1,59 +0,0 @@
-on:
-  push:
-    tags:
-      - 'v*'
-
-env:
-  GHCR_NAMESPACE: sigb.us
-  PLATFORMS: linux/amd64
-  FORGEJO_USER: signaryk
-
-jobs:
-  monolith:
-    name: Monolith image
-    runs-on: docker
-    container:
-      image: ghcr.io/catthehacker/ubuntu:act-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Get release tag & build flags
-        if: github.event_name == 'release' # Only for GitHub releases
-        run: |
-          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Login to sigb.us container registry
-        uses: docker/login-action@v3
-        with:
-          registry: git.sigb.us
-          username: ${{ env.FORGEJO_USER }}
-          password: ${{ secrets.FORGEJO_TOKEN }}
-
-      - name: Build main monolith image
-        id: docker_build_monolith
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:${{ github.ref_name }}
-            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:latest
-            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:devel
-
-      - name: Build release monolith image
-        if: github.event_name == 'release' # Only for GitHub releases
-        id: docker_build_monolith_release
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          platforms: ${{ env.PLATFORMS }}
-          push: true
-          tags: |
-            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:latest
-            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:stable
-            git.sigb.us/${{ env.GHCR_NAMESPACE }}/dendrite:${{ env.RELEASE_VERSION }}

.gitignore

@@ -5,7 +5,6 @@
 
 # Allow GitHub config
 !.github
-!.forgejo
 
 # Downloads
 /.downloads

clientapi/auth/login.go

@@ -61,8 +61,8 @@ func LoginFromJSONReader(
 	switch header.Type {
 	case authtypes.LoginTypePassword:
 		typ = &LoginTypePassword{
-			UserAPI: useraccountAPI,
+			GetAccountByPassword: useraccountAPI.QueryAccountByPassword,
-			Config:  cfg,
+			Config:               cfg,
 		}
 	case authtypes.LoginTypeToken:
 		typ = &LoginTypeToken{

clientapi/auth/login_test.go

@@ -292,14 +292,6 @@ func (ua *fakeUserInternalAPI) QueryAccountByPassword(ctx context.Context, req *
 	return nil
 }
 
-func (ua *fakeUserInternalAPI) QueryAccountByLocalpart(ctx context.Context, req *uapi.QueryAccountByLocalpartRequest, res *uapi.QueryAccountByLocalpartResponse) error {
-	return nil
-}
-
-func (ua *fakeUserInternalAPI) PerformAccountCreation(ctx context.Context, req *uapi.PerformAccountCreationRequest, res *uapi.PerformAccountCreationResponse) error {
-	return nil
-}
-
 func (ua *fakeUserInternalAPI) PerformLoginTokenDeletion(ctx context.Context, req *uapi.PerformLoginTokenDeletionRequest, res *uapi.PerformLoginTokenDeletionResponse) error {
 	ua.DeletedTokens = append(ua.DeletedTokens, req.Token)
 	return nil

clientapi/auth/password.go

@@ -16,9 +16,6 @@ package auth
 
 import (
 	"context"
-	"database/sql"
-	"github.com/go-ldap/ldap/v3"
-	"github.com/google/uuid"
 	"net/http"
 	"strings"
 
@@ -31,6 +28,8 @@ import (
 	"github.com/matrix-org/util"
 )
 
+type GetAccountByPassword func(ctx context.Context, req *api.QueryAccountByPasswordRequest, res *api.QueryAccountByPasswordResponse) error
+
 type PasswordRequest struct {
 	Login
 	Password string `json:"password"`
@@ -38,8 +37,8 @@ type PasswordRequest struct {
 
 // LoginTypePassword implements https://matrix.org/docs/spec/client_server/r0.6.1#password-based
 type LoginTypePassword struct {
-	Config  *config.ClientAPI
+	GetAccountByPassword GetAccountByPassword
-	UserAPI api.UserLoginAPI
+	Config               *config.ClientAPI
 }
 
 func (t *LoginTypePassword) Name() string {
@@ -60,21 +59,22 @@ func (t *LoginTypePassword) LoginFromJSON(ctx context.Context, reqBytes []byte)
 	return login, func(context.Context, *util.JSONResponse) {}, nil
 }
 
-func (t *LoginTypePassword) Login(ctx context.Context, request *PasswordRequest) (*Login, *util.JSONResponse) {
+func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) {
-	fullUsername := request.Username()
+	r := req.(*PasswordRequest)
-	if fullUsername == "" {
+	username := r.Username()
+	if username == "" {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
 			JSON: spec.BadJSON("A username must be supplied."),
 		}
 	}
-	if len(request.Password) == 0 {
+	if len(r.Password) == 0 {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
 			JSON: spec.BadJSON("A password must be supplied."),
 		}
 	}
-	username, domain, err := userutil.ParseUsernameParam(fullUsername, t.Config.Matrix)
+	localpart, domain, err := userutil.ParseUsernameParam(username, t.Config.Matrix)
 	if err != nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
@@ -87,38 +87,12 @@ func (t *LoginTypePassword) Login(ctx context.Context, request *PasswordRequest)
 			JSON: spec.InvalidUsername("The server name is not known."),
 		}
 	}
-
+	// Squash username to all lowercase letters
-	var account *api.Account
-	if t.Config.Ldap.Enabled {
-		isAdmin, err := t.authenticateLdap(username, request.Password)
-		if err != nil {
-			return nil, err
-		}
-		acc, err := t.getOrCreateAccount(ctx, username, domain, isAdmin)
-		if err != nil {
-			return nil, err
-		}
-		account = acc
-	} else {
-		acc, err := t.authenticateDb(ctx, username, domain, request.Password)
-		if err != nil {
-			return nil, err
-		}
-		account = acc
-	}
-
-	// Set the user, so login.Username() can do the right thing
-	request.Identifier.User = account.UserID
-	request.User = account.UserID
-	return &request.Login, nil
-}
-
-func (t *LoginTypePassword) authenticateDb(ctx context.Context, username string, domain spec.ServerName, password string) (*api.Account, *util.JSONResponse) {
 	res := &api.QueryAccountByPasswordResponse{}
-	err := t.UserAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
+	err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
-		Localpart:         strings.ToLower(username),
+		Localpart:         strings.ToLower(localpart),
 		ServerName:        domain,
-		PlaintextPassword: password,
+		PlaintextPassword: r.Password,
 	}, res)
 	if err != nil {
 		return nil, &util.JSONResponse{
@@ -127,11 +101,13 @@ func (t *LoginTypePassword) authenticateDb(ctx context.Context, username string,
 		}
 	}
 
+	// If we couldn't find the user by the lower cased localpart, try the provided
+	// localpart as is.
 	if !res.Exists {
-		err = t.UserAPI.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
+		err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
-			Localpart:         username,
+			Localpart:         localpart,
 			ServerName:        domain,
-			PlaintextPassword: password,
+			PlaintextPassword: r.Password,
 		}, res)
 		if err != nil {
 			return nil, &util.JSONResponse{
@@ -139,6 +115,8 @@ func (t *LoginTypePassword) authenticateDb(ctx context.Context, username string,
 				JSON: spec.Unknown("Unable to fetch account by password."),
 			}
 		}
+		// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
+		// but that would leak the existence of the user.
 		if !res.Exists {
 			return nil, &util.JSONResponse{
 				Code: http.StatusForbidden,
@@ -146,141 +124,8 @@ func (t *LoginTypePassword) authenticateDb(ctx context.Context, username string,
 			}
 		}
 	}
-	return res.Account, nil
+	// Set the user, so login.Username() can do the right thing
-}
+	r.Identifier.User = res.Account.UserID
-func (t *LoginTypePassword) authenticateLdap(username, password string) (bool, *util.JSONResponse) {
+	r.User = res.Account.UserID
-	var conn *ldap.Conn
+	return &r.Login, nil
-	conn, err := ldap.DialURL(t.Config.Ldap.Uri)
-	if err != nil {
-		return false, &util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("unable to connect to ldap: " + err.Error()),
-		}
-	}
-	defer conn.Close()
-
-	if t.Config.Ldap.AdminBindEnabled {
-		err = conn.Bind(t.Config.Ldap.AdminBindDn, t.Config.Ldap.AdminBindPassword)
-		if err != nil {
-			return false, &util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: spec.Unknown("unable to bind to ldap: " + err.Error()),
-			}
-		}
-		filter := strings.ReplaceAll(t.Config.Ldap.SearchFilter, "{username}", username)
-		searchRequest := ldap.NewSearchRequest(
-			t.Config.Ldap.BaseDn, ldap.ScopeWholeSubtree, ldap.NeverDerefAliases,
-			0, 0, false, filter, []string{t.Config.Ldap.SearchAttribute}, nil,
-		)
-		result, err := conn.Search(searchRequest)
-		if err != nil {
-			return false, &util.JSONResponse{
-				Code: http.StatusInternalServerError,
-				JSON: spec.Unknown("unable to bind to search ldap: " + err.Error()),
-			}
-		}
-		if len(result.Entries) > 1 {
-			return false, &util.JSONResponse{
-				Code: http.StatusUnauthorized,
-				JSON: spec.BadJSON("'user' must be duplicated."),
-			}
-		}
-		if len(result.Entries) < 1 {
-			return false, &util.JSONResponse{
-				Code: http.StatusUnauthorized,
-				JSON: spec.BadJSON("'user' not found."),
-			}
-		}
-
-		userDN := result.Entries[0].DN
-		err = conn.Bind(userDN, password)
-		if err != nil {
-			return false, &util.JSONResponse{
-				Code: http.StatusUnauthorized,
-				JSON: spec.InvalidUsername(err.Error()),
-			}
-		}
-	} else {
-		bindDn := strings.ReplaceAll(t.Config.Ldap.UserBindDn, "{username}", username)
-		err = conn.Bind(bindDn, password)
-		if err != nil {
-			return false, &util.JSONResponse{
-				Code: http.StatusUnauthorized,
-				JSON: spec.InvalidUsername(err.Error()),
-			}
-		}
-	}
-
-	isAdmin, err := t.isLdapAdmin(conn, username)
-	if err != nil {
-		return false, &util.JSONResponse{
-			Code: http.StatusUnauthorized,
-			JSON: spec.InvalidUsername(err.Error()),
-		}
-	}
-	return isAdmin, nil
-}
-
-func (t *LoginTypePassword) isLdapAdmin(conn *ldap.Conn, username string) (bool, error) {
-	searchRequest := ldap.NewSearchRequest(
-		t.Config.Ldap.AdminGroupDn,
-		ldap.ScopeWholeSubtree, ldap.DerefAlways, 0, 0, false,
-		strings.ReplaceAll(t.Config.Ldap.AdminGroupFilter, "{username}", username),
-		[]string{t.Config.Ldap.AdminGroupAttribute},
-		nil)
-
-	sr, err := conn.Search(searchRequest)
-	if err != nil {
-		return false, err
-	}
-
-	if len(sr.Entries) < 1 {
-		return false, nil
-	}
-	return true, nil
-}
-
-func (t *LoginTypePassword) getOrCreateAccount(ctx context.Context, username string, domain spec.ServerName, admin bool) (*api.Account, *util.JSONResponse) {
-	var existing api.QueryAccountByLocalpartResponse
-	err := t.UserAPI.QueryAccountByLocalpart(ctx, &api.QueryAccountByLocalpartRequest{
-		Localpart:  username,
-		ServerName: domain,
-	}, &existing)
-
-	if err == nil {
-		return existing.Account, nil
-	}
-	if err != sql.ErrNoRows {
-		return nil, &util.JSONResponse{
-			Code: http.StatusUnauthorized,
-			JSON: spec.InvalidUsername(err.Error()),
-		}
-	}
-
-	accountType := api.AccountTypeUser
-	if admin {
-		accountType = api.AccountTypeAdmin
-	}
-	var created api.PerformAccountCreationResponse
-	err = t.UserAPI.PerformAccountCreation(ctx, &api.PerformAccountCreationRequest{
-		AppServiceID: "ldap",
-		Localpart:    username,
-		Password:     uuid.New().String(),
-		AccountType:  accountType,
-		OnConflict:   api.ConflictAbort,
-	}, &created)
-
-	if err != nil {
-		if _, ok := err.(*api.ErrorConflict); ok {
-			return nil, &util.JSONResponse{
-				Code: http.StatusBadRequest,
-				JSON: spec.UserInUse("Desired user ID is already taken."),
-			}
-		}
-		return nil, &util.JSONResponse{
-			Code: http.StatusInternalServerError,
-			JSON: spec.Unknown("failed to create account: " + err.Error()),
-		}
-	}
-	return created.Account, nil
 }

clientapi/auth/user_interactive.go

@@ -113,8 +113,8 @@ type UserInteractive struct {
 
 func NewUserInteractive(userAccountAPI api.UserLoginAPI, cfg *config.ClientAPI) *UserInteractive {
 	typePassword := &LoginTypePassword{
-		UserAPI: userAccountAPI,
+		GetAccountByPassword: userAccountAPI.QueryAccountByPassword,
-		Config:  cfg,
+		Config:               cfg,
 	}
 	return &UserInteractive{
 		Flows: []userInteractiveFlow{

clientapi/auth/user_interactive_test.go

@@ -45,14 +45,6 @@ func (d *fakeAccountDatabase) QueryAccountByPassword(ctx context.Context, req *a
 	return nil
 }
 
-func (d *fakeAccountDatabase) QueryAccountByLocalpart(ctx context.Context, req *api.QueryAccountByLocalpartRequest, res *api.QueryAccountByLocalpartResponse) error {
-	return nil
-}
-
-func (d *fakeAccountDatabase) PerformAccountCreation(ctx context.Context, req *api.PerformAccountCreationRequest, res *api.PerformAccountCreationResponse) error {
-	return nil
-}
-
 func setup() *UserInteractive {
 	cfg := &config.ClientAPI{
 		Matrix: &config.Global{

clientapi/routing/key_crosssigning.go

@@ -32,7 +32,7 @@ type crossSigningRequest struct {
 }
 
 func UploadCrossSigningDeviceKeys(
-	req *http.Request,
+	req *http.Request, userInteractiveAuth *auth.UserInteractive,
 	keyserverAPI api.ClientKeyAPI, device *api.Device,
 	accountAPI api.ClientUserAPI, cfg *config.ClientAPI,
 ) util.JSONResponse {
@@ -62,8 +62,8 @@ func UploadCrossSigningDeviceKeys(
 		}
 	}
 	typePassword := auth.LoginTypePassword{
-		UserAPI: accountAPI,
+		GetAccountByPassword: accountAPI.QueryAccountByPassword,
-		Config:  cfg,
+		Config:               cfg,
 	}
 	if _, authErr := typePassword.Login(req.Context(), &uploadReq.Auth.PasswordRequest); authErr != nil {
 		return *authErr

clientapi/routing/password.go

@@ -73,8 +73,8 @@ func Password(
 
 	// Check if the existing password is correct.
 	typePassword := auth.LoginTypePassword{
-		UserAPI: userAPI,
+		GetAccountByPassword: userAPI.QueryAccountByPassword,
-		Config:  cfg,
+		Config:               cfg,
 	}
 	if _, authErr := typePassword.Login(req.Context(), &r.Auth.PasswordRequest); authErr != nil {
 		return *authErr

clientapi/routing/routing.go

@@ -1448,7 +1448,7 @@ func Setup(
 	// Cross-signing device keys
 
 	postDeviceSigningKeys := httputil.MakeAuthAPI("post_device_signing_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
-		return UploadCrossSigningDeviceKeys(req, userAPI, device, userAPI, cfg)
+		return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, userAPI, device, userAPI, cfg)
 	})
 
 	postDeviceSigningSignatures := httputil.MakeAuthAPI("post_device_signing_signatures", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {

go.mod

@@ -12,7 +12,6 @@ require (
 	github.com/docker/docker v24.0.9+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/getsentry/sentry-go v0.14.0
-	github.com/go-ldap/ldap/v3 v3.4.4
 	github.com/gologme/log v1.3.0
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.3.0
@@ -58,7 +57,6 @@ require (
 )
 
 require (
-	github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect
 	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/RoaringBitmap/roaring v1.2.3 // indirect
@@ -86,7 +84,6 @@ require (
 	github.com/docker/distribution v2.8.2+incompatible // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect

go.sum

@@ -5,8 +5,6 @@ github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3
 github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
-github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e h1:NeAW1fUYUEWhft7pkxDf6WoUvEZJ/uOKsvtpjLnn8MU=
-github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
 github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
@@ -117,12 +115,8 @@ github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwv
 github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8=
 github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
-github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
-github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs=
-github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
@@ -389,7 +383,6 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
@@ -413,7 +406,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -435,7 +427,6 @@ golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=

setup/config/config_clientapi.go

@@ -56,26 +56,9 @@ type ClientAPI struct {
 	RateLimiting RateLimiting `yaml:"rate_limiting"`
 
 	MSCs *MSCs `yaml:"-"`
-
-	Ldap Ldap `yaml:"ldap"`
 }
 
-type Ldap struct {
+func (c *ClientAPI) Defaults(opts DefaultOpts) {
-	Enabled             bool   `yaml:"enabled"`
-	Uri                 string `yaml:"uri"`
-	BaseDn              string `yaml:"base_dn"`
-	SearchFilter        string `yaml:"search_filter"`
-	SearchAttribute     string `yaml:"search_attribute"`
-	AdminBindEnabled    bool   `yaml:"admin_bind_enabled"`
-	AdminBindDn         string `yaml:"admin_bind_dn"`
-	AdminBindPassword   string `yaml:"admin_bind_password"`
-	UserBindDn          string `yaml:"user_bind_dn"`
-	AdminGroupDn        string `yaml:"admin_group_dn"`
-	AdminGroupFilter    string `yaml:"admin_group_filter"`
-	AdminGroupAttribute string `yaml:"admin_group_attribute"`
-}
-
-func (c *ClientAPI) Defaults(_ DefaultOpts) {
 	c.RegistrationSharedSecret = ""
 	c.RegistrationRequiresToken = false
 	c.RecaptchaPublicKey = ""

syncapi/storage/shared/storage_consumer.go

@@ -508,7 +508,7 @@ func (d *Database) CleanSendToDeviceUpdates(
 // getMembershipFromEvent returns the value of content.membership iff the event is a state event
 // with type 'm.room.member' and state_key of userID. Otherwise, an empty string is returned.
 func getMembershipFromEvent(ctx context.Context, ev gomatrixserverlib.PDU, userID string, rsAPI api.SyncRoomserverAPI) (string, string) {
-	if ev.StateKey() == nil || *ev.StateKey() == "" {
+	if ev.Type() != "m.room.member" || ev.StateKey() == nil || *ev.StateKey() == "" {
 		return "", ""
 	}
 	fullUser, err := spec.NewUserID(userID, true)
@@ -520,7 +520,7 @@ func getMembershipFromEvent(ctx context.Context, ev gomatrixserverlib.PDU, userI
 		return "", ""
 	}
 
-	if ev.Type() != "m.room.member" || !ev.StateKeyEquals(string(*senderID)) {
+	if !ev.StateKeyEquals(string(*senderID)) {
 		return "", ""
 	}
 	membership, err := ev.Membership()

syncapi/sync/requestpool.go

@@ -397,13 +397,19 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 			}
 		} else {
 			// Incremental sync
+			reasonablePositions := findReasonableIncrementalSyncWindow(
+				syncReq.Since, rp.Notifier.CurrentPosition(),
+			)
+			// Also update the currentPos, which is used for the retry logic below.
+			// Otherwise we may skip over some events.
+			currentPos = reasonablePositions
 			syncReq.Response.NextBatch = types.StreamingToken{
 				PDUPosition: withTransaction(
 					syncReq.Since.PDUPosition,
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.PDUStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.PDUPosition, rp.Notifier.CurrentPosition().PDUPosition,
+							syncReq.Since.PDUPosition, reasonablePositions.PDUPosition,
 						)
 					},
 				),
@@ -412,7 +418,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.TypingStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.TypingPosition, rp.Notifier.CurrentPosition().TypingPosition,
+							syncReq.Since.TypingPosition, reasonablePositions.TypingPosition,
 						)
 					},
 				),
@@ -421,7 +427,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.ReceiptStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.ReceiptPosition, rp.Notifier.CurrentPosition().ReceiptPosition,
+							syncReq.Since.ReceiptPosition, reasonablePositions.ReceiptPosition,
 						)
 					},
 				),
@@ -430,7 +436,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.InviteStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.InvitePosition, rp.Notifier.CurrentPosition().InvitePosition,
+							syncReq.Since.InvitePosition, reasonablePositions.InvitePosition,
 						)
 					},
 				),
@@ -439,7 +445,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.SendToDeviceStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.SendToDevicePosition, rp.Notifier.CurrentPosition().SendToDevicePosition,
+							syncReq.Since.SendToDevicePosition, reasonablePositions.SendToDevicePosition,
 						)
 					},
 				),
@@ -448,7 +454,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.AccountDataStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.AccountDataPosition, rp.Notifier.CurrentPosition().AccountDataPosition,
+							syncReq.Since.AccountDataPosition, reasonablePositions.AccountDataPosition,
 						)
 					},
 				),
@@ -457,7 +463,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.NotificationDataStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.NotificationDataPosition, rp.Notifier.CurrentPosition().NotificationDataPosition,
+							syncReq.Since.NotificationDataPosition, reasonablePositions.NotificationDataPosition,
 						)
 					},
 				),
@@ -466,7 +472,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.DeviceListStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.DeviceListPosition, rp.Notifier.CurrentPosition().DeviceListPosition,
+							syncReq.Since.DeviceListPosition, reasonablePositions.DeviceListPosition,
 						)
 					},
 				),
@@ -475,7 +481,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
 					func(txn storage.DatabaseTransaction) types.StreamPosition {
 						return rp.streams.PresenceStreamProvider.IncrementalSync(
 							syncReq.Context, txn, syncReq,
-							syncReq.Since.PresencePosition, rp.Notifier.CurrentPosition().PresencePosition,
+							syncReq.Since.PresencePosition, reasonablePositions.PresencePosition,
 						)
 					},
 				),
@@ -585,3 +591,23 @@ func (rp *RequestPool) shouldReturnImmediately(syncReq *types.SyncRequest, curre
 	}
 	return false
 }
+
+func findReasonableIncrementalSyncWindow(since, limit types.StreamingToken) types.StreamingToken {
+	const windowSize = 100 // TODO: reasonable number?
+	for s, l := range map[*types.StreamPosition]types.StreamPosition{
+		&since.AccountDataPosition:      limit.AccountDataPosition,
+		&since.DeviceListPosition:       limit.DeviceListPosition,
+		&since.InvitePosition:           limit.InvitePosition,
+		&since.NotificationDataPosition: limit.NotificationDataPosition,
+		&since.PDUPosition:              limit.PDUPosition,
+		&since.PresencePosition:         limit.PresencePosition,
+		&since.ReceiptPosition:          limit.ReceiptPosition,
+		&since.SendToDevicePosition:     limit.SendToDevicePosition,
+		&since.TypingPosition:           limit.TypingPosition,
+	} {
+		if *s += windowSize; *s > l {
+			*s = l
+		}
+	}
+	return since
+}

sytest-whitelist

@@ -540,6 +540,7 @@ Will not back up to an old backup version
 Can create more than 10 backup versions
 Can delete backup
 Deleted & recreated backups are empty
+Can upload self-signing keys
 Fails to upload self-signing keys with no auth
 Fails to upload self-signing key without master key
 can fetch self-signing keys over federation
@@ -632,6 +633,7 @@ Trying to add push rule with no scope fails with 400
 Trying to add push rule with invalid scope fails with 400
 Forward extremities remain so even after the next events are populated as outliers
 uploading self-signing key notifies over federation
+uploading signed devices gets propagated over federation
 Device list doesn't change if remote server is down
 /context/ on joined room works
 /context/ on non world readable room does not work

userapi/api/api.go

@@ -144,8 +144,6 @@ type QueryAcccessTokenAPI interface {
 
 type UserLoginAPI interface {
 	QueryAccountByPassword(ctx context.Context, req *QueryAccountByPasswordRequest, res *QueryAccountByPasswordResponse) error
-	QueryAccountByLocalpart(ctx context.Context, req *QueryAccountByLocalpartRequest, res *QueryAccountByLocalpartResponse) error
-	PerformAccountCreation(ctx context.Context, req *PerformAccountCreationRequest, res *PerformAccountCreationResponse) error
 }
 
 type PerformKeyBackupRequest struct {