Merge branch 'main' into gh-pages

Dockerfile

@@ -27,6 +27,7 @@ RUN --mount=target=. \
 # The dendrite base image
 #
 FROM alpine:latest AS dendrite-base
+RUN apk --update --no-cache add curl
 LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
 LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
 LABEL org.opencontainers.image.licenses="Apache-2.0"

build/docker/Dockerfile.demo-pinecone

@@ -17,6 +17,7 @@ RUN go build -trimpath -o bin/ ./cmd/create-account
 RUN go build -trimpath -o bin/ ./cmd/generate-keys
 
 FROM alpine:latest
+RUN apk --update --no-cache add curl
 LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)"
 LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
 LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"

clientapi/auth/login_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/dendrite/setup/config"
 	uapi "github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/gomatrixserverlib"
@@ -47,7 +48,7 @@ func TestLoginFromJSONReader(t *testing.T) {
 				"password": "herpassword",
 				"device_id": "adevice"
             }`,
-			WantUsername: "alice",
+			WantUsername: "@alice:example.com",
 			WantDeviceID: "adevice",
 		},
 		{
@@ -174,7 +175,7 @@ func (ua *fakeUserInternalAPI) QueryAccountByPassword(ctx context.Context, req *
 		return nil
 	}
 	res.Exists = true
-	res.Account = &uapi.Account{}
+	res.Account = &uapi.Account{UserID: userutil.MakeUserID(req.Localpart, req.ServerName)}
 	return nil
 }
 

clientapi/auth/password.go

@@ -101,6 +101,8 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 		}
 	}
 
+	// If we couldn't find the user by the lower cased localpart, try the provided
+	// localpart as is.
 	if !res.Exists {
 		err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
 			Localpart:         localpart,
@@ -122,5 +124,8 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
 			}
 		}
 	}
+	// Set the user, so login.Username() can do the right thing
+	r.Identifier.User = res.Account.UserID
+	r.User = res.Account.UserID
 	return &r.Login, nil
 }

clientapi/routing/login_test.go

@@ -0,0 +1,152 @@
+package routing
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/keyserver"
+	"github.com/matrix-org/dendrite/roomserver"
+	"github.com/matrix-org/dendrite/setup/config"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+
+	"github.com/matrix-org/dendrite/test"
+	"github.com/matrix-org/dendrite/test/testrig"
+	"github.com/matrix-org/dendrite/userapi"
+	uapi "github.com/matrix-org/dendrite/userapi/api"
+)
+
+func TestLogin(t *testing.T) {
+	aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
+	bobUser := &test.User{ID: "@bob:test", AccountType: uapi.AccountTypeUser}
+	charlie := &test.User{ID: "@Charlie:test", AccountType: uapi.AccountTypeUser}
+	vhUser := &test.User{ID: "@vhuser:vh1"}
+
+	ctx := context.Background()
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+		base.Cfg.ClientAPI.RateLimiting.Enabled = false
+		// add a vhost
+		base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{
+			SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"},
+		})
+
+		rsAPI := roomserver.NewInternalAPI(base)
+		// Needed for /login
+		keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
+		userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
+		keyAPI.SetUserAPI(userAPI)
+
+		// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
+		Setup(base, &base.Cfg.ClientAPI, nil, nil, userAPI, nil, nil, nil, nil, nil, keyAPI, nil, &base.Cfg.MSCs, nil)
+
+		// Create password
+		password := util.RandomString(8)
+
+		// create the users
+		for _, u := range []*test.User{aliceAdmin, bobUser, vhUser, charlie} {
+			localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
+			userRes := &uapi.PerformAccountCreationResponse{}
+
+			if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
+				AccountType: u.AccountType,
+				Localpart:   localpart,
+				ServerName:  serverName,
+				Password:    password,
+			}, userRes); err != nil {
+				t.Errorf("failed to create account: %s", err)
+			}
+			if !userRes.AccountCreated {
+				t.Fatalf("account not created")
+			}
+		}
+
+		testCases := []struct {
+			name   string
+			userID string
+			wantOK bool
+		}{
+			{
+				name:   "aliceAdmin can login",
+				userID: aliceAdmin.ID,
+				wantOK: true,
+			},
+			{
+				name:   "bobUser can login",
+				userID: bobUser.ID,
+				wantOK: true,
+			},
+			{
+				name:   "vhuser can login",
+				userID: vhUser.ID,
+				wantOK: true,
+			},
+			{
+				name:   "bob with uppercase can login",
+				userID: "@Bob:test",
+				wantOK: true,
+			},
+			{
+				name:   "Charlie can login (existing uppercase)",
+				userID: charlie.ID,
+				wantOK: true,
+			},
+			{
+				name:   "Charlie can not login with lowercase userID",
+				userID: strings.ToLower(charlie.ID),
+				wantOK: false,
+			},
+		}
+
+		ctx := context.Background()
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
+					"type": authtypes.LoginTypePassword,
+					"identifier": map[string]interface{}{
+						"type": "m.id.user",
+						"user": tc.userID,
+					},
+					"password": password,
+				}))
+				rec := httptest.NewRecorder()
+				base.PublicClientAPIMux.ServeHTTP(rec, req)
+				if tc.wantOK && rec.Code != http.StatusOK {
+					t.Fatalf("failed to login: %s", rec.Body.String())
+				}
+
+				t.Logf("Response: %s", rec.Body.String())
+				// get the response
+				resp := loginResponse{}
+				if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
+					t.Fatal(err)
+				}
+				// everything OK
+				if !tc.wantOK && resp.AccessToken == "" {
+					return
+				}
+				if tc.wantOK && resp.AccessToken == "" {
+					t.Fatalf("expected accessToken after successful login but got none: %+v", resp)
+				}
+
+				devicesResp := &uapi.QueryDevicesResponse{}
+				if err := userAPI.QueryDevices(ctx, &uapi.QueryDevicesRequest{UserID: resp.UserID}, devicesResp); err != nil {
+					t.Fatal(err)
+				}
+				for _, dev := range devicesResp.Devices {
+					// We expect the userID on the device to be the same as resp.UserID
+					if dev.UserID != resp.UserID {
+						t.Fatalf("unexpected userID on device: %s", dev.UserID)
+					}
+				}
+			})
+		}
+	})
+}

clientapi/routing/register.go

@@ -780,7 +780,7 @@ func handleApplicationServiceRegistration(
 	// Don't need to worry about appending to registration stages as
 	// application service registration is entirely separate.
 	return completeRegistration(
-		req.Context(), userAPI, r.Username, r.ServerName, "", appserviceID, req.RemoteAddr,
+		req.Context(), userAPI, r.Username, r.ServerName, "", "", appserviceID, req.RemoteAddr,
 		req.UserAgent(), r.Auth.Session, r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
 		userapi.AccountTypeAppService,
 	)
@@ -800,7 +800,7 @@ func checkAndCompleteFlow(
 	if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
 		// This flow was completed, registration can continue
 		return completeRegistration(
-			req.Context(), userAPI, r.Username, r.ServerName, r.Password, "", req.RemoteAddr,
+			req.Context(), userAPI, r.Username, r.ServerName, "", r.Password, "", req.RemoteAddr,
 			req.UserAgent(), sessionID, r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
 			userapi.AccountTypeUser,
 		)
@@ -824,10 +824,10 @@ func checkAndCompleteFlow(
 func completeRegistration(
 	ctx context.Context,
 	userAPI userapi.ClientUserAPI,
-	username string, serverName gomatrixserverlib.ServerName,
+	username string, serverName gomatrixserverlib.ServerName, displayName string,
 	password, appserviceID, ipAddr, userAgent, sessionID string,
 	inhibitLogin eventutil.WeakBoolean,
-	displayName, deviceID *string,
+	deviceDisplayName, deviceID *string,
 	accType userapi.AccountType,
 ) util.JSONResponse {
 	if username == "" {
@@ -887,12 +887,28 @@ func completeRegistration(
 		}
 	}
 
+	if displayName != "" {
+		nameReq := userapi.PerformUpdateDisplayNameRequest{
+			Localpart:   username,
+			ServerName:  serverName,
+			DisplayName: displayName,
+		}
+		var nameRes userapi.PerformUpdateDisplayNameResponse
+		err = userAPI.SetDisplayName(ctx, &nameReq, &nameRes)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: jsonerror.Unknown("failed to set display name: " + err.Error()),
+			}
+		}
+	}
+
 	var devRes userapi.PerformDeviceCreationResponse
 	err = userAPI.PerformDeviceCreation(ctx, &userapi.PerformDeviceCreationRequest{
 		Localpart:         username,
 		ServerName:        serverName,
 		AccessToken:       token,
-		DeviceDisplayName: displayName,
+		DeviceDisplayName: deviceDisplayName,
 		DeviceID:          deviceID,
 		IPAddr:            ipAddr,
 		UserAgent:         userAgent,
@@ -1077,5 +1093,5 @@ func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.Clien
 	if ssrr.Admin {
 		accType = userapi.AccountTypeAdmin
 	}
-	return completeRegistration(req.Context(), userAPI, ssrr.User, cfg.Matrix.ServerName, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType)
+	return completeRegistration(req.Context(), userAPI, ssrr.User, cfg.Matrix.ServerName, ssrr.DisplayName, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType)
 }

clientapi/routing/register_secret.go

@@ -18,12 +18,13 @@ import (
 )
 
 type SharedSecretRegistrationRequest struct {
-	User     string `json:"username"`
+	User        string `json:"username"`
-	Password string `json:"password"`
+	Password    string `json:"password"`
-	Nonce    string `json:"nonce"`
+	Nonce       string `json:"nonce"`
-	MacBytes []byte
+	MacBytes    []byte
-	MacStr   string `json:"mac"`
+	MacStr      string `json:"mac"`
-	Admin    bool   `json:"admin"`
+	Admin       bool   `json:"admin"`
+	DisplayName string `json:"displayname,omitempty"`
 }
 
 func NewSharedSecretRegistrationRequest(reader io.ReadCloser) (*SharedSecretRegistrationRequest, error) {

clientapi/routing/register_secret_test.go

@@ -10,7 +10,7 @@ import (
 
 func TestSharedSecretRegister(t *testing.T) {
 	// these values have come from a local synapse instance to ensure compatibility
-	jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice"}`)
+	jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`)
 	sharedSecret := "dendritetest"
 
 	req, err := NewSharedSecretRegistrationRequest(io.NopCloser(bytes.NewBuffer(jsonStr)))

clientapi/routing/register_test.go

@@ -18,6 +18,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
@@ -35,7 +36,10 @@ import (
 	"github.com/matrix-org/dendrite/test"
 	"github.com/matrix-org/dendrite/test/testrig"
 	"github.com/matrix-org/dendrite/userapi"
+	"github.com/matrix-org/dendrite/userapi/api"
 	"github.com/matrix-org/util"
+	"github.com/patrickmn/go-cache"
+	"github.com/stretchr/testify/assert"
 )
 
 var (
@@ -570,3 +574,92 @@ func Test_register(t *testing.T) {
 		}
 	})
 }
+
+func TestRegisterUserWithDisplayName(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+		base.Cfg.Global.ServerName = "server"
+
+		rsAPI := roomserver.NewInternalAPI(base)
+		keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
+		userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
+		keyAPI.SetUserAPI(userAPI)
+		deviceName, deviceID := "deviceName", "deviceID"
+		expectedDisplayName := "DisplayName"
+		response := completeRegistration(
+			base.Context(),
+			userAPI,
+			"user",
+			"server",
+			expectedDisplayName,
+			"password",
+			"",
+			"localhost",
+			"user agent",
+			"session",
+			false,
+			&deviceName,
+			&deviceID,
+			api.AccountTypeAdmin,
+		)
+
+		assert.Equal(t, http.StatusOK, response.Code)
+
+		req := api.QueryProfileRequest{UserID: "@user:server"}
+		var res api.QueryProfileResponse
+		err := userAPI.QueryProfile(base.Context(), &req, &res)
+		assert.NoError(t, err)
+		assert.Equal(t, expectedDisplayName, res.DisplayName)
+	})
+}
+
+func TestRegisterAdminUsingSharedSecret(t *testing.T) {
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
+		base, baseClose := testrig.CreateBaseDendrite(t, dbType)
+		defer baseClose()
+		base.Cfg.Global.ServerName = "server"
+		sharedSecret := "dendritetest"
+		base.Cfg.ClientAPI.RegistrationSharedSecret = sharedSecret
+
+		rsAPI := roomserver.NewInternalAPI(base)
+		keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
+		userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
+		keyAPI.SetUserAPI(userAPI)
+
+		expectedDisplayName := "rabbit"
+		jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`)
+		req, err := NewSharedSecretRegistrationRequest(io.NopCloser(bytes.NewBuffer(jsonStr)))
+		assert.NoError(t, err)
+		if err != nil {
+			t.Fatalf("failed to read request: %s", err)
+		}
+
+		r := NewSharedSecretRegistration(sharedSecret)
+
+		// force the nonce to be known
+		r.nonces.Set(req.Nonce, true, cache.DefaultExpiration)
+
+		_, err = r.IsValidMacLogin(req.Nonce, req.User, req.Password, req.Admin, req.MacBytes)
+		assert.NoError(t, err)
+
+		body := &bytes.Buffer{}
+		err = json.NewEncoder(body).Encode(req)
+		assert.NoError(t, err)
+		ssrr := httptest.NewRequest(http.MethodPost, "/", body)
+
+		response := handleSharedSecretRegistration(
+			&base.Cfg.ClientAPI,
+			userAPI,
+			r,
+			ssrr,
+		)
+		assert.Equal(t, http.StatusOK, response.Code)
+
+		profilReq := api.QueryProfileRequest{UserID: "@alice:server"}
+		var profileRes api.QueryProfileResponse
+		err = userAPI.QueryProfile(base.Context(), &profilReq, &profileRes)
+		assert.NoError(t, err)
+		assert.Equal(t, expectedDisplayName, profileRes.DisplayName)
+	})
+}

docs/FAQ.md

@@ -6,6 +6,12 @@ permalink: /faq
 
 # FAQ
 
+## Why does Dendrite exist?
+
+Dendrite aims to provide a matrix compatible server that has low resource usage compared to [Synapse](https://github.com/matrix-org/synapse).
+It also aims to provide more flexibility when scaling either up or down.
+Dendrite's code is also very easy to hack on which makes it suitable for experimenting with new matrix features such as peer-to-peer.
+
 ## Is Dendrite stable?
 
 Mostly, although there are still bugs and missing features. If you are a confident power user and you are happy to spend some time debugging things when they go wrong, then please try out Dendrite. If you are a community, organisation or business that demands stability and uptime, then Dendrite is not for you yet - please install Synapse instead.
@@ -34,6 +40,10 @@ No, Dendrite has a very different database schema to Synapse and the two are not
 Monolith deployments are always preferred where possible, and at this time, are far better tested than polylith deployments are. The only reason to consider a polylith deployment is if you wish to run different Dendrite components on separate physical machines, but this is an advanced configuration which we don't
 recommend.
 
+## Can I configure which port Dendrite listens on?
+
+Yes, use the cli flag `-http-bind-address`. 
+
 ## I've installed Dendrite but federation isn't working
 
 Check the [Federation Tester](https://federationtester.matrix.org). You need at least:
@@ -42,6 +52,10 @@ Check the [Federation Tester](https://federationtester.matrix.org). You need at
 * A valid TLS certificate for that DNS name
 * Either DNS SRV records or well-known files
 
+## Whenever I try to connect from Element it says unable to connect to homeserver
+
+Check that your dendrite instance is running. Otherwise this is most likely due to a reverse proxy misconfiguration.
+
 ## Does Dendrite work with my favourite client?
 
 It should do, although we are aware of some minor issues:
@@ -49,6 +63,10 @@ It should do, although we are aware of some minor issues:
 * **Element Android**: registration does not work, but logging in with an existing account does
 * **Hydrogen**: occasionally sync can fail due to gaps in the `since` parameter, but clearing the cache fixes this
 
+## Is there a public instance of Dendrite I can try out?
+
+Use [dendrite.matrix.org](https://dendrite.matrix.org) which we officially support.
+
 ## Does Dendrite support Space Summaries?
 
 Yes, [Space Summaries](https://github.com/matrix-org/matrix-spec-proposals/pull/2946) were merged into the Matrix Spec as of 2022-01-17 however, they are still treated as an MSC (Matrix Specification Change) in Dendrite. In order to enable Space Summaries in Dendrite, you must add the MSC to the MSC configuration section in the configuration YAML. If the MSC is not enabled, a user will typically see a perpetual loading icon on the summary page. See below for a demonstration of how to add to the Dendrite configuration:
@@ -84,10 +102,42 @@ Remember to add the config file(s) to the `app_service_api` section of the confi
 
 Yes, you can do this by disabling federation - set `disable_federation` to `true` in the `global` section of the Dendrite configuration file.
 
+## How can I migrate a room in order to change the internal ID?
+
+This can be done by performing a room upgrade. Use the command `/upgraderoom <version>` in Element to do this.
+
+## How do I reset somebody's password on my server?
+
+Use the admin endpoint [resetpassword](https://matrix-org.github.io/dendrite/administration/adminapi#post-_dendriteadminresetpassworduserid)
+
 ## Should I use PostgreSQL or SQLite for my databases?
 
 Please use PostgreSQL wherever possible, especially if you are planning to run a homeserver that caters to more than a couple of users.
 
+## What data needs to be kept if transferring/backing up Dendrite?
+
+The list of files that need to be stored is:
+- matrix-key.pem
+- dendrite.yaml
+- the postgres or sqlite DB
+- the media store
+- the search index (although this can be regenerated)
+
+Note that this list may change / be out of date. We don't officially maintain instructions for migrations like this.
+
+## How can I prepare enough storage for media caches?
+
+This might be what you want: [matrix-media-repo](https://github.com/turt2live/matrix-media-repo)
+We don't officially support this or any other dedicated media storage solutions.
+
+## Is there an upgrade guide for Dendrite?
+
+Run a newer docker image. We don't officially support deployments other than Docker. 
+Most of the time you should be able to just 
+- stop 
+- replace binary
+- start
+
 ## Dendrite is using a lot of CPU
 
 Generally speaking, you should expect to see some CPU spikes, particularly if you are joining or participating in large rooms. However, constant/sustained high CPU usage is not expected - if you are experiencing that, please join `#dendrite-dev:matrix.org` and let us know what you were doing when the
@@ -102,6 +152,10 @@ not expected. Join `#dendrite-dev:matrix.org` and let us know what you were doin
 ballooned, or file a GitHub issue if you can. If you can take a [memory profile](development/PROFILING.md) then that
 would be a huge help too, as that will help us to understand where the memory usage is happening.
 
+## Do I need to generate the self-signed certificate if I'm going to use a reverse proxy?
+
+No, if you already have a proper certificate from some provider, like Let's Encrypt, and use that on your reverse proxy, and the reverse proxy does TLS termination, then you’re good and can use HTTP to the dendrite process.
+
 ## Dendrite is running out of PostgreSQL database connections
 
 You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode!