diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml new file mode 100644 index 000000000..b5a8f0bbd --- /dev/null +++ b/.github/workflows/gh-pages.yml @@ -0,0 +1,52 @@ +# Sample workflow for building and deploying a Jekyll site to GitHub Pages +name: Deploy GitHub Pages dependencies preinstalled + +on: + # Runs on pushes targeting the default branch + push: + branches: ["main"] + paths: + - 'docs/**' # only execute if we have docs changes + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow one concurrent deployment +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + # Build job + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Pages + uses: actions/configure-pages@v2 + - name: Build with Jekyll + uses: actions/jekyll-build-pages@v1 + with: + source: ./docs + destination: ./_site + - name: Upload artifact + uses: actions/upload-pages-artifact@v1 + + # Deployment job + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v1 diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml new file mode 100644 index 000000000..7cdc369ba --- /dev/null +++ b/.github/workflows/helm.yml @@ -0,0 +1,39 @@ +name: Release Charts + +on: + push: + branches: + - main + paths: + - 'helm/**' # only execute if we have helm chart changes + +jobs: + release: + # depending on default permission settings for your org (contents being read-only or read-write for workloads), you will have to add permissions + # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token + permissions: + contents: write + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Install Helm + uses: azure/setup-helm@v3 + with: + version: v3.10.0 + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.4.1 + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + with: + config: helm/cr.yaml + charts_dir: helm/ diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml new file mode 100644 index 000000000..fc5e8c906 --- /dev/null +++ b/.github/workflows/k8s.yml @@ -0,0 +1,90 @@ +name: k8s + +on: + push: + branches: ["main"] + paths: + - 'helm/**' # only execute if we have helm chart changes + pull_request: + branches: ["main"] + paths: + - 'helm/**' + +jobs: + lint: + name: Lint Helm chart + runs-on: ubuntu-latest + outputs: + changed: ${{ steps.list-changed.outputs.changed }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: azure/setup-helm@v3 + with: + version: v3.10.0 + - uses: actions/setup-python@v4 + with: + python-version: 3.11 + check-latest: true + - uses: helm/chart-testing-action@v2.3.1 + - name: Get changed status + id: list-changed + run: | + changed=$(ct list-changed --config helm/ct.yaml --target-branch ${{ github.event.repository.default_branch }}) + if [[ -n "$changed" ]]; then + echo "::set-output name=changed::true" + fi + + - name: Run lint + run: ct lint --config helm/ct.yaml + + # only bother to run if lint step reports a change to the helm chart + install: + needs: + - lint + if: ${{ needs.lint.outputs.changed == 'true' }} + name: Install Helm charts + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.checkoutCommit }} + - name: Install Kubernetes tools + uses: yokawasa/action-setup-kube-tools@v0.8.2 + with: + setup-tools: | + helmv3 + helm: "3.10.3" + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.3.1 + - name: Create k3d cluster + uses: nolar/setup-k3d-k3s@v1 + with: + version: v1.21 + - name: Remove node taints + run: | + kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true + - name: Run chart-testing (install) + run: ct install --config helm/ct.yaml + + # Install the chart using helm directly and test with create-account + - name: Install chart + run: | + helm install --values helm/dendrite/ci/ct-postgres-sharedsecret-values.yaml dendrite helm/dendrite + - name: Wait for Postgres and Dendrite to be up + run: | + kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=postgresql || kubectl get pods -A + kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=dendrite || kubectl get pods -A + kubectl get pods -A + kubectl get services + kubectl get ingress + - name: Run create account + run: | + podName=$(kubectl get pods -l app.kubernetes.io/name=dendrite -o name) + kubectl exec "${podName}" -- /usr/bin/create-account -username alice -password somerandompassword \ No newline at end of file diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index d2a1f6e1f..5636c4cf9 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -73,3 +73,41 @@ jobs: path: | /logs/results.tap /logs/**/*.log* + + element_web: + timeout-minutes: 120 + runs-on: ubuntu-latest + steps: + - uses: tecolicom/actions-use-apt-tools@v1 + with: + # Our test suite includes some screenshot tests with unusual diacritics, which are + # supposed to be covered by STIXGeneral. + tools: fonts-stix + - uses: actions/checkout@v2 + with: + repository: matrix-org/matrix-react-sdk + - uses: actions/setup-node@v3 + with: + cache: 'yarn' + - name: Fetch layered build + run: scripts/ci/layered.sh + - name: Copy config + run: cp element.io/develop/config.json config.json + working-directory: ./element-web + - name: Build + env: + CI_PACKAGE: true + run: yarn build + working-directory: ./element-web + - name: Edit Test Config + run: | + sed -i '/HOMESERVER/c\ HOMESERVER: "dendrite",' cypress.config.ts + - name: "Run cypress tests" + uses: cypress-io/github-action@v4.1.1 + with: + browser: chrome + start: npx serve -p 8080 ./element-web/webapp + wait-on: 'http://localhost:8080' + env: + PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true + TMPDIR: ${{ runner.temp }} diff --git a/Dockerfile b/Dockerfile index ede33e635..6da555c04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,7 @@ RUN --mount=target=. \ # The dendrite base image # FROM alpine:latest AS dendrite-base +RUN apk --update --no-cache add curl LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" LABEL org.opencontainers.image.licenses="Apache-2.0" diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone index facd1e3af..90f515167 100644 --- a/build/docker/Dockerfile.demo-pinecone +++ b/build/docker/Dockerfile.demo-pinecone @@ -17,6 +17,7 @@ RUN go build -trimpath -o bin/ ./cmd/create-account RUN go build -trimpath -o bin/ ./cmd/generate-keys FROM alpine:latest +RUN apk --update --no-cache add curl LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)" LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" diff --git a/clientapi/auth/login_test.go b/clientapi/auth/login_test.go index b79c573aa..044062c42 100644 --- a/clientapi/auth/login_test.go +++ b/clientapi/auth/login_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/setup/config" uapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -47,7 +48,7 @@ func TestLoginFromJSONReader(t *testing.T) { "password": "herpassword", "device_id": "adevice" }`, - WantUsername: "alice", + WantUsername: "@alice:example.com", WantDeviceID: "adevice", }, { @@ -174,7 +175,7 @@ func (ua *fakeUserInternalAPI) QueryAccountByPassword(ctx context.Context, req * return nil } res.Exists = true - res.Account = &uapi.Account{} + res.Account = &uapi.Account{UserID: userutil.MakeUserID(req.Localpart, req.ServerName)} return nil } diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index 4de2b443c..f2b0383ab 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -101,6 +101,8 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, } } + // If we couldn't find the user by the lower cased localpart, try the provided + // localpart as is. if !res.Exists { err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{ Localpart: localpart, @@ -122,5 +124,8 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, } } } + // Set the user, so login.Username() can do the right thing + r.Identifier.User = res.Account.UserID + r.User = res.Account.UserID return &r.Login, nil } diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go new file mode 100644 index 000000000..d429d7f8c --- /dev/null +++ b/clientapi/routing/login_test.go @@ -0,0 +1,152 @@ +package routing + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/keyserver" + "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/test/testrig" + "github.com/matrix-org/dendrite/userapi" + uapi "github.com/matrix-org/dendrite/userapi/api" +) + +func TestLogin(t *testing.T) { + aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin)) + bobUser := &test.User{ID: "@bob:test", AccountType: uapi.AccountTypeUser} + charlie := &test.User{ID: "@Charlie:test", AccountType: uapi.AccountTypeUser} + vhUser := &test.User{ID: "@vhuser:vh1"} + + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + base, baseClose := testrig.CreateBaseDendrite(t, dbType) + defer baseClose() + base.Cfg.ClientAPI.RateLimiting.Enabled = false + // add a vhost + base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{ + SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"}, + }) + + rsAPI := roomserver.NewInternalAPI(base) + // Needed for /login + keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI) + userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil) + keyAPI.SetUserAPI(userAPI) + + // We mostly need the userAPI for this test, so nil for other APIs/caches etc. + Setup(base, &base.Cfg.ClientAPI, nil, nil, userAPI, nil, nil, nil, nil, nil, keyAPI, nil, &base.Cfg.MSCs, nil) + + // Create password + password := util.RandomString(8) + + // create the users + for _, u := range []*test.User{aliceAdmin, bobUser, vhUser, charlie} { + localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID) + userRes := &uapi.PerformAccountCreationResponse{} + + if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{ + AccountType: u.AccountType, + Localpart: localpart, + ServerName: serverName, + Password: password, + }, userRes); err != nil { + t.Errorf("failed to create account: %s", err) + } + if !userRes.AccountCreated { + t.Fatalf("account not created") + } + } + + testCases := []struct { + name string + userID string + wantOK bool + }{ + { + name: "aliceAdmin can login", + userID: aliceAdmin.ID, + wantOK: true, + }, + { + name: "bobUser can login", + userID: bobUser.ID, + wantOK: true, + }, + { + name: "vhuser can login", + userID: vhUser.ID, + wantOK: true, + }, + { + name: "bob with uppercase can login", + userID: "@Bob:test", + wantOK: true, + }, + { + name: "Charlie can login (existing uppercase)", + userID: charlie.ID, + wantOK: true, + }, + { + name: "Charlie can not login with lowercase userID", + userID: strings.ToLower(charlie.ID), + wantOK: false, + }, + } + + ctx := context.Background() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{ + "type": authtypes.LoginTypePassword, + "identifier": map[string]interface{}{ + "type": "m.id.user", + "user": tc.userID, + }, + "password": password, + })) + rec := httptest.NewRecorder() + base.PublicClientAPIMux.ServeHTTP(rec, req) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("failed to login: %s", rec.Body.String()) + } + + t.Logf("Response: %s", rec.Body.String()) + // get the response + resp := loginResponse{} + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + // everything OK + if !tc.wantOK && resp.AccessToken == "" { + return + } + if tc.wantOK && resp.AccessToken == "" { + t.Fatalf("expected accessToken after successful login but got none: %+v", resp) + } + + devicesResp := &uapi.QueryDevicesResponse{} + if err := userAPI.QueryDevices(ctx, &uapi.QueryDevicesRequest{UserID: resp.UserID}, devicesResp); err != nil { + t.Fatal(err) + } + for _, dev := range devicesResp.Devices { + // We expect the userID on the device to be the same as resp.UserID + if dev.UserID != resp.UserID { + t.Fatalf("unexpected userID on device: %s", dev.UserID) + } + } + }) + } + }) +} diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 6087bda0c..ff6a0900e 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -780,7 +780,7 @@ func handleApplicationServiceRegistration( // Don't need to worry about appending to registration stages as // application service registration is entirely separate. return completeRegistration( - req.Context(), userAPI, r.Username, r.ServerName, "", appserviceID, req.RemoteAddr, + req.Context(), userAPI, r.Username, r.ServerName, "", "", appserviceID, req.RemoteAddr, req.UserAgent(), r.Auth.Session, r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeAppService, ) @@ -800,7 +800,7 @@ func checkAndCompleteFlow( if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) { // This flow was completed, registration can continue return completeRegistration( - req.Context(), userAPI, r.Username, r.ServerName, r.Password, "", req.RemoteAddr, + req.Context(), userAPI, r.Username, r.ServerName, "", r.Password, "", req.RemoteAddr, req.UserAgent(), sessionID, r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeUser, ) @@ -824,10 +824,10 @@ func checkAndCompleteFlow( func completeRegistration( ctx context.Context, userAPI userapi.ClientUserAPI, - username string, serverName gomatrixserverlib.ServerName, + username string, serverName gomatrixserverlib.ServerName, displayName string, password, appserviceID, ipAddr, userAgent, sessionID string, inhibitLogin eventutil.WeakBoolean, - displayName, deviceID *string, + deviceDisplayName, deviceID *string, accType userapi.AccountType, ) util.JSONResponse { if username == "" { @@ -887,12 +887,28 @@ func completeRegistration( } } + if displayName != "" { + nameReq := userapi.PerformUpdateDisplayNameRequest{ + Localpart: username, + ServerName: serverName, + DisplayName: displayName, + } + var nameRes userapi.PerformUpdateDisplayNameResponse + err = userAPI.SetDisplayName(ctx, &nameReq, &nameRes) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.Unknown("failed to set display name: " + err.Error()), + } + } + } + var devRes userapi.PerformDeviceCreationResponse err = userAPI.PerformDeviceCreation(ctx, &userapi.PerformDeviceCreationRequest{ Localpart: username, ServerName: serverName, AccessToken: token, - DeviceDisplayName: displayName, + DeviceDisplayName: deviceDisplayName, DeviceID: deviceID, IPAddr: ipAddr, UserAgent: userAgent, @@ -1077,5 +1093,5 @@ func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.Clien if ssrr.Admin { accType = userapi.AccountTypeAdmin } - return completeRegistration(req.Context(), userAPI, ssrr.User, cfg.Matrix.ServerName, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType) + return completeRegistration(req.Context(), userAPI, ssrr.User, cfg.Matrix.ServerName, ssrr.DisplayName, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType) } diff --git a/clientapi/routing/register_secret.go b/clientapi/routing/register_secret.go index 1a974b77a..f384b604a 100644 --- a/clientapi/routing/register_secret.go +++ b/clientapi/routing/register_secret.go @@ -18,12 +18,13 @@ import ( ) type SharedSecretRegistrationRequest struct { - User string `json:"username"` - Password string `json:"password"` - Nonce string `json:"nonce"` - MacBytes []byte - MacStr string `json:"mac"` - Admin bool `json:"admin"` + User string `json:"username"` + Password string `json:"password"` + Nonce string `json:"nonce"` + MacBytes []byte + MacStr string `json:"mac"` + Admin bool `json:"admin"` + DisplayName string `json:"displayname,omitempty"` } func NewSharedSecretRegistrationRequest(reader io.ReadCloser) (*SharedSecretRegistrationRequest, error) { diff --git a/clientapi/routing/register_secret_test.go b/clientapi/routing/register_secret_test.go index a2ed35853..ca265d237 100644 --- a/clientapi/routing/register_secret_test.go +++ b/clientapi/routing/register_secret_test.go @@ -10,7 +10,7 @@ import ( func TestSharedSecretRegister(t *testing.T) { // these values have come from a local synapse instance to ensure compatibility - jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice"}`) + jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`) sharedSecret := "dendritetest" req, err := NewSharedSecretRegistrationRequest(io.NopCloser(bytes.NewBuffer(jsonStr))) diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index b8fd19e90..bccc1b79b 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -18,6 +18,7 @@ import ( "bytes" "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" "reflect" @@ -35,7 +36,10 @@ import ( "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/userapi" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" + "github.com/patrickmn/go-cache" + "github.com/stretchr/testify/assert" ) var ( @@ -570,3 +574,92 @@ func Test_register(t *testing.T) { } }) } + +func TestRegisterUserWithDisplayName(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + base, baseClose := testrig.CreateBaseDendrite(t, dbType) + defer baseClose() + base.Cfg.Global.ServerName = "server" + + rsAPI := roomserver.NewInternalAPI(base) + keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI) + userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil) + keyAPI.SetUserAPI(userAPI) + deviceName, deviceID := "deviceName", "deviceID" + expectedDisplayName := "DisplayName" + response := completeRegistration( + base.Context(), + userAPI, + "user", + "server", + expectedDisplayName, + "password", + "", + "localhost", + "user agent", + "session", + false, + &deviceName, + &deviceID, + api.AccountTypeAdmin, + ) + + assert.Equal(t, http.StatusOK, response.Code) + + req := api.QueryProfileRequest{UserID: "@user:server"} + var res api.QueryProfileResponse + err := userAPI.QueryProfile(base.Context(), &req, &res) + assert.NoError(t, err) + assert.Equal(t, expectedDisplayName, res.DisplayName) + }) +} + +func TestRegisterAdminUsingSharedSecret(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + base, baseClose := testrig.CreateBaseDendrite(t, dbType) + defer baseClose() + base.Cfg.Global.ServerName = "server" + sharedSecret := "dendritetest" + base.Cfg.ClientAPI.RegistrationSharedSecret = sharedSecret + + rsAPI := roomserver.NewInternalAPI(base) + keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI) + userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil) + keyAPI.SetUserAPI(userAPI) + + expectedDisplayName := "rabbit" + jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`) + req, err := NewSharedSecretRegistrationRequest(io.NopCloser(bytes.NewBuffer(jsonStr))) + assert.NoError(t, err) + if err != nil { + t.Fatalf("failed to read request: %s", err) + } + + r := NewSharedSecretRegistration(sharedSecret) + + // force the nonce to be known + r.nonces.Set(req.Nonce, true, cache.DefaultExpiration) + + _, err = r.IsValidMacLogin(req.Nonce, req.User, req.Password, req.Admin, req.MacBytes) + assert.NoError(t, err) + + body := &bytes.Buffer{} + err = json.NewEncoder(body).Encode(req) + assert.NoError(t, err) + ssrr := httptest.NewRequest(http.MethodPost, "/", body) + + response := handleSharedSecretRegistration( + &base.Cfg.ClientAPI, + userAPI, + r, + ssrr, + ) + assert.Equal(t, http.StatusOK, response.Code) + + profilReq := api.QueryProfileRequest{UserID: "@alice:server"} + var profileRes api.QueryProfileResponse + err = userAPI.QueryProfile(base.Context(), &profilReq, &profileRes) + assert.NoError(t, err) + assert.Equal(t, expectedDisplayName, profileRes.DisplayName) + }) +} diff --git a/docs/FAQ.md b/docs/FAQ.md index 816130515..4047bfffc 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -6,6 +6,12 @@ permalink: /faq # FAQ +## Why does Dendrite exist? + +Dendrite aims to provide a matrix compatible server that has low resource usage compared to [Synapse](https://github.com/matrix-org/synapse). +It also aims to provide more flexibility when scaling either up or down. +Dendrite's code is also very easy to hack on which makes it suitable for experimenting with new matrix features such as peer-to-peer. + ## Is Dendrite stable? Mostly, although there are still bugs and missing features. If you are a confident power user and you are happy to spend some time debugging things when they go wrong, then please try out Dendrite. If you are a community, organisation or business that demands stability and uptime, then Dendrite is not for you yet - please install Synapse instead. @@ -34,6 +40,10 @@ No, Dendrite has a very different database schema to Synapse and the two are not Monolith deployments are always preferred where possible, and at this time, are far better tested than polylith deployments are. The only reason to consider a polylith deployment is if you wish to run different Dendrite components on separate physical machines, but this is an advanced configuration which we don't recommend. +## Can I configure which port Dendrite listens on? + +Yes, use the cli flag `-http-bind-address`. + ## I've installed Dendrite but federation isn't working Check the [Federation Tester](https://federationtester.matrix.org). You need at least: @@ -42,6 +52,10 @@ Check the [Federation Tester](https://federationtester.matrix.org). You need at * A valid TLS certificate for that DNS name * Either DNS SRV records or well-known files +## Whenever I try to connect from Element it says unable to connect to homeserver + +Check that your dendrite instance is running. Otherwise this is most likely due to a reverse proxy misconfiguration. + ## Does Dendrite work with my favourite client? It should do, although we are aware of some minor issues: @@ -49,6 +63,10 @@ It should do, although we are aware of some minor issues: * **Element Android**: registration does not work, but logging in with an existing account does * **Hydrogen**: occasionally sync can fail due to gaps in the `since` parameter, but clearing the cache fixes this +## Is there a public instance of Dendrite I can try out? + +Use [dendrite.matrix.org](https://dendrite.matrix.org) which we officially support. + ## Does Dendrite support Space Summaries? Yes, [Space Summaries](https://github.com/matrix-org/matrix-spec-proposals/pull/2946) were merged into the Matrix Spec as of 2022-01-17 however, they are still treated as an MSC (Matrix Specification Change) in Dendrite. In order to enable Space Summaries in Dendrite, you must add the MSC to the MSC configuration section in the configuration YAML. If the MSC is not enabled, a user will typically see a perpetual loading icon on the summary page. See below for a demonstration of how to add to the Dendrite configuration: @@ -84,10 +102,42 @@ Remember to add the config file(s) to the `app_service_api` section of the confi Yes, you can do this by disabling federation - set `disable_federation` to `true` in the `global` section of the Dendrite configuration file. +## How can I migrate a room in order to change the internal ID? + +This can be done by performing a room upgrade. Use the command `/upgraderoom ` in Element to do this. + +## How do I reset somebody's password on my server? + +Use the admin endpoint [resetpassword](https://matrix-org.github.io/dendrite/administration/adminapi#post-_dendriteadminresetpassworduserid) + ## Should I use PostgreSQL or SQLite for my databases? Please use PostgreSQL wherever possible, especially if you are planning to run a homeserver that caters to more than a couple of users. +## What data needs to be kept if transferring/backing up Dendrite? + +The list of files that need to be stored is: +- matrix-key.pem +- dendrite.yaml +- the postgres or sqlite DB +- the media store +- the search index (although this can be regenerated) + +Note that this list may change / be out of date. We don't officially maintain instructions for migrations like this. + +## How can I prepare enough storage for media caches? + +This might be what you want: [matrix-media-repo](https://github.com/turt2live/matrix-media-repo) +We don't officially support this or any other dedicated media storage solutions. + +## Is there an upgrade guide for Dendrite? + +Run a newer docker image. We don't officially support deployments other than Docker. +Most of the time you should be able to just +- stop +- replace binary +- start + ## Dendrite is using a lot of CPU Generally speaking, you should expect to see some CPU spikes, particularly if you are joining or participating in large rooms. However, constant/sustained high CPU usage is not expected - if you are experiencing that, please join `#dendrite-dev:matrix.org` and let us know what you were doing when the @@ -102,6 +152,10 @@ not expected. Join `#dendrite-dev:matrix.org` and let us know what you were doin ballooned, or file a GitHub issue if you can. If you can take a [memory profile](development/PROFILING.md) then that would be a huge help too, as that will help us to understand where the memory usage is happening. +## Do I need to generate the self-signed certificate if I'm going to use a reverse proxy? + +No, if you already have a proper certificate from some provider, like Let's Encrypt, and use that on your reverse proxy, and the reverse proxy does TLS termination, then you’re good and can use HTTP to the dendrite process. + ## Dendrite is running out of PostgreSQL database connections You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode! diff --git a/federationapi/statistics/statistics.go b/federationapi/statistics/statistics.go index 3698880a1..2803c891d 100644 --- a/federationapi/statistics/statistics.go +++ b/federationapi/statistics/statistics.go @@ -45,18 +45,13 @@ func NewStatistics( FailuresUntilBlacklist: failuresUntilBlacklist, FailuresUntilAssumedOffline: failuresUntilAssumedOffline, backoffTimers: make(map[gomatrixserverlib.ServerName]*time.Timer), + servers: make(map[gomatrixserverlib.ServerName]*ServerStatistics), } } // ForServer returns server statistics for the given server name. If it // does not exist, it will create empty statistics and return those. func (s *Statistics) ForServer(serverName gomatrixserverlib.ServerName) *ServerStatistics { - // If the map hasn't been initialised yet then do that. - if s.servers == nil { - s.mutex.Lock() - s.servers = make(map[gomatrixserverlib.ServerName]*ServerStatistics) - s.mutex.Unlock() - } // Look up if we have statistics for this server already. s.mutex.RLock() server, found := s.servers[serverName] diff --git a/helm/cr.yaml b/helm/cr.yaml new file mode 100644 index 000000000..884c2b46b --- /dev/null +++ b/helm/cr.yaml @@ -0,0 +1,2 @@ +release-name-template: "helm-{{ .Name }}-{{ .Version }}" +pages-index-path: docs/index.yaml \ No newline at end of file diff --git a/helm/ct.yaml b/helm/ct.yaml new file mode 100644 index 000000000..af706fa3d --- /dev/null +++ b/helm/ct.yaml @@ -0,0 +1,7 @@ +remote: origin +target-branch: main +chart-repos: + - bitnami=https://charts.bitnami.com/bitnami +chart-dirs: + - helm +validate-maintainers: false \ No newline at end of file diff --git a/helm/dendrite/.helm-docs/about.gotmpl b/helm/dendrite/.helm-docs/about.gotmpl new file mode 100644 index 000000000..a92c6be42 --- /dev/null +++ b/helm/dendrite/.helm-docs/about.gotmpl @@ -0,0 +1,5 @@ +{{ define "chart.about" }} +## About + +This chart creates a monolith deployment, including an optionally enabled PostgreSQL dependency to connect to. +{{ end }} \ No newline at end of file diff --git a/helm/dendrite/.helm-docs/appservices.gotmpl b/helm/dendrite/.helm-docs/appservices.gotmpl new file mode 100644 index 000000000..8a79a0780 --- /dev/null +++ b/helm/dendrite/.helm-docs/appservices.gotmpl @@ -0,0 +1,5 @@ +{{ define "chart.appservices" }} +## Usage with appservices + +Create a folder `appservices` and place your configurations in there. The configurations will be read and placed in a secret `dendrite-appservices-conf`. +{{ end }} \ No newline at end of file diff --git a/helm/dendrite/.helm-docs/database.gotmpl b/helm/dendrite/.helm-docs/database.gotmpl new file mode 100644 index 000000000..85ef01ecc --- /dev/null +++ b/helm/dendrite/.helm-docs/database.gotmpl @@ -0,0 +1,18 @@ +{{ define "chart.dbCreation" }} +## Manual database creation + +(You can skip this, if you're deploying the PostgreSQL dependency) + +You'll need to create the following database before starting Dendrite (see [installation](https://matrix-org.github.io/dendrite/installation/database#single-database-creation)): + +```postgres +create database dendrite +``` + +or + +```bash +sudo -u postgres createdb -O dendrite -E UTF-8 dendrite +``` + +{{ end }} \ No newline at end of file diff --git a/helm/dendrite/.helm-docs/state.gotmpl b/helm/dendrite/.helm-docs/state.gotmpl new file mode 100644 index 000000000..2fe987ddd --- /dev/null +++ b/helm/dendrite/.helm-docs/state.gotmpl @@ -0,0 +1,3 @@ +{{ define "chart.state" }} +Status: **NOT PRODUCTION READY** +{{ end }} \ No newline at end of file diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml new file mode 100644 index 000000000..15d1e6d19 --- /dev/null +++ b/helm/dendrite/Chart.yaml @@ -0,0 +1,19 @@ +apiVersion: v2 +name: dendrite +version: "0.10.8" +appVersion: "0.10.8" +description: Dendrite Matrix Homeserver +type: application +keywords: + - matrix + - chat + - homeserver + - dendrite +home: https://github.com/matrix-org/dendrite +sources: + - https://github.com/matrix-org/dendrite +dependencies: +- name: postgresql + version: 12.1.7 + repository: https://charts.bitnami.com/bitnami + condition: postgresql.enabled diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md new file mode 100644 index 000000000..cb850d655 --- /dev/null +++ b/helm/dendrite/README.md @@ -0,0 +1,147 @@ +# dendrite + +![Version: 0.10.8](https://img.shields.io/badge/Version-0.10.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.10.8](https://img.shields.io/badge/AppVersion-0.10.8-informational?style=flat-square) +Dendrite Matrix Homeserver + +Status: **NOT PRODUCTION READY** + +## About + +This chart creates a monolith deployment, including an optionally enabled PostgreSQL dependency to connect to. + +## Manual database creation + +(You can skip this, if you're deploying the PostgreSQL dependency) + +You'll need to create the following database before starting Dendrite (see [installation](https://matrix-org.github.io/dendrite/installation/database#single-database-creation)): + +```postgres +create database dendrite +``` + +or + +```bash +sudo -u postgres createdb -O dendrite -E UTF-8 dendrite +``` + +## Usage with appservices + +Create a folder `appservices` and place your configurations in there. The configurations will be read and placed in a secret `dendrite-appservices-conf`. + +## Source Code + +* +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | postgresql | 12.1.7 | +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| image.name | string | `"ghcr.io/matrix-org/dendrite-monolith:v0.10.8"` | Docker repository/image to use | +| image.pullPolicy | string | `"IfNotPresent"` | Kubernetes pullPolicy | +| signing_key.create | bool | `true` | Create a new signing key, if not exists | +| signing_key.existingSecret | string | `""` | Use an existing secret | +| resources | object | sets some sane default values | Default resource requests/limits. | +| persistence.storageClass | string | `""` | The storage class to use for volume claims. Defaults to the cluster default storage class. | +| persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream | +| persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume | +| persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files | +| persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume | +| persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index | +| persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume | +| dendrite_config.version | int | `2` | | +| dendrite_config.global.server_name | string | `""` | **REQUIRED** Servername for this Dendrite deployment. | +| dendrite_config.global.private_key | string | `"/etc/dendrite/secrets/signing.key"` | The private key to use. (**NOTE**: This is overriden in Helm) | +| dendrite_config.global.well_known_server_name | string | `""` | The server name to delegate server-server communications to, with optional port e.g. localhost:443 | +| dendrite_config.global.well_known_client_name | string | `""` | The server name to delegate client-server communications to, with optional port e.g. localhost:443 | +| dendrite_config.global.trusted_third_party_id_servers | list | `["matrix.org","vector.im"]` | Lists of domains that the server will trust as identity servers to verify third party identifiers such as phone numbers and email addresses. | +| dendrite_config.global.old_private_keys | string | `nil` | The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) to old signing keys that were formerly in use on this domain name. These keys will not be used for federation request or event signing, but will be provided to any other homeserver that asks when trying to verify old events. | +| dendrite_config.global.disable_federation | bool | `false` | Disable federation. Dendrite will not be able to make any outbound HTTP requests to other servers and the federation API will not be exposed. | +| dendrite_config.global.key_validity_period | string | `"168h0m0s"` | | +| dendrite_config.global.database.connection_string | string | `""` | The connection string for connections to Postgres. This will be set automatically if using the Postgres dependency | +| dendrite_config.global.database.max_open_conns | int | `90` | Default database maximum open connections | +| dendrite_config.global.database.max_idle_conns | int | `5` | Default database maximum idle connections | +| dendrite_config.global.database.conn_max_lifetime | int | `-1` | Default database maximum lifetime | +| dendrite_config.global.jetstream.storage_path | string | `"/data/jetstream"` | Persistent directory to store JetStream streams in. | +| dendrite_config.global.jetstream.addresses | list | `[]` | NATS JetStream server addresses if not using internal NATS. | +| dendrite_config.global.jetstream.topic_prefix | string | `"Dendrite"` | The prefix for JetStream streams | +| dendrite_config.global.jetstream.in_memory | bool | `false` | Keep all data in memory. (**NOTE**: This is overriden in Helm to `false`) | +| dendrite_config.global.jetstream.disable_tls_validation | bool | `true` | Disables TLS validation. This should **NOT** be used in production. | +| dendrite_config.global.cache.max_size_estimated | string | `"1gb"` | The estimated maximum size for the global cache in bytes, or in terabytes, gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or 'kb' suffix is specified. Note that this is not a hard limit, nor is it a memory limit for the entire process. A cache that is too small may ultimately provide little or no benefit. | +| dendrite_config.global.cache.max_age | string | `"1h"` | The maximum amount of time that a cache entry can live for in memory before it will be evicted and/or refreshed from the database. Lower values result in easier admission of new cache entries but may also increase database load in comparison to higher values, so adjust conservatively. Higher values may make it harder for new items to make it into the cache, e.g. if new rooms suddenly become popular. | +| dendrite_config.global.report_stats.enabled | bool | `false` | Configures phone-home statistics reporting. These statistics contain the server name, number of active users and some information on your deployment config. We use this information to understand how Dendrite is being used in the wild. | +| dendrite_config.global.report_stats.endpoint | string | `"https://matrix.org/report-usage-stats/push"` | Endpoint to report statistics to. | +| dendrite_config.global.presence.enable_inbound | bool | `false` | Controls whether we receive presence events from other servers | +| dendrite_config.global.presence.enable_outbound | bool | `false` | Controls whether we send presence events for our local users to other servers. (_May increase CPU/memory usage_) | +| dendrite_config.global.server_notices.enabled | bool | `false` | Server notices allows server admins to send messages to all users on the server. | +| dendrite_config.global.server_notices.local_part | string | `"_server"` | The local part for the user sending server notices. | +| dendrite_config.global.server_notices.display_name | string | `"Server Alerts"` | The display name for the user sending server notices. | +| dendrite_config.global.server_notices.avatar_url | string | `""` | The avatar URL (as a mxc:// URL) name for the user sending server notices. | +| dendrite_config.global.server_notices.room_name | string | `"Server Alerts"` | | +| dendrite_config.global.metrics.enabled | bool | `false` | Whether or not Prometheus metrics are enabled. | +| dendrite_config.global.metrics.basic_auth.user | string | `"metrics"` | HTTP basic authentication username | +| dendrite_config.global.metrics.basic_auth.password | string | `"metrics"` | HTTP basic authentication password | +| dendrite_config.global.dns_cache.enabled | bool | `false` | Whether or not the DNS cache is enabled. | +| dendrite_config.global.dns_cache.cache_size | int | `256` | Maximum number of entries to hold in the DNS cache | +| dendrite_config.global.dns_cache.cache_lifetime | string | `"10m"` | Duration for how long DNS cache items should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) | +| dendrite_config.global.profiling.enabled | bool | `false` | Enable pprof. You will need to manually create a port forwarding to the deployment to access PPROF, as it will only listen on localhost and the defined port. e.g. `kubectl port-forward deployments/dendrite 65432:65432` | +| dendrite_config.global.profiling.port | int | `65432` | pprof port, if enabled | +| dendrite_config.mscs | object | `{"mscs":["msc2946"]}` | Configuration for experimental MSC's. (Valid values are: msc2836 and msc2946) | +| dendrite_config.app_service_api.disable_tls_validation | bool | `false` | Disable the validation of TLS certificates of appservices. This is not recommended in production since it may allow appservice traffic to be sent to an insecure endpoint. | +| dendrite_config.app_service_api.config_files | list | `[]` | Appservice config files to load on startup. (**NOTE**: This is overriden by Helm, if a folder `./appservices/` exists) | +| dendrite_config.client_api.registration_disabled | bool | `true` | Prevents new users from being able to register on this homeserver, except when using the registration shared secret below. | +| dendrite_config.client_api.guests_disabled | bool | `true` | | +| dendrite_config.client_api.registration_shared_secret | string | `""` | If set, allows registration by anyone who knows the shared secret, regardless of whether registration is otherwise disabled. | +| dendrite_config.client_api.enable_registration_captcha | bool | `false` | enable reCAPTCHA registration | +| dendrite_config.client_api.recaptcha_public_key | string | `""` | reCAPTCHA public key | +| dendrite_config.client_api.recaptcha_private_key | string | `""` | reCAPTCHA private key | +| dendrite_config.client_api.recaptcha_bypass_secret | string | `""` | reCAPTCHA bypass secret | +| dendrite_config.client_api.recaptcha_siteverify_api | string | `""` | | +| dendrite_config.client_api.turn.turn_user_lifetime | string | `"24h"` | Duration for how long users should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) | +| dendrite_config.client_api.turn.turn_uris | list | `[]` | | +| dendrite_config.client_api.turn.turn_shared_secret | string | `""` | | +| dendrite_config.client_api.turn.turn_username | string | `""` | The TURN username | +| dendrite_config.client_api.turn.turn_password | string | `""` | The TURN password | +| dendrite_config.client_api.rate_limiting.enabled | bool | `true` | Enable rate limiting | +| dendrite_config.client_api.rate_limiting.threshold | int | `20` | After how many requests a rate limit should be activated | +| dendrite_config.client_api.rate_limiting.cooloff_ms | int | `500` | Cooloff time in milliseconds | +| dendrite_config.client_api.rate_limiting.exempt_user_ids | string | `nil` | Users which should be exempt from rate limiting | +| dendrite_config.federation_api.send_max_retries | int | `16` | Federation failure threshold. How many consecutive failures that we should tolerate when sending federation requests to a specific server. The backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. The default value is 16 if not specified, which is circa 18 hours. | +| dendrite_config.federation_api.disable_tls_validation | bool | `false` | Disable TLS validation. This should **NOT** be used in production. | +| dendrite_config.federation_api.prefer_direct_fetch | bool | `false` | | +| dendrite_config.federation_api.disable_http_keepalives | bool | `false` | Prevents Dendrite from keeping HTTP connections open for reuse for future requests. Connections will be closed quicker but we may spend more time on TLS handshakes instead. | +| dendrite_config.federation_api.key_perspectives | list | See value.yaml | Perspective keyservers, to use as a backup when direct key fetch requests don't succeed. | +| dendrite_config.media_api.base_path | string | `"/data/media_store"` | The path to store media files (e.g. avatars) in | +| dendrite_config.media_api.max_file_size_bytes | int | `10485760` | The max file size for uploaded media files | +| dendrite_config.media_api.dynamic_thumbnails | bool | `false` | | +| dendrite_config.media_api.max_thumbnail_generators | int | `10` | The maximum number of simultaneous thumbnail generators to run. | +| dendrite_config.media_api.thumbnail_sizes | list | See value.yaml | A list of thumbnail sizes to be generated for media content. | +| dendrite_config.sync_api.real_ip_header | string | `"X-Real-IP"` | This option controls which HTTP header to inspect to find the real remote IP address of the client. This is likely required if Dendrite is running behind a reverse proxy server. | +| dendrite_config.sync_api.search | object | `{"enabled":true,"index_path":"/data/search","language":"en"}` | Configuration for the full-text search engine. | +| dendrite_config.sync_api.search.enabled | bool | `true` | Whether fulltext search is enabled. | +| dendrite_config.sync_api.search.index_path | string | `"/data/search"` | The path to store the search index in. | +| dendrite_config.sync_api.search.language | string | `"en"` | The language most likely to be used on the server - used when indexing, to ensure the returned results match expectations. A full list of possible languages can be found [here](https://github.com/matrix-org/dendrite/blob/76db8e90defdfb9e61f6caea8a312c5d60bcc005/internal/fulltext/bleve.go#L25-L46) | +| dendrite_config.user_api.bcrypt_cost | int | `10` | bcrypt cost to use when hashing passwords. (ranges from 4-31; 4 being least secure, 31 being most secure; _NOTE: Using a too high value can cause clients to timeout and uses more CPU._) | +| dendrite_config.user_api.openid_token_lifetime_ms | int | `3600000` | OpenID Token lifetime in milliseconds. | +| dendrite_config.user_api.push_gateway_disable_tls_validation | bool | `false` | | +| dendrite_config.user_api.auto_join_rooms | list | `[]` | Rooms to join users to after registration | +| dendrite_config.logging | list | `[{"level":"info","type":"std"}]` | Default logging configuration | +| postgresql.enabled | bool | See value.yaml | Enable and configure postgres as the database for dendrite. | +| postgresql.image.repository | string | `"bitnami/postgresql"` | | +| postgresql.image.tag | string | `"15.1.0"` | | +| postgresql.auth.username | string | `"dendrite"` | | +| postgresql.auth.password | string | `"changeme"` | | +| postgresql.auth.database | string | `"dendrite"` | | +| postgresql.persistence.enabled | bool | `false` | | +| ingress.enabled | bool | `false` | Create an ingress for a monolith deployment | +| ingress.hosts | list | `[]` | | +| ingress.className | string | `""` | | +| ingress.hostName | string | `""` | | +| ingress.annotations | object | `{}` | Extra, custom annotations | +| ingress.tls | list | `[]` | | +| service.type | string | `"ClusterIP"` | | +| service.port | int | `80` | | diff --git a/helm/dendrite/README.md.gotmpl b/helm/dendrite/README.md.gotmpl new file mode 100644 index 000000000..7c32f7b02 --- /dev/null +++ b/helm/dendrite/README.md.gotmpl @@ -0,0 +1,13 @@ +{{ template "chart.header" . }} +{{ template "chart.deprecationWarning" . }} +{{ template "chart.badgesSection" . }} +{{ template "chart.description" . }} +{{ template "chart.state" . }} +{{ template "chart.about" . }} +{{ template "chart.dbCreation" . }} +{{ template "chart.appservices" . }} +{{ template "chart.maintainersSection" . }} +{{ template "chart.sourcesSection" . }} +{{ template "chart.requirementsSection" . }} +{{ template "chart.valuesSection" . }} +{{ template "helm-docs.versionFooter" . }} \ No newline at end of file diff --git a/helm/dendrite/ci/ct-ingress-values.yaml b/helm/dendrite/ci/ct-ingress-values.yaml new file mode 100644 index 000000000..28311d33e --- /dev/null +++ b/helm/dendrite/ci/ct-ingress-values.yaml @@ -0,0 +1,13 @@ +--- +postgresql: + enabled: true + primary: + persistence: + size: 1Gi + +dendrite_config: + global: + server_name: "localhost" + +ingress: + enabled: true diff --git a/helm/dendrite/ci/ct-postgres-sharedsecret-values.yaml b/helm/dendrite/ci/ct-postgres-sharedsecret-values.yaml new file mode 100644 index 000000000..55e652c63 --- /dev/null +++ b/helm/dendrite/ci/ct-postgres-sharedsecret-values.yaml @@ -0,0 +1,16 @@ +--- +postgresql: + enabled: true + primary: + persistence: + size: 1Gi + +dendrite_config: + global: + server_name: "localhost" + + client_api: + registration_shared_secret: "d233f2fcb0470845a8e150a20ef594ddbe0b4cf7fe482fb9d5120c198557acbf" # echo "dendrite" | sha256sum + +ingress: + enabled: true diff --git a/helm/dendrite/templates/_helpers.tpl b/helm/dendrite/templates/_helpers.tpl new file mode 100644 index 000000000..291f351bc --- /dev/null +++ b/helm/dendrite/templates/_helpers.tpl @@ -0,0 +1,72 @@ +{{- define "validate.config" }} +{{- if not .Values.signing_key.create -}} +{{- fail "You must create a signing key for configuration.signing_key. (see https://github.com/matrix-org/dendrite/blob/master/docs/INSTALL.md#server-key-generation)" -}} +{{- end -}} +{{- if not (or .Values.dendrite_config.global.database.host .Values.postgresql.enabled) -}} +{{- fail "Database server must be set." -}} +{{- end -}} +{{- if not (or .Values.dendrite_config.global.database.user .Values.postgresql.enabled) -}} +{{- fail "Database user must be set." -}} +{{- end -}} +{{- if not (or .Values.dendrite_config.global.database.password .Values.postgresql.enabled) -}} +{{- fail "Database password must be set." -}} +{{- end -}} +{{- end -}} + + +{{- define "image.name" -}} +image: {{ .name }} +imagePullPolicy: {{ .pullPolicy }} +{{- end -}} + +{{/* +Expand the name of the chart. +*/}} +{{- define "dendrite.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "dendrite.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "dendrite.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "dendrite.labels" -}} +helm.sh/chart: {{ include "dendrite.chart" . }} +{{ include "dendrite.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "dendrite.selectorLabels" -}} +app.kubernetes.io/name: {{ include "dendrite.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/helm/dendrite/templates/_overrides.yaml b/helm/dendrite/templates/_overrides.yaml new file mode 100644 index 000000000..edb8ba83a --- /dev/null +++ b/helm/dendrite/templates/_overrides.yaml @@ -0,0 +1,16 @@ +{{- define "override.config" }} +{{- if .Values.postgresql.enabled }} +{{- $_ := set .Values.dendrite_config.global.database "connection_string" (print "postgresql://" .Values.postgresql.auth.username ":" .Values.postgresql.auth.password "@" .Release.Name "-postgresql/dendrite?sslmode=disable") -}} +{{ end }} +global: + private_key: /etc/dendrite/secrets/signing.key + jetstream: + in_memory: false +{{ if (gt (len (.Files.Glob "appservices/*")) 0) }} +app_service_api: + config_files: + {{- range $x, $y := .Files.Glob "appservices/*" }} + - /etc/dendrite/appservices/{{ base $x }} + {{ end }} +{{ end }} +{{ end }} diff --git a/helm/dendrite/templates/deployment.yaml b/helm/dendrite/templates/deployment.yaml new file mode 100644 index 000000000..629ffe528 --- /dev/null +++ b/helm/dendrite/templates/deployment.yaml @@ -0,0 +1,103 @@ +{{ template "validate.config" . }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: {{ $.Release.Namespace }} + name: {{ include "dendrite.fullname" . }} + labels: + {{- include "dendrite.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "dendrite.selectorLabels" . | nindent 6 }} + replicas: 1 + template: + metadata: + labels: + {{- include "dendrite.selectorLabels" . | nindent 8 }} + annotations: + confighash-global: secret-{{ .Values.global | toYaml | sha256sum | trunc 32 }} + confighash-clientapi: clientapi-{{ .Values.clientapi | toYaml | sha256sum | trunc 32 }} + confighash-federationapi: federationapi-{{ .Values.federationapi | toYaml | sha256sum | trunc 32 }} + confighash-mediaapi: mediaapi-{{ .Values.mediaapi | toYaml | sha256sum | trunc 32 }} + confighash-syncapi: syncapi-{{ .Values.syncapi | toYaml | sha256sum | trunc 32 }} + spec: + volumes: + - name: {{ include "dendrite.fullname" . }}-conf-vol + secret: + secretName: {{ include "dendrite.fullname" . }}-conf + - name: {{ include "dendrite.fullname" . }}-signing-key + secret: + secretName: {{ default (print ( include "dendrite.fullname" . ) "-signing-key") $.Values.signing_key.existingSecret | quote }} + {{- if (gt (len ($.Files.Glob "appservices/*")) 0) }} + - name: {{ include "dendrite.fullname" . }}-appservices + secret: + secretName: {{ include "dendrite.fullname" . }}-appservices-conf + {{- end }} + - name: {{ include "dendrite.fullname" . }}-jetstream + persistentVolumeClaim: + claimName: {{ default (print ( include "dendrite.fullname" . ) "-jetstream-pvc") $.Values.persistence.jetstream.existingClaim | quote }} + - name: {{ include "dendrite.fullname" . }}-media + persistentVolumeClaim: + claimName: {{ default (print ( include "dendrite.fullname" . ) "-media-pvc") $.Values.persistence.media.existingClaim | quote }} + - name: {{ include "dendrite.fullname" . }}-search + persistentVolumeClaim: + claimName: {{ default (print ( include "dendrite.fullname" . ) "-search-pvc") $.Values.persistence.search.existingClaim | quote }} + containers: + - name: {{ $.Chart.Name }} + {{- include "image.name" $.Values.image | nindent 8 }} + args: + - '--config' + - '/etc/dendrite/dendrite.yaml' + ports: + - name: http + containerPort: 8008 + protocol: TCP + {{- if $.Values.dendrite_config.global.profiling.enabled }} + env: + - name: PPROFLISTEN + value: "localhost:{{- $.Values.global.profiling.port -}}" + {{- end }} + resources: + {{- toYaml $.Values.resources | nindent 10 }} + volumeMounts: + - mountPath: /etc/dendrite/ + name: {{ include "dendrite.fullname" . }}-conf-vol + - mountPath: /etc/dendrite/secrets/ + name: {{ include "dendrite.fullname" . }}-signing-key + {{- if (gt (len ($.Files.Glob "appservices/*")) 0) }} + - mountPath: /etc/dendrite/appservices + name: {{ include "dendrite.fullname" . }}-appservices + readOnly: true + {{ end }} + - mountPath: {{ .Values.dendrite_config.media_api.base_path }} + name: {{ include "dendrite.fullname" . }}-media + - mountPath: {{ .Values.dendrite_config.global.jetstream.storage_path }} + name: {{ include "dendrite.fullname" . }}-jetstream + - mountPath: {{ .Values.dendrite_config.sync_api.search.index_path }} + name: {{ include "dendrite.fullname" . }}-search + livenessProbe: + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 10 + httpGet: + path: /_dendrite/monitor/health + port: http + readinessProbe: + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 10 + httpGet: + path: /_dendrite/monitor/health + port: http + startupProbe: + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 10 + httpGet: + path: /_dendrite/monitor/up + port: http \ No newline at end of file diff --git a/helm/dendrite/templates/ingress.yaml b/helm/dendrite/templates/ingress.yaml new file mode 100644 index 000000000..8f86ad723 --- /dev/null +++ b/helm/dendrite/templates/ingress.yaml @@ -0,0 +1,55 @@ +{{- if .Values.ingress.enabled -}} + {{- $fullName := include "dendrite.fullname" . -}} + {{- $svcPort := .Values.service.port -}} + {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} + {{- end }} + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 + {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 + {{- else -}} +apiVersion: extensions/v1beta1 + {{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "dendrite.labels" . | nindent 4 }} + annotations: + {{- with .Values.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + - host: {{ .Values.ingress.hostName | quote }} + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} \ No newline at end of file diff --git a/helm/dendrite/templates/jobs.yaml b/helm/dendrite/templates/jobs.yaml new file mode 100644 index 000000000..76915694d --- /dev/null +++ b/helm/dendrite/templates/jobs.yaml @@ -0,0 +1,99 @@ +{{ if and .Values.signing_key.create (not .Values.signing_key.existingSecret ) }} +{{ $name := (print ( include "dendrite.fullname" . ) "-signing-key") }} +{{ $secretName := (print ( include "dendrite.fullname" . ) "-signing-key") }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/component: signingkey-job +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/component: signingkey-job + {{- include "dendrite.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - {{ $secretName }} + verbs: + - get + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/component: signingkey-job + {{- include "dendrite.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $name }} +subjects: + - kind: ServiceAccount + name: {{ $name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: generate-signing-key + labels: + {{- include "dendrite.labels" . | nindent 4 }} +spec: + template: + spec: + restartPolicy: "Never" + serviceAccount: {{ $name }} + containers: + - name: upload-key + image: bitnami/kubectl + command: + - sh + - -c + - | + # check if key already exists + key=$(kubectl get secret {{ $secretName }} -o jsonpath="{.data['signing\.key']}" 2> /dev/null) + [ $? -ne 0 ] && echo "Failed to get existing secret" && exit 1 + [ -n "$key" ] && echo "Key already created, exiting." && exit 0 + # wait for signing key + while [ ! -f /etc/dendrite/signing-key.pem ]; do + echo "Waiting for signing key.." + sleep 5; + done + # update secret + kubectl patch secret {{ $secretName }} -p "{\"data\":{\"signing.key\":\"$(base64 /etc/dendrite/signing-key.pem | tr -d '\n')\"}}" + [ $? -ne 0 ] && echo "Failed to update secret." && exit 1 + echo "Signing key successfully created." + volumeMounts: + - mountPath: /etc/dendrite/ + name: signing-key + readOnly: true + - name: generate-key + {{- include "image.name" $.Values.image | nindent 8 }} + command: + - sh + - -c + - | + /usr/bin/generate-keys -private-key /etc/dendrite/signing-key.pem + chown 1001:1001 /etc/dendrite/signing-key.pem + volumeMounts: + - mountPath: /etc/dendrite/ + name: signing-key + volumes: + - name: signing-key + emptyDir: {} + parallelism: 1 + completions: 1 + backoffLimit: 1 +{{ end }} \ No newline at end of file diff --git a/helm/dendrite/templates/pvc.yaml b/helm/dendrite/templates/pvc.yaml new file mode 100644 index 000000000..897957e60 --- /dev/null +++ b/helm/dendrite/templates/pvc.yaml @@ -0,0 +1,48 @@ +{{ if not .Values.persistence.media.existingClaim }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + annotations: + helm.sh/resource-policy: keep + name: {{ include "dendrite.fullname" . }}-media-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistence.media.capacity }} + storageClassName: {{ .Values.persistence.storageClass }} +{{ end }} +{{ if not .Values.persistence.jetstream.existingClaim }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + annotations: + helm.sh/resource-policy: keep + name: {{ include "dendrite.fullname" . }}-jetstream-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistence.jetstream.capacity }} + storageClassName: {{ .Values.persistence.storageClass }} +{{ end }} +{{ if not .Values.persistence.search.existingClaim }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + annotations: + helm.sh/resource-policy: keep + name: {{ include "dendrite.fullname" . }}-search-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistence.search.capacity }} + storageClassName: {{ .Values.persistence.storageClass }} +{{ end }} \ No newline at end of file diff --git a/helm/dendrite/templates/secrets.yaml b/helm/dendrite/templates/secrets.yaml new file mode 100644 index 000000000..d4b8ecbf2 --- /dev/null +++ b/helm/dendrite/templates/secrets.yaml @@ -0,0 +1,33 @@ +{{ if (gt (len (.Files.Glob "appservices/*")) 0) }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "dendrite.fullname" . }}-appservices-conf + namespace: {{ .Release.Namespace }} +type: Opaque +data: +{{ (.Files.Glob "appservices/*").AsSecrets | indent 2 }} +{{ end }} +{{ if and .Values.signing_key.create (not .Values.signing_key.existingSecret) }} +--- +apiVersion: v1 +kind: Secret +metadata: + annotations: + helm.sh/resource-policy: keep + name: {{ include "dendrite.fullname" . }}-signing-key + namespace: {{ .Release.Namespace }} +type: Opaque +{{ end }} + +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: {{ include "dendrite.fullname" . }}-conf + namespace: {{ .Release.Namespace }} +stringData: + dendrite.yaml: | + {{ toYaml ( mustMergeOverwrite .Values.dendrite_config ( fromYaml (include "override.config" .) ) .Values.dendrite_config ) | nindent 4 }} \ No newline at end of file diff --git a/helm/dendrite/templates/service.yaml b/helm/dendrite/templates/service.yaml new file mode 100644 index 000000000..365a43f04 --- /dev/null +++ b/helm/dendrite/templates/service.yaml @@ -0,0 +1,17 @@ +{{ template "validate.config" . }} +--- +apiVersion: v1 +kind: Service +metadata: + namespace: {{ $.Release.Namespace }} + name: {{ include "dendrite.fullname" . }} + labels: + {{- include "dendrite.labels" . | nindent 4 }} +spec: + selector: + {{- include "dendrite.selectorLabels" . | nindent 4 }} + ports: + - name: http + protocol: TCP + port: 8008 + targetPort: 8008 \ No newline at end of file diff --git a/helm/dendrite/templates/tests/test-version.yaml b/helm/dendrite/templates/tests/test-version.yaml new file mode 100644 index 000000000..d88751325 --- /dev/null +++ b/helm/dendrite/templates/tests/test-version.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "dendrite.fullname" . }}-test-version" + labels: + {{- include "dendrite.selectorLabels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: curl + image: curlimages/curl + imagePullPolicy: IfNotPresent + args: + - 'http://{{- include "dendrite.fullname" . -}}:8008/_matrix/client/versions' + restartPolicy: Never diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml new file mode 100644 index 000000000..2c6e80942 --- /dev/null +++ b/helm/dendrite/values.yaml @@ -0,0 +1,348 @@ +image: + # -- Docker repository/image to use + name: "ghcr.io/matrix-org/dendrite-monolith:v0.10.8" + # -- Kubernetes pullPolicy + pullPolicy: IfNotPresent + + +# signing key to use +signing_key: + # -- Create a new signing key, if not exists + create: true + # -- Use an existing secret + existingSecret: "" + +# -- Default resource requests/limits. +# @default -- sets some sane default values +resources: + requests: + memory: "512Mi" + + limits: + memory: "4096Mi" + +persistence: + # -- The storage class to use for volume claims. Defaults to the + # cluster default storage class. + storageClass: "" + jetstream: + # -- Use an existing volume claim for jetstream + existingClaim: "" + # -- PVC Storage Request for the jetstream volume + capacity: "1Gi" + media: + # -- Use an existing volume claim for media files + existingClaim: "" + # -- PVC Storage Request for the media volume + capacity: "1Gi" + search: + # -- Use an existing volume claim for the fulltext search index + existingClaim: "" + # -- PVC Storage Request for the search volume + capacity: "1Gi" + +dendrite_config: + version: 2 + global: + # -- **REQUIRED** Servername for this Dendrite deployment. + server_name: "" + + # -- The private key to use. (**NOTE**: This is overriden in Helm) + private_key: /etc/dendrite/secrets/signing.key + + # -- The server name to delegate server-server communications to, with optional port + # e.g. localhost:443 + well_known_server_name: "" + + # -- The server name to delegate client-server communications to, with optional port + # e.g. localhost:443 + well_known_client_name: "" + + # -- Lists of domains that the server will trust as identity servers to verify third + # party identifiers such as phone numbers and email addresses. + trusted_third_party_id_servers: + - matrix.org + - vector.im + + # -- The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) + # to old signing keys that were formerly in use on this domain name. These + # keys will not be used for federation request or event signing, but will be + # provided to any other homeserver that asks when trying to verify old events. + old_private_keys: + # If the old private key file is available: + # - private_key: old_matrix_key.pem + # expired_at: 1601024554498 + # If only the public key (in base64 format) and key ID are known: + # - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM= + # key_id: ed25519:mykeyid + # expired_at: 1601024554498 + + # -- Disable federation. Dendrite will not be able to make any outbound HTTP requests + # to other servers and the federation API will not be exposed. + disable_federation: false + + key_validity_period: 168h0m0s + + database: + # -- The connection string for connections to Postgres. + # This will be set automatically if using the Postgres dependency + connection_string: "" + + # -- Default database maximum open connections + max_open_conns: 90 + # -- Default database maximum idle connections + max_idle_conns: 5 + # -- Default database maximum lifetime + conn_max_lifetime: -1 + + jetstream: + # -- Persistent directory to store JetStream streams in. + storage_path: "/data/jetstream" + # -- NATS JetStream server addresses if not using internal NATS. + addresses: [] + # -- The prefix for JetStream streams + topic_prefix: "Dendrite" + # -- Keep all data in memory. (**NOTE**: This is overriden in Helm to `false`) + in_memory: false + # -- Disables TLS validation. This should **NOT** be used in production. + disable_tls_validation: true + + cache: + # -- The estimated maximum size for the global cache in bytes, or in terabytes, + # gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or + # 'kb' suffix is specified. Note that this is not a hard limit, nor is it a + # memory limit for the entire process. A cache that is too small may ultimately + # provide little or no benefit. + max_size_estimated: 1gb + # -- The maximum amount of time that a cache entry can live for in memory before + # it will be evicted and/or refreshed from the database. Lower values result in + # easier admission of new cache entries but may also increase database load in + # comparison to higher values, so adjust conservatively. Higher values may make + # it harder for new items to make it into the cache, e.g. if new rooms suddenly + # become popular. + max_age: 1h + + report_stats: + # -- Configures phone-home statistics reporting. These statistics contain the server + # name, number of active users and some information on your deployment config. + # We use this information to understand how Dendrite is being used in the wild. + enabled: false + # -- Endpoint to report statistics to. + endpoint: https://matrix.org/report-usage-stats/push + + presence: + # -- Controls whether we receive presence events from other servers + enable_inbound: false + # -- Controls whether we send presence events for our local users to other servers. + # (_May increase CPU/memory usage_) + enable_outbound: false + + server_notices: + # -- Server notices allows server admins to send messages to all users on the server. + enabled: false + # -- The local part for the user sending server notices. + local_part: "_server" + # -- The display name for the user sending server notices. + display_name: "Server Alerts" + # -- The avatar URL (as a mxc:// URL) name for the user sending server notices. + avatar_url: "" + # The room name to be used when sending server notices. This room name will + # appear in user clients. + room_name: "Server Alerts" + + # prometheus metrics + metrics: + # -- Whether or not Prometheus metrics are enabled. + enabled: false + # HTTP basic authentication to protect access to monitoring. + basic_auth: + # -- HTTP basic authentication username + user: "metrics" + # -- HTTP basic authentication password + password: metrics + + dns_cache: + # -- Whether or not the DNS cache is enabled. + enabled: false + # -- Maximum number of entries to hold in the DNS cache + cache_size: 256 + # -- Duration for how long DNS cache items should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) + cache_lifetime: "10m" + + profiling: + # -- Enable pprof. You will need to manually create a port forwarding to the deployment to access PPROF, + # as it will only listen on localhost and the defined port. + # e.g. `kubectl port-forward deployments/dendrite 65432:65432` + enabled: false + # -- pprof port, if enabled + port: 65432 + + # -- Configuration for experimental MSC's. (Valid values are: msc2836 and msc2946) + mscs: + mscs: + - msc2946 + # A list of enabled MSC's + # Currently valid values are: + # - msc2836 (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) + # - msc2946 (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946) + + + app_service_api: + # -- Disable the validation of TLS certificates of appservices. This is + # not recommended in production since it may allow appservice traffic + # to be sent to an insecure endpoint. + disable_tls_validation: false + # -- Appservice config files to load on startup. (**NOTE**: This is overriden by Helm, if a folder `./appservices/` exists) + config_files: [] + + client_api: + # -- Prevents new users from being able to register on this homeserver, except when + # using the registration shared secret below. + registration_disabled: true + + # Prevents new guest accounts from being created. Guest registration is also + # disabled implicitly by setting 'registration_disabled' above. + guests_disabled: true + + # -- If set, allows registration by anyone who knows the shared secret, regardless of + # whether registration is otherwise disabled. + registration_shared_secret: "" + + # -- enable reCAPTCHA registration + enable_registration_captcha: false + # -- reCAPTCHA public key + recaptcha_public_key: "" + # -- reCAPTCHA private key + recaptcha_private_key: "" + # -- reCAPTCHA bypass secret + recaptcha_bypass_secret: "" + recaptcha_siteverify_api: "" + + # TURN server information that this homeserver should send to clients. + turn: + # -- Duration for how long users should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) + turn_user_lifetime: "24h" + turn_uris: [] + turn_shared_secret: "" + # -- The TURN username + turn_username: "" + # -- The TURN password + turn_password: "" + + rate_limiting: + # -- Enable rate limiting + enabled: true + # -- After how many requests a rate limit should be activated + threshold: 20 + # -- Cooloff time in milliseconds + cooloff_ms: 500 + # -- Users which should be exempt from rate limiting + exempt_user_ids: + + federation_api: + # -- Federation failure threshold. How many consecutive failures that we should + # tolerate when sending federation requests to a specific server. The backoff + # is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. + # The default value is 16 if not specified, which is circa 18 hours. + send_max_retries: 16 + # -- Disable TLS validation. This should **NOT** be used in production. + disable_tls_validation: false + prefer_direct_fetch: false + # -- Prevents Dendrite from keeping HTTP connections + # open for reuse for future requests. Connections will be closed quicker + # but we may spend more time on TLS handshakes instead. + disable_http_keepalives: false + # -- Perspective keyservers, to use as a backup when direct key fetch + # requests don't succeed. + # @default -- See value.yaml + key_perspectives: + - server_name: matrix.org + keys: + - key_id: ed25519:auto + public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw + - key_id: ed25519:a_RXGa + public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ + + media_api: + # -- The path to store media files (e.g. avatars) in + base_path: "/data/media_store" + # -- The max file size for uploaded media files + max_file_size_bytes: 10485760 + # Whether to dynamically generate thumbnails if needed. + dynamic_thumbnails: false + # -- The maximum number of simultaneous thumbnail generators to run. + max_thumbnail_generators: 10 + # -- A list of thumbnail sizes to be generated for media content. + # @default -- See value.yaml + thumbnail_sizes: + - width: 32 + height: 32 + method: crop + - width: 96 + height: 96 + method: crop + - width: 640 + height: 480 + method: scale + + sync_api: + # -- This option controls which HTTP header to inspect to find the real remote IP + # address of the client. This is likely required if Dendrite is running behind + # a reverse proxy server. + real_ip_header: X-Real-IP + # -- Configuration for the full-text search engine. + search: + # -- Whether fulltext search is enabled. + enabled: true + # -- The path to store the search index in. + index_path: "/data/search" + # -- The language most likely to be used on the server - used when indexing, to + # ensure the returned results match expectations. A full list of possible languages + # can be found [here](https://github.com/matrix-org/dendrite/blob/76db8e90defdfb9e61f6caea8a312c5d60bcc005/internal/fulltext/bleve.go#L25-L46) + language: "en" + + user_api: + # -- bcrypt cost to use when hashing passwords. + # (ranges from 4-31; 4 being least secure, 31 being most secure; _NOTE: Using a too high value can cause clients to timeout and uses more CPU._) + bcrypt_cost: 10 + # -- OpenID Token lifetime in milliseconds. + openid_token_lifetime_ms: 3600000 + # - Disable TLS validation when hitting push gateways. This should **NOT** be used in production. + push_gateway_disable_tls_validation: false + # -- Rooms to join users to after registration + auto_join_rooms: [] + + # -- Default logging configuration + logging: + - type: std + level: info + +postgresql: + # -- Enable and configure postgres as the database for dendrite. + # @default -- See value.yaml + enabled: false + image: + repository: bitnami/postgresql + tag: "15.1.0" + auth: + username: dendrite + password: changeme + database: dendrite + + persistence: + enabled: false + +ingress: + # -- Create an ingress for a monolith deployment + enabled: false + hosts: [] + className: "" + hostName: "" + # -- Extra, custom annotations + annotations: {} + + tls: [] + +service: + type: ClusterIP + port: 80 diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go index 069f017a9..d9214fdc6 100644 --- a/roomserver/internal/perform/perform_backfill.go +++ b/roomserver/internal/perform/perform_backfill.go @@ -122,11 +122,14 @@ func (r *Backfiller) backfillViaFederation(ctx context.Context, req *api.Perform ctx, req.VirtualHost, requester, r.KeyRing, req.RoomID, info.RoomVersion, req.PrevEventIDs(), 100, ) - if err != nil { + // Only return an error if we really couldn't get any events. + if err != nil && len(events) == 0 { logrus.WithError(err).Errorf("gomatrixserverlib.RequestBackfill failed") return err } - logrus.WithField("room_id", req.RoomID).Infof("backfilled %d events", len(events)) + // If we got an error but still got events, that's fine, because a server might have returned a 404 (or something) + // but other servers could provide the missing event. + logrus.WithError(err).WithField("room_id", req.RoomID).Infof("backfilled %d events", len(events)) // persist these new events - auth checks have already been done roomNID, backfilledEventMap := persistEvents(ctx, r.DB, events) @@ -319,6 +322,7 @@ FederationHit: FedClient: b.fsAPI, RememberAuthEvents: false, Server: srv, + Origin: b.virtualHost, } res, err := c.StateIDsBeforeEvent(ctx, targetEvent) if err != nil { @@ -394,6 +398,7 @@ func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatr FedClient: b.fsAPI, RememberAuthEvents: false, Server: srv, + Origin: b.virtualHost, } result, err := c.StateBeforeEvent(ctx, roomVer, event, eventIDs) if err != nil { diff --git a/setup/config/config.go b/setup/config/config.go index fa0aa15ae..2b38cd512 100644 --- a/setup/config/config.go +++ b/setup/config/config.go @@ -229,7 +229,7 @@ func loadConfig( privateKeyPath := absPath(basePath, c.Global.PrivateKeyPath) if c.Global.KeyID, c.Global.PrivateKey, err = LoadMatrixKey(privateKeyPath, readFile); err != nil { - return nil, err + return nil, fmt.Errorf("failed to load private_key: %w", err) } for _, v := range c.Global.VirtualHosts { @@ -243,7 +243,7 @@ func loadConfig( } privateKeyPath := absPath(basePath, v.PrivateKeyPath) if v.KeyID, v.PrivateKey, err = LoadMatrixKey(privateKeyPath, readFile); err != nil { - return nil, err + return nil, fmt.Errorf("failed to load private_key for virtualhost %s: %w", v.ServerName, err) } } diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 75afbce15..4e22f8a6f 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -45,7 +45,7 @@ type DatabaseTransaction interface { GetStateDeltas(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter) ([]types.StateDelta, []string, error) RoomIDsWithMembership(ctx context.Context, userID string, membership string) ([]string, error) MembershipCount(ctx context.Context, roomID, membership string, pos types.StreamPosition) (int, error) - GetRoomHeroes(ctx context.Context, roomID, userID string, memberships []string) ([]string, error) + GetRoomSummary(ctx context.Context, roomID, userID string) (summary *types.Summary, err error) RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) GetBackwardTopologyPos(ctx context.Context, events []*gomatrixserverlib.HeaderedEvent) (types.TopologyToken, error) PositionInTopology(ctx context.Context, eventID string) (pos types.StreamPosition, spos types.StreamPosition, err error) diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 48ed20021..3caafa14b 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -19,6 +19,7 @@ import ( "context" "database/sql" "encoding/json" + "errors" "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" @@ -110,6 +111,15 @@ const selectSharedUsersSQL = "" + " SELECT DISTINCT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" + ") AND type = 'm.room.member' AND state_key = ANY($2) AND membership IN ('join', 'invite');" +const selectMembershipCount = `SELECT count(*) FROM syncapi_current_room_state WHERE type = 'm.room.member' AND room_id = $1 AND membership = $2` + +const selectRoomHeroes = ` +SELECT state_key FROM syncapi_current_room_state +WHERE type = 'm.room.member' AND room_id = $1 AND membership = ANY($2) AND state_key != $3 +ORDER BY added_at, state_key +LIMIT 5 +` + type currentRoomStateStatements struct { upsertRoomStateStmt *sql.Stmt deleteRoomStateByEventIDStmt *sql.Stmt @@ -122,6 +132,8 @@ type currentRoomStateStatements struct { selectEventsWithEventIDsStmt *sql.Stmt selectStateEventStmt *sql.Stmt selectSharedUsersStmt *sql.Stmt + selectMembershipCountStmt *sql.Stmt + selectRoomHeroesStmt *sql.Stmt } func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) { @@ -141,40 +153,21 @@ func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, erro return nil, err } - if s.upsertRoomStateStmt, err = db.Prepare(upsertRoomStateSQL); err != nil { - return nil, err - } - if s.deleteRoomStateByEventIDStmt, err = db.Prepare(deleteRoomStateByEventIDSQL); err != nil { - return nil, err - } - if s.deleteRoomStateForRoomStmt, err = db.Prepare(deleteRoomStateForRoomSQL); err != nil { - return nil, err - } - if s.selectRoomIDsWithMembershipStmt, err = db.Prepare(selectRoomIDsWithMembershipSQL); err != nil { - return nil, err - } - if s.selectRoomIDsWithAnyMembershipStmt, err = db.Prepare(selectRoomIDsWithAnyMembershipSQL); err != nil { - return nil, err - } - if s.selectCurrentStateStmt, err = db.Prepare(selectCurrentStateSQL); err != nil { - return nil, err - } - if s.selectJoinedUsersStmt, err = db.Prepare(selectJoinedUsersSQL); err != nil { - return nil, err - } - if s.selectJoinedUsersInRoomStmt, err = db.Prepare(selectJoinedUsersInRoomSQL); err != nil { - return nil, err - } - if s.selectEventsWithEventIDsStmt, err = db.Prepare(selectEventsWithEventIDsSQL); err != nil { - return nil, err - } - if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil { - return nil, err - } - if s.selectSharedUsersStmt, err = db.Prepare(selectSharedUsersSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.upsertRoomStateStmt, upsertRoomStateSQL}, + {&s.deleteRoomStateByEventIDStmt, deleteRoomStateByEventIDSQL}, + {&s.deleteRoomStateForRoomStmt, deleteRoomStateForRoomSQL}, + {&s.selectRoomIDsWithMembershipStmt, selectRoomIDsWithMembershipSQL}, + {&s.selectRoomIDsWithAnyMembershipStmt, selectRoomIDsWithAnyMembershipSQL}, + {&s.selectCurrentStateStmt, selectCurrentStateSQL}, + {&s.selectJoinedUsersStmt, selectJoinedUsersSQL}, + {&s.selectJoinedUsersInRoomStmt, selectJoinedUsersInRoomSQL}, + {&s.selectEventsWithEventIDsStmt, selectEventsWithEventIDsSQL}, + {&s.selectStateEventStmt, selectStateEventSQL}, + {&s.selectSharedUsersStmt, selectSharedUsersSQL}, + {&s.selectMembershipCountStmt, selectMembershipCount}, + {&s.selectRoomHeroesStmt, selectRoomHeroes}, + }.Prepare(db) } // SelectJoinedUsers returns a map of room ID to a list of joined user IDs. @@ -447,3 +440,34 @@ func (s *currentRoomStateStatements) SelectSharedUsers( } return result, rows.Err() } + +func (s *currentRoomStateStatements) SelectRoomHeroes(ctx context.Context, txn *sql.Tx, roomID, excludeUserID string, memberships []string) ([]string, error) { + stmt := sqlutil.TxStmt(txn, s.selectRoomHeroesStmt) + rows, err := stmt.QueryContext(ctx, roomID, pq.StringArray(memberships), excludeUserID) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectRoomHeroesStmt: rows.close() failed") + + var stateKey string + result := make([]string, 0, 5) + for rows.Next() { + if err = rows.Scan(&stateKey); err != nil { + return nil, err + } + result = append(result, stateKey) + } + return result, rows.Err() +} + +func (s *currentRoomStateStatements) SelectMembershipCount(ctx context.Context, txn *sql.Tx, roomID, membership string) (count int, err error) { + stmt := sqlutil.TxStmt(txn, s.selectMembershipCountStmt) + err = stmt.QueryRowContext(ctx, roomID, membership).Scan(&count) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return 0, nil + } + return 0, err + } + return count, nil +} diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go index b555e8456..ac44b235f 100644 --- a/syncapi/storage/postgres/memberships_table.go +++ b/syncapi/storage/postgres/memberships_table.go @@ -19,10 +19,8 @@ import ( "database/sql" "fmt" - "github.com/lib/pq" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" @@ -64,9 +62,6 @@ const selectMembershipCountSQL = "" + " SELECT DISTINCT ON (room_id, user_id) room_id, user_id, membership FROM syncapi_memberships WHERE room_id = $1 AND stream_pos <= $2 ORDER BY room_id, user_id, stream_pos DESC" + ") t WHERE t.membership = $3" -const selectHeroesSQL = "" + - "SELECT DISTINCT user_id FROM syncapi_memberships WHERE room_id = $1 AND user_id != $2 AND membership = ANY($3) LIMIT 5" - const selectMembershipBeforeSQL = "" + "SELECT membership, topological_pos FROM syncapi_memberships WHERE room_id = $1 and user_id = $2 AND topological_pos <= $3 ORDER BY topological_pos DESC LIMIT 1" @@ -81,7 +76,6 @@ WHERE ($3::text IS NULL OR t.membership = $3) type membershipsStatements struct { upsertMembershipStmt *sql.Stmt selectMembershipCountStmt *sql.Stmt - selectHeroesStmt *sql.Stmt selectMembershipForUserStmt *sql.Stmt selectMembersStmt *sql.Stmt } @@ -95,7 +89,6 @@ func NewPostgresMembershipsTable(db *sql.DB) (tables.Memberships, error) { return s, sqlutil.StatementList{ {&s.upsertMembershipStmt, upsertMembershipSQL}, {&s.selectMembershipCountStmt, selectMembershipCountSQL}, - {&s.selectHeroesStmt, selectHeroesSQL}, {&s.selectMembershipForUserStmt, selectMembershipBeforeSQL}, {&s.selectMembersStmt, selectMembersSQL}, }.Prepare(db) @@ -129,26 +122,6 @@ func (s *membershipsStatements) SelectMembershipCount( return } -func (s *membershipsStatements) SelectHeroes( - ctx context.Context, txn *sql.Tx, roomID, userID string, memberships []string, -) (heroes []string, err error) { - stmt := sqlutil.TxStmt(txn, s.selectHeroesStmt) - var rows *sql.Rows - rows, err = stmt.QueryContext(ctx, roomID, userID, pq.StringArray(memberships)) - if err != nil { - return - } - defer internal.CloseAndLogIfError(ctx, rows, "SelectHeroes: rows.close() failed") - var hero string - for rows.Next() { - if err = rows.Scan(&hero); err != nil { - return - } - heroes = append(heroes, hero) - } - return heroes, rows.Err() -} - // SelectMembershipForUser returns the membership of the user before and including the given position. If no membership can be found // returns "leave", the topological position and no error. If an error occurs, other than sql.ErrNoRows, returns that and an empty // string as the membership. diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index 77afa0290..c6933486c 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/matrix-org/gomatrixserverlib" + "github.com/tidwall/gjson" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/syncapi/types" @@ -92,8 +93,61 @@ func (d *DatabaseTransaction) MembershipCount(ctx context.Context, roomID, membe return d.Memberships.SelectMembershipCount(ctx, d.txn, roomID, membership, pos) } -func (d *DatabaseTransaction) GetRoomHeroes(ctx context.Context, roomID, userID string, memberships []string) ([]string, error) { - return d.Memberships.SelectHeroes(ctx, d.txn, roomID, userID, memberships) +func (d *DatabaseTransaction) GetRoomSummary(ctx context.Context, roomID, userID string) (*types.Summary, error) { + summary := &types.Summary{Heroes: []string{}} + + joinCount, err := d.CurrentRoomState.SelectMembershipCount(ctx, d.txn, roomID, gomatrixserverlib.Join) + if err != nil { + return summary, err + } + inviteCount, err := d.CurrentRoomState.SelectMembershipCount(ctx, d.txn, roomID, gomatrixserverlib.Invite) + if err != nil { + return summary, err + } + summary.InvitedMemberCount = &inviteCount + summary.JoinedMemberCount = &joinCount + + // Get the room name and canonical alias, if any + filter := gomatrixserverlib.DefaultStateFilter() + filterTypes := []string{gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomCanonicalAlias} + filterRooms := []string{roomID} + + filter.Types = &filterTypes + filter.Rooms = &filterRooms + evs, err := d.CurrentRoomState.SelectCurrentState(ctx, d.txn, roomID, &filter, nil) + if err != nil { + return summary, err + } + + for _, ev := range evs { + switch ev.Type() { + case gomatrixserverlib.MRoomName: + if gjson.GetBytes(ev.Content(), "name").Str != "" { + return summary, nil + } + case gomatrixserverlib.MRoomCanonicalAlias: + if gjson.GetBytes(ev.Content(), "alias").Str != "" { + return summary, nil + } + } + } + + // If there's no room name or canonical alias, get the room heroes, excluding the user + heroes, err := d.CurrentRoomState.SelectRoomHeroes(ctx, d.txn, roomID, userID, []string{gomatrixserverlib.Join, gomatrixserverlib.Invite}) + if err != nil { + return summary, err + } + + // "When no joined or invited members are available, this should consist of the banned and left users" + if len(heroes) == 0 { + heroes, err = d.CurrentRoomState.SelectRoomHeroes(ctx, d.txn, roomID, userID, []string{gomatrixserverlib.Leave, gomatrixserverlib.Ban}) + if err != nil { + return summary, err + } + } + summary.Heroes = heroes + + return summary, nil } func (d *DatabaseTransaction) RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) { diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 7a381f68b..6bc1b267a 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -19,6 +19,7 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "strings" @@ -95,6 +96,15 @@ const selectSharedUsersSQL = "" + " SELECT DISTINCT room_id FROM syncapi_current_room_state WHERE state_key = $1 AND membership='join'" + ") AND type = 'm.room.member' AND state_key IN ($2) AND membership IN ('join', 'invite');" +const selectMembershipCount = `SELECT count(*) FROM syncapi_current_room_state WHERE type = 'm.room.member' AND room_id = $1 AND membership = $2` + +const selectRoomHeroes = ` +SELECT state_key FROM syncapi_current_room_state +WHERE type = 'm.room.member' AND room_id = $1 AND state_key != $2 AND membership IN ($3) +ORDER BY added_at, state_key +LIMIT 5 +` + type currentRoomStateStatements struct { db *sql.DB streamIDStatements *StreamIDStatements @@ -107,6 +117,8 @@ type currentRoomStateStatements struct { //selectJoinedUsersInRoomStmt *sql.Stmt - prepared at runtime due to variadic selectStateEventStmt *sql.Stmt //selectSharedUsersSQL *sql.Stmt - prepared at runtime due to variadic + selectMembershipCountStmt *sql.Stmt + //selectRoomHeroes *sql.Stmt - prepared at runtime due to variadic } func NewSqliteCurrentRoomStateTable(db *sql.DB, streamID *StreamIDStatements) (tables.CurrentRoomState, error) { @@ -129,31 +141,16 @@ func NewSqliteCurrentRoomStateTable(db *sql.DB, streamID *StreamIDStatements) (t return nil, err } - if s.upsertRoomStateStmt, err = db.Prepare(upsertRoomStateSQL); err != nil { - return nil, err - } - if s.deleteRoomStateByEventIDStmt, err = db.Prepare(deleteRoomStateByEventIDSQL); err != nil { - return nil, err - } - if s.deleteRoomStateForRoomStmt, err = db.Prepare(deleteRoomStateForRoomSQL); err != nil { - return nil, err - } - if s.selectRoomIDsWithMembershipStmt, err = db.Prepare(selectRoomIDsWithMembershipSQL); err != nil { - return nil, err - } - if s.selectRoomIDsWithAnyMembershipStmt, err = db.Prepare(selectRoomIDsWithAnyMembershipSQL); err != nil { - return nil, err - } - if s.selectJoinedUsersStmt, err = db.Prepare(selectJoinedUsersSQL); err != nil { - return nil, err - } - //if s.selectJoinedUsersInRoomStmt, err = db.Prepare(selectJoinedUsersInRoomSQL); err != nil { - // return nil, err - //} - if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.upsertRoomStateStmt, upsertRoomStateSQL}, + {&s.deleteRoomStateByEventIDStmt, deleteRoomStateByEventIDSQL}, + {&s.deleteRoomStateForRoomStmt, deleteRoomStateForRoomSQL}, + {&s.selectRoomIDsWithMembershipStmt, selectRoomIDsWithMembershipSQL}, + {&s.selectRoomIDsWithAnyMembershipStmt, selectRoomIDsWithAnyMembershipSQL}, + {&s.selectJoinedUsersStmt, selectJoinedUsersSQL}, + {&s.selectStateEventStmt, selectStateEventSQL}, + {&s.selectMembershipCountStmt, selectMembershipCount}, + }.Prepare(db) } // SelectJoinedUsers returns a map of room ID to a list of joined user IDs. @@ -485,3 +482,53 @@ func (s *currentRoomStateStatements) SelectSharedUsers( return result, err } + +func (s *currentRoomStateStatements) SelectRoomHeroes(ctx context.Context, txn *sql.Tx, roomID, excludeUserID string, memberships []string) ([]string, error) { + params := make([]interface{}, len(memberships)+2) + params[0] = roomID + params[1] = excludeUserID + for k, v := range memberships { + params[k+2] = v + } + + query := strings.Replace(selectRoomHeroes, "($3)", sqlutil.QueryVariadicOffset(len(memberships), 2), 1) + var stmt *sql.Stmt + var err error + if txn != nil { + stmt, err = txn.Prepare(query) + } else { + stmt, err = s.db.Prepare(query) + } + if err != nil { + return []string{}, err + } + defer internal.CloseAndLogIfError(ctx, stmt, "selectRoomHeroes: stmt.close() failed") + + rows, err := stmt.QueryContext(ctx, params...) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectRoomHeroes: rows.close() failed") + + var stateKey string + result := make([]string, 0, 5) + for rows.Next() { + if err = rows.Scan(&stateKey); err != nil { + return nil, err + } + result = append(result, stateKey) + } + return result, rows.Err() +} + +func (s *currentRoomStateStatements) SelectMembershipCount(ctx context.Context, txn *sql.Tx, roomID, membership string) (count int, err error) { + stmt := sqlutil.TxStmt(txn, s.selectMembershipCountStmt) + err = stmt.QueryRowContext(ctx, roomID, membership).Scan(&count) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return 0, nil + } + return 0, err + } + return count, nil +} diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go index 7e54fac17..905a1e1a8 100644 --- a/syncapi/storage/sqlite3/memberships_table.go +++ b/syncapi/storage/sqlite3/memberships_table.go @@ -18,11 +18,9 @@ import ( "context" "database/sql" "fmt" - "strings" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" @@ -64,9 +62,6 @@ const selectMembershipCountSQL = "" + " SELECT * FROM syncapi_memberships WHERE room_id = $1 AND stream_pos <= $2 GROUP BY user_id HAVING(max(stream_pos))" + ") t WHERE t.membership = $3" -const selectHeroesSQL = "" + - "SELECT DISTINCT user_id FROM syncapi_memberships WHERE room_id = $1 AND user_id != $2 AND membership IN ($3) LIMIT 5" - const selectMembershipBeforeSQL = "" + "SELECT membership, topological_pos FROM syncapi_memberships WHERE room_id = $1 and user_id = $2 AND topological_pos <= $3 ORDER BY topological_pos DESC LIMIT 1" @@ -99,7 +94,6 @@ func NewSqliteMembershipsTable(db *sql.DB) (tables.Memberships, error) { {&s.selectMembershipCountStmt, selectMembershipCountSQL}, {&s.selectMembershipForUserStmt, selectMembershipBeforeSQL}, {&s.selectMembersStmt, selectMembersSQL}, - // {&s.selectHeroesStmt, selectHeroesSQL}, - prepared at runtime due to variadic }.Prepare(db) } @@ -131,39 +125,6 @@ func (s *membershipsStatements) SelectMembershipCount( return } -func (s *membershipsStatements) SelectHeroes( - ctx context.Context, txn *sql.Tx, roomID, userID string, memberships []string, -) (heroes []string, err error) { - stmtSQL := strings.Replace(selectHeroesSQL, "($3)", sqlutil.QueryVariadicOffset(len(memberships), 2), 1) - stmt, err := s.db.PrepareContext(ctx, stmtSQL) - if err != nil { - return - } - defer internal.CloseAndLogIfError(ctx, stmt, "SelectHeroes: stmt.close() failed") - params := []interface{}{ - roomID, userID, - } - for _, membership := range memberships { - params = append(params, membership) - } - - stmt = sqlutil.TxStmt(txn, stmt) - var rows *sql.Rows - rows, err = stmt.QueryContext(ctx, params...) - if err != nil { - return - } - defer internal.CloseAndLogIfError(ctx, rows, "SelectHeroes: rows.close() failed") - var hero string - for rows.Next() { - if err = rows.Scan(&hero); err != nil { - return - } - heroes = append(heroes, hero) - } - return heroes, rows.Err() -} - // SelectMembershipForUser returns the membership of the user before and including the given position. If no membership can be found // returns "leave", the topological position and no error. If an error occurs, other than sql.ErrNoRows, returns that and an empty // string as the membership. diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 5ff185a32..166ddd233 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -14,6 +14,7 @@ import ( "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/gomatrixserverlib" + "github.com/stretchr/testify/assert" ) var ctx = context.Background() @@ -664,3 +665,181 @@ func topologyTokenBefore(t *testing.T, db storage.Database, eventID string) *typ return &tok } */ + +func pointer[t any](s t) *t { + return &s +} + +func TestRoomSummary(t *testing.T) { + + alice := test.NewUser(t) + bob := test.NewUser(t) + charlie := test.NewUser(t) + + // Create some dummy users + moreUsers := []*test.User{} + moreUserIDs := []string{} + for i := 0; i < 10; i++ { + u := test.NewUser(t) + moreUsers = append(moreUsers, u) + moreUserIDs = append(moreUserIDs, u.ID) + } + + testCases := []struct { + name string + wantSummary *types.Summary + additionalEvents func(t *testing.T, room *test.Room) + }{ + { + name: "after initial creation", + wantSummary: &types.Summary{JoinedMemberCount: pointer(1), InvitedMemberCount: pointer(0), Heroes: []string{}}, + }, + { + name: "invited user", + wantSummary: &types.Summary{JoinedMemberCount: pointer(1), InvitedMemberCount: pointer(1), Heroes: []string{bob.ID}}, + additionalEvents: func(t *testing.T, room *test.Room) { + room.CreateAndInsert(t, alice, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "invite", + }, test.WithStateKey(bob.ID)) + }, + }, + { + name: "invited user, but declined", + wantSummary: &types.Summary{JoinedMemberCount: pointer(1), InvitedMemberCount: pointer(0), Heroes: []string{bob.ID}}, + additionalEvents: func(t *testing.T, room *test.Room) { + room.CreateAndInsert(t, alice, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "invite", + }, test.WithStateKey(bob.ID)) + room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "leave", + }, test.WithStateKey(bob.ID)) + }, + }, + { + name: "joined user after invitation", + wantSummary: &types.Summary{JoinedMemberCount: pointer(2), InvitedMemberCount: pointer(0), Heroes: []string{bob.ID}}, + additionalEvents: func(t *testing.T, room *test.Room) { + room.CreateAndInsert(t, alice, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "invite", + }, test.WithStateKey(bob.ID)) + room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID)) + }, + }, + { + name: "multiple joined user", + wantSummary: &types.Summary{JoinedMemberCount: pointer(3), InvitedMemberCount: pointer(0), Heroes: []string{charlie.ID, bob.ID}}, + additionalEvents: func(t *testing.T, room *test.Room) { + room.CreateAndInsert(t, charlie, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(charlie.ID)) + room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID)) + }, + }, + { + name: "multiple joined/invited user", + wantSummary: &types.Summary{JoinedMemberCount: pointer(2), InvitedMemberCount: pointer(1), Heroes: []string{charlie.ID, bob.ID}}, + additionalEvents: func(t *testing.T, room *test.Room) { + room.CreateAndInsert(t, alice, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "invite", + }, test.WithStateKey(charlie.ID)) + room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID)) + }, + }, + { + name: "multiple joined/invited/left user", + wantSummary: &types.Summary{JoinedMemberCount: pointer(1), InvitedMemberCount: pointer(1), Heroes: []string{charlie.ID}}, + additionalEvents: func(t *testing.T, room *test.Room) { + room.CreateAndInsert(t, alice, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "invite", + }, test.WithStateKey(charlie.ID)) + room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID)) + room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "leave", + }, test.WithStateKey(bob.ID)) + }, + }, + { + name: "leaving user after joining", + wantSummary: &types.Summary{JoinedMemberCount: pointer(1), InvitedMemberCount: pointer(0), Heroes: []string{bob.ID}}, + additionalEvents: func(t *testing.T, room *test.Room) { + room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID)) + room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "leave", + }, test.WithStateKey(bob.ID)) + }, + }, + { + name: "many users", // heroes ordered by stream id + wantSummary: &types.Summary{JoinedMemberCount: pointer(len(moreUserIDs) + 1), InvitedMemberCount: pointer(0), Heroes: moreUserIDs[:5]}, + additionalEvents: func(t *testing.T, room *test.Room) { + for _, x := range moreUsers { + room.CreateAndInsert(t, x, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(x.ID)) + } + }, + }, + { + name: "canonical alias set", + wantSummary: &types.Summary{JoinedMemberCount: pointer(2), InvitedMemberCount: pointer(0), Heroes: []string{}}, + additionalEvents: func(t *testing.T, room *test.Room) { + room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID)) + room.CreateAndInsert(t, alice, gomatrixserverlib.MRoomCanonicalAlias, map[string]interface{}{ + "alias": "myalias", + }, test.WithStateKey("")) + }, + }, + { + name: "room name set", + wantSummary: &types.Summary{JoinedMemberCount: pointer(2), InvitedMemberCount: pointer(0), Heroes: []string{}}, + additionalEvents: func(t *testing.T, room *test.Room) { + room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID)) + room.CreateAndInsert(t, alice, gomatrixserverlib.MRoomName, map[string]interface{}{ + "name": "my room name", + }, test.WithStateKey("")) + }, + }, + } + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close, closeBase := MustCreateDatabase(t, dbType) + defer close() + defer closeBase() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + r := test.NewRoom(t, alice) + + if tc.additionalEvents != nil { + tc.additionalEvents(t, r) + } + + // write the room before creating a transaction + MustWriteEvents(t, db, r.Events()) + + transaction, err := db.NewDatabaseTransaction(ctx) + assert.NoError(t, err) + defer transaction.Rollback() + + summary, err := transaction.GetRoomSummary(ctx, r.ID, alice.ID) + assert.NoError(t, err) + assert.Equal(t, tc.wantSummary, summary) + }) + } + }) +} diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index a0574b257..c02e4ecc5 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -115,6 +115,9 @@ type CurrentRoomState interface { SelectJoinedUsersInRoom(ctx context.Context, txn *sql.Tx, roomIDs []string) (map[string][]string, error) // SelectSharedUsers returns a subset of otherUserIDs that share a room with userID. SelectSharedUsers(ctx context.Context, txn *sql.Tx, userID string, otherUserIDs []string) ([]string, error) + + SelectRoomHeroes(ctx context.Context, txn *sql.Tx, roomID, excludeUserID string, memberships []string) ([]string, error) + SelectMembershipCount(ctx context.Context, txn *sql.Tx, roomID, membership string) (int, error) } // BackwardsExtremities keeps track of backwards extremities for a room. @@ -185,7 +188,6 @@ type Receipts interface { type Memberships interface { UpsertMembership(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, streamPos, topologicalPos types.StreamPosition) error SelectMembershipCount(ctx context.Context, txn *sql.Tx, roomID, membership string, pos types.StreamPosition) (count int, err error) - SelectHeroes(ctx context.Context, txn *sql.Tx, roomID, userID string, memberships []string) (heroes []string, err error) SelectMembershipForUser(ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) SelectMemberships( ctx context.Context, txn *sql.Tx, diff --git a/syncapi/storage/tables/memberships_test.go b/syncapi/storage/tables/memberships_test.go index 0cee7f5a5..df593ae78 100644 --- a/syncapi/storage/tables/memberships_test.go +++ b/syncapi/storage/tables/memberships_test.go @@ -3,8 +3,6 @@ package tables_test import ( "context" "database/sql" - "reflect" - "sort" "testing" "time" @@ -88,43 +86,9 @@ func TestMembershipsTable(t *testing.T) { testUpsert(t, ctx, table, userEvents[0], alice, room) testMembershipCount(t, ctx, table, room) - testHeroes(t, ctx, table, alice, room, users) }) } -func testHeroes(t *testing.T, ctx context.Context, table tables.Memberships, user *test.User, room *test.Room, users []string) { - - // Re-slice and sort the expected users - users = users[1:] - sort.Strings(users) - type testCase struct { - name string - memberships []string - wantHeroes []string - } - - testCases := []testCase{ - {name: "no memberships queried", memberships: []string{}}, - {name: "joined memberships queried should be limited", memberships: []string{gomatrixserverlib.Join}, wantHeroes: users[:5]}, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - got, err := table.SelectHeroes(ctx, nil, room.ID, user.ID, tc.memberships) - if err != nil { - t.Fatalf("unable to select heroes: %s", err) - } - if gotLen := len(got); gotLen != len(tc.wantHeroes) { - t.Fatalf("expected %d heroes, got %d", len(tc.wantHeroes), gotLen) - } - - if !reflect.DeepEqual(got, tc.wantHeroes) { - t.Fatalf("expected heroes to be %+v, got %+v", tc.wantHeroes, got) - } - }) - } -} - func testMembershipCount(t *testing.T, ctx context.Context, table tables.Memberships, room *test.Room) { t.Run("membership counts are correct", func(t *testing.T) { // After 10 events, we should have 6 users (5 create related [incl. one member event], 5 member events = 6 users) diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index dd7845574..4664276cf 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "fmt" - "sort" "time" "github.com/matrix-org/dendrite/internal/caching" @@ -14,11 +13,9 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/syncapi/notifier" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" - "github.com/tidwall/gjson" - - "github.com/matrix-org/dendrite/syncapi/notifier" ) // The max number of per-room goroutines to have running. @@ -339,7 +336,10 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( case gomatrixserverlib.Join: jr := types.NewJoinResponse() if hasMembershipChange { - p.addRoomSummary(ctx, snapshot, jr, delta.RoomID, device.UserID, latestPosition) + jr.Summary, err = snapshot.GetRoomSummary(ctx, delta.RoomID, device.UserID) + if err != nil { + logrus.WithError(err).Warn("failed to get room summary") + } } jr.Timeline.PrevBatch = &prevBatch jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatSync) @@ -411,45 +411,6 @@ func applyHistoryVisibilityFilter( return events, nil } -func (p *PDUStreamProvider) addRoomSummary(ctx context.Context, snapshot storage.DatabaseTransaction, jr *types.JoinResponse, roomID, userID string, latestPosition types.StreamPosition) { - // Work out how many members are in the room. - joinedCount, _ := snapshot.MembershipCount(ctx, roomID, gomatrixserverlib.Join, latestPosition) - invitedCount, _ := snapshot.MembershipCount(ctx, roomID, gomatrixserverlib.Invite, latestPosition) - - jr.Summary.JoinedMemberCount = &joinedCount - jr.Summary.InvitedMemberCount = &invitedCount - - fetchStates := []gomatrixserverlib.StateKeyTuple{ - {EventType: gomatrixserverlib.MRoomName}, - {EventType: gomatrixserverlib.MRoomCanonicalAlias}, - } - // Check if the room has a name or a canonical alias - latestState := &roomserverAPI.QueryLatestEventsAndStateResponse{} - err := p.rsAPI.QueryLatestEventsAndState(ctx, &roomserverAPI.QueryLatestEventsAndStateRequest{StateToFetch: fetchStates, RoomID: roomID}, latestState) - if err != nil { - return - } - // Check if the room has a name or canonical alias, if so, return. - for _, ev := range latestState.StateEvents { - switch ev.Type() { - case gomatrixserverlib.MRoomName: - if gjson.GetBytes(ev.Content(), "name").Str != "" { - return - } - case gomatrixserverlib.MRoomCanonicalAlias: - if gjson.GetBytes(ev.Content(), "alias").Str != "" { - return - } - } - } - heroes, err := snapshot.GetRoomHeroes(ctx, roomID, userID, []string{"join", "invite"}) - if err != nil { - return - } - sort.Strings(heroes) - jr.Summary.Heroes = heroes -} - func (p *PDUStreamProvider) getJoinResponseForCompleteSync( ctx context.Context, snapshot storage.DatabaseTransaction, @@ -493,7 +454,10 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( return } - p.addRoomSummary(ctx, snapshot, jr, roomID, device.UserID, r.From) + jr.Summary, err = snapshot.GetRoomSummary(ctx, roomID, device.UserID) + if err != nil { + logrus.WithError(err).Warn("failed to get room summary") + } // We don't include a device here as we don't need to send down // transaction IDs for complete syncs, but we do it anyway because Sytest demands it for: