From d579ddb8e7c1a7e118797bcef08113379535e6fb Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 6 Jan 2023 15:44:10 +0100 Subject: [PATCH 01/13] Add simplified helm chart (#2905) As discussed yesterday, a simplified version of [my helm](https://github.com/S7evinK/dendrite-helm) which deploys a monolith with internal NATS and an optionally enabled PostgreSQL server. If the PostgreSQL dependency is not enabled, a user specified connection string is constructed. Co-authored-by: kegsay --- .github/workflows/gh-pages.yml | 52 +++ .github/workflows/helm.yml | 39 ++ .github/workflows/k8s.yml | 90 +++++ helm/cr.yaml | 1 + helm/ct.yaml | 7 + helm/dendrite/.helm-docs/about.gotmpl | 5 + helm/dendrite/.helm-docs/appservices.gotmpl | 5 + helm/dendrite/.helm-docs/database.gotmpl | 18 + helm/dendrite/.helm-docs/state.gotmpl | 3 + helm/dendrite/Chart.yaml | 19 + helm/dendrite/README.md | 147 ++++++++ helm/dendrite/README.md.gotmpl | 13 + helm/dendrite/ci/ct-ingress-values.yaml | 13 + .../ci/ct-postgres-sharedsecret-values.yaml | 16 + helm/dendrite/templates/_helpers.tpl | 72 ++++ helm/dendrite/templates/_overrides.yaml | 16 + helm/dendrite/templates/deployment.yaml | 103 ++++++ helm/dendrite/templates/ingress.yaml | 55 +++ helm/dendrite/templates/jobs.yaml | 99 +++++ helm/dendrite/templates/pvc.yaml | 48 +++ helm/dendrite/templates/secrets.yaml | 33 ++ helm/dendrite/templates/service.yaml | 17 + .../templates/tests/test-version.yaml | 17 + helm/dendrite/values.yaml | 348 ++++++++++++++++++ setup/config/config.go | 4 +- 25 files changed, 1238 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/gh-pages.yml create mode 100644 .github/workflows/helm.yml create mode 100644 .github/workflows/k8s.yml create mode 100644 helm/cr.yaml create mode 100644 helm/ct.yaml create mode 100644 helm/dendrite/.helm-docs/about.gotmpl create mode 100644 helm/dendrite/.helm-docs/appservices.gotmpl create mode 100644 helm/dendrite/.helm-docs/database.gotmpl create mode 100644 helm/dendrite/.helm-docs/state.gotmpl create mode 100644 helm/dendrite/Chart.yaml create mode 100644 helm/dendrite/README.md create mode 100644 helm/dendrite/README.md.gotmpl create mode 100644 helm/dendrite/ci/ct-ingress-values.yaml create mode 100644 helm/dendrite/ci/ct-postgres-sharedsecret-values.yaml create mode 100644 helm/dendrite/templates/_helpers.tpl create mode 100644 helm/dendrite/templates/_overrides.yaml create mode 100644 helm/dendrite/templates/deployment.yaml create mode 100644 helm/dendrite/templates/ingress.yaml create mode 100644 helm/dendrite/templates/jobs.yaml create mode 100644 helm/dendrite/templates/pvc.yaml create mode 100644 helm/dendrite/templates/secrets.yaml create mode 100644 helm/dendrite/templates/service.yaml create mode 100644 helm/dendrite/templates/tests/test-version.yaml create mode 100644 helm/dendrite/values.yaml diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml new file mode 100644 index 000000000..b5a8f0bbd --- /dev/null +++ b/.github/workflows/gh-pages.yml @@ -0,0 +1,52 @@ +# Sample workflow for building and deploying a Jekyll site to GitHub Pages +name: Deploy GitHub Pages dependencies preinstalled + +on: + # Runs on pushes targeting the default branch + push: + branches: ["main"] + paths: + - 'docs/**' # only execute if we have docs changes + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow one concurrent deployment +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + # Build job + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Pages + uses: actions/configure-pages@v2 + - name: Build with Jekyll + uses: actions/jekyll-build-pages@v1 + with: + source: ./docs + destination: ./_site + - name: Upload artifact + uses: actions/upload-pages-artifact@v1 + + # Deployment job + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v1 diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml new file mode 100644 index 000000000..7cdc369ba --- /dev/null +++ b/.github/workflows/helm.yml @@ -0,0 +1,39 @@ +name: Release Charts + +on: + push: + branches: + - main + paths: + - 'helm/**' # only execute if we have helm chart changes + +jobs: + release: + # depending on default permission settings for your org (contents being read-only or read-write for workloads), you will have to add permissions + # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token + permissions: + contents: write + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Install Helm + uses: azure/setup-helm@v3 + with: + version: v3.10.0 + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.4.1 + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + with: + config: helm/cr.yaml + charts_dir: helm/ diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml new file mode 100644 index 000000000..fc5e8c906 --- /dev/null +++ b/.github/workflows/k8s.yml @@ -0,0 +1,90 @@ +name: k8s + +on: + push: + branches: ["main"] + paths: + - 'helm/**' # only execute if we have helm chart changes + pull_request: + branches: ["main"] + paths: + - 'helm/**' + +jobs: + lint: + name: Lint Helm chart + runs-on: ubuntu-latest + outputs: + changed: ${{ steps.list-changed.outputs.changed }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: azure/setup-helm@v3 + with: + version: v3.10.0 + - uses: actions/setup-python@v4 + with: + python-version: 3.11 + check-latest: true + - uses: helm/chart-testing-action@v2.3.1 + - name: Get changed status + id: list-changed + run: | + changed=$(ct list-changed --config helm/ct.yaml --target-branch ${{ github.event.repository.default_branch }}) + if [[ -n "$changed" ]]; then + echo "::set-output name=changed::true" + fi + + - name: Run lint + run: ct lint --config helm/ct.yaml + + # only bother to run if lint step reports a change to the helm chart + install: + needs: + - lint + if: ${{ needs.lint.outputs.changed == 'true' }} + name: Install Helm charts + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.checkoutCommit }} + - name: Install Kubernetes tools + uses: yokawasa/action-setup-kube-tools@v0.8.2 + with: + setup-tools: | + helmv3 + helm: "3.10.3" + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.3.1 + - name: Create k3d cluster + uses: nolar/setup-k3d-k3s@v1 + with: + version: v1.21 + - name: Remove node taints + run: | + kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true + - name: Run chart-testing (install) + run: ct install --config helm/ct.yaml + + # Install the chart using helm directly and test with create-account + - name: Install chart + run: | + helm install --values helm/dendrite/ci/ct-postgres-sharedsecret-values.yaml dendrite helm/dendrite + - name: Wait for Postgres and Dendrite to be up + run: | + kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=postgresql || kubectl get pods -A + kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=dendrite || kubectl get pods -A + kubectl get pods -A + kubectl get services + kubectl get ingress + - name: Run create account + run: | + podName=$(kubectl get pods -l app.kubernetes.io/name=dendrite -o name) + kubectl exec "${podName}" -- /usr/bin/create-account -username alice -password somerandompassword \ No newline at end of file diff --git a/helm/cr.yaml b/helm/cr.yaml new file mode 100644 index 000000000..f895ab8d6 --- /dev/null +++ b/helm/cr.yaml @@ -0,0 +1 @@ +release-name-template: "helm-{{ .Name }}-{{ .Version }}" \ No newline at end of file diff --git a/helm/ct.yaml b/helm/ct.yaml new file mode 100644 index 000000000..af706fa3d --- /dev/null +++ b/helm/ct.yaml @@ -0,0 +1,7 @@ +remote: origin +target-branch: main +chart-repos: + - bitnami=https://charts.bitnami.com/bitnami +chart-dirs: + - helm +validate-maintainers: false \ No newline at end of file diff --git a/helm/dendrite/.helm-docs/about.gotmpl b/helm/dendrite/.helm-docs/about.gotmpl new file mode 100644 index 000000000..a92c6be42 --- /dev/null +++ b/helm/dendrite/.helm-docs/about.gotmpl @@ -0,0 +1,5 @@ +{{ define "chart.about" }} +## About + +This chart creates a monolith deployment, including an optionally enabled PostgreSQL dependency to connect to. +{{ end }} \ No newline at end of file diff --git a/helm/dendrite/.helm-docs/appservices.gotmpl b/helm/dendrite/.helm-docs/appservices.gotmpl new file mode 100644 index 000000000..8a79a0780 --- /dev/null +++ b/helm/dendrite/.helm-docs/appservices.gotmpl @@ -0,0 +1,5 @@ +{{ define "chart.appservices" }} +## Usage with appservices + +Create a folder `appservices` and place your configurations in there. The configurations will be read and placed in a secret `dendrite-appservices-conf`. +{{ end }} \ No newline at end of file diff --git a/helm/dendrite/.helm-docs/database.gotmpl b/helm/dendrite/.helm-docs/database.gotmpl new file mode 100644 index 000000000..85ef01ecc --- /dev/null +++ b/helm/dendrite/.helm-docs/database.gotmpl @@ -0,0 +1,18 @@ +{{ define "chart.dbCreation" }} +## Manual database creation + +(You can skip this, if you're deploying the PostgreSQL dependency) + +You'll need to create the following database before starting Dendrite (see [installation](https://matrix-org.github.io/dendrite/installation/database#single-database-creation)): + +```postgres +create database dendrite +``` + +or + +```bash +sudo -u postgres createdb -O dendrite -E UTF-8 dendrite +``` + +{{ end }} \ No newline at end of file diff --git a/helm/dendrite/.helm-docs/state.gotmpl b/helm/dendrite/.helm-docs/state.gotmpl new file mode 100644 index 000000000..2fe987ddd --- /dev/null +++ b/helm/dendrite/.helm-docs/state.gotmpl @@ -0,0 +1,3 @@ +{{ define "chart.state" }} +Status: **NOT PRODUCTION READY** +{{ end }} \ No newline at end of file diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml new file mode 100644 index 000000000..15d1e6d19 --- /dev/null +++ b/helm/dendrite/Chart.yaml @@ -0,0 +1,19 @@ +apiVersion: v2 +name: dendrite +version: "0.10.8" +appVersion: "0.10.8" +description: Dendrite Matrix Homeserver +type: application +keywords: + - matrix + - chat + - homeserver + - dendrite +home: https://github.com/matrix-org/dendrite +sources: + - https://github.com/matrix-org/dendrite +dependencies: +- name: postgresql + version: 12.1.7 + repository: https://charts.bitnami.com/bitnami + condition: postgresql.enabled diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md new file mode 100644 index 000000000..cb850d655 --- /dev/null +++ b/helm/dendrite/README.md @@ -0,0 +1,147 @@ +# dendrite + +![Version: 0.10.8](https://img.shields.io/badge/Version-0.10.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.10.8](https://img.shields.io/badge/AppVersion-0.10.8-informational?style=flat-square) +Dendrite Matrix Homeserver + +Status: **NOT PRODUCTION READY** + +## About + +This chart creates a monolith deployment, including an optionally enabled PostgreSQL dependency to connect to. + +## Manual database creation + +(You can skip this, if you're deploying the PostgreSQL dependency) + +You'll need to create the following database before starting Dendrite (see [installation](https://matrix-org.github.io/dendrite/installation/database#single-database-creation)): + +```postgres +create database dendrite +``` + +or + +```bash +sudo -u postgres createdb -O dendrite -E UTF-8 dendrite +``` + +## Usage with appservices + +Create a folder `appservices` and place your configurations in there. The configurations will be read and placed in a secret `dendrite-appservices-conf`. + +## Source Code + +* +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | postgresql | 12.1.7 | +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| image.name | string | `"ghcr.io/matrix-org/dendrite-monolith:v0.10.8"` | Docker repository/image to use | +| image.pullPolicy | string | `"IfNotPresent"` | Kubernetes pullPolicy | +| signing_key.create | bool | `true` | Create a new signing key, if not exists | +| signing_key.existingSecret | string | `""` | Use an existing secret | +| resources | object | sets some sane default values | Default resource requests/limits. | +| persistence.storageClass | string | `""` | The storage class to use for volume claims. Defaults to the cluster default storage class. | +| persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream | +| persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume | +| persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files | +| persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume | +| persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index | +| persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume | +| dendrite_config.version | int | `2` | | +| dendrite_config.global.server_name | string | `""` | **REQUIRED** Servername for this Dendrite deployment. | +| dendrite_config.global.private_key | string | `"/etc/dendrite/secrets/signing.key"` | The private key to use. (**NOTE**: This is overriden in Helm) | +| dendrite_config.global.well_known_server_name | string | `""` | The server name to delegate server-server communications to, with optional port e.g. localhost:443 | +| dendrite_config.global.well_known_client_name | string | `""` | The server name to delegate client-server communications to, with optional port e.g. localhost:443 | +| dendrite_config.global.trusted_third_party_id_servers | list | `["matrix.org","vector.im"]` | Lists of domains that the server will trust as identity servers to verify third party identifiers such as phone numbers and email addresses. | +| dendrite_config.global.old_private_keys | string | `nil` | The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) to old signing keys that were formerly in use on this domain name. These keys will not be used for federation request or event signing, but will be provided to any other homeserver that asks when trying to verify old events. | +| dendrite_config.global.disable_federation | bool | `false` | Disable federation. Dendrite will not be able to make any outbound HTTP requests to other servers and the federation API will not be exposed. | +| dendrite_config.global.key_validity_period | string | `"168h0m0s"` | | +| dendrite_config.global.database.connection_string | string | `""` | The connection string for connections to Postgres. This will be set automatically if using the Postgres dependency | +| dendrite_config.global.database.max_open_conns | int | `90` | Default database maximum open connections | +| dendrite_config.global.database.max_idle_conns | int | `5` | Default database maximum idle connections | +| dendrite_config.global.database.conn_max_lifetime | int | `-1` | Default database maximum lifetime | +| dendrite_config.global.jetstream.storage_path | string | `"/data/jetstream"` | Persistent directory to store JetStream streams in. | +| dendrite_config.global.jetstream.addresses | list | `[]` | NATS JetStream server addresses if not using internal NATS. | +| dendrite_config.global.jetstream.topic_prefix | string | `"Dendrite"` | The prefix for JetStream streams | +| dendrite_config.global.jetstream.in_memory | bool | `false` | Keep all data in memory. (**NOTE**: This is overriden in Helm to `false`) | +| dendrite_config.global.jetstream.disable_tls_validation | bool | `true` | Disables TLS validation. This should **NOT** be used in production. | +| dendrite_config.global.cache.max_size_estimated | string | `"1gb"` | The estimated maximum size for the global cache in bytes, or in terabytes, gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or 'kb' suffix is specified. Note that this is not a hard limit, nor is it a memory limit for the entire process. A cache that is too small may ultimately provide little or no benefit. | +| dendrite_config.global.cache.max_age | string | `"1h"` | The maximum amount of time that a cache entry can live for in memory before it will be evicted and/or refreshed from the database. Lower values result in easier admission of new cache entries but may also increase database load in comparison to higher values, so adjust conservatively. Higher values may make it harder for new items to make it into the cache, e.g. if new rooms suddenly become popular. | +| dendrite_config.global.report_stats.enabled | bool | `false` | Configures phone-home statistics reporting. These statistics contain the server name, number of active users and some information on your deployment config. We use this information to understand how Dendrite is being used in the wild. | +| dendrite_config.global.report_stats.endpoint | string | `"https://matrix.org/report-usage-stats/push"` | Endpoint to report statistics to. | +| dendrite_config.global.presence.enable_inbound | bool | `false` | Controls whether we receive presence events from other servers | +| dendrite_config.global.presence.enable_outbound | bool | `false` | Controls whether we send presence events for our local users to other servers. (_May increase CPU/memory usage_) | +| dendrite_config.global.server_notices.enabled | bool | `false` | Server notices allows server admins to send messages to all users on the server. | +| dendrite_config.global.server_notices.local_part | string | `"_server"` | The local part for the user sending server notices. | +| dendrite_config.global.server_notices.display_name | string | `"Server Alerts"` | The display name for the user sending server notices. | +| dendrite_config.global.server_notices.avatar_url | string | `""` | The avatar URL (as a mxc:// URL) name for the user sending server notices. | +| dendrite_config.global.server_notices.room_name | string | `"Server Alerts"` | | +| dendrite_config.global.metrics.enabled | bool | `false` | Whether or not Prometheus metrics are enabled. | +| dendrite_config.global.metrics.basic_auth.user | string | `"metrics"` | HTTP basic authentication username | +| dendrite_config.global.metrics.basic_auth.password | string | `"metrics"` | HTTP basic authentication password | +| dendrite_config.global.dns_cache.enabled | bool | `false` | Whether or not the DNS cache is enabled. | +| dendrite_config.global.dns_cache.cache_size | int | `256` | Maximum number of entries to hold in the DNS cache | +| dendrite_config.global.dns_cache.cache_lifetime | string | `"10m"` | Duration for how long DNS cache items should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) | +| dendrite_config.global.profiling.enabled | bool | `false` | Enable pprof. You will need to manually create a port forwarding to the deployment to access PPROF, as it will only listen on localhost and the defined port. e.g. `kubectl port-forward deployments/dendrite 65432:65432` | +| dendrite_config.global.profiling.port | int | `65432` | pprof port, if enabled | +| dendrite_config.mscs | object | `{"mscs":["msc2946"]}` | Configuration for experimental MSC's. (Valid values are: msc2836 and msc2946) | +| dendrite_config.app_service_api.disable_tls_validation | bool | `false` | Disable the validation of TLS certificates of appservices. This is not recommended in production since it may allow appservice traffic to be sent to an insecure endpoint. | +| dendrite_config.app_service_api.config_files | list | `[]` | Appservice config files to load on startup. (**NOTE**: This is overriden by Helm, if a folder `./appservices/` exists) | +| dendrite_config.client_api.registration_disabled | bool | `true` | Prevents new users from being able to register on this homeserver, except when using the registration shared secret below. | +| dendrite_config.client_api.guests_disabled | bool | `true` | | +| dendrite_config.client_api.registration_shared_secret | string | `""` | If set, allows registration by anyone who knows the shared secret, regardless of whether registration is otherwise disabled. | +| dendrite_config.client_api.enable_registration_captcha | bool | `false` | enable reCAPTCHA registration | +| dendrite_config.client_api.recaptcha_public_key | string | `""` | reCAPTCHA public key | +| dendrite_config.client_api.recaptcha_private_key | string | `""` | reCAPTCHA private key | +| dendrite_config.client_api.recaptcha_bypass_secret | string | `""` | reCAPTCHA bypass secret | +| dendrite_config.client_api.recaptcha_siteverify_api | string | `""` | | +| dendrite_config.client_api.turn.turn_user_lifetime | string | `"24h"` | Duration for how long users should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) | +| dendrite_config.client_api.turn.turn_uris | list | `[]` | | +| dendrite_config.client_api.turn.turn_shared_secret | string | `""` | | +| dendrite_config.client_api.turn.turn_username | string | `""` | The TURN username | +| dendrite_config.client_api.turn.turn_password | string | `""` | The TURN password | +| dendrite_config.client_api.rate_limiting.enabled | bool | `true` | Enable rate limiting | +| dendrite_config.client_api.rate_limiting.threshold | int | `20` | After how many requests a rate limit should be activated | +| dendrite_config.client_api.rate_limiting.cooloff_ms | int | `500` | Cooloff time in milliseconds | +| dendrite_config.client_api.rate_limiting.exempt_user_ids | string | `nil` | Users which should be exempt from rate limiting | +| dendrite_config.federation_api.send_max_retries | int | `16` | Federation failure threshold. How many consecutive failures that we should tolerate when sending federation requests to a specific server. The backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. The default value is 16 if not specified, which is circa 18 hours. | +| dendrite_config.federation_api.disable_tls_validation | bool | `false` | Disable TLS validation. This should **NOT** be used in production. | +| dendrite_config.federation_api.prefer_direct_fetch | bool | `false` | | +| dendrite_config.federation_api.disable_http_keepalives | bool | `false` | Prevents Dendrite from keeping HTTP connections open for reuse for future requests. Connections will be closed quicker but we may spend more time on TLS handshakes instead. | +| dendrite_config.federation_api.key_perspectives | list | See value.yaml | Perspective keyservers, to use as a backup when direct key fetch requests don't succeed. | +| dendrite_config.media_api.base_path | string | `"/data/media_store"` | The path to store media files (e.g. avatars) in | +| dendrite_config.media_api.max_file_size_bytes | int | `10485760` | The max file size for uploaded media files | +| dendrite_config.media_api.dynamic_thumbnails | bool | `false` | | +| dendrite_config.media_api.max_thumbnail_generators | int | `10` | The maximum number of simultaneous thumbnail generators to run. | +| dendrite_config.media_api.thumbnail_sizes | list | See value.yaml | A list of thumbnail sizes to be generated for media content. | +| dendrite_config.sync_api.real_ip_header | string | `"X-Real-IP"` | This option controls which HTTP header to inspect to find the real remote IP address of the client. This is likely required if Dendrite is running behind a reverse proxy server. | +| dendrite_config.sync_api.search | object | `{"enabled":true,"index_path":"/data/search","language":"en"}` | Configuration for the full-text search engine. | +| dendrite_config.sync_api.search.enabled | bool | `true` | Whether fulltext search is enabled. | +| dendrite_config.sync_api.search.index_path | string | `"/data/search"` | The path to store the search index in. | +| dendrite_config.sync_api.search.language | string | `"en"` | The language most likely to be used on the server - used when indexing, to ensure the returned results match expectations. A full list of possible languages can be found [here](https://github.com/matrix-org/dendrite/blob/76db8e90defdfb9e61f6caea8a312c5d60bcc005/internal/fulltext/bleve.go#L25-L46) | +| dendrite_config.user_api.bcrypt_cost | int | `10` | bcrypt cost to use when hashing passwords. (ranges from 4-31; 4 being least secure, 31 being most secure; _NOTE: Using a too high value can cause clients to timeout and uses more CPU._) | +| dendrite_config.user_api.openid_token_lifetime_ms | int | `3600000` | OpenID Token lifetime in milliseconds. | +| dendrite_config.user_api.push_gateway_disable_tls_validation | bool | `false` | | +| dendrite_config.user_api.auto_join_rooms | list | `[]` | Rooms to join users to after registration | +| dendrite_config.logging | list | `[{"level":"info","type":"std"}]` | Default logging configuration | +| postgresql.enabled | bool | See value.yaml | Enable and configure postgres as the database for dendrite. | +| postgresql.image.repository | string | `"bitnami/postgresql"` | | +| postgresql.image.tag | string | `"15.1.0"` | | +| postgresql.auth.username | string | `"dendrite"` | | +| postgresql.auth.password | string | `"changeme"` | | +| postgresql.auth.database | string | `"dendrite"` | | +| postgresql.persistence.enabled | bool | `false` | | +| ingress.enabled | bool | `false` | Create an ingress for a monolith deployment | +| ingress.hosts | list | `[]` | | +| ingress.className | string | `""` | | +| ingress.hostName | string | `""` | | +| ingress.annotations | object | `{}` | Extra, custom annotations | +| ingress.tls | list | `[]` | | +| service.type | string | `"ClusterIP"` | | +| service.port | int | `80` | | diff --git a/helm/dendrite/README.md.gotmpl b/helm/dendrite/README.md.gotmpl new file mode 100644 index 000000000..7c32f7b02 --- /dev/null +++ b/helm/dendrite/README.md.gotmpl @@ -0,0 +1,13 @@ +{{ template "chart.header" . }} +{{ template "chart.deprecationWarning" . }} +{{ template "chart.badgesSection" . }} +{{ template "chart.description" . }} +{{ template "chart.state" . }} +{{ template "chart.about" . }} +{{ template "chart.dbCreation" . }} +{{ template "chart.appservices" . }} +{{ template "chart.maintainersSection" . }} +{{ template "chart.sourcesSection" . }} +{{ template "chart.requirementsSection" . }} +{{ template "chart.valuesSection" . }} +{{ template "helm-docs.versionFooter" . }} \ No newline at end of file diff --git a/helm/dendrite/ci/ct-ingress-values.yaml b/helm/dendrite/ci/ct-ingress-values.yaml new file mode 100644 index 000000000..28311d33e --- /dev/null +++ b/helm/dendrite/ci/ct-ingress-values.yaml @@ -0,0 +1,13 @@ +--- +postgresql: + enabled: true + primary: + persistence: + size: 1Gi + +dendrite_config: + global: + server_name: "localhost" + +ingress: + enabled: true diff --git a/helm/dendrite/ci/ct-postgres-sharedsecret-values.yaml b/helm/dendrite/ci/ct-postgres-sharedsecret-values.yaml new file mode 100644 index 000000000..55e652c63 --- /dev/null +++ b/helm/dendrite/ci/ct-postgres-sharedsecret-values.yaml @@ -0,0 +1,16 @@ +--- +postgresql: + enabled: true + primary: + persistence: + size: 1Gi + +dendrite_config: + global: + server_name: "localhost" + + client_api: + registration_shared_secret: "d233f2fcb0470845a8e150a20ef594ddbe0b4cf7fe482fb9d5120c198557acbf" # echo "dendrite" | sha256sum + +ingress: + enabled: true diff --git a/helm/dendrite/templates/_helpers.tpl b/helm/dendrite/templates/_helpers.tpl new file mode 100644 index 000000000..291f351bc --- /dev/null +++ b/helm/dendrite/templates/_helpers.tpl @@ -0,0 +1,72 @@ +{{- define "validate.config" }} +{{- if not .Values.signing_key.create -}} +{{- fail "You must create a signing key for configuration.signing_key. (see https://github.com/matrix-org/dendrite/blob/master/docs/INSTALL.md#server-key-generation)" -}} +{{- end -}} +{{- if not (or .Values.dendrite_config.global.database.host .Values.postgresql.enabled) -}} +{{- fail "Database server must be set." -}} +{{- end -}} +{{- if not (or .Values.dendrite_config.global.database.user .Values.postgresql.enabled) -}} +{{- fail "Database user must be set." -}} +{{- end -}} +{{- if not (or .Values.dendrite_config.global.database.password .Values.postgresql.enabled) -}} +{{- fail "Database password must be set." -}} +{{- end -}} +{{- end -}} + + +{{- define "image.name" -}} +image: {{ .name }} +imagePullPolicy: {{ .pullPolicy }} +{{- end -}} + +{{/* +Expand the name of the chart. +*/}} +{{- define "dendrite.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "dendrite.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "dendrite.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "dendrite.labels" -}} +helm.sh/chart: {{ include "dendrite.chart" . }} +{{ include "dendrite.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "dendrite.selectorLabels" -}} +app.kubernetes.io/name: {{ include "dendrite.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/helm/dendrite/templates/_overrides.yaml b/helm/dendrite/templates/_overrides.yaml new file mode 100644 index 000000000..edb8ba83a --- /dev/null +++ b/helm/dendrite/templates/_overrides.yaml @@ -0,0 +1,16 @@ +{{- define "override.config" }} +{{- if .Values.postgresql.enabled }} +{{- $_ := set .Values.dendrite_config.global.database "connection_string" (print "postgresql://" .Values.postgresql.auth.username ":" .Values.postgresql.auth.password "@" .Release.Name "-postgresql/dendrite?sslmode=disable") -}} +{{ end }} +global: + private_key: /etc/dendrite/secrets/signing.key + jetstream: + in_memory: false +{{ if (gt (len (.Files.Glob "appservices/*")) 0) }} +app_service_api: + config_files: + {{- range $x, $y := .Files.Glob "appservices/*" }} + - /etc/dendrite/appservices/{{ base $x }} + {{ end }} +{{ end }} +{{ end }} diff --git a/helm/dendrite/templates/deployment.yaml b/helm/dendrite/templates/deployment.yaml new file mode 100644 index 000000000..629ffe528 --- /dev/null +++ b/helm/dendrite/templates/deployment.yaml @@ -0,0 +1,103 @@ +{{ template "validate.config" . }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: {{ $.Release.Namespace }} + name: {{ include "dendrite.fullname" . }} + labels: + {{- include "dendrite.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "dendrite.selectorLabels" . | nindent 6 }} + replicas: 1 + template: + metadata: + labels: + {{- include "dendrite.selectorLabels" . | nindent 8 }} + annotations: + confighash-global: secret-{{ .Values.global | toYaml | sha256sum | trunc 32 }} + confighash-clientapi: clientapi-{{ .Values.clientapi | toYaml | sha256sum | trunc 32 }} + confighash-federationapi: federationapi-{{ .Values.federationapi | toYaml | sha256sum | trunc 32 }} + confighash-mediaapi: mediaapi-{{ .Values.mediaapi | toYaml | sha256sum | trunc 32 }} + confighash-syncapi: syncapi-{{ .Values.syncapi | toYaml | sha256sum | trunc 32 }} + spec: + volumes: + - name: {{ include "dendrite.fullname" . }}-conf-vol + secret: + secretName: {{ include "dendrite.fullname" . }}-conf + - name: {{ include "dendrite.fullname" . }}-signing-key + secret: + secretName: {{ default (print ( include "dendrite.fullname" . ) "-signing-key") $.Values.signing_key.existingSecret | quote }} + {{- if (gt (len ($.Files.Glob "appservices/*")) 0) }} + - name: {{ include "dendrite.fullname" . }}-appservices + secret: + secretName: {{ include "dendrite.fullname" . }}-appservices-conf + {{- end }} + - name: {{ include "dendrite.fullname" . }}-jetstream + persistentVolumeClaim: + claimName: {{ default (print ( include "dendrite.fullname" . ) "-jetstream-pvc") $.Values.persistence.jetstream.existingClaim | quote }} + - name: {{ include "dendrite.fullname" . }}-media + persistentVolumeClaim: + claimName: {{ default (print ( include "dendrite.fullname" . ) "-media-pvc") $.Values.persistence.media.existingClaim | quote }} + - name: {{ include "dendrite.fullname" . }}-search + persistentVolumeClaim: + claimName: {{ default (print ( include "dendrite.fullname" . ) "-search-pvc") $.Values.persistence.search.existingClaim | quote }} + containers: + - name: {{ $.Chart.Name }} + {{- include "image.name" $.Values.image | nindent 8 }} + args: + - '--config' + - '/etc/dendrite/dendrite.yaml' + ports: + - name: http + containerPort: 8008 + protocol: TCP + {{- if $.Values.dendrite_config.global.profiling.enabled }} + env: + - name: PPROFLISTEN + value: "localhost:{{- $.Values.global.profiling.port -}}" + {{- end }} + resources: + {{- toYaml $.Values.resources | nindent 10 }} + volumeMounts: + - mountPath: /etc/dendrite/ + name: {{ include "dendrite.fullname" . }}-conf-vol + - mountPath: /etc/dendrite/secrets/ + name: {{ include "dendrite.fullname" . }}-signing-key + {{- if (gt (len ($.Files.Glob "appservices/*")) 0) }} + - mountPath: /etc/dendrite/appservices + name: {{ include "dendrite.fullname" . }}-appservices + readOnly: true + {{ end }} + - mountPath: {{ .Values.dendrite_config.media_api.base_path }} + name: {{ include "dendrite.fullname" . }}-media + - mountPath: {{ .Values.dendrite_config.global.jetstream.storage_path }} + name: {{ include "dendrite.fullname" . }}-jetstream + - mountPath: {{ .Values.dendrite_config.sync_api.search.index_path }} + name: {{ include "dendrite.fullname" . }}-search + livenessProbe: + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 10 + httpGet: + path: /_dendrite/monitor/health + port: http + readinessProbe: + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 10 + httpGet: + path: /_dendrite/monitor/health + port: http + startupProbe: + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 10 + httpGet: + path: /_dendrite/monitor/up + port: http \ No newline at end of file diff --git a/helm/dendrite/templates/ingress.yaml b/helm/dendrite/templates/ingress.yaml new file mode 100644 index 000000000..8f86ad723 --- /dev/null +++ b/helm/dendrite/templates/ingress.yaml @@ -0,0 +1,55 @@ +{{- if .Values.ingress.enabled -}} + {{- $fullName := include "dendrite.fullname" . -}} + {{- $svcPort := .Values.service.port -}} + {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} + {{- end }} + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 + {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 + {{- else -}} +apiVersion: extensions/v1beta1 + {{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "dendrite.labels" . | nindent 4 }} + annotations: + {{- with .Values.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + - host: {{ .Values.ingress.hostName | quote }} + http: + paths: + - path: / + pathType: ImplementationSpecific + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} \ No newline at end of file diff --git a/helm/dendrite/templates/jobs.yaml b/helm/dendrite/templates/jobs.yaml new file mode 100644 index 000000000..76915694d --- /dev/null +++ b/helm/dendrite/templates/jobs.yaml @@ -0,0 +1,99 @@ +{{ if and .Values.signing_key.create (not .Values.signing_key.existingSecret ) }} +{{ $name := (print ( include "dendrite.fullname" . ) "-signing-key") }} +{{ $secretName := (print ( include "dendrite.fullname" . ) "-signing-key") }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/component: signingkey-job +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/component: signingkey-job + {{- include "dendrite.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - {{ $secretName }} + verbs: + - get + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name }} + labels: + app.kubernetes.io/component: signingkey-job + {{- include "dendrite.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $name }} +subjects: + - kind: ServiceAccount + name: {{ $name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: generate-signing-key + labels: + {{- include "dendrite.labels" . | nindent 4 }} +spec: + template: + spec: + restartPolicy: "Never" + serviceAccount: {{ $name }} + containers: + - name: upload-key + image: bitnami/kubectl + command: + - sh + - -c + - | + # check if key already exists + key=$(kubectl get secret {{ $secretName }} -o jsonpath="{.data['signing\.key']}" 2> /dev/null) + [ $? -ne 0 ] && echo "Failed to get existing secret" && exit 1 + [ -n "$key" ] && echo "Key already created, exiting." && exit 0 + # wait for signing key + while [ ! -f /etc/dendrite/signing-key.pem ]; do + echo "Waiting for signing key.." + sleep 5; + done + # update secret + kubectl patch secret {{ $secretName }} -p "{\"data\":{\"signing.key\":\"$(base64 /etc/dendrite/signing-key.pem | tr -d '\n')\"}}" + [ $? -ne 0 ] && echo "Failed to update secret." && exit 1 + echo "Signing key successfully created." + volumeMounts: + - mountPath: /etc/dendrite/ + name: signing-key + readOnly: true + - name: generate-key + {{- include "image.name" $.Values.image | nindent 8 }} + command: + - sh + - -c + - | + /usr/bin/generate-keys -private-key /etc/dendrite/signing-key.pem + chown 1001:1001 /etc/dendrite/signing-key.pem + volumeMounts: + - mountPath: /etc/dendrite/ + name: signing-key + volumes: + - name: signing-key + emptyDir: {} + parallelism: 1 + completions: 1 + backoffLimit: 1 +{{ end }} \ No newline at end of file diff --git a/helm/dendrite/templates/pvc.yaml b/helm/dendrite/templates/pvc.yaml new file mode 100644 index 000000000..897957e60 --- /dev/null +++ b/helm/dendrite/templates/pvc.yaml @@ -0,0 +1,48 @@ +{{ if not .Values.persistence.media.existingClaim }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + annotations: + helm.sh/resource-policy: keep + name: {{ include "dendrite.fullname" . }}-media-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistence.media.capacity }} + storageClassName: {{ .Values.persistence.storageClass }} +{{ end }} +{{ if not .Values.persistence.jetstream.existingClaim }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + annotations: + helm.sh/resource-policy: keep + name: {{ include "dendrite.fullname" . }}-jetstream-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistence.jetstream.capacity }} + storageClassName: {{ .Values.persistence.storageClass }} +{{ end }} +{{ if not .Values.persistence.search.existingClaim }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + annotations: + helm.sh/resource-policy: keep + name: {{ include "dendrite.fullname" . }}-search-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistence.search.capacity }} + storageClassName: {{ .Values.persistence.storageClass }} +{{ end }} \ No newline at end of file diff --git a/helm/dendrite/templates/secrets.yaml b/helm/dendrite/templates/secrets.yaml new file mode 100644 index 000000000..d4b8ecbf2 --- /dev/null +++ b/helm/dendrite/templates/secrets.yaml @@ -0,0 +1,33 @@ +{{ if (gt (len (.Files.Glob "appservices/*")) 0) }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "dendrite.fullname" . }}-appservices-conf + namespace: {{ .Release.Namespace }} +type: Opaque +data: +{{ (.Files.Glob "appservices/*").AsSecrets | indent 2 }} +{{ end }} +{{ if and .Values.signing_key.create (not .Values.signing_key.existingSecret) }} +--- +apiVersion: v1 +kind: Secret +metadata: + annotations: + helm.sh/resource-policy: keep + name: {{ include "dendrite.fullname" . }}-signing-key + namespace: {{ .Release.Namespace }} +type: Opaque +{{ end }} + +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: {{ include "dendrite.fullname" . }}-conf + namespace: {{ .Release.Namespace }} +stringData: + dendrite.yaml: | + {{ toYaml ( mustMergeOverwrite .Values.dendrite_config ( fromYaml (include "override.config" .) ) .Values.dendrite_config ) | nindent 4 }} \ No newline at end of file diff --git a/helm/dendrite/templates/service.yaml b/helm/dendrite/templates/service.yaml new file mode 100644 index 000000000..365a43f04 --- /dev/null +++ b/helm/dendrite/templates/service.yaml @@ -0,0 +1,17 @@ +{{ template "validate.config" . }} +--- +apiVersion: v1 +kind: Service +metadata: + namespace: {{ $.Release.Namespace }} + name: {{ include "dendrite.fullname" . }} + labels: + {{- include "dendrite.labels" . | nindent 4 }} +spec: + selector: + {{- include "dendrite.selectorLabels" . | nindent 4 }} + ports: + - name: http + protocol: TCP + port: 8008 + targetPort: 8008 \ No newline at end of file diff --git a/helm/dendrite/templates/tests/test-version.yaml b/helm/dendrite/templates/tests/test-version.yaml new file mode 100644 index 000000000..d88751325 --- /dev/null +++ b/helm/dendrite/templates/tests/test-version.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "dendrite.fullname" . }}-test-version" + labels: + {{- include "dendrite.selectorLabels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: curl + image: curlimages/curl + imagePullPolicy: IfNotPresent + args: + - 'http://{{- include "dendrite.fullname" . -}}:8008/_matrix/client/versions' + restartPolicy: Never diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml new file mode 100644 index 000000000..2c6e80942 --- /dev/null +++ b/helm/dendrite/values.yaml @@ -0,0 +1,348 @@ +image: + # -- Docker repository/image to use + name: "ghcr.io/matrix-org/dendrite-monolith:v0.10.8" + # -- Kubernetes pullPolicy + pullPolicy: IfNotPresent + + +# signing key to use +signing_key: + # -- Create a new signing key, if not exists + create: true + # -- Use an existing secret + existingSecret: "" + +# -- Default resource requests/limits. +# @default -- sets some sane default values +resources: + requests: + memory: "512Mi" + + limits: + memory: "4096Mi" + +persistence: + # -- The storage class to use for volume claims. Defaults to the + # cluster default storage class. + storageClass: "" + jetstream: + # -- Use an existing volume claim for jetstream + existingClaim: "" + # -- PVC Storage Request for the jetstream volume + capacity: "1Gi" + media: + # -- Use an existing volume claim for media files + existingClaim: "" + # -- PVC Storage Request for the media volume + capacity: "1Gi" + search: + # -- Use an existing volume claim for the fulltext search index + existingClaim: "" + # -- PVC Storage Request for the search volume + capacity: "1Gi" + +dendrite_config: + version: 2 + global: + # -- **REQUIRED** Servername for this Dendrite deployment. + server_name: "" + + # -- The private key to use. (**NOTE**: This is overriden in Helm) + private_key: /etc/dendrite/secrets/signing.key + + # -- The server name to delegate server-server communications to, with optional port + # e.g. localhost:443 + well_known_server_name: "" + + # -- The server name to delegate client-server communications to, with optional port + # e.g. localhost:443 + well_known_client_name: "" + + # -- Lists of domains that the server will trust as identity servers to verify third + # party identifiers such as phone numbers and email addresses. + trusted_third_party_id_servers: + - matrix.org + - vector.im + + # -- The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) + # to old signing keys that were formerly in use on this domain name. These + # keys will not be used for federation request or event signing, but will be + # provided to any other homeserver that asks when trying to verify old events. + old_private_keys: + # If the old private key file is available: + # - private_key: old_matrix_key.pem + # expired_at: 1601024554498 + # If only the public key (in base64 format) and key ID are known: + # - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM= + # key_id: ed25519:mykeyid + # expired_at: 1601024554498 + + # -- Disable federation. Dendrite will not be able to make any outbound HTTP requests + # to other servers and the federation API will not be exposed. + disable_federation: false + + key_validity_period: 168h0m0s + + database: + # -- The connection string for connections to Postgres. + # This will be set automatically if using the Postgres dependency + connection_string: "" + + # -- Default database maximum open connections + max_open_conns: 90 + # -- Default database maximum idle connections + max_idle_conns: 5 + # -- Default database maximum lifetime + conn_max_lifetime: -1 + + jetstream: + # -- Persistent directory to store JetStream streams in. + storage_path: "/data/jetstream" + # -- NATS JetStream server addresses if not using internal NATS. + addresses: [] + # -- The prefix for JetStream streams + topic_prefix: "Dendrite" + # -- Keep all data in memory. (**NOTE**: This is overriden in Helm to `false`) + in_memory: false + # -- Disables TLS validation. This should **NOT** be used in production. + disable_tls_validation: true + + cache: + # -- The estimated maximum size for the global cache in bytes, or in terabytes, + # gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or + # 'kb' suffix is specified. Note that this is not a hard limit, nor is it a + # memory limit for the entire process. A cache that is too small may ultimately + # provide little or no benefit. + max_size_estimated: 1gb + # -- The maximum amount of time that a cache entry can live for in memory before + # it will be evicted and/or refreshed from the database. Lower values result in + # easier admission of new cache entries but may also increase database load in + # comparison to higher values, so adjust conservatively. Higher values may make + # it harder for new items to make it into the cache, e.g. if new rooms suddenly + # become popular. + max_age: 1h + + report_stats: + # -- Configures phone-home statistics reporting. These statistics contain the server + # name, number of active users and some information on your deployment config. + # We use this information to understand how Dendrite is being used in the wild. + enabled: false + # -- Endpoint to report statistics to. + endpoint: https://matrix.org/report-usage-stats/push + + presence: + # -- Controls whether we receive presence events from other servers + enable_inbound: false + # -- Controls whether we send presence events for our local users to other servers. + # (_May increase CPU/memory usage_) + enable_outbound: false + + server_notices: + # -- Server notices allows server admins to send messages to all users on the server. + enabled: false + # -- The local part for the user sending server notices. + local_part: "_server" + # -- The display name for the user sending server notices. + display_name: "Server Alerts" + # -- The avatar URL (as a mxc:// URL) name for the user sending server notices. + avatar_url: "" + # The room name to be used when sending server notices. This room name will + # appear in user clients. + room_name: "Server Alerts" + + # prometheus metrics + metrics: + # -- Whether or not Prometheus metrics are enabled. + enabled: false + # HTTP basic authentication to protect access to monitoring. + basic_auth: + # -- HTTP basic authentication username + user: "metrics" + # -- HTTP basic authentication password + password: metrics + + dns_cache: + # -- Whether or not the DNS cache is enabled. + enabled: false + # -- Maximum number of entries to hold in the DNS cache + cache_size: 256 + # -- Duration for how long DNS cache items should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) + cache_lifetime: "10m" + + profiling: + # -- Enable pprof. You will need to manually create a port forwarding to the deployment to access PPROF, + # as it will only listen on localhost and the defined port. + # e.g. `kubectl port-forward deployments/dendrite 65432:65432` + enabled: false + # -- pprof port, if enabled + port: 65432 + + # -- Configuration for experimental MSC's. (Valid values are: msc2836 and msc2946) + mscs: + mscs: + - msc2946 + # A list of enabled MSC's + # Currently valid values are: + # - msc2836 (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) + # - msc2946 (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946) + + + app_service_api: + # -- Disable the validation of TLS certificates of appservices. This is + # not recommended in production since it may allow appservice traffic + # to be sent to an insecure endpoint. + disable_tls_validation: false + # -- Appservice config files to load on startup. (**NOTE**: This is overriden by Helm, if a folder `./appservices/` exists) + config_files: [] + + client_api: + # -- Prevents new users from being able to register on this homeserver, except when + # using the registration shared secret below. + registration_disabled: true + + # Prevents new guest accounts from being created. Guest registration is also + # disabled implicitly by setting 'registration_disabled' above. + guests_disabled: true + + # -- If set, allows registration by anyone who knows the shared secret, regardless of + # whether registration is otherwise disabled. + registration_shared_secret: "" + + # -- enable reCAPTCHA registration + enable_registration_captcha: false + # -- reCAPTCHA public key + recaptcha_public_key: "" + # -- reCAPTCHA private key + recaptcha_private_key: "" + # -- reCAPTCHA bypass secret + recaptcha_bypass_secret: "" + recaptcha_siteverify_api: "" + + # TURN server information that this homeserver should send to clients. + turn: + # -- Duration for how long users should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) + turn_user_lifetime: "24h" + turn_uris: [] + turn_shared_secret: "" + # -- The TURN username + turn_username: "" + # -- The TURN password + turn_password: "" + + rate_limiting: + # -- Enable rate limiting + enabled: true + # -- After how many requests a rate limit should be activated + threshold: 20 + # -- Cooloff time in milliseconds + cooloff_ms: 500 + # -- Users which should be exempt from rate limiting + exempt_user_ids: + + federation_api: + # -- Federation failure threshold. How many consecutive failures that we should + # tolerate when sending federation requests to a specific server. The backoff + # is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds, etc. + # The default value is 16 if not specified, which is circa 18 hours. + send_max_retries: 16 + # -- Disable TLS validation. This should **NOT** be used in production. + disable_tls_validation: false + prefer_direct_fetch: false + # -- Prevents Dendrite from keeping HTTP connections + # open for reuse for future requests. Connections will be closed quicker + # but we may spend more time on TLS handshakes instead. + disable_http_keepalives: false + # -- Perspective keyservers, to use as a backup when direct key fetch + # requests don't succeed. + # @default -- See value.yaml + key_perspectives: + - server_name: matrix.org + keys: + - key_id: ed25519:auto + public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw + - key_id: ed25519:a_RXGa + public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ + + media_api: + # -- The path to store media files (e.g. avatars) in + base_path: "/data/media_store" + # -- The max file size for uploaded media files + max_file_size_bytes: 10485760 + # Whether to dynamically generate thumbnails if needed. + dynamic_thumbnails: false + # -- The maximum number of simultaneous thumbnail generators to run. + max_thumbnail_generators: 10 + # -- A list of thumbnail sizes to be generated for media content. + # @default -- See value.yaml + thumbnail_sizes: + - width: 32 + height: 32 + method: crop + - width: 96 + height: 96 + method: crop + - width: 640 + height: 480 + method: scale + + sync_api: + # -- This option controls which HTTP header to inspect to find the real remote IP + # address of the client. This is likely required if Dendrite is running behind + # a reverse proxy server. + real_ip_header: X-Real-IP + # -- Configuration for the full-text search engine. + search: + # -- Whether fulltext search is enabled. + enabled: true + # -- The path to store the search index in. + index_path: "/data/search" + # -- The language most likely to be used on the server - used when indexing, to + # ensure the returned results match expectations. A full list of possible languages + # can be found [here](https://github.com/matrix-org/dendrite/blob/76db8e90defdfb9e61f6caea8a312c5d60bcc005/internal/fulltext/bleve.go#L25-L46) + language: "en" + + user_api: + # -- bcrypt cost to use when hashing passwords. + # (ranges from 4-31; 4 being least secure, 31 being most secure; _NOTE: Using a too high value can cause clients to timeout and uses more CPU._) + bcrypt_cost: 10 + # -- OpenID Token lifetime in milliseconds. + openid_token_lifetime_ms: 3600000 + # - Disable TLS validation when hitting push gateways. This should **NOT** be used in production. + push_gateway_disable_tls_validation: false + # -- Rooms to join users to after registration + auto_join_rooms: [] + + # -- Default logging configuration + logging: + - type: std + level: info + +postgresql: + # -- Enable and configure postgres as the database for dendrite. + # @default -- See value.yaml + enabled: false + image: + repository: bitnami/postgresql + tag: "15.1.0" + auth: + username: dendrite + password: changeme + database: dendrite + + persistence: + enabled: false + +ingress: + # -- Create an ingress for a monolith deployment + enabled: false + hosts: [] + className: "" + hostName: "" + # -- Extra, custom annotations + annotations: {} + + tls: [] + +service: + type: ClusterIP + port: 80 diff --git a/setup/config/config.go b/setup/config/config.go index 6523a2452..41d2b6674 100644 --- a/setup/config/config.go +++ b/setup/config/config.go @@ -228,7 +228,7 @@ func loadConfig( privateKeyPath := absPath(basePath, c.Global.PrivateKeyPath) if c.Global.KeyID, c.Global.PrivateKey, err = LoadMatrixKey(privateKeyPath, readFile); err != nil { - return nil, err + return nil, fmt.Errorf("failed to load private_key: %w", err) } for _, v := range c.Global.VirtualHosts { @@ -242,7 +242,7 @@ func loadConfig( } privateKeyPath := absPath(basePath, v.PrivateKeyPath) if v.KeyID, v.PrivateKey, err = LoadMatrixKey(privateKeyPath, readFile); err != nil { - return nil, err + return nil, fmt.Errorf("failed to load private_key for virtualhost %s: %w", v.ServerName, err) } } From 002310390f874f69e52be1a8d20b17a3cb11d126 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Fri, 6 Jan 2023 15:51:07 +0100 Subject: [PATCH 02/13] Output to docs folder, hopefully --- helm/cr.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helm/cr.yaml b/helm/cr.yaml index f895ab8d6..014803cf1 100644 --- a/helm/cr.yaml +++ b/helm/cr.yaml @@ -1 +1,2 @@ -release-name-template: "helm-{{ .Name }}-{{ .Version }}" \ No newline at end of file +release-name-template: "helm-{{ .Name }}-{{ .Version }}" +package-path: docs/ \ No newline at end of file From 3fd95e60cc5fc3ae0610d0aca2177d8436a65ee1 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Fri, 6 Jan 2023 15:54:04 +0100 Subject: [PATCH 03/13] Try that again --- helm/cr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/cr.yaml b/helm/cr.yaml index 014803cf1..884c2b46b 100644 --- a/helm/cr.yaml +++ b/helm/cr.yaml @@ -1,2 +1,2 @@ release-name-template: "helm-{{ .Name }}-{{ .Version }}" -package-path: docs/ \ No newline at end of file +pages-index-path: docs/index.yaml \ No newline at end of file From 54b47a98e57cf210b568fa99ae159fd000012eb2 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 6 Jan 2023 11:49:59 -0700 Subject: [PATCH 04/13] Add curl to dendrite docker containers --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index ede33e635..6da555c04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,7 @@ RUN --mount=target=. \ # The dendrite base image # FROM alpine:latest AS dendrite-base +RUN apk --update --no-cache add curl LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" LABEL org.opencontainers.image.licenses="Apache-2.0" From 0995dc48224b90432e38fa92345cf5735bca6090 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 6 Jan 2023 12:02:43 -0700 Subject: [PATCH 05/13] Add curl to dendrite-demo-pinecone docker container --- build/docker/Dockerfile.demo-pinecone | 1 + 1 file changed, 1 insertion(+) diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone index facd1e3af..90f515167 100644 --- a/build/docker/Dockerfile.demo-pinecone +++ b/build/docker/Dockerfile.demo-pinecone @@ -17,6 +17,7 @@ RUN go build -trimpath -o bin/ ./cmd/create-account RUN go build -trimpath -o bin/ ./cmd/generate-keys FROM alpine:latest +RUN apk --update --no-cache add curl LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)" LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" From b0c5af6674465a3384a1b55c84325e7989ce1eb5 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 10 Jan 2023 17:02:38 +0100 Subject: [PATCH 06/13] Fix `/login` issue causing wrong device list updates (#2922) Fixes https://github.com/matrix-org/dendrite/issues/2914 and possibly https://github.com/matrix-org/dendrite/issues/2073? --- clientapi/auth/login_test.go | 5 +- clientapi/auth/password.go | 5 ++ clientapi/routing/login_test.go | 152 ++++++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 clientapi/routing/login_test.go diff --git a/clientapi/auth/login_test.go b/clientapi/auth/login_test.go index b79c573aa..044062c42 100644 --- a/clientapi/auth/login_test.go +++ b/clientapi/auth/login_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/setup/config" uapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -47,7 +48,7 @@ func TestLoginFromJSONReader(t *testing.T) { "password": "herpassword", "device_id": "adevice" }`, - WantUsername: "alice", + WantUsername: "@alice:example.com", WantDeviceID: "adevice", }, { @@ -174,7 +175,7 @@ func (ua *fakeUserInternalAPI) QueryAccountByPassword(ctx context.Context, req * return nil } res.Exists = true - res.Account = &uapi.Account{} + res.Account = &uapi.Account{UserID: userutil.MakeUserID(req.Localpart, req.ServerName)} return nil } diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index 4de2b443c..f2b0383ab 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -101,6 +101,8 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, } } + // If we couldn't find the user by the lower cased localpart, try the provided + // localpart as is. if !res.Exists { err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{ Localpart: localpart, @@ -122,5 +124,8 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, } } } + // Set the user, so login.Username() can do the right thing + r.Identifier.User = res.Account.UserID + r.User = res.Account.UserID return &r.Login, nil } diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go new file mode 100644 index 000000000..d429d7f8c --- /dev/null +++ b/clientapi/routing/login_test.go @@ -0,0 +1,152 @@ +package routing + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/keyserver" + "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/test/testrig" + "github.com/matrix-org/dendrite/userapi" + uapi "github.com/matrix-org/dendrite/userapi/api" +) + +func TestLogin(t *testing.T) { + aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin)) + bobUser := &test.User{ID: "@bob:test", AccountType: uapi.AccountTypeUser} + charlie := &test.User{ID: "@Charlie:test", AccountType: uapi.AccountTypeUser} + vhUser := &test.User{ID: "@vhuser:vh1"} + + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + base, baseClose := testrig.CreateBaseDendrite(t, dbType) + defer baseClose() + base.Cfg.ClientAPI.RateLimiting.Enabled = false + // add a vhost + base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{ + SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"}, + }) + + rsAPI := roomserver.NewInternalAPI(base) + // Needed for /login + keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI) + userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil) + keyAPI.SetUserAPI(userAPI) + + // We mostly need the userAPI for this test, so nil for other APIs/caches etc. + Setup(base, &base.Cfg.ClientAPI, nil, nil, userAPI, nil, nil, nil, nil, nil, keyAPI, nil, &base.Cfg.MSCs, nil) + + // Create password + password := util.RandomString(8) + + // create the users + for _, u := range []*test.User{aliceAdmin, bobUser, vhUser, charlie} { + localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID) + userRes := &uapi.PerformAccountCreationResponse{} + + if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{ + AccountType: u.AccountType, + Localpart: localpart, + ServerName: serverName, + Password: password, + }, userRes); err != nil { + t.Errorf("failed to create account: %s", err) + } + if !userRes.AccountCreated { + t.Fatalf("account not created") + } + } + + testCases := []struct { + name string + userID string + wantOK bool + }{ + { + name: "aliceAdmin can login", + userID: aliceAdmin.ID, + wantOK: true, + }, + { + name: "bobUser can login", + userID: bobUser.ID, + wantOK: true, + }, + { + name: "vhuser can login", + userID: vhUser.ID, + wantOK: true, + }, + { + name: "bob with uppercase can login", + userID: "@Bob:test", + wantOK: true, + }, + { + name: "Charlie can login (existing uppercase)", + userID: charlie.ID, + wantOK: true, + }, + { + name: "Charlie can not login with lowercase userID", + userID: strings.ToLower(charlie.ID), + wantOK: false, + }, + } + + ctx := context.Background() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{ + "type": authtypes.LoginTypePassword, + "identifier": map[string]interface{}{ + "type": "m.id.user", + "user": tc.userID, + }, + "password": password, + })) + rec := httptest.NewRecorder() + base.PublicClientAPIMux.ServeHTTP(rec, req) + if tc.wantOK && rec.Code != http.StatusOK { + t.Fatalf("failed to login: %s", rec.Body.String()) + } + + t.Logf("Response: %s", rec.Body.String()) + // get the response + resp := loginResponse{} + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + // everything OK + if !tc.wantOK && resp.AccessToken == "" { + return + } + if tc.wantOK && resp.AccessToken == "" { + t.Fatalf("expected accessToken after successful login but got none: %+v", resp) + } + + devicesResp := &uapi.QueryDevicesResponse{} + if err := userAPI.QueryDevices(ctx, &uapi.QueryDevicesRequest{UserID: resp.UserID}, devicesResp); err != nil { + t.Fatal(err) + } + for _, dev := range devicesResp.Devices { + // We expect the userID on the device to be the same as resp.UserID + if dev.UserID != resp.UserID { + t.Fatalf("unexpected userID on device: %s", dev.UserID) + } + } + }) + } + }) +} From 7482cd2b47cf19f4da9121461950409ea8f07a12 Mon Sep 17 00:00:00 2001 From: devonh Date: Tue, 10 Jan 2023 18:09:25 +0000 Subject: [PATCH 07/13] Handle DisplayName field in admin user registration endpoint (#2935) `/_synapse/admin/v1/register` has a `displayname` field that we were previously ignoring. This handles that field and adds the displayname to the new user if one was provided. --- clientapi/routing/register.go | 28 +++++-- clientapi/routing/register_secret.go | 13 ++-- clientapi/routing/register_secret_test.go | 2 +- clientapi/routing/register_test.go | 93 +++++++++++++++++++++++ 4 files changed, 123 insertions(+), 13 deletions(-) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 6087bda0c..ff6a0900e 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -780,7 +780,7 @@ func handleApplicationServiceRegistration( // Don't need to worry about appending to registration stages as // application service registration is entirely separate. return completeRegistration( - req.Context(), userAPI, r.Username, r.ServerName, "", appserviceID, req.RemoteAddr, + req.Context(), userAPI, r.Username, r.ServerName, "", "", appserviceID, req.RemoteAddr, req.UserAgent(), r.Auth.Session, r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeAppService, ) @@ -800,7 +800,7 @@ func checkAndCompleteFlow( if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) { // This flow was completed, registration can continue return completeRegistration( - req.Context(), userAPI, r.Username, r.ServerName, r.Password, "", req.RemoteAddr, + req.Context(), userAPI, r.Username, r.ServerName, "", r.Password, "", req.RemoteAddr, req.UserAgent(), sessionID, r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeUser, ) @@ -824,10 +824,10 @@ func checkAndCompleteFlow( func completeRegistration( ctx context.Context, userAPI userapi.ClientUserAPI, - username string, serverName gomatrixserverlib.ServerName, + username string, serverName gomatrixserverlib.ServerName, displayName string, password, appserviceID, ipAddr, userAgent, sessionID string, inhibitLogin eventutil.WeakBoolean, - displayName, deviceID *string, + deviceDisplayName, deviceID *string, accType userapi.AccountType, ) util.JSONResponse { if username == "" { @@ -887,12 +887,28 @@ func completeRegistration( } } + if displayName != "" { + nameReq := userapi.PerformUpdateDisplayNameRequest{ + Localpart: username, + ServerName: serverName, + DisplayName: displayName, + } + var nameRes userapi.PerformUpdateDisplayNameResponse + err = userAPI.SetDisplayName(ctx, &nameReq, &nameRes) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.Unknown("failed to set display name: " + err.Error()), + } + } + } + var devRes userapi.PerformDeviceCreationResponse err = userAPI.PerformDeviceCreation(ctx, &userapi.PerformDeviceCreationRequest{ Localpart: username, ServerName: serverName, AccessToken: token, - DeviceDisplayName: displayName, + DeviceDisplayName: deviceDisplayName, DeviceID: deviceID, IPAddr: ipAddr, UserAgent: userAgent, @@ -1077,5 +1093,5 @@ func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.Clien if ssrr.Admin { accType = userapi.AccountTypeAdmin } - return completeRegistration(req.Context(), userAPI, ssrr.User, cfg.Matrix.ServerName, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType) + return completeRegistration(req.Context(), userAPI, ssrr.User, cfg.Matrix.ServerName, ssrr.DisplayName, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType) } diff --git a/clientapi/routing/register_secret.go b/clientapi/routing/register_secret.go index 1a974b77a..f384b604a 100644 --- a/clientapi/routing/register_secret.go +++ b/clientapi/routing/register_secret.go @@ -18,12 +18,13 @@ import ( ) type SharedSecretRegistrationRequest struct { - User string `json:"username"` - Password string `json:"password"` - Nonce string `json:"nonce"` - MacBytes []byte - MacStr string `json:"mac"` - Admin bool `json:"admin"` + User string `json:"username"` + Password string `json:"password"` + Nonce string `json:"nonce"` + MacBytes []byte + MacStr string `json:"mac"` + Admin bool `json:"admin"` + DisplayName string `json:"displayname,omitempty"` } func NewSharedSecretRegistrationRequest(reader io.ReadCloser) (*SharedSecretRegistrationRequest, error) { diff --git a/clientapi/routing/register_secret_test.go b/clientapi/routing/register_secret_test.go index a2ed35853..ca265d237 100644 --- a/clientapi/routing/register_secret_test.go +++ b/clientapi/routing/register_secret_test.go @@ -10,7 +10,7 @@ import ( func TestSharedSecretRegister(t *testing.T) { // these values have come from a local synapse instance to ensure compatibility - jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice"}`) + jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`) sharedSecret := "dendritetest" req, err := NewSharedSecretRegistrationRequest(io.NopCloser(bytes.NewBuffer(jsonStr))) diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index b8fd19e90..bccc1b79b 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -18,6 +18,7 @@ import ( "bytes" "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" "reflect" @@ -35,7 +36,10 @@ import ( "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/userapi" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" + "github.com/patrickmn/go-cache" + "github.com/stretchr/testify/assert" ) var ( @@ -570,3 +574,92 @@ func Test_register(t *testing.T) { } }) } + +func TestRegisterUserWithDisplayName(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + base, baseClose := testrig.CreateBaseDendrite(t, dbType) + defer baseClose() + base.Cfg.Global.ServerName = "server" + + rsAPI := roomserver.NewInternalAPI(base) + keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI) + userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil) + keyAPI.SetUserAPI(userAPI) + deviceName, deviceID := "deviceName", "deviceID" + expectedDisplayName := "DisplayName" + response := completeRegistration( + base.Context(), + userAPI, + "user", + "server", + expectedDisplayName, + "password", + "", + "localhost", + "user agent", + "session", + false, + &deviceName, + &deviceID, + api.AccountTypeAdmin, + ) + + assert.Equal(t, http.StatusOK, response.Code) + + req := api.QueryProfileRequest{UserID: "@user:server"} + var res api.QueryProfileResponse + err := userAPI.QueryProfile(base.Context(), &req, &res) + assert.NoError(t, err) + assert.Equal(t, expectedDisplayName, res.DisplayName) + }) +} + +func TestRegisterAdminUsingSharedSecret(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + base, baseClose := testrig.CreateBaseDendrite(t, dbType) + defer baseClose() + base.Cfg.Global.ServerName = "server" + sharedSecret := "dendritetest" + base.Cfg.ClientAPI.RegistrationSharedSecret = sharedSecret + + rsAPI := roomserver.NewInternalAPI(base) + keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI) + userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil) + keyAPI.SetUserAPI(userAPI) + + expectedDisplayName := "rabbit" + jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`) + req, err := NewSharedSecretRegistrationRequest(io.NopCloser(bytes.NewBuffer(jsonStr))) + assert.NoError(t, err) + if err != nil { + t.Fatalf("failed to read request: %s", err) + } + + r := NewSharedSecretRegistration(sharedSecret) + + // force the nonce to be known + r.nonces.Set(req.Nonce, true, cache.DefaultExpiration) + + _, err = r.IsValidMacLogin(req.Nonce, req.User, req.Password, req.Admin, req.MacBytes) + assert.NoError(t, err) + + body := &bytes.Buffer{} + err = json.NewEncoder(body).Encode(req) + assert.NoError(t, err) + ssrr := httptest.NewRequest(http.MethodPost, "/", body) + + response := handleSharedSecretRegistration( + &base.Cfg.ClientAPI, + userAPI, + r, + ssrr, + ) + assert.Equal(t, http.StatusOK, response.Code) + + profilReq := api.QueryProfileRequest{UserID: "@alice:server"} + var profileRes api.QueryProfileResponse + err = userAPI.QueryProfile(base.Context(), &profilReq, &profileRes) + assert.NoError(t, err) + assert.Equal(t, expectedDisplayName, profileRes.DisplayName) + }) +} From 97ebd72b5a731decdf8f67742179e1adc0f9f30d Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 10 Jan 2023 16:26:41 -0700 Subject: [PATCH 08/13] Add FAQs based on commonly asked questions from the community --- docs/FAQ.md | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/docs/FAQ.md b/docs/FAQ.md index 816130515..4047bfffc 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -6,6 +6,12 @@ permalink: /faq # FAQ +## Why does Dendrite exist? + +Dendrite aims to provide a matrix compatible server that has low resource usage compared to [Synapse](https://github.com/matrix-org/synapse). +It also aims to provide more flexibility when scaling either up or down. +Dendrite's code is also very easy to hack on which makes it suitable for experimenting with new matrix features such as peer-to-peer. + ## Is Dendrite stable? Mostly, although there are still bugs and missing features. If you are a confident power user and you are happy to spend some time debugging things when they go wrong, then please try out Dendrite. If you are a community, organisation or business that demands stability and uptime, then Dendrite is not for you yet - please install Synapse instead. @@ -34,6 +40,10 @@ No, Dendrite has a very different database schema to Synapse and the two are not Monolith deployments are always preferred where possible, and at this time, are far better tested than polylith deployments are. The only reason to consider a polylith deployment is if you wish to run different Dendrite components on separate physical machines, but this is an advanced configuration which we don't recommend. +## Can I configure which port Dendrite listens on? + +Yes, use the cli flag `-http-bind-address`. + ## I've installed Dendrite but federation isn't working Check the [Federation Tester](https://federationtester.matrix.org). You need at least: @@ -42,6 +52,10 @@ Check the [Federation Tester](https://federationtester.matrix.org). You need at * A valid TLS certificate for that DNS name * Either DNS SRV records or well-known files +## Whenever I try to connect from Element it says unable to connect to homeserver + +Check that your dendrite instance is running. Otherwise this is most likely due to a reverse proxy misconfiguration. + ## Does Dendrite work with my favourite client? It should do, although we are aware of some minor issues: @@ -49,6 +63,10 @@ It should do, although we are aware of some minor issues: * **Element Android**: registration does not work, but logging in with an existing account does * **Hydrogen**: occasionally sync can fail due to gaps in the `since` parameter, but clearing the cache fixes this +## Is there a public instance of Dendrite I can try out? + +Use [dendrite.matrix.org](https://dendrite.matrix.org) which we officially support. + ## Does Dendrite support Space Summaries? Yes, [Space Summaries](https://github.com/matrix-org/matrix-spec-proposals/pull/2946) were merged into the Matrix Spec as of 2022-01-17 however, they are still treated as an MSC (Matrix Specification Change) in Dendrite. In order to enable Space Summaries in Dendrite, you must add the MSC to the MSC configuration section in the configuration YAML. If the MSC is not enabled, a user will typically see a perpetual loading icon on the summary page. See below for a demonstration of how to add to the Dendrite configuration: @@ -84,10 +102,42 @@ Remember to add the config file(s) to the `app_service_api` section of the confi Yes, you can do this by disabling federation - set `disable_federation` to `true` in the `global` section of the Dendrite configuration file. +## How can I migrate a room in order to change the internal ID? + +This can be done by performing a room upgrade. Use the command `/upgraderoom ` in Element to do this. + +## How do I reset somebody's password on my server? + +Use the admin endpoint [resetpassword](https://matrix-org.github.io/dendrite/administration/adminapi#post-_dendriteadminresetpassworduserid) + ## Should I use PostgreSQL or SQLite for my databases? Please use PostgreSQL wherever possible, especially if you are planning to run a homeserver that caters to more than a couple of users. +## What data needs to be kept if transferring/backing up Dendrite? + +The list of files that need to be stored is: +- matrix-key.pem +- dendrite.yaml +- the postgres or sqlite DB +- the media store +- the search index (although this can be regenerated) + +Note that this list may change / be out of date. We don't officially maintain instructions for migrations like this. + +## How can I prepare enough storage for media caches? + +This might be what you want: [matrix-media-repo](https://github.com/turt2live/matrix-media-repo) +We don't officially support this or any other dedicated media storage solutions. + +## Is there an upgrade guide for Dendrite? + +Run a newer docker image. We don't officially support deployments other than Docker. +Most of the time you should be able to just +- stop +- replace binary +- start + ## Dendrite is using a lot of CPU Generally speaking, you should expect to see some CPU spikes, particularly if you are joining or participating in large rooms. However, constant/sustained high CPU usage is not expected - if you are experiencing that, please join `#dendrite-dev:matrix.org` and let us know what you were doing when the @@ -102,6 +152,10 @@ not expected. Join `#dendrite-dev:matrix.org` and let us know what you were doin ballooned, or file a GitHub issue if you can. If you can take a [memory profile](development/PROFILING.md) then that would be a huge help too, as that will help us to understand where the memory usage is happening. +## Do I need to generate the self-signed certificate if I'm going to use a reverse proxy? + +No, if you already have a proper certificate from some provider, like Let's Encrypt, and use that on your reverse proxy, and the reverse proxy does TLS termination, then you’re good and can use HTTP to the dendrite process. + ## Dendrite is running out of PostgreSQL database connections You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode! From 11a07d855dd7f08fcd386cb778cbdd353ddd5aa4 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 11 Jan 2023 09:52:58 -0700 Subject: [PATCH 09/13] Initial attempt at adding cypress tests to ci --- .github/workflows/schedules.yaml | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index d2a1f6e1f..63a60a241 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -73,3 +73,38 @@ jobs: path: | /logs/results.tap /logs/**/*.log* + + element_web: + runs-on: ubuntu-latest + steps: + - uses: tecolicom/actions-use-apt-tools@v1 + with: + # Our test suite includes some screenshot tests with unusual diacritics, which are + # supposed to be covered by STIXGeneral. + tools: fonts-stix + - uses: actions/checkout@v2 + with: + repository: matrix-org/matrix-react-sdk + - uses: actions/setup-node@v3 + with: + cache: 'yarn' + - name: Fetch layered build + run: scripts/ci/layered.sh + - name: Copy config + run: cp element.io/develop/config.json config.json + working-directory: ./element-web + - name: Build + env: + CI_PACKAGE: true + run: yarn build + working-directory: ./element-web + - name: "Run cypress tests" + uses: cypress-io/github-action@v4.1.1 + with: + browser: chrome + start: npx serve -p 8080 ./element-web/webapp + wait-on: 'http://localhost:8080' + env: + PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true + TMPDIR: ${{ runner.temp }} + HOMESERVER: 'dendrite' From 8fef692741f2e228c04b5f986ab65036e89947d2 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 11 Jan 2023 10:10:24 -0700 Subject: [PATCH 10/13] Edit cypress config before running tests --- .github/workflows/schedules.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index 63a60a241..ba05f2083 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -98,6 +98,9 @@ jobs: CI_PACKAGE: true run: yarn build working-directory: ./element-web + - name: Edit Test Config + run: | + sed -i '/HOMESERVER/c\ HOMESERVER: "dendrite",' cypress.config.ts - name: "Run cypress tests" uses: cypress-io/github-action@v4.1.1 with: @@ -107,4 +110,3 @@ jobs: env: PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true TMPDIR: ${{ runner.temp }} - HOMESERVER: 'dendrite' From b297ea7379d6d5b953a810fe2475b549a917cc9a Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 11 Jan 2023 10:40:38 -0700 Subject: [PATCH 11/13] Add cypress cloud recording --- .github/workflows/schedules.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index ba05f2083..45098925f 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -75,6 +75,7 @@ jobs: /logs/**/*.log* element_web: + timeout-minutes: 120 runs-on: ubuntu-latest steps: - uses: tecolicom/actions-use-apt-tools@v1 @@ -107,6 +108,13 @@ jobs: browser: chrome start: npx serve -p 8080 ./element-web/webapp wait-on: 'http://localhost:8080' + record: + true env: + # pass the Dashboard record key as an environment variable + CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} + # pass GitHub token to allow accurately detecting a build vs a re-run build + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true TMPDIR: ${{ runner.temp }} From 6ae1dd565c739efb1f847558e4170cdc0cb4085a Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 11 Jan 2023 10:46:52 -0700 Subject: [PATCH 12/13] Revert "Add cypress cloud recording" This reverts commit b297ea7379d6d5b953a810fe2475b549a917cc9a. --- .github/workflows/schedules.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index 45098925f..ba05f2083 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -75,7 +75,6 @@ jobs: /logs/**/*.log* element_web: - timeout-minutes: 120 runs-on: ubuntu-latest steps: - uses: tecolicom/actions-use-apt-tools@v1 @@ -108,13 +107,6 @@ jobs: browser: chrome start: npx serve -p 8080 ./element-web/webapp wait-on: 'http://localhost:8080' - record: - true env: - # pass the Dashboard record key as an environment variable - CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} - # pass GitHub token to allow accurately detecting a build vs a re-run build - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true TMPDIR: ${{ runner.temp }} From 25dfbc6ec3991ba04f317cbae4a4dd51bab6013e Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 11 Jan 2023 10:47:37 -0700 Subject: [PATCH 13/13] Extend cypress test timeout in ci --- .github/workflows/schedules.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index ba05f2083..5636c4cf9 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -75,6 +75,7 @@ jobs: /logs/**/*.log* element_web: + timeout-minutes: 120 runs-on: ubuntu-latest steps: - uses: tecolicom/actions-use-apt-tools@v1