From f956a8c1d9172f6bbfb9f7515feacd477a0e35f5 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 30 May 2023 10:02:53 +0200 Subject: [PATCH 1/5] Docs restructure (#2953) Needs to be merged into `gh-pages` later on. --- .github/workflows/helm.yml | 1 + README.md | 8 +- build.cmd | 51 ------ build.sh | 24 --- build/docker/README.md | 17 +- build/docker/docker-compose.monolith.yml | 44 ----- build/docker/docker-compose.yml | 52 ++++++ build/docker/postgres/create_db.sh | 5 - clientapi/admin_test.go | 6 +- clientapi/routing/admin.go | 5 +- dendrite-sample.yaml | 3 +- docs/FAQ.md | 6 +- docs/INSTALL.md | 10 +- docs/administration/1_createusers.md | 5 +- docs/administration/4_adminapi.md | 18 +- .../5_optimisation.md} | 11 +- ...roubleshooting.md => 6_troubleshooting.md} | 12 +- docs/caddy/{monolith => }/Caddyfile | 0 docs/caddy/polylith/Caddyfile | 85 --------- docs/development/CONTRIBUTING.md | 1 + docs/development/PROFILING.md | 1 + docs/development/coverage.md | 164 ++++++++++++------ docs/development/sytest.md | 70 +------- docs/development/tracing/opentracing.md | 114 ------------ docs/development/tracing/setup.md | 57 ------ ...olith-sample.conf => dendrite-sample.conf} | 0 docs/hiawatha/polylith-sample.conf | 35 ---- docs/installation/1_planning.md | 21 +-- docs/installation/2_domainname.md | 2 +- docs/installation/5_install_monolith.md | 21 --- docs/installation/9_starting_monolith.md | 42 ----- docs/installation/docker.md | 11 ++ docs/installation/docker/1_docker.md | 57 ++++++ docs/installation/helm.md | 11 ++ docs/installation/helm/1_helm.md | 58 +++++++ docs/installation/manual.md | 11 ++ .../{3_build.md => manual/1_build.md} | 23 +-- .../{4_database.md => manual/2_database.md} | 52 ++---- .../3_configuration.md} | 62 ++----- .../4_signingkey.md} | 9 +- .../manual/5_starting_dendrite.md | 26 +++ ...olith-sample.conf => dendrite-sample.conf} | 0 docs/nginx/polylith-sample.conf | 58 ------- docs/systemd/monolith-example.service | 19 -- roomserver/internal/perform/perform_admin.go | 12 +- syncapi/storage/sqlite3/account_data_table.go | 2 +- 46 files changed, 447 insertions(+), 855 deletions(-) delete mode 100644 build.cmd delete mode 100755 build.sh delete mode 100644 build/docker/docker-compose.monolith.yml create mode 100644 build/docker/docker-compose.yml delete mode 100755 build/docker/postgres/create_db.sh rename docs/{installation/11_optimisation.md => administration/5_optimisation.md} (90%) rename docs/administration/{5_troubleshooting.md => 6_troubleshooting.md} (88%) rename docs/caddy/{monolith => }/Caddyfile (100%) delete mode 100644 docs/caddy/polylith/Caddyfile delete mode 100644 docs/development/tracing/opentracing.md delete mode 100644 docs/development/tracing/setup.md rename docs/hiawatha/{monolith-sample.conf => dendrite-sample.conf} (100%) delete mode 100644 docs/hiawatha/polylith-sample.conf delete mode 100644 docs/installation/5_install_monolith.md delete mode 100644 docs/installation/9_starting_monolith.md create mode 100644 docs/installation/docker.md create mode 100644 docs/installation/docker/1_docker.md create mode 100644 docs/installation/helm.md create mode 100644 docs/installation/helm/1_helm.md create mode 100644 docs/installation/manual.md rename docs/installation/{3_build.md => manual/1_build.md} (53%) rename docs/installation/{4_database.md => manual/2_database.md} (57%) rename docs/installation/{7_configuration.md => manual/3_configuration.md} (67%) rename docs/installation/{8_signingkey.md => manual/4_signingkey.md} (92%) create mode 100644 docs/installation/manual/5_starting_dendrite.md rename docs/nginx/{monolith-sample.conf => dendrite-sample.conf} (100%) delete mode 100644 docs/nginx/polylith-sample.conf delete mode 100644 docs/systemd/monolith-example.service diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index a9c1718a0..bf62a1c19 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -38,3 +38,4 @@ jobs: with: config: helm/cr.yaml charts_dir: helm/ + mark_as_latest: false diff --git a/README.md b/README.md index 295203eb4..0b9788768 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ It intends to provide an **efficient**, **reliable** and **scalable** alternativ Dendrite is **beta** software, which means: -- Dendrite is ready for early adopters. We recommend running in Monolith mode with a PostgreSQL database. +- Dendrite is ready for early adopters. We recommend running Dendrite with a PostgreSQL database. - Dendrite has periodic releases. We intend to release new versions as we fix bugs and land significant features. - Dendrite supports database schema upgrades between releases. This means you should never lose your messages when upgrading Dendrite. @@ -21,7 +21,7 @@ This does not mean: - Dendrite is bug-free. It has not yet been battle-tested in the real world and so will be error prone initially. - Dendrite is feature-complete. There may be client or federation APIs that are not implemented. -- Dendrite is ready for massive homeserver deployments. There is no sharding of microservices (although it is possible to run them on separate machines) and there is no high-availability/clustering support. +- Dendrite is ready for massive homeserver deployments. There is no high-availability/clustering support. Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices. @@ -60,7 +60,7 @@ The following instructions are enough to get Dendrite started as a non-federatin ```bash $ git clone https://github.com/matrix-org/dendrite $ cd dendrite -$ ./build.sh +$ go build -o bin/ ./cmd/... # Generate a Matrix signing key for federation (required) $ ./bin/generate-keys --private-key matrix_key.pem @@ -85,7 +85,7 @@ Then point your favourite Matrix client at `http://localhost:8008` or `https://l ## Progress -We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver +We use a script called "Are We Synapse Yet" which checks Sytest compliance rates. Sytest is a black-box homeserver test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it updates with CI. As of January 2023, we have 100% server-server parity with Synapse, and the client-server parity is at 93% , though check CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse diff --git a/build.cmd b/build.cmd deleted file mode 100644 index 9e90622c8..000000000 --- a/build.cmd +++ /dev/null @@ -1,51 +0,0 @@ -@echo off - -:ENTRY_POINT - setlocal EnableDelayedExpansion - - REM script base dir - set SCRIPTDIR=%~dp0 - set PROJDIR=%SCRIPTDIR:~0,-1% - - REM Put installed packages into ./bin - set GOBIN=%PROJDIR%\bin - - set FLAGS= - - REM Check if sources are under Git control - if not exist ".git" goto :CHECK_BIN - - REM set BUILD=`git rev-parse --short HEAD \\ ""` - FOR /F "tokens=*" %%X IN ('git rev-parse --short HEAD') DO ( - set BUILD=%%X - ) - - REM set BRANCH=`(git symbolic-ref --short HEAD \ tr -d \/ ) \\ ""` - FOR /F "tokens=*" %%X IN ('git symbolic-ref --short HEAD') DO ( - set BRANCHRAW=%%X - set BRANCH=!BRANCHRAW:/=! - ) - if "%BRANCH%" == "main" set BRANCH= - - set FLAGS=-X github.com/matrix-org/dendrite/internal.branch=%BRANCH% -X github.com/matrix-org/dendrite/internal.build=%BUILD% - -:CHECK_BIN - if exist "bin" goto :ALL_SET - mkdir "bin" - -:ALL_SET - set CGO_ENABLED=1 - for /D %%P in (cmd\*) do ( - go build -trimpath -ldflags "%FLAGS%" -v -o ".\bin" ".\%%P" - ) - - set CGO_ENABLED=0 - set GOOS=js - set GOARCH=wasm - go build -trimpath -ldflags "%FLAGS%" -o bin\main.wasm .\cmd\dendritejs-pinecone - - goto :DONE - -:DONE - echo Done - endlocal \ No newline at end of file diff --git a/build.sh b/build.sh deleted file mode 100755 index f8b5001bf..000000000 --- a/build.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -eu - -# Put installed packages into ./bin -export GOBIN=$PWD/`dirname $0`/bin - -if [ -d ".git" ] -then - export BUILD=`git rev-parse --short HEAD || ""` - export BRANCH=`(git symbolic-ref --short HEAD | tr -d \/ ) || ""` - if [ "$BRANCH" = main ] - then - export BRANCH="" - fi - - export FLAGS="-X github.com/matrix-org/dendrite/internal.branch=$BRANCH -X github.com/matrix-org/dendrite/internal.build=$BUILD" -else - export FLAGS="" -fi - -mkdir -p bin - -CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/... - -# CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs-pinecone diff --git a/build/docker/README.md b/build/docker/README.md index b66cb864b..8d69b9af1 100644 --- a/build/docker/README.md +++ b/build/docker/README.md @@ -6,23 +6,20 @@ They can be found on Docker Hub: - [matrixdotorg/dendrite-monolith](https://hub.docker.com/r/matrixdotorg/dendrite-monolith) for monolith deployments -## Dockerfiles +## Dockerfile -The `Dockerfile` is a multistage file which can build all four Dendrite -images depending on the supplied `--target`. From the root of the Dendrite +The `Dockerfile` is a multistage file which can build Dendrite. From the root of the Dendrite repository, run: ``` -docker build . --target monolith -t matrixdotorg/dendrite-monolith -docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone -docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil +docker build . -t matrixdotorg/dendrite-monolith ``` -## Compose files +## Compose file -There are two sample `docker-compose` files: +There is one sample `docker-compose` files: -- `docker-compose.monolith.yml` which runs a monolith Dendrite deployment +- `docker-compose.yml` which runs a Dendrite deployment with Postgres ## Configuration @@ -55,7 +52,7 @@ Create your config based on the [`dendrite-sample.yaml`](https://github.com/matr Then start the deployment: ``` -docker-compose -f docker-compose.monolith.yml up +docker-compose -f docker-compose.yml up ``` ## Building the images diff --git a/build/docker/docker-compose.monolith.yml b/build/docker/docker-compose.monolith.yml deleted file mode 100644 index 1a8fe4eee..000000000 --- a/build/docker/docker-compose.monolith.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: "3.4" -services: - postgres: - hostname: postgres - image: postgres:14 - restart: always - volumes: - - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh - # To persist your PostgreSQL databases outside of the Docker image, - # to prevent data loss, modify the following ./path_to path: - - ./path_to/postgresql:/var/lib/postgresql/data - environment: - POSTGRES_PASSWORD: itsasecret - POSTGRES_USER: dendrite - healthcheck: - test: ["CMD-SHELL", "pg_isready -U dendrite"] - interval: 5s - timeout: 5s - retries: 5 - networks: - - internal - - monolith: - hostname: monolith - image: matrixdotorg/dendrite-monolith:latest - command: [ - "--tls-cert=server.crt", - "--tls-key=server.key" - ] - ports: - - 8008:8008 - - 8448:8448 - volumes: - - ./config:/etc/dendrite - - ./media:/var/dendrite/media - depends_on: - - postgres - networks: - - internal - restart: unless-stopped - -networks: - internal: - attachable: true diff --git a/build/docker/docker-compose.yml b/build/docker/docker-compose.yml new file mode 100644 index 000000000..9397673f8 --- /dev/null +++ b/build/docker/docker-compose.yml @@ -0,0 +1,52 @@ +version: "3.4" + +services: + postgres: + hostname: postgres + image: postgres:15-alpine + restart: always + volumes: + # This will create a docker volume to persist the database files in. + # If you prefer those files to be outside of docker, you'll need to change this. + - dendrite_postgres_data:/var/lib/postgresql/data + environment: + POSTGRES_PASSWORD: itsasecret + POSTGRES_USER: dendrite + POSTGRES_DATABASE: dendrite + healthcheck: + test: ["CMD-SHELL", "pg_isready -U dendrite"] + interval: 5s + timeout: 5s + retries: 5 + networks: + - internal + + monolith: + hostname: monolith + image: matrixdotorg/dendrite-monolith:latest + ports: + - 8008:8008 + - 8448:8448 + volumes: + - ./config:/etc/dendrite + # The following volumes use docker volumes, change this + # if you prefer to have those files outside of docker. + - dendrite_media:/var/dendrite/media + - dendrite_jetstream:/var/dendrite/jetstream + - dendrite_search_index:/var/dendrite/searchindex + depends_on: + postgres: + condition: service_healthy + networks: + - internal + restart: unless-stopped + +networks: + internal: + attachable: true + +volumes: + dendrite_postgres_data: + dendrite_media: + dendrite_jetstream: + dendrite_search_index: \ No newline at end of file diff --git a/build/docker/postgres/create_db.sh b/build/docker/postgres/create_db.sh deleted file mode 100755 index 27d2a4df4..000000000 --- a/build/docker/postgres/create_db.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -for db in userapi_accounts mediaapi syncapi roomserver keyserver federationapi appservice mscs; do - createdb -U dendrite -O dendrite dendrite_$db -done diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index 69a815321..1145cb12d 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -133,7 +133,11 @@ func TestPurgeRoom(t *testing.T) { cfg, processCtx, close := testrig.CreateConfig(t, dbType) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) natsInstance := jetstream.NATSInstance{} - defer close() + defer func() { + // give components the time to process purge requests + time.Sleep(time.Millisecond * 50) + close() + }() routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go index 8dd662a1b..3d64454c4 100644 --- a/clientapi/routing/admin.go +++ b/clientapi/routing/admin.go @@ -123,7 +123,8 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.De } } request := struct { - Password string `json:"password"` + Password string `json:"password"` + LogoutDevices bool `json:"logout_devices"` }{} if err = json.NewDecoder(req.Body).Decode(&request); err != nil { return util.JSONResponse{ @@ -146,7 +147,7 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.De Localpart: localpart, ServerName: serverName, Password: request.Password, - LogoutDevices: true, + LogoutDevices: request.LogoutDevices, } updateRes := &api.PerformPasswordUpdateResponse{} if err := userAPI.PerformPasswordUpdate(req.Context(), updateReq, updateRes); err != nil { diff --git a/dendrite-sample.yaml b/dendrite-sample.yaml index 6b3ea74f2..96143d85f 100644 --- a/dendrite-sample.yaml +++ b/dendrite-sample.yaml @@ -69,8 +69,7 @@ global: # e.g. localhost:443 well_known_server_name: "" - # The server name to delegate client-server communications to, with optional port - # e.g. localhost:443 + # The base URL to delegate client-server communications to e.g. https://localhost well_known_client_name: "" # Lists of domains that the server will trust as identity servers to verify third diff --git a/docs/FAQ.md b/docs/FAQ.md index 200020726..757bf9625 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -24,7 +24,7 @@ No, although a good portion of the Matrix specification has been implemented. Mo Dendrite development is currently supported by a small team of developers and due to those limited resources, the majority of the effort is focused on getting Dendrite to be specification complete. If there are major features you're requesting (e.g. new administration endpoints), we'd like to strongly encourage you to join the community in supporting -the development efforts through [contributing](https://matrix-org.github.io/dendrite/development/contributing). +the development efforts through [contributing](../development/contributing). ## Is there a migration path from Synapse to Dendrite? @@ -103,7 +103,7 @@ This can be done by performing a room upgrade. Use the command `/upgraderoom " } diff --git a/docs/installation/11_optimisation.md b/docs/administration/5_optimisation.md similarity index 90% rename from docs/installation/11_optimisation.md rename to docs/administration/5_optimisation.md index 686ec2eb9..b327171eb 100644 --- a/docs/installation/11_optimisation.md +++ b/docs/administration/5_optimisation.md @@ -1,9 +1,9 @@ --- title: Optimise your installation -parent: Installation +parent: Administration has_toc: true -nav_order: 11 -permalink: /installation/start/optimisation +nav_order: 5 +permalink: /administration/optimisation --- # Optimise your installation @@ -36,11 +36,6 @@ connections it will open to the database. **If you are using the `global` database pool** then you only need to configure the `max_open_conns` setting once in the `global` section. -**If you are defining a `database` config per component** then you will need to ensure that -the **sum total** of all configured `max_open_conns` to a given database server do not exceed -the connection limit. If you configure a total that adds up to more connections than are available -then this will cause database queries to fail. - You may wish to raise the `max_connections` limit on your PostgreSQL server to accommodate additional connections, in which case you should also update the `max_open_conns` in your Dendrite configuration accordingly. However be aware that this is only advisable on particularly diff --git a/docs/administration/5_troubleshooting.md b/docs/administration/6_troubleshooting.md similarity index 88% rename from docs/administration/5_troubleshooting.md rename to docs/administration/6_troubleshooting.md index 8ba510ef6..5f11f9931 100644 --- a/docs/administration/5_troubleshooting.md +++ b/docs/administration/6_troubleshooting.md @@ -1,6 +1,7 @@ --- title: Troubleshooting parent: Administration +nav_order: 6 permalink: /administration/troubleshooting --- @@ -18,7 +19,7 @@ be clues in the logs. You can increase this log level to the more verbose `debug` level if necessary by adding this to the config and restarting Dendrite: -``` +```yaml logging: - type: std level: debug @@ -56,12 +57,7 @@ number of database connections does not exceed the maximum allowed by PostgreSQL Open your `postgresql.conf` configuration file and check the value of `max_connections` (which is typically `100` by default). Then open your `dendrite.yaml` configuration file -and ensure that: - -1. If you are using the `global.database` section, that `max_open_conns` does not exceed - that number; -2. If you are **not** using the `global.database` section, that the sum total of all - `max_open_conns` across all `database` blocks does not exceed that number. +and ensure that in the `global.database` section, `max_open_conns` does not exceed that number. ## 5. File descriptors @@ -77,7 +73,7 @@ If there aren't, you will see a log lines like this: level=warning msg="IMPORTANT: Process file descriptor limit is currently 65535, it is recommended to raise the limit for Dendrite to at least 65535 to avoid issues" ``` -Follow the [Optimisation](../installation/11_optimisation.md) instructions to correct the +Follow the [Optimisation](5_optimisation.md) instructions to correct the available number of file descriptors. ## 6. STUN/TURN Server tester diff --git a/docs/caddy/monolith/Caddyfile b/docs/caddy/Caddyfile similarity index 100% rename from docs/caddy/monolith/Caddyfile rename to docs/caddy/Caddyfile diff --git a/docs/caddy/polylith/Caddyfile b/docs/caddy/polylith/Caddyfile deleted file mode 100644 index c2d81b49b..000000000 --- a/docs/caddy/polylith/Caddyfile +++ /dev/null @@ -1,85 +0,0 @@ -# Sample Caddyfile for using Caddy in front of Dendrite - -# - -# Customize email address and domain names - -# Optional settings commented out - -# - -# BE SURE YOUR DOMAINS ARE POINTED AT YOUR SERVER FIRST - -# Documentation: - -# - -# Bonus tip: If your IP address changes, use Caddy's - -# dynamic DNS plugin to update your DNS records to - -# point to your new IP automatically - -# - -# - -# Global options block - -{ - # In case there is a problem with your certificates. - # email example@example.com - - # Turn off the admin endpoint if you don't need graceful config - # changes and/or are running untrusted code on your machine. - # admin off - - # Enable this if your clients don't send ServerName in TLS handshakes. - # default_sni example.com - - # Enable debug mode for verbose logging. - # debug - - # Use Let's Encrypt's staging endpoint for testing. - # acme_ca https://acme-staging-v02.api.letsencrypt.org/directory - - # If you're port-forwarding HTTP/HTTPS ports from 80/443 to something - # else, enable these and put the alternate port numbers here. - # http_port 8080 - # https_port 8443 -} - -# The server name of your matrix homeserver. This example shows - -# "well-known delegation" from the registered domain to a subdomain - -# which is only needed if your server_name doesn't match your Matrix - -# homeserver URL (i.e. you can show users a vanity domain that looks - -# nice and is easy to remember but still have your Matrix server on - -# its own subdomain or hosted service) - -example.com { - header /.well-known/matrix/*Content-Type application/json - header /.well-known/matrix/* Access-Control-Allow-Origin * - respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}` - respond /.well-known/matrix/client `{"m.homeserver": {"base_url": "https://matrix.example.com"}}` -} - -# The actual domain name whereby your Matrix server is accessed - -matrix.example.com { - # Change the end of each reverse_proxy line to the correct - # address for your various services. - @sync_api { - path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|.*?_?members|context/.*?|relations/.*?|event/.*?))$ - } - reverse_proxy @sync_api sync_api:8073 - - reverse_proxy /_matrix/client* client_api:8071 - reverse_proxy /_matrix/federation* federation_api:8071 - reverse_proxy /_matrix/key* federation_api:8071 - reverse_proxy /_matrix/media* media_api:8071 -} diff --git a/docs/development/CONTRIBUTING.md b/docs/development/CONTRIBUTING.md index 2aec4c363..71e7516a2 100644 --- a/docs/development/CONTRIBUTING.md +++ b/docs/development/CONTRIBUTING.md @@ -1,6 +1,7 @@ --- title: Contributing parent: Development +nav_order: 1 permalink: /development/contributing --- diff --git a/docs/development/PROFILING.md b/docs/development/PROFILING.md index 57c37a900..dc4eca7b7 100644 --- a/docs/development/PROFILING.md +++ b/docs/development/PROFILING.md @@ -1,6 +1,7 @@ --- title: Profiling parent: Development +nav_order: 4 permalink: /development/profiling --- diff --git a/docs/development/coverage.md b/docs/development/coverage.md index c4a8a1174..1b15f71a2 100644 --- a/docs/development/coverage.md +++ b/docs/development/coverage.md @@ -1,78 +1,130 @@ --- title: Coverage parent: Development +nav_order: 3 permalink: /development/coverage --- -To generate a test coverage report for Sytest, a small patch needs to be applied to the Sytest repository to compile and use the instrumented binary: -```patch -diff --git a/lib/SyTest/Homeserver/Dendrite.pm b/lib/SyTest/Homeserver/Dendrite.pm -index 8f0e209c..ad057e52 100644 ---- a/lib/SyTest/Homeserver/Dendrite.pm -+++ b/lib/SyTest/Homeserver/Dendrite.pm -@@ -337,7 +337,7 @@ sub _start_monolith - - $output->diag( "Starting monolith server" ); - my @command = ( -- $self->{bindir} . '/dendrite', -+ $self->{bindir} . '/dendrite', '--test.coverprofile=' . $self->{hs_dir} . '/integrationcover.log', "DEVEL", - '--config', $self->{paths}{config}, - '--http-bind-address', $self->{bind_host} . ':' . $self->unsecure_port, - '--https-bind-address', $self->{bind_host} . ':' . $self->secure_port, -diff --git a/scripts/dendrite_sytest.sh b/scripts/dendrite_sytest.sh -index f009332b..7ea79869 100755 ---- a/scripts/dendrite_sytest.sh -+++ b/scripts/dendrite_sytest.sh -@@ -34,7 +34,8 @@ export GOBIN=/tmp/bin - echo >&2 "--- Building dendrite from source" - cd /src - mkdir -p $GOBIN --go install -v ./cmd/dendrite -+# go install -v ./cmd/dendrite -+go test -c -cover -covermode=atomic -o $GOBIN/dendrite -coverpkg "github.com/matrix-org/..." ./cmd/dendrite - go install -v ./cmd/generate-keys - cd - - ``` +## Running unit tests with coverage enabled + +Running unit tests with coverage enabled can be done with the following commands, this will generate a `integrationcover.log` +```bash +go test -covermode=atomic -coverpkg=./... -coverprofile=integrationcover.log $(go list ./... | grep -v '/cmd/') +go tool cover -func=integrationcover.log +``` + +## Running Sytest with coverage enabled + +To run Sytest with coverage enabled: + +```bash +docker run --rm --name sytest -v "/Users/kegan/github/sytest:/sytest" \ + -v "/Users/kegan/github/dendrite:/src" -v "$(pwd)/sytest_logs:/logs" \ + -v "/Users/kegan/go/:/gopath" -e "POSTGRES=1" \ + -e "COVER=1" \ + matrixdotorg/sytest-dendrite:latest + +# to get a more accurate coverage you may also need to run Sytest using SQLite as the database: +docker run --rm --name sytest -v "/Users/kegan/github/sytest:/sytest" \ + -v "/Users/kegan/github/dendrite:/src" -v "$(pwd)/sytest_logs:/logs" \ + -v "/Users/kegan/go/:/gopath" \ + -e "COVER=1" \ + matrixdotorg/sytest-dendrite:latest +``` + +This will generate a folder `covdatafiles` in each server's directory, e.g `server-0/covdatafiles`. To parse them, +ensure your working directory is under the Dendrite repository then run: - Then run Sytest. This will generate a new file `integrationcover.log` in each server's directory e.g `server-0/integrationcover.log`. To parse it, - ensure your working directory is under the Dendrite repository then run: ```bash - go tool cover -func=/path/to/server-0/integrationcover.log + go tool covdata func -i="$(find -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" ``` which will produce an output like: ``` ... - github.com/matrix-org/util/json.go:83: NewJSONRequestHandler 100.0% -github.com/matrix-org/util/json.go:90: Protect 57.1% -github.com/matrix-org/util/json.go:110: RequestWithLogging 100.0% -github.com/matrix-org/util/json.go:132: MakeJSONAPI 70.0% -github.com/matrix-org/util/json.go:151: respond 61.5% -github.com/matrix-org/util/json.go:180: WithCORSOptions 0.0% -github.com/matrix-org/util/json.go:191: SetCORSHeaders 100.0% -github.com/matrix-org/util/json.go:202: RandomString 100.0% -github.com/matrix-org/util/json.go:210: init 100.0% -github.com/matrix-org/util/unique.go:13: Unique 91.7% -github.com/matrix-org/util/unique.go:48: SortAndUnique 100.0% -github.com/matrix-org/util/unique.go:55: UniqueStrings 100.0% -total: (statements) 53.7% +github.com/matrix-org/util/json.go:132: MakeJSONAPI 70.0% +github.com/matrix-org/util/json.go:151: respond 84.6% +github.com/matrix-org/util/json.go:180: WithCORSOptions 0.0% +github.com/matrix-org/util/json.go:191: SetCORSHeaders 100.0% +github.com/matrix-org/util/json.go:202: RandomString 100.0% +github.com/matrix-org/util/json.go:210: init 100.0% +github.com/matrix-org/util/unique.go:13: Unique 91.7% +github.com/matrix-org/util/unique.go:48: SortAndUnique 100.0% +github.com/matrix-org/util/unique.go:55: UniqueStrings 100.0% +total (statements) 64.0% ``` -The total coverage for this run is the last line at the bottom. However, this value is misleading because Dendrite can run in many different configurations, -which will never be tested in a single test run (e.g sqlite or postgres). To get a more accurate value, additional processing is required -to remove packages which will never be tested and extension MSCs: +(after running Sytest for Postgres _and_ SQLite) + +The total coverage for this run is the last line at the bottom. However, this value is misleading because Dendrite can run in different configurations, +which will never be tested in a single test run (e.g sqlite or postgres). To get a more accurate value, you'll need run Sytest for Postgres and SQLite (see commands above). +Additional processing is required also to remove packages which will never be tested and extension MSCs: + ```bash -# These commands are all similar but change which package paths are _removed_ from the output. +# If you executed both commands from above, you can get the total coverage using the following commands +go tool covdata textfmt -i="$(find -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov +grep -Ev 'relayapi|setup/mscs' sytest.cov > final.cov +go tool cover -func=final.cov -# For Postgres -go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'inthttp|sqlite|setup/mscs|api_trace' > coverage.txt +# If you only executed the one for Postgres: +go tool covdata textfmt -i="$(find -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov +grep -Ev 'relayapi|sqlite|setup/mscs' sytest.cov > final.cov +go tool cover -func=final.cov -# For SQLite -go tool cover -func=/path/to/server-0/integrationcover.log | grep 'github.com/matrix-org/dendrite' | grep -Ev 'inthttp|postgres|setup/mscs|api_trace' > coverage.txt +# If you only executed the one for SQLite: +go tool covdata textfmt -i="$(find -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov +grep -Ev 'relayapi|postgres|setup/mscs' sytest.cov > final.cov +go tool cover -func=final.cov ``` -A total value can then be calculated using: +## Getting coverage from Complement + +Getting the coverage for Complement runs is a bit more involved. + +First you'll need a docker image compatible with Complement, one can be built using ```bash -cat coverage.txt | awk -F '\t+' '{x = x + $3} END {print x/NR}' +docker build -t complement-dendrite -f build/scripts/Complement.Dockerfile . +``` +from within the Dendrite repository. + +Clone complement to a directory of your liking: +```bash +git clone https://github.com/matrix-org/complement.git +cd complement ``` +Next we'll need a script to execute after a test finishes, create a new file `posttest.sh`, make the file executable (`chmod +x posttest.sh`) +and add the following content: +```bash +#!/bin/bash -We currently do not have a way to combine Sytest/Complement/Unit Tests into a single coverage report. \ No newline at end of file +mkdir -p /tmp/Complement/logs/$2/$1/ +docker cp $1:/tmp/covdatafiles/. /tmp/Complement/logs/$2/$1/ +``` +This will copy the `covdatafiles` files from each container to something like +`/tmp/Complement/logs/TestLogin/94f9c428de95779d2b62a3ccd8eab9d5ddcf65cc259a40ece06bdc61687ffed3/`. (`$1` is the containerID, `$2` the test name) + +Now that we have set up everything we need, we can finally execute Complement: +```bash +COMPLEMENT_BASE_IMAGE=complement-dendrite \ +COMPLEMENT_SHARE_ENV_PREFIX=COMPLEMENT_DENDRITE_ \ +COMPLEMENT_DENDRITE_COVER=1 \ +COMPLEMENT_POST_TEST_SCRIPT=$(pwd)/posttest.sh \ + go test -tags dendrite_blacklist ./tests/... -count=1 -v -timeout=30m -failfast=false +``` + +Once this is done, you can copy the resulting `covdatafiles` files to your Dendrite repository for the next step. +```bash +cp -pr /tmp/Complement/logs PathToYourDendriteRepository +``` + +You can also run the following to get the coverage for Complement runs alone: +```bash +go tool covdata func -i="$(find /tmp/Complement -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" +``` + +## Combining the results of (almost) all runs + +Now that we have all our `covdatafiles` files within the Dendrite repository, you can now execute the following command, to get the coverage +overall (excluding unit tests): +```bash +go tool covdata func -i="$(find -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" +``` \ No newline at end of file diff --git a/docs/development/sytest.md b/docs/development/sytest.md index 4fae2ea3d..2f681f3e5 100644 --- a/docs/development/sytest.md +++ b/docs/development/sytest.md @@ -1,6 +1,7 @@ --- title: SyTest parent: Development +nav_order: 2 permalink: /development/sytest --- @@ -23,7 +24,7 @@ After running the tests, a script will print the tests you need to add to You should proceed after you see no build problems for dendrite after running: ```sh -./build.sh +go build -o bin/ ./cmd/... ``` If you are fixing an issue marked with @@ -61,6 +62,8 @@ When debugging, the following Docker `run` options may also be useful: * `-e "DENDRITE_TRACE_HTTP=1"`: Adds HTTP tracing to server logs. * `-e "DENDRITE_TRACE_INTERNAL=1"`: Adds roomserver internal API tracing to server logs. +* `-e "COVER=1"`: Run Sytest with an instrumented binary, producing a Go coverage file per server. +* `-e "RACE_DETECTION=1"`: Build the binaries with the `-race` flag (Note: This will significantly slow down test runs) The docker command also supports a single positional argument for the test file to run, so you can run a single `.pl` file rather than the whole test suite. For example: @@ -71,68 +74,3 @@ docker run --rm --name sytest -v "/Users/kegan/github/sytest:/sytest" -v "/Users/kegan/go/:/gopath" -e "POSTGRES=1" -e "DENDRITE_TRACE_HTTP=1" matrixdotorg/sytest-dendrite:latest tests/50federation/40devicelists.pl ``` - -### Manually Setting up SyTest - -**We advise AGAINST using manual SyTest setups.** - -If you don't want to use the Docker image, you can also run SyTest by hand. Make -sure you have Perl 5 or above, and get SyTest with: - -(Note that this guide assumes your SyTest checkout is next to your -`dendrite` checkout.) - -```sh -git clone -b develop https://github.com/matrix-org/sytest -cd sytest -./install-deps.pl -``` - -Set up the database: - -```sh -sudo -u postgres psql -c "CREATE USER dendrite PASSWORD 'itsasecret'" -sudo -u postgres psql -c "ALTER USER dendrite CREATEDB" -for i in dendrite0 dendrite1 sytest_template; do sudo -u postgres psql -c "CREATE DATABASE $i OWNER dendrite;"; done -mkdir -p "server-0" -cat > "server-0/database.yaml" << EOF -args: - user: dendrite - password: itsasecret - database: dendrite0 - host: 127.0.0.1 - sslmode: disable -type: pg -EOF -mkdir -p "server-1" -cat > "server-1/database.yaml" << EOF -args: - user: dendrite - password: itsasecret - database: dendrite1 - host: 127.0.0.1 - sslmode: disable -type: pg -EOF -``` - -Run the tests: - -```sh -POSTGRES=1 ./run-tests.pl -I Dendrite::Monolith -d ../dendrite/bin -W ../dendrite/sytest-whitelist -O tap --all | tee results.tap -``` - -where `tee` lets you see the results while they're being piped to the file, and -`POSTGRES=1` enables testing with PostgeSQL. If the `POSTGRES` environment -variable is not set or is set to 0, SyTest will fall back to SQLite 3. For more -flags and options, see . - -Once the tests are complete, run the helper script to see if you need to add -any newly passing test names to `sytest-whitelist` in the project's root -directory: - -```sh -../dendrite/show-expected-fail-tests.sh results.tap ../dendrite/sytest-whitelist ../dendrite/sytest-blacklist -``` - -If the script prints nothing/exits with 0, then you're good to go. diff --git a/docs/development/tracing/opentracing.md b/docs/development/tracing/opentracing.md deleted file mode 100644 index 8528c2ba3..000000000 --- a/docs/development/tracing/opentracing.md +++ /dev/null @@ -1,114 +0,0 @@ ---- -title: OpenTracing -has_children: true -parent: Development -permalink: /development/opentracing ---- - -# OpenTracing - -Dendrite extensively uses the [opentracing.io](http://opentracing.io) framework -to trace work across the different logical components. - -At its most basic opentracing tracks "spans" of work; recording start and end -times as well as any parent span that caused the piece of work. - -A typical example would be a new span being created on an incoming request that -finishes when the response is sent. When the code needs to hit out to a -different component a new span is created with the initial span as its parent. -This would end up looking roughly like: - -``` -Received request Sent response - |<───────────────────────────────────────>| - |<────────────────────>| - RPC call RPC call returns -``` - -This is useful to see where the time is being spent processing a request on a -component. However, opentracing allows tracking of spans across components. This -makes it possible to see exactly what work goes into processing a request: - -``` -Component 1 |<─────────────────── HTTP ────────────────────>| - |<──────────────── RPC ─────────────────>| -Component 2 |<─ SQL ─>| |<── RPC ───>| -Component 3 |<─ SQL ─>| -``` - -This is achieved by serializing span information during all communication -between components. For HTTP requests, this is achieved by the sender -serializing the span into a HTTP header, and the receiver deserializing the span -on receipt. (Generally a new span is then immediately created with the -deserialized span as the parent). - -A collection of spans that are related is called a trace. - -Spans are passed through the code via contexts, rather than manually. It is -therefore important that all spans that are created are immediately added to the -current context. Thankfully the opentracing library gives helper functions for -doing this: - -```golang -span, ctx := opentracing.StartSpanFromContext(ctx, spanName) -defer span.Finish() -``` - -This will create a new span, adding any span already in `ctx` as a parent to the -new span. - -Adding Information ------------------- - -Opentracing allows adding information to a trace via three mechanisms: - -- "tags" ─ A span can be tagged with a key/value pair. This is typically - information that relates to the span, e.g. for spans created for incoming HTTP - requests could include the request path and response codes as tags, spans for - SQL could include the query being executed. -- "logs" ─ Key/value pairs can be looged at a particular instance in a trace. - This can be useful to log e.g. any errors that happen. -- "baggage" ─ Arbitrary key/value pairs can be added to a span to which all - child spans have access. Baggage isn't saved and so isn't available when - inspecting the traces, but can be used to add context to logs or tags in child - spans. - -See -[specification.md](https://github.com/opentracing/specification/blob/master/specification.md) -for some of the common tags and log fields used. - -Span Relationships ------------------- - -Spans can be related to each other. The most common relation is `childOf`, which -indicates the child span somehow depends on the parent span ─ typically the -parent span cannot complete until all child spans are completed. - -A second relation type is `followsFrom`, where the parent has no dependence on -the child span. This usually indicates some sort of fire and forget behaviour, -e.g. adding a message to a pipeline or inserting into a kafka topic. - -Jaeger ------- - -Opentracing is just a framework. We use -[jaeger](https://github.com/jaegertracing/jaeger) as the actual implementation. - -Jaeger is responsible for recording, sending and saving traces, as well as -giving a UI for viewing and interacting with traces. - -To enable jaeger a `Tracer` object must be instansiated from the config (as well -as having a jaeger server running somewhere, usually locally). A `Tracer` does -several things: - -- Decides which traces to save and send to the server. There are multiple - schemes for doing this, with a simple example being to save a certain fraction - of traces. -- Communicating with the jaeger backend. If not explicitly specified uses the - default port on localhost. -- Associates a service name to all spans created by the tracer. This service - name equates to a logical component, e.g. spans created by clientapi will have - a different service name than ones created by the syncapi. Database access - will also typically use a different service name. - - This means that there is a tracer per service name/component. diff --git a/docs/development/tracing/setup.md b/docs/development/tracing/setup.md deleted file mode 100644 index cef1089e4..000000000 --- a/docs/development/tracing/setup.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -title: Setup -parent: OpenTracing -grand_parent: Development -permalink: /development/opentracing/setup ---- - -# OpenTracing Setup - -Dendrite uses [Jaeger](https://www.jaegertracing.io/) for tracing between microservices. -Tracing shows the nesting of logical spans which provides visibility on how the microservices interact. -This document explains how to set up Jaeger locally on a single machine. - -## Set up the Jaeger backend - -The [easiest way](https://www.jaegertracing.io/docs/1.18/getting-started/) is to use the all-in-one Docker image: - -``` -$ docker run -d --name jaeger \ - -e COLLECTOR_ZIPKIN_HTTP_PORT=9411 \ - -p 5775:5775/udp \ - -p 6831:6831/udp \ - -p 6832:6832/udp \ - -p 5778:5778 \ - -p 16686:16686 \ - -p 14268:14268 \ - -p 14250:14250 \ - -p 9411:9411 \ - jaegertracing/all-in-one:1.18 -``` - -## Configuring Dendrite to talk to Jaeger - -Modify your config to look like: (this will send every single span to Jaeger which will be slow on large instances, but for local testing it's fine) - -``` -tracing: - enabled: true - jaeger: - serviceName: "dendrite" - disabled: false - rpc_metrics: true - tags: [] - sampler: - type: const - param: 1 -``` - -then run the monolith server: - -``` -./dendrite --tls-cert server.crt --tls-key server.key --config dendrite.yaml -``` - -## Checking traces - -Visit to see traces under `DendriteMonolith`. diff --git a/docs/hiawatha/monolith-sample.conf b/docs/hiawatha/dendrite-sample.conf similarity index 100% rename from docs/hiawatha/monolith-sample.conf rename to docs/hiawatha/dendrite-sample.conf diff --git a/docs/hiawatha/polylith-sample.conf b/docs/hiawatha/polylith-sample.conf deleted file mode 100644 index eb1dd4f9a..000000000 --- a/docs/hiawatha/polylith-sample.conf +++ /dev/null @@ -1,35 +0,0 @@ -# Depending on which port is used for federation (.well-known/matrix/server or SRV record), -# ensure there's a binding for that port in the configuration. Replace "FEDPORT" with port -# number, (e.g. "8448"), and "IPV4" with your server's ipv4 address (separate binding for -# each ip address, e.g. if you use both ipv4 and ipv6 addresses). - -Binding { - Port = FEDPORT - Interface = IPV4 - TLScertFile = /path/to/fullchainandprivkey.pem -} - - -VirtualHost { - ... - # route requests to: - # /_matrix/client/.*/sync - # /_matrix/client/.*/user/{userId}/filter - # /_matrix/client/.*/user/{userId}/filter/{filterID} - # /_matrix/client/.*/keys/changes - # /_matrix/client/.*/rooms/{roomId}/messages - # /_matrix/client/.*/rooms/{roomId}/context/{eventID} - # /_matrix/client/.*/rooms/{roomId}/event/{eventID} - # /_matrix/client/.*/rooms/{roomId}/relations/{eventID} - # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType} - # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}/{eventType} - # /_matrix/client/.*/rooms/{roomId}/members - # /_matrix/client/.*/rooms/{roomId}/joined_members - # to sync_api - ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|.*?_?members|context/.*?|relations/.*?|event/.*?))$ http://localhost:8073 600 - ReverseProxy = /_matrix/client http://localhost:8071 600 - ReverseProxy = /_matrix/federation http://localhost:8072 600 - ReverseProxy = /_matrix/key http://localhost:8072 600 - ReverseProxy = /_matrix/media http://localhost:8074 600 - ... -} diff --git a/docs/installation/1_planning.md b/docs/installation/1_planning.md index 36d90abda..354003aef 100644 --- a/docs/installation/1_planning.md +++ b/docs/installation/1_planning.md @@ -7,23 +7,13 @@ permalink: /installation/planning # Planning your installation -## Modes - -Dendrite consists of several components, each responsible for a different aspect of the Matrix protocol. -Users can run Dendrite in one of two modes which dictate how these components are executed and communicate. - -* **Monolith mode** runs all components in a single process. Components communicate through an internal NATS - server with generally low overhead. This mode dramatically simplifies deployment complexity and offers the - best balance between performance and resource usage for low-to-mid volume deployments. - - -## Databases +## Database Dendrite can run with either a PostgreSQL or a SQLite backend. There are considerable tradeoffs to consider: * **PostgreSQL**: Needs to run separately to Dendrite, needs to be installed and configured separately - and and will use more resources over all, but will be **considerably faster** than SQLite. PostgreSQL + and will use more resources over all, but will be **considerably faster** than SQLite. PostgreSQL has much better write concurrency which will allow Dendrite to process more tasks in parallel. This will be necessary for federated deployments to perform adequately. @@ -80,18 +70,17 @@ If using the PostgreSQL database engine, you should install PostgreSQL 12 or lat ### NATS Server Dendrite comes with a built-in [NATS Server](https://github.com/nats-io/nats-server) and -therefore does not need this to be manually installed. If you are planning a monolith installation, you -do not need to do anything. +therefore does not need this to be manually installed. ### Reverse proxy A reverse proxy such as [Caddy](https://caddyserver.com), [NGINX](https://www.nginx.com) or -[HAProxy](http://www.haproxy.org) is useful for deployments. Configuring those is not covered in this documentation, although sample configurations +[HAProxy](http://www.haproxy.org) is useful for deployments. Configuring this is not covered in this documentation, although sample configurations for [Caddy](https://github.com/matrix-org/dendrite/blob/main/docs/caddy) and [NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx) are provided. ### Windows -Finally, if you want to build Dendrite on Windows, you will need need `gcc` in the path. The best +Finally, if you want to build Dendrite on Windows, you will need `gcc` in the path. The best way to achieve this is by installing and building Dendrite under [MinGW-w64](https://www.mingw-w64.org/). diff --git a/docs/installation/2_domainname.md b/docs/installation/2_domainname.md index 545a2daf6..d86a664cb 100644 --- a/docs/installation/2_domainname.md +++ b/docs/installation/2_domainname.md @@ -20,7 +20,7 @@ Matrix servers usually discover each other when federating using the following m well-known file to connect to the remote homeserver; 2. If a DNS SRV delegation exists on `example.com`, use the IP address and port from the DNS SRV record to connect to the remote homeserver; -3. If neither well-known or DNS SRV delegation are configured, attempt to connect to the remote +3. If neither well-known nor DNS SRV delegation are configured, attempt to connect to the remote homeserver by connecting to `example.com` port TCP/8448 using HTTPS. The exact details of how server name resolution works can be found in diff --git a/docs/installation/5_install_monolith.md b/docs/installation/5_install_monolith.md deleted file mode 100644 index 901975a65..000000000 --- a/docs/installation/5_install_monolith.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -title: Installing as a monolith -parent: Installation -has_toc: true -nav_order: 5 -permalink: /installation/install/monolith ---- - -# Installing as a monolith - -You can install the Dendrite monolith binary into `$GOPATH/bin` by using `go install`: - -```sh -go install ./cmd/dendrite -``` - -Alternatively, you can specify a custom path for the binary to be written to using `go build`: - -```sh -go build -o /usr/local/bin/ ./cmd/dendrite -``` diff --git a/docs/installation/9_starting_monolith.md b/docs/installation/9_starting_monolith.md deleted file mode 100644 index d7e8c0b8b..000000000 --- a/docs/installation/9_starting_monolith.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: Starting the monolith -parent: Installation -has_toc: true -nav_order: 9 -permalink: /installation/start/monolith ---- - -# Starting the monolith - -Once you have completed all of the preparation and installation steps, -you can start your Dendrite monolith deployment by starting `dendrite`: - -```bash -./dendrite -config /path/to/dendrite.yaml -``` - -By default, Dendrite will listen HTTP on port 8008. If you want to change the addresses -or ports that Dendrite listens on, you can use the `-http-bind-address` and -`-https-bind-address` command line arguments: - -```bash -./dendrite -config /path/to/dendrite.yaml \ - -http-bind-address 1.2.3.4:12345 \ - -https-bind-address 1.2.3.4:54321 -``` - -## Running under systemd - -A common deployment pattern is to run the monolith under systemd. For this, you -will need to create a service unit file. An example service unit file is available -in the [GitHub repository](https://github.com/matrix-org/dendrite/blob/main/docs/systemd/monolith-example.service). - -Once you have installed the service unit, you can notify systemd, enable and start -the service: - -```bash -systemctl daemon-reload -systemctl enable dendrite -systemctl start dendrite -journalctl -fu dendrite -``` diff --git a/docs/installation/docker.md b/docs/installation/docker.md new file mode 100644 index 000000000..1ecc7c6ee --- /dev/null +++ b/docs/installation/docker.md @@ -0,0 +1,11 @@ +--- +title: Docker +parent: Installation +has_children: true +nav_order: 4 +permalink: /docker +--- + +# Installation using Docker + +This section contains documentation how to install Dendrite using Docker diff --git a/docs/installation/docker/1_docker.md b/docs/installation/docker/1_docker.md new file mode 100644 index 000000000..1fe792636 --- /dev/null +++ b/docs/installation/docker/1_docker.md @@ -0,0 +1,57 @@ +--- +title: Installation +parent: Docker +grand_parent: Installation +has_toc: true +nav_order: 1 +permalink: /installation/docker/install +--- + +# Installing Dendrite using Docker Compose + +Dendrite provides an [example](https://github.com/matrix-org/dendrite/blob/main/build/docker/docker-compose.yml) +Docker compose file, which needs some preparation to start successfully. +Please note that this compose file only has Postgres as a dependency, and you need to configure +a [reverse proxy](../planning#reverse-proxy). + +## Preparations + +### Generate a private key + +First we'll generate private key, which is used to sign events, the following will create one in `./config`: + +```bash +mkdir -p ./config +docker run --rm --entrypoint="/usr/bin/generate-keys" \ + -v $(pwd)/config:/mnt \ + matrixdotorg/dendrite-monolith:latest \ + -private-key /mnt/matrix_key.pem +``` +(**NOTE**: This only needs to be executed **once**, as you otherwise overwrite the key) + +### Generate a config + +Similar to the command above, we can generate a config to be used, which will use the correct paths +as specified in the example docker-compose file. Change `server` to your domain and `db` according to your changes +to the docker-compose file (`services.postgres.environment` values): + +```bash +mkdir -p ./config +docker run --rm --entrypoint="/bin/sh" \ + -v $(pwd)/config:/mnt \ + matrixdotorg/dendrite-monolith:latest \ + -c "/usr/bin/generate-config \ + -dir /var/dendrite/ \ + -db postgres://dendrite:itsasecret@postgres/dendrite?sslmode=disable \ + -server YourDomainHere > /mnt/dendrite.yaml" +``` + +You can then change `config/dendrite.yaml` to your liking. + +## Starting Dendrite + +Once you're done changing the config, you can now start up Dendrite with + +```bash +docker-compose -f docker-compose.yml up +``` diff --git a/docs/installation/helm.md b/docs/installation/helm.md new file mode 100644 index 000000000..dd20e0261 --- /dev/null +++ b/docs/installation/helm.md @@ -0,0 +1,11 @@ +--- +title: Helm +parent: Installation +has_children: true +nav_order: 3 +permalink: /helm +--- + +# Helm + +This section contains documentation how to use [Helm](https://helm.sh/) to install Dendrite on a [Kubernetes](https://kubernetes.io/) cluster. diff --git a/docs/installation/helm/1_helm.md b/docs/installation/helm/1_helm.md new file mode 100644 index 000000000..00fe4fdca --- /dev/null +++ b/docs/installation/helm/1_helm.md @@ -0,0 +1,58 @@ +--- +title: Installation +parent: Helm +grand_parent: Installation +has_toc: true +nav_order: 1 +permalink: /installation/helm/install +--- + +# Installing Dendrite using Helm + +To install Dendrite using the Helm chart, you first have to add the repository using the following commands: + +```bash +helm repo add dendrite https://matrix-org.github.io/dendrite/ +helm repo update +``` + +Next you'll need to create a `values.yaml` file and configure it to your liking. All possible values can be found +[here](https://github.com/matrix-org/dendrite/blob/main/helm/dendrite/values.yaml), but at least you need to configure +a `server_name`, otherwise the chart will complain about it: + +```yaml +dendrite_config: + global: + server_name: "localhost" +``` + +If you are going to use an existing Postgres database, you'll also need to configure this connection: + +```yaml +dendrite_config: + global: + database: + connection_string: "postgresql://PostgresUser:PostgresPassword@PostgresHostName/DendriteDatabaseName" + max_open_conns: 90 + max_idle_conns: 5 + conn_max_lifetime: -1 +``` + +## Installing with PostgreSQL + +The chart comes with a dependency on Postgres, which can be installed alongside Dendrite, this needs to be enabled in +the `values.yaml`: + +```yaml +postgresql: + enabled: true # this installs Postgres + primary: + persistence: + size: 1Gi # defines the size for $PGDATA + +dendrite_config: + global: + server_name: "localhost" +``` + +Using this option, the `database.connection_string` will be set for you automatically. \ No newline at end of file diff --git a/docs/installation/manual.md b/docs/installation/manual.md new file mode 100644 index 000000000..3ab1fd627 --- /dev/null +++ b/docs/installation/manual.md @@ -0,0 +1,11 @@ +--- +title: Manual +parent: Installation +has_children: true +nav_order: 5 +permalink: /manual +--- + +# Manual Installation + +This section contains documentation how to manually install Dendrite diff --git a/docs/installation/3_build.md b/docs/installation/manual/1_build.md similarity index 53% rename from docs/installation/3_build.md rename to docs/installation/manual/1_build.md index 824c81d37..73a626882 100644 --- a/docs/installation/3_build.md +++ b/docs/installation/manual/1_build.md @@ -1,31 +1,26 @@ --- -title: Building Dendrite -parent: Installation +title: Building/Installing Dendrite +parent: Manual +grand_parent: Installation has_toc: true -nav_order: 3 -permalink: /installation/build +nav_order: 1 +permalink: /installation/manual/build --- # Build all Dendrite commands Dendrite has numerous utility commands in addition to the actual server binaries. -Build them all from the root of the source repo with `build.sh` (Linux/Mac): +Build them all from the root of the source repo with: ```sh -./build.sh -``` - -or `build.cmd` (Windows): - -```powershell -build.cmd +go build -o bin/ ./cmd/... ``` The resulting binaries will be placed in the `bin` subfolder. -# Installing as a monolith +# Installing Dendrite -You can install the Dendrite monolith binary into `$GOPATH/bin` by using `go install`: +You can install the Dendrite binary into `$GOPATH/bin` by using `go install`: ```sh go install ./cmd/dendrite diff --git a/docs/installation/4_database.md b/docs/installation/manual/2_database.md similarity index 57% rename from docs/installation/4_database.md rename to docs/installation/manual/2_database.md index d64ee6615..1be602c66 100644 --- a/docs/installation/4_database.md +++ b/docs/installation/manual/2_database.md @@ -1,8 +1,10 @@ --- title: Preparing database storage parent: Installation -nav_order: 3 -permalink: /installation/database +nav_order: 2 +parent: Manual +grand_parent: Installation +permalink: /installation/manual/database --- # Preparing database storage @@ -13,31 +15,22 @@ may need to perform some manual steps outlined below. ## PostgreSQL Dendrite can automatically populate the database with the relevant tables and indexes, but -it is not capable of creating the databases themselves. You will need to create the databases +it is not capable of creating the database itself. You will need to create the database manually. -The databases **must** be created with UTF-8 encoding configured or you will likely run into problems +The database **must** be created with UTF-8 encoding configured, or you will likely run into problems with your Dendrite deployment. -At this point, you can choose to either use a single database for all Dendrite components, -or you can run each component with its own separate database: +You will need to create a single PostgreSQL database. Deployments +can use a single global connection pool, which makes updating the configuration file much easier. +Only one database connection string to manage and likely simpler to back up the database. All +components will be sharing the same database resources (CPU, RAM, storage). -* **Single database**: You will need to create a single PostgreSQL database. Monolith deployments - can use a single global connection pool, which makes updating the configuration file much easier. - Only one database connection string to manage and likely simpler to back up the database. All - components will be sharing the same database resources (CPU, RAM, storage). - -* **Separate databases**: You will need to create a separate PostgreSQL database for each - component. You will need to configure each component that has storage in the Dendrite - configuration file with its own connection parameters. Allows running a different database engine - for each component on a different machine if needs be, each with their own CPU, RAM and storage — - almost certainly overkill unless you are running a very large Dendrite deployment. - -For either configuration, you will want to: +You will most likely want to: 1. Configure a role (with a username and password) which Dendrite can use to connect to the database; -2. Create the database(s) themselves, ensuring that the Dendrite role has privileges over them. +2. Create the database itself, ensuring that the Dendrite role has privileges over them. As Dendrite will create and manage the database tables, indexes and sequences by itself, the Dendrite role must have suitable privileges over the database. @@ -71,27 +64,6 @@ Create the database itself, using the `dendrite` role from above: sudo -u postgres createdb -O dendrite -E UTF-8 dendrite ``` -### Multiple database creation - -The following eight components require a database. In this example they will be named: - -| Appservice API | `dendrite_appservice` | -| Federation API | `dendrite_federationapi` | -| Media API | `dendrite_mediaapi` | -| MSCs | `dendrite_mscs` | -| Roomserver | `dendrite_roomserver` | -| Sync API | `dendrite_syncapi` | -| Key server | `dendrite_keyserver` | -| User API | `dendrite_userapi` | - -... therefore you will need to create eight different databases: - -```bash -for i in appservice federationapi mediaapi mscs roomserver syncapi keyserver userapi; do - sudo -u postgres createdb -O dendrite -E UTF-8 dendrite_$i -done -``` - ## SQLite **WARNING:** The Dendrite SQLite backend is slower, less reliable and not recommended for diff --git a/docs/installation/7_configuration.md b/docs/installation/manual/3_configuration.md similarity index 67% rename from docs/installation/7_configuration.md rename to docs/installation/manual/3_configuration.md index 0cc67b156..a9dd81c87 100644 --- a/docs/installation/7_configuration.md +++ b/docs/installation/manual/3_configuration.md @@ -1,8 +1,9 @@ --- title: Configuring Dendrite -parent: Installation -nav_order: 7 -permalink: /installation/configuration +parent: Manual +grand_parent: Installation +nav_order: 3 +permalink: /installation/manual/configuration --- # Configuring Dendrite @@ -20,7 +21,7 @@ sections: First of all, you will need to configure the server name of your Matrix homeserver. This must match the domain name that you have selected whilst [configuring the domain -name delegation](domainname). +name delegation](domainname#delegation). In the `global` section, set the `server_name` to your delegated domain name: @@ -44,7 +45,7 @@ global: ## JetStream configuration -Monolith deployments can use the built-in NATS Server rather than running a standalone +Dendrite deployments can use the built-in NATS Server rather than running a standalone server. If you want to use a standalone NATS Server anyway, you can also configure that too. ### Built-in NATS Server @@ -56,7 +57,6 @@ configured and set a `storage_path` to a persistent folder on the filesystem: global: # ... jetstream: - in_memory: false storage_path: /path/to/storage/folder topic_prefix: Dendrite ``` @@ -79,22 +79,17 @@ You do not need to configure the `storage_path` when using a standalone NATS Ser In the case that you are connecting to a multi-node NATS cluster, you can configure more than one address in the `addresses` field. -## Database connections +## Database connection using a global connection pool -Configuring database connections varies based on the [database configuration](database) -that you chose. - -### Global connection pool - -If you want to use a single connection pool to a single PostgreSQL database, then you must -uncomment and configure the `database` section within the `global` section: +If you want to use a single connection pool to a single PostgreSQL database, +then you must uncomment and configure the `database` section within the `global` section: ```yaml global: # ... database: connection_string: postgres://user:pass@hostname/database?sslmode=disable - max_open_conns: 100 + max_open_conns: 90 max_idle_conns: 5 conn_max_lifetime: -1 ``` @@ -104,42 +99,13 @@ configuration file, e.g. under the `app_service_api`, `federation_api`, `key_ser `media_api`, `mscs`, `relay_api`, `room_server`, `sync_api` and `user_api` blocks, otherwise these will override the `global` database configuration. -### Per-component connections (all other configurations) - -If you are are using SQLite databases or separate PostgreSQL -databases per component, then you must instead configure the `database` sections under each -of the component blocks ,e.g. under the `app_service_api`, `federation_api`, `key_server`, -`media_api`, `mscs`, `relay_api`, `room_server`, `sync_api` and `user_api` blocks. - -For example, with PostgreSQL: - -```yaml -room_server: - # ... - database: - connection_string: postgres://user:pass@hostname/dendrite_component?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 -``` - -... or with SQLite: - -```yaml -room_server: - # ... - database: - connection_string: file:roomserver.db - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 -``` - ## Full-text search -Dendrite supports experimental full-text indexing using [Bleve](https://github.com/blevesearch/bleve). It is configured in the `sync_api` section as follows. +Dendrite supports full-text indexing using [Bleve](https://github.com/blevesearch/bleve). It is configured in the `sync_api` section as follows. -Depending on the language most likely to be used on the server, it might make sense to change the `language` used when indexing, to ensure the returned results match the expectations. A full list of possible languages can be found [here](https://github.com/blevesearch/bleve/tree/master/analysis/lang). +Depending on the language most likely to be used on the server, it might make sense to change the `language` used when indexing, +to ensure the returned results match the expectations. A full list of possible languages +can be found [here](https://github.com/matrix-org/dendrite/blob/5b73592f5a4dddf64184fcbe33f4c1835c656480/internal/fulltext/bleve.go#L25-L46). ```yaml sync_api: diff --git a/docs/installation/8_signingkey.md b/docs/installation/manual/4_signingkey.md similarity index 92% rename from docs/installation/8_signingkey.md rename to docs/installation/manual/4_signingkey.md index 323759a88..bd9c242ab 100644 --- a/docs/installation/8_signingkey.md +++ b/docs/installation/manual/4_signingkey.md @@ -1,8 +1,9 @@ --- title: Generating signing keys -parent: Installation -nav_order: 8 -permalink: /installation/signingkeys +parent: Manual +grand_parent: Installation +nav_order: 4 +permalink: /installation/manual/signingkeys --- # Generating signing keys @@ -11,7 +12,7 @@ All Matrix homeservers require a signing private key, which will be used to auth federation requests and events. The `generate-keys` utility can be used to generate a private key. Assuming that Dendrite was -built using `build.sh`, you should find the `generate-keys` utility in the `bin` folder. +built using `go build -o bin/ ./cmd/...`, you should find the `generate-keys` utility in the `bin` folder. To generate a Matrix signing private key: diff --git a/docs/installation/manual/5_starting_dendrite.md b/docs/installation/manual/5_starting_dendrite.md new file mode 100644 index 000000000..d13504372 --- /dev/null +++ b/docs/installation/manual/5_starting_dendrite.md @@ -0,0 +1,26 @@ +--- +title: Starting Dendrite +parent: Manual +grand_parent: Installation +nav_order: 5 +permalink: /installation/manual/start +--- + +# Starting Dendrite + +Once you have completed all preparation and installation steps, +you can start your Dendrite deployment by executing the `dendrite` binary: + +```bash +./dendrite -config /path/to/dendrite.yaml +``` + +By default, Dendrite will listen HTTP on port 8008. If you want to change the addresses +or ports that Dendrite listens on, you can use the `-http-bind-address` and +`-https-bind-address` command line arguments: + +```bash +./dendrite -config /path/to/dendrite.yaml \ + -http-bind-address 1.2.3.4:12345 \ + -https-bind-address 1.2.3.4:54321 +``` diff --git a/docs/nginx/monolith-sample.conf b/docs/nginx/dendrite-sample.conf similarity index 100% rename from docs/nginx/monolith-sample.conf rename to docs/nginx/dendrite-sample.conf diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf deleted file mode 100644 index 0ad24509a..000000000 --- a/docs/nginx/polylith-sample.conf +++ /dev/null @@ -1,58 +0,0 @@ -server { - listen 443 ssl; # IPv4 - listen [::]:443 ssl; # IPv6 - server_name my.hostname.com; - - ssl_certificate /path/to/fullchain.pem; - ssl_certificate_key /path/to/privkey.pem; - ssl_dhparam /path/to/ssl-dhparams.pem; - - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_read_timeout 600; - - location /.well-known/matrix/server { - return 200 '{ "m.server": "my.hostname.com:443" }'; - } - - location /.well-known/matrix/client { - # If your sever_name here doesn't match your matrix homeserver URL - # (e.g. hostname.com as server_name and matrix.hostname.com as homeserver URL) - # add_header Access-Control-Allow-Origin '*'; - return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }'; - } - - # route requests to: - # /_matrix/client/.*/sync - # /_matrix/client/.*/user/{userId}/filter - # /_matrix/client/.*/user/{userId}/filter/{filterID} - # /_matrix/client/.*/keys/changes - # /_matrix/client/.*/rooms/{roomId}/messages - # /_matrix/client/.*/rooms/{roomId}/context/{eventID} - # /_matrix/client/.*/rooms/{roomId}/event/{eventID} - # /_matrix/client/.*/rooms/{roomId}/relations/{eventID} - # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType} - # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}/{eventType} - # /_matrix/client/.*/rooms/{roomId}/members - # /_matrix/client/.*/rooms/{roomId}/joined_members - # to sync_api - location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|.*?_?members|context/.*?|relations/.*?|event/.*?))$ { - proxy_pass http://sync_api:8073; - } - - location /_matrix/client { - proxy_pass http://client_api:8071; - } - - location /_matrix/federation { - proxy_pass http://federation_api:8072; - } - - location /_matrix/key { - proxy_pass http://federation_api:8072; - } - - location /_matrix/media { - proxy_pass http://media_api:8074; - } -} diff --git a/docs/systemd/monolith-example.service b/docs/systemd/monolith-example.service deleted file mode 100644 index 8a948a3fa..000000000 --- a/docs/systemd/monolith-example.service +++ /dev/null @@ -1,19 +0,0 @@ -[Unit] -Description=Dendrite (Matrix Homeserver) -After=syslog.target -After=network.target -After=postgresql.service - -[Service] -Environment=GODEBUG=madvdontneed=1 -RestartSec=2s -Type=simple -User=dendrite -Group=dendrite -WorkingDirectory=/opt/dendrite/ -ExecStart=/opt/dendrite/bin/dendrite -Restart=always -LimitNOFILE=65535 - -[Install] -WantedBy=multi-user.target diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go index 17296febc..8d21b7829 100644 --- a/roomserver/internal/perform/perform_admin.go +++ b/roomserver/internal/perform/perform_admin.go @@ -136,7 +136,7 @@ func (r *Admin) PerformAdminEvacuateRoom( inputReq := &api.InputRoomEventsRequest{ InputRoomEvents: inputEvents, - Asynchronous: true, + Asynchronous: false, } inputRes := &api.InputRoomEventsResponse{} r.Inputer.InputRoomEvents(ctx, inputReq, inputRes) @@ -200,18 +200,24 @@ func (r *Admin) PerformAdminPurgeRoom( } // Evacuate the room before purging it from the database - if _, err := r.PerformAdminEvacuateRoom(ctx, roomID); err != nil { + evacAffected, err := r.PerformAdminEvacuateRoom(ctx, roomID) + if err != nil { logrus.WithField("room_id", roomID).WithError(err).Warn("Failed to evacuate room before purging") return err } + logrus.WithFields(logrus.Fields{ + "room_id": roomID, + "evacuated_users": len(evacAffected), + }).Warn("Evacuated room, purging room from roomserver now") + logrus.WithField("room_id", roomID).Warn("Purging room from roomserver") if err := r.DB.PurgeRoom(ctx, roomID); err != nil { logrus.WithField("room_id", roomID).WithError(err).Warn("Failed to purge room from roomserver") return err } - logrus.WithField("room_id", roomID).Warn("Room purged from roomserver") + logrus.WithField("room_id", roomID).Warn("Room purged from roomserver, informing other components") return r.Inputer.OutputProducer.ProduceRoomEvents(roomID, []api.OutputEvent{ { diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index b49c2f701..132bd80c8 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -130,7 +130,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( if pos == 0 { pos = r.High() } - return data, pos, nil + return data, pos, rows.Err() } func (s *accountDataStatements) SelectMaxAccountDataID( From 3dcca4017cb919fb249784d9cf9b83ea60a77f15 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 30 May 2023 15:27:11 +0200 Subject: [PATCH 2/5] Fix potential state reset when trying to join a room (#3040) When trying to join a room in short sequence, it is possible that a state reset occurs. This fixes it by using `singleflight`. --- clientapi/routing/routing.go | 42 +++++++++++++++++++++++++++++------- go.mod | 1 + go.sum | 1 + 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 2a2fa6655..d3f19cae1 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -20,14 +20,16 @@ import ( "strings" "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/setup/base" - userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" "github.com/nats-io/nats.go" "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" + "golang.org/x/sync/singleflight" + + "github.com/matrix-org/dendrite/setup/base" + userapi "github.com/matrix-org/dendrite/userapi/api" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/api" @@ -84,6 +86,14 @@ func Setup( unstableFeatures["org.matrix."+msc] = true } + // singleflight protects /join endpoints from being invoked + // multiple times from the same user and room, otherwise + // a state reset can occur. This also avoids unneeded + // state calculations. + // TODO: actually fix this in the roomserver, as there are + // possibly other ways that can result in a stat reset. + sf := singleflight.Group{} + if cfg.Matrix.WellKnownClientName != "" { logrus.Infof("Setting m.homeserver base_url as %s at /.well-known/matrix/client", cfg.Matrix.WellKnownClientName) wkMux.Handle("/client", httputil.MakeExternalAPI("wellknown", func(r *http.Request) util.JSONResponse { @@ -264,9 +274,17 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return JoinRoomByIDOrAlias( - req, device, rsAPI, userAPI, vars["roomIDOrAlias"], - ) + // Only execute a join for roomIDOrAlias and UserID once. If there is a join in progress + // it waits for it to complete and returns that result for subsequent requests. + resp, _, _ := sf.Do(vars["roomIDOrAlias"]+device.UserID, func() (any, error) { + return JoinRoomByIDOrAlias( + req, device, rsAPI, userAPI, vars["roomIDOrAlias"], + ), nil + }) + // once all joins are processed, drop them from the cache. Further requests + // will be processed as usual. + sf.Forget(vars["roomIDOrAlias"] + device.UserID) + return resp.(util.JSONResponse) }, httputil.WithAllowGuests()), ).Methods(http.MethodPost, http.MethodOptions) @@ -300,9 +318,17 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return JoinRoomByIDOrAlias( - req, device, rsAPI, userAPI, vars["roomID"], - ) + // Only execute a join for roomID and UserID once. If there is a join in progress + // it waits for it to complete and returns that result for subsequent requests. + resp, _, _ := sf.Do(vars["roomID"]+device.UserID, func() (any, error) { + return JoinRoomByIDOrAlias( + req, device, rsAPI, userAPI, vars["roomID"], + ), nil + }) + // once all joins are processed, drop them from the cache. Further requests + // will be processed as usual. + sf.Forget(vars["roomID"] + device.UserID) + return resp.(util.JSONResponse) }, httputil.WithAllowGuests()), ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/leave", diff --git a/go.mod b/go.mod index 16e5adc8c..360ddf5b2 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( golang.org/x/crypto v0.9.0 golang.org/x/image v0.5.0 golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e + golang.org/x/sync v0.1.0 golang.org/x/term v0.8.0 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index 98e7e839d..4a6054af8 100644 --- a/go.sum +++ b/go.sum @@ -614,6 +614,7 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= From 61341aca500ec4d87e5b6d4c3f965c3836d6e6d6 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 30 May 2023 18:05:48 +0200 Subject: [PATCH 3/5] Add tests for the `UpDropEventReferenceSHAPrevEvents` migration (#3087) ... as they could fail if there are duplicate events in `roomserver_previous_events`. This fixes the migration by trying to combine the `event_nids` if possible (same room) as mentioned by @kegsay in https://github.com/matrix-org/dendrite/pull/3083#discussion_r1195508963 --- .../20230516154000_drop_reference_sha.go | 86 ++++++++++++++++--- .../20230516154000_drop_reference_sha_test.go | 60 +++++++++++++ .../20230516154000_drop_reference_sha.go | 78 ++++++++++++++++- .../20230516154000_drop_reference_sha_test.go | 59 +++++++++++++ 4 files changed, 271 insertions(+), 12 deletions(-) create mode 100644 roomserver/storage/postgres/deltas/20230516154000_drop_reference_sha_test.go create mode 100644 roomserver/storage/sqlite3/deltas/20230516154000_drop_reference_sha_test.go diff --git a/roomserver/storage/postgres/deltas/20230516154000_drop_reference_sha.go b/roomserver/storage/postgres/deltas/20230516154000_drop_reference_sha.go index c19577713..1b1dd44d3 100644 --- a/roomserver/storage/postgres/deltas/20230516154000_drop_reference_sha.go +++ b/roomserver/storage/postgres/deltas/20230516154000_drop_reference_sha.go @@ -18,19 +18,14 @@ import ( "context" "database/sql" "fmt" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/util" ) func UpDropEventReferenceSHAEvents(ctx context.Context, tx *sql.Tx) error { - var count int - err := tx.QueryRowContext(ctx, `SELECT count(*) FROM roomserver_events GROUP BY event_id HAVING count(event_id) > 1`). - Scan(&count) - if err != nil && err != sql.ErrNoRows { - return fmt.Errorf("failed to query duplicate event ids") - } - if count > 0 { - return fmt.Errorf("unable to drop column, as there are duplicate event ids") - } - _, err = tx.ExecContext(ctx, `ALTER TABLE roomserver_events DROP COLUMN IF EXISTS reference_sha256;`) + _, err := tx.ExecContext(ctx, `ALTER TABLE roomserver_events DROP COLUMN IF EXISTS reference_sha256;`) if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) } @@ -46,9 +41,80 @@ func UpDropEventReferenceSHAPrevEvents(ctx context.Context, tx *sql.Tx) error { if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) } + + // figure out if there are duplicates + dupeRows, err := tx.QueryContext(ctx, `SELECT previous_event_id FROM roomserver_previous_events GROUP BY previous_event_id HAVING count(previous_event_id) > 1`) + if err != nil { + return fmt.Errorf("failed to query duplicate event ids") + } + defer internal.CloseAndLogIfError(ctx, dupeRows, "failed to close rows") + + var prevEvents []string + var prevEventID string + for dupeRows.Next() { + if err = dupeRows.Scan(&prevEventID); err != nil { + return err + } + prevEvents = append(prevEvents, prevEventID) + } + if dupeRows.Err() != nil { + return dupeRows.Err() + } + + // if we found duplicates, check if we can combine them, e.g. they are in the same room + for _, dupeID := range prevEvents { + var dupeNIDsRows *sql.Rows + dupeNIDsRows, err = tx.QueryContext(ctx, `SELECT event_nids FROM roomserver_previous_events WHERE previous_event_id = $1`, dupeID) + if err != nil { + return fmt.Errorf("failed to query duplicate event ids") + } + defer internal.CloseAndLogIfError(ctx, dupeNIDsRows, "failed to close rows") + var dupeNIDs []int64 + for dupeNIDsRows.Next() { + var nids pq.Int64Array + if err = dupeNIDsRows.Scan(&nids); err != nil { + return err + } + dupeNIDs = append(dupeNIDs, nids...) + } + + if dupeNIDsRows.Err() != nil { + return dupeNIDsRows.Err() + } + // dedupe NIDs + dupeNIDs = dupeNIDs[:util.SortAndUnique(nids(dupeNIDs))] + // now that we have all NIDs, check which room they belong to + var roomCount int + err = tx.QueryRowContext(ctx, `SELECT count(distinct room_nid) FROM roomserver_events WHERE event_nid = ANY($1)`, pq.Array(dupeNIDs)).Scan(&roomCount) + if err != nil { + return err + } + // if the events are from different rooms, that's bad and we can't continue + if roomCount > 1 { + return fmt.Errorf("detected events (%v) referenced for different rooms (%v)", dupeNIDs, roomCount) + } + // otherwise delete the dupes + _, err = tx.ExecContext(ctx, "DELETE FROM roomserver_previous_events WHERE previous_event_id = $1", dupeID) + if err != nil { + return fmt.Errorf("unable to delete duplicates: %w", err) + } + + // insert combined values + _, err = tx.ExecContext(ctx, "INSERT INTO roomserver_previous_events (previous_event_id, event_nids) VALUES ($1, $2)", dupeID, pq.Array(dupeNIDs)) + if err != nil { + return fmt.Errorf("unable to insert new event NIDs: %w", err) + } + } + _, err = tx.ExecContext(ctx, `ALTER TABLE roomserver_previous_events ADD CONSTRAINT roomserver_previous_event_id_unique UNIQUE (previous_event_id);`) if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) } return nil } + +type nids []int64 + +func (s nids) Len() int { return len(s) } +func (s nids) Less(i, j int) bool { return s[i] < s[j] } +func (s nids) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/roomserver/storage/postgres/deltas/20230516154000_drop_reference_sha_test.go b/roomserver/storage/postgres/deltas/20230516154000_drop_reference_sha_test.go new file mode 100644 index 000000000..c79daac5f --- /dev/null +++ b/roomserver/storage/postgres/deltas/20230516154000_drop_reference_sha_test.go @@ -0,0 +1,60 @@ +package deltas + +import ( + "testing" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/test/testrig" + "github.com/stretchr/testify/assert" +) + +func TestUpDropEventReferenceSHAPrevEvents(t *testing.T) { + + cfg, ctx, close := testrig.CreateConfig(t, test.DBTypePostgres) + defer close() + + db, err := sqlutil.Open(&cfg.Global.DatabaseOptions, sqlutil.NewDummyWriter()) + assert.Nil(t, err) + assert.NotNil(t, db) + defer db.Close() + + // create the table in the old layout + _, err = db.ExecContext(ctx.Context(), ` +CREATE TABLE IF NOT EXISTS roomserver_previous_events ( + previous_event_id TEXT NOT NULL, + event_nids BIGINT[] NOT NULL, + previous_reference_sha256 BYTEA NOT NULL, + CONSTRAINT roomserver_previous_event_id_unique UNIQUE (previous_event_id, previous_reference_sha256) +);`) + assert.Nil(t, err) + + // create the events table as well, slimmed down with one eventNID + _, err = db.ExecContext(ctx.Context(), ` +CREATE SEQUENCE IF NOT EXISTS roomserver_event_nid_seq; +CREATE TABLE IF NOT EXISTS roomserver_events ( + event_nid BIGINT PRIMARY KEY DEFAULT nextval('roomserver_event_nid_seq'), + room_nid BIGINT NOT NULL +); + +INSERT INTO roomserver_events (event_nid, room_nid) VALUES (1, 1) +`) + assert.Nil(t, err) + + // insert duplicate prev events with different event_nids + stmt, err := db.PrepareContext(ctx.Context(), `INSERT INTO roomserver_previous_events (previous_event_id, event_nids, previous_reference_sha256) VALUES ($1, $2, $3)`) + assert.Nil(t, err) + assert.NotNil(t, stmt) + _, err = stmt.ExecContext(ctx.Context(), "1", pq.Array([]int64{1, 2}), "a") + assert.Nil(t, err) + _, err = stmt.ExecContext(ctx.Context(), "1", pq.Array([]int64{1, 2, 3}), "b") + assert.Nil(t, err) + // execute the migration + txn, err := db.Begin() + assert.Nil(t, err) + assert.NotNil(t, txn) + defer txn.Rollback() + err = UpDropEventReferenceSHAPrevEvents(ctx.Context(), txn) + assert.NoError(t, err) +} diff --git a/roomserver/storage/sqlite3/deltas/20230516154000_drop_reference_sha.go b/roomserver/storage/sqlite3/deltas/20230516154000_drop_reference_sha.go index 452d72ace..515bccc37 100644 --- a/roomserver/storage/sqlite3/deltas/20230516154000_drop_reference_sha.go +++ b/roomserver/storage/sqlite3/deltas/20230516154000_drop_reference_sha.go @@ -18,6 +18,10 @@ import ( "context" "database/sql" "fmt" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/util" ) func UpDropEventReferenceSHA(ctx context.Context, tx *sql.Tx) error { @@ -52,8 +56,72 @@ func UpDropEventReferenceSHAPrevEvents(ctx context.Context, tx *sql.Tx) error { return fmt.Errorf("tx.ExecContext: %w", err) } + // figure out if there are duplicates + dupeRows, err := tx.QueryContext(ctx, `SELECT previous_event_id FROM _roomserver_previous_events GROUP BY previous_event_id HAVING count(previous_event_id) > 1`) + if err != nil { + return fmt.Errorf("failed to query duplicate event ids") + } + defer internal.CloseAndLogIfError(ctx, dupeRows, "failed to close rows") + + var prevEvents []string + var prevEventID string + for dupeRows.Next() { + if err = dupeRows.Scan(&prevEventID); err != nil { + return err + } + prevEvents = append(prevEvents, prevEventID) + } + if dupeRows.Err() != nil { + return dupeRows.Err() + } + + // if we found duplicates, check if we can combine them, e.g. they are in the same room + for _, dupeID := range prevEvents { + var dupeNIDsRows *sql.Rows + dupeNIDsRows, err = tx.QueryContext(ctx, `SELECT event_nids FROM _roomserver_previous_events WHERE previous_event_id = $1`, dupeID) + if err != nil { + return fmt.Errorf("failed to query duplicate event ids") + } + defer internal.CloseAndLogIfError(ctx, dupeNIDsRows, "failed to close rows") + var dupeNIDs []int64 + for dupeNIDsRows.Next() { + var nids pq.Int64Array + if err = dupeNIDsRows.Scan(&nids); err != nil { + return err + } + dupeNIDs = append(dupeNIDs, nids...) + } + + if dupeNIDsRows.Err() != nil { + return dupeNIDsRows.Err() + } + // dedupe NIDs + dupeNIDs = dupeNIDs[:util.SortAndUnique(nids(dupeNIDs))] + // now that we have all NIDs, check which room they belong to + var roomCount int + err = tx.QueryRowContext(ctx, `SELECT count(distinct room_nid) FROM roomserver_events WHERE event_nid IN ($1)`, pq.Array(dupeNIDs)).Scan(&roomCount) + if err != nil { + return err + } + // if the events are from different rooms, that's bad and we can't continue + if roomCount > 1 { + return fmt.Errorf("detected events (%v) referenced for different rooms (%v)", dupeNIDs, roomCount) + } + // otherwise delete the dupes + _, err = tx.ExecContext(ctx, "DELETE FROM _roomserver_previous_events WHERE previous_event_id = $1", dupeID) + if err != nil { + return fmt.Errorf("unable to delete duplicates: %w", err) + } + + // insert combined values + _, err = tx.ExecContext(ctx, "INSERT INTO _roomserver_previous_events (previous_event_id, event_nids) VALUES ($1, $2)", dupeID, pq.Array(dupeNIDs)) + if err != nil { + return fmt.Errorf("unable to insert new event NIDs: %w", err) + } + } + // move data - if _, err := tx.ExecContext(ctx, ` + if _, err = tx.ExecContext(ctx, ` INSERT INTO roomserver_previous_events ( previous_event_id, event_nids @@ -64,9 +132,15 @@ INSERT return fmt.Errorf("tx.ExecContext: %w", err) } // drop old table - _, err := tx.ExecContext(ctx, `DROP TABLE _roomserver_previous_events;`) + _, err = tx.ExecContext(ctx, `DROP TABLE _roomserver_previous_events;`) if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) } return nil } + +type nids []int64 + +func (s nids) Len() int { return len(s) } +func (s nids) Less(i, j int) bool { return s[i] < s[j] } +func (s nids) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/roomserver/storage/sqlite3/deltas/20230516154000_drop_reference_sha_test.go b/roomserver/storage/sqlite3/deltas/20230516154000_drop_reference_sha_test.go new file mode 100644 index 000000000..547d9703b --- /dev/null +++ b/roomserver/storage/sqlite3/deltas/20230516154000_drop_reference_sha_test.go @@ -0,0 +1,59 @@ +package deltas + +import ( + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/test/testrig" + "github.com/stretchr/testify/assert" +) + +func TestUpDropEventReferenceSHAPrevEvents(t *testing.T) { + + cfg, ctx, close := testrig.CreateConfig(t, test.DBTypeSQLite) + defer close() + + db, err := sqlutil.Open(&cfg.RoomServer.Database, sqlutil.NewExclusiveWriter()) + assert.Nil(t, err) + assert.NotNil(t, db) + defer db.Close() + + // create the table in the old layout + _, err = db.ExecContext(ctx.Context(), ` + CREATE TABLE IF NOT EXISTS roomserver_previous_events ( + previous_event_id TEXT NOT NULL, + previous_reference_sha256 BLOB, + event_nids TEXT NOT NULL, + UNIQUE (previous_event_id, previous_reference_sha256) + );`) + assert.Nil(t, err) + + // create the events table as well, slimmed down with one eventNID + _, err = db.ExecContext(ctx.Context(), ` + CREATE TABLE IF NOT EXISTS roomserver_events ( + event_nid INTEGER PRIMARY KEY AUTOINCREMENT, + room_nid INTEGER NOT NULL +); + +INSERT INTO roomserver_events (event_nid, room_nid) VALUES (1, 1) +`) + assert.Nil(t, err) + + // insert duplicate prev events with different event_nids + stmt, err := db.PrepareContext(ctx.Context(), `INSERT INTO roomserver_previous_events (previous_event_id, event_nids, previous_reference_sha256) VALUES ($1, $2, $3)`) + assert.Nil(t, err) + assert.NotNil(t, stmt) + _, err = stmt.ExecContext(ctx.Context(), "1", "{1,2}", "a") + assert.Nil(t, err) + _, err = stmt.ExecContext(ctx.Context(), "1", "{1,2,3}", "b") + assert.Nil(t, err) + + // execute the migration + txn, err := db.Begin() + assert.Nil(t, err) + assert.NotNil(t, txn) + err = UpDropEventReferenceSHAPrevEvents(ctx.Context(), txn) + defer txn.Rollback() + assert.NoError(t, err) +} From cbdc601f1b6d1c2a648b69ff44b3a49916f4d31a Mon Sep 17 00:00:00 2001 From: devonh Date: Wed, 31 May 2023 15:27:08 +0000 Subject: [PATCH 4/5] Move CreateRoom logic to Roomserver (#3093) Move create room logic over to roomserver. --- clientapi/routing/createroom.go | 550 +++--------------- clientapi/routing/joinroom_test.go | 5 +- clientapi/routing/membership.go | 59 +- clientapi/routing/profile.go | 2 +- clientapi/routing/redaction.go | 2 +- clientapi/routing/sendevent.go | 2 +- clientapi/routing/server_notices.go | 2 +- clientapi/threepid/invites.go | 2 +- cmd/dendrite-upgrade-tests/tests.go | 5 +- federationapi/routing/join.go | 2 +- federationapi/routing/leave.go | 2 +- go.mod | 2 +- go.sum | 4 +- internal/eventutil/events.go | 7 +- roomserver/api/api.go | 2 + roomserver/api/perform.go | 24 + roomserver/internal/alias.go | 2 +- roomserver/internal/api.go | 13 + roomserver/internal/input/input_events.go | 2 +- roomserver/internal/perform/perform_admin.go | 4 +- .../internal/perform/perform_create_room.go | 498 ++++++++++++++++ roomserver/internal/perform/perform_join.go | 2 +- roomserver/internal/perform/perform_leave.go | 2 +- .../internal/perform/perform_upgrade.go | 39 +- 24 files changed, 684 insertions(+), 550 deletions(-) create mode 100644 roomserver/internal/perform/perform_create_room.go diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 7a7a85e85..aaa305f06 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -22,17 +22,13 @@ import ( "strings" "time" - "github.com/getsentry/sentry-go" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/roomserver/types" roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/dendrite/clientapi/httputil" - "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -41,32 +37,19 @@ import ( // https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-createroom type createRoomRequest struct { - Invite []string `json:"invite"` - Name string `json:"name"` - Visibility string `json:"visibility"` - Topic string `json:"topic"` - Preset string `json:"preset"` - CreationContent json.RawMessage `json:"creation_content"` - InitialState []fledglingEvent `json:"initial_state"` - RoomAliasName string `json:"room_alias_name"` - RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` - PowerLevelContentOverride json.RawMessage `json:"power_level_content_override"` - IsDirect bool `json:"is_direct"` + Invite []string `json:"invite"` + Name string `json:"name"` + Visibility string `json:"visibility"` + Topic string `json:"topic"` + Preset string `json:"preset"` + CreationContent json.RawMessage `json:"creation_content"` + InitialState []gomatrixserverlib.FledglingEvent `json:"initial_state"` + RoomAliasName string `json:"room_alias_name"` + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` + PowerLevelContentOverride json.RawMessage `json:"power_level_content_override"` + IsDirect bool `json:"is_direct"` } -const ( - presetPrivateChat = "private_chat" - presetTrustedPrivateChat = "trusted_private_chat" - presetPublicChat = "public_chat" -) - -const ( - historyVisibilityShared = "shared" - // TODO: These should be implemented once history visibility is implemented - // historyVisibilityWorldReadable = "world_readable" - // historyVisibilityInvited = "invited" -) - func (r createRoomRequest) Validate() *util.JSONResponse { whitespace := "\t\n\x0b\x0c\r " // https://docs.python.org/2/library/string.html#string.whitespace // https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/handlers/room.py#L81 @@ -78,12 +61,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse { } } for _, userID := range r.Invite { - // TODO: We should put user ID parsing code into gomatrixserverlib and use that instead - // (see https://github.com/matrix-org/gomatrixserverlib/blob/3394e7c7003312043208aa73727d2256eea3d1f6/eventcontent.go#L347 ) - // It should be a struct (with pointers into a single string to avoid copying) and - // we should update all refs to use UserID types rather than strings. - // https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/types.py#L92 - if _, _, err := gomatrixserverlib.SplitID('@', userID); err != nil { + if _, err := spec.NewUserID(userID, true); err != nil { return &util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.BadJSON("user id must be in the form @localpart:domain"), @@ -91,7 +69,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse { } } switch r.Preset { - case presetPrivateChat, presetTrustedPrivateChat, presetPublicChat, "": + case spec.PresetPrivateChat, spec.PresetTrustedPrivateChat, spec.PresetPublicChat, "": default: return &util.JSONResponse{ Code: http.StatusBadRequest, @@ -129,13 +107,6 @@ type createRoomResponse struct { RoomAlias string `json:"room_alias,omitempty"` // in synapse not spec } -// fledglingEvent is a helper representation of an event used when creating many events in succession. -type fledglingEvent struct { - Type string `json:"type"` - StateKey string `json:"state_key"` - Content interface{} `json:"content"` -} - // CreateRoom implements /createRoom func CreateRoom( req *http.Request, device *api.Device, @@ -143,12 +114,12 @@ func CreateRoom( profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI, ) util.JSONResponse { - var r createRoomRequest - resErr := httputil.UnmarshalJSONRequest(req, &r) + var createRequest createRoomRequest + resErr := httputil.UnmarshalJSONRequest(req, &createRequest) if resErr != nil { return *resErr } - if resErr = r.Validate(); resErr != nil { + if resErr = createRequest.Validate(); resErr != nil { return *resErr } evTime, err := httputil.ParseTSParam(req) @@ -158,46 +129,52 @@ func CreateRoom( JSON: spec.InvalidParam(err.Error()), } } - return createRoom(req.Context(), r, device, cfg, profileAPI, rsAPI, asAPI, evTime) + return createRoom(req.Context(), createRequest, device, cfg, profileAPI, rsAPI, asAPI, evTime) } // createRoom implements /createRoom -// nolint: gocyclo func createRoom( ctx context.Context, - r createRoomRequest, device *api.Device, + // TODO: remove dependency on createRoomRequest + createRequest createRoomRequest, device *api.Device, cfg *config.ClientAPI, profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time, ) util.JSONResponse { - _, userDomain, err := gomatrixserverlib.SplitID('@', device.UserID) + userID, err := spec.NewUserID(device.UserID, true) if err != nil { - util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.SplitID failed") + util.GetLogger(ctx).WithError(err).Error("invalid userID") return util.JSONResponse{ Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}, } } - if !cfg.Matrix.IsLocalServerName(userDomain) { + if !cfg.Matrix.IsLocalServerName(userID.Domain()) { return util.JSONResponse{ Code: http.StatusForbidden, - JSON: spec.Forbidden(fmt.Sprintf("User domain %q not configured locally", userDomain)), + JSON: spec.Forbidden(fmt.Sprintf("User domain %q not configured locally", userID.Domain())), } } - // TODO (#267): Check room ID doesn't clash with an existing one, and we - // probably shouldn't be using pseudo-random strings, maybe GUIDs? - roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), userDomain) - logger := util.GetLogger(ctx) - userID := device.UserID + + // TODO: Check room ID doesn't clash with an existing one, and we + // probably shouldn't be using pseudo-random strings, maybe GUIDs? + roomID, err := spec.NewRoomID(fmt.Sprintf("!%s:%s", util.RandomString(16), userID.Domain())) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("invalid roomID") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } // Clobber keys: creator, room_version roomVersion := roomserverVersion.DefaultRoomVersion() - if r.RoomVersion != "" { - candidateVersion := gomatrixserverlib.RoomVersion(r.RoomVersion) + if createRequest.RoomVersion != "" { + candidateVersion := gomatrixserverlib.RoomVersion(createRequest.RoomVersion) _, roomVersionError := roomserverVersion.SupportedRoomVersion(candidateVersion) if roomVersionError != nil { return util.JSONResponse{ @@ -208,17 +185,13 @@ func createRoom( roomVersion = candidateVersion } - // TODO: visibility/presets/raw initial state - // TODO: Create room alias association - // Make sure this doesn't fall into an application service's namespace though! - logger.WithFields(log.Fields{ - "userID": userID, - "roomID": roomID, + "userID": userID.String(), + "roomID": roomID.String(), "roomVersion": roomVersion, }).Info("Creating new room") - profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, profileAPI) + profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID.String(), asAPI, profileAPI) if err != nil { util.GetLogger(ctx).WithError(err).Error("appserviceAPI.RetrieveUserProfile failed") return util.JSONResponse{ @@ -227,427 +200,38 @@ func createRoom( } } - createContent := map[string]interface{}{} - if len(r.CreationContent) > 0 { - if err = json.Unmarshal(r.CreationContent, &createContent); err != nil { - util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for creation_content failed") - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.BadJSON("invalid create content"), - } - } + userDisplayName := profile.DisplayName + userAvatarURL := profile.AvatarURL + + keyID := cfg.Matrix.KeyID + privateKey := cfg.Matrix.PrivateKey + + req := roomserverAPI.PerformCreateRoomRequest{ + InvitedUsers: createRequest.Invite, + RoomName: createRequest.Name, + Visibility: createRequest.Visibility, + Topic: createRequest.Topic, + StatePreset: createRequest.Preset, + CreationContent: createRequest.CreationContent, + InitialState: createRequest.InitialState, + RoomAliasName: createRequest.RoomAliasName, + RoomVersion: roomVersion, + PowerLevelContentOverride: createRequest.PowerLevelContentOverride, + IsDirect: createRequest.IsDirect, + + UserDisplayName: userDisplayName, + UserAvatarURL: userAvatarURL, + KeyID: keyID, + PrivateKey: privateKey, + EventTime: evTime, } - createContent["creator"] = userID - createContent["room_version"] = roomVersion - powerLevelContent := eventutil.InitialPowerLevelsContent(userID) - joinRuleContent := gomatrixserverlib.JoinRuleContent{ - JoinRule: spec.Invite, - } - historyVisibilityContent := gomatrixserverlib.HistoryVisibilityContent{ - HistoryVisibility: historyVisibilityShared, - } - - if r.PowerLevelContentOverride != nil { - // Merge powerLevelContentOverride fields by unmarshalling it atop the defaults - err = json.Unmarshal(r.PowerLevelContentOverride, &powerLevelContent) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for power_level_content_override failed") - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.BadJSON("malformed power_level_content_override"), - } - } - } - - var guestsCanJoin bool - switch r.Preset { - case presetPrivateChat: - joinRuleContent.JoinRule = spec.Invite - historyVisibilityContent.HistoryVisibility = historyVisibilityShared - guestsCanJoin = true - case presetTrustedPrivateChat: - joinRuleContent.JoinRule = spec.Invite - historyVisibilityContent.HistoryVisibility = historyVisibilityShared - for _, invitee := range r.Invite { - powerLevelContent.Users[invitee] = 100 - } - guestsCanJoin = true - case presetPublicChat: - joinRuleContent.JoinRule = spec.Public - historyVisibilityContent.HistoryVisibility = historyVisibilityShared - } - - createEvent := fledglingEvent{ - Type: spec.MRoomCreate, - Content: createContent, - } - powerLevelEvent := fledglingEvent{ - Type: spec.MRoomPowerLevels, - Content: powerLevelContent, - } - joinRuleEvent := fledglingEvent{ - Type: spec.MRoomJoinRules, - Content: joinRuleContent, - } - historyVisibilityEvent := fledglingEvent{ - Type: spec.MRoomHistoryVisibility, - Content: historyVisibilityContent, - } - membershipEvent := fledglingEvent{ - Type: spec.MRoomMember, - StateKey: userID, - Content: gomatrixserverlib.MemberContent{ - Membership: spec.Join, - DisplayName: profile.DisplayName, - AvatarURL: profile.AvatarURL, - }, - } - - var nameEvent *fledglingEvent - var topicEvent *fledglingEvent - var guestAccessEvent *fledglingEvent - var aliasEvent *fledglingEvent - - if r.Name != "" { - nameEvent = &fledglingEvent{ - Type: spec.MRoomName, - Content: eventutil.NameContent{ - Name: r.Name, - }, - } - } - - if r.Topic != "" { - topicEvent = &fledglingEvent{ - Type: spec.MRoomTopic, - Content: eventutil.TopicContent{ - Topic: r.Topic, - }, - } - } - - if guestsCanJoin { - guestAccessEvent = &fledglingEvent{ - Type: spec.MRoomGuestAccess, - Content: eventutil.GuestAccessContent{ - GuestAccess: "can_join", - }, - } - } - - var roomAlias string - if r.RoomAliasName != "" { - roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, userDomain) - // check it's free TODO: This races but is better than nothing - hasAliasReq := roomserverAPI.GetRoomIDForAliasRequest{ - Alias: roomAlias, - IncludeAppservices: false, - } - - var aliasResp roomserverAPI.GetRoomIDForAliasResponse - err = rsAPI.GetRoomIDForAlias(ctx, &hasAliasReq, &aliasResp) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - if aliasResp.RoomID != "" { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.RoomInUse("Room ID already exists."), - } - } - - aliasEvent = &fledglingEvent{ - Type: spec.MRoomCanonicalAlias, - Content: eventutil.CanonicalAlias{ - Alias: roomAlias, - }, - } - } - - var initialStateEvents []fledglingEvent - for i := range r.InitialState { - if r.InitialState[i].StateKey != "" { - initialStateEvents = append(initialStateEvents, r.InitialState[i]) - continue - } - - switch r.InitialState[i].Type { - case spec.MRoomCreate: - continue - - case spec.MRoomPowerLevels: - powerLevelEvent = r.InitialState[i] - - case spec.MRoomJoinRules: - joinRuleEvent = r.InitialState[i] - - case spec.MRoomHistoryVisibility: - historyVisibilityEvent = r.InitialState[i] - - case spec.MRoomGuestAccess: - guestAccessEvent = &r.InitialState[i] - - case spec.MRoomName: - nameEvent = &r.InitialState[i] - - case spec.MRoomTopic: - topicEvent = &r.InitialState[i] - - default: - initialStateEvents = append(initialStateEvents, r.InitialState[i]) - } - } - - // send events into the room in order of: - // 1- m.room.create - // 2- room creator join member - // 3- m.room.power_levels - // 4- m.room.join_rules - // 5- m.room.history_visibility - // 6- m.room.canonical_alias (opt) - // 7- m.room.guest_access (opt) - // 8- other initial state items - // 9- m.room.name (opt) - // 10- m.room.topic (opt) - // 11- invite events (opt) - with is_direct flag if applicable TODO - // 12- 3pid invite events (opt) TODO - // This differs from Synapse slightly. Synapse would vary the ordering of 3-7 - // depending on if those events were in "initial_state" or not. This made it - // harder to reason about, hence sticking to a strict static ordering. - // TODO: Synapse has txn/token ID on each event. Do we need to do this here? - eventsToMake := []fledglingEvent{ - createEvent, membershipEvent, powerLevelEvent, joinRuleEvent, historyVisibilityEvent, - } - if guestAccessEvent != nil { - eventsToMake = append(eventsToMake, *guestAccessEvent) - } - eventsToMake = append(eventsToMake, initialStateEvents...) - if nameEvent != nil { - eventsToMake = append(eventsToMake, *nameEvent) - } - if topicEvent != nil { - eventsToMake = append(eventsToMake, *topicEvent) - } - if aliasEvent != nil { - // TODO: bit of a chicken and egg problem here as the alias doesn't exist and cannot until we have made the room. - // This means we might fail creating the alias but say the canonical alias is something that doesn't exist. - eventsToMake = append(eventsToMake, *aliasEvent) - } - - // TODO: invite events - // TODO: 3pid invite events - - verImpl, err := gomatrixserverlib.GetRoomVersion(roomVersion) - if err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.BadJSON("unknown room version"), - } - } - - var builtEvents []*types.HeaderedEvent - authEvents := gomatrixserverlib.NewAuthEvents(nil) - for i, e := range eventsToMake { - depth := i + 1 // depth starts at 1 - - builder := verImpl.NewEventBuilderFromProtoEvent(&gomatrixserverlib.ProtoEvent{ - Sender: userID, - RoomID: roomID, - Type: e.Type, - StateKey: &e.StateKey, - Depth: int64(depth), - }) - err = builder.SetContent(e.Content) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("builder.SetContent failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - if i > 0 { - builder.PrevEvents = []string{builtEvents[i-1].EventID()} - } - var ev gomatrixserverlib.PDU - if err = builder.AddAuthEvents(&authEvents); err != nil { - util.GetLogger(ctx).WithError(err).Error("AddAuthEvents failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - ev, err = builder.Build(evTime, userDomain, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("buildEvent failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - - if err = gomatrixserverlib.Allowed(ev, &authEvents); err != nil { - util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.Allowed failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - - // Add the event to the list of auth events - builtEvents = append(builtEvents, &types.HeaderedEvent{PDU: ev}) - err = authEvents.AddEvent(ev) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("authEvents.AddEvent failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - } - - inputs := make([]roomserverAPI.InputRoomEvent, 0, len(builtEvents)) - for _, event := range builtEvents { - inputs = append(inputs, roomserverAPI.InputRoomEvent{ - Kind: roomserverAPI.KindNew, - Event: event, - Origin: userDomain, - SendAsServer: roomserverAPI.DoNotSendToOtherServers, - }) - } - if err = roomserverAPI.SendInputRoomEvents(ctx, rsAPI, device.UserDomain(), inputs, false); err != nil { - util.GetLogger(ctx).WithError(err).Error("roomserverAPI.SendInputRoomEvents failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - - // TODO(#269): Reserve room alias while we create the room. This stops us - // from creating the room but still failing due to the alias having already - // been taken. - if roomAlias != "" { - aliasReq := roomserverAPI.SetRoomAliasRequest{ - Alias: roomAlias, - RoomID: roomID, - UserID: userID, - } - - var aliasResp roomserverAPI.SetRoomAliasResponse - err = rsAPI.SetRoomAlias(ctx, &aliasReq, &aliasResp) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("aliasAPI.SetRoomAlias failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - - if aliasResp.AliasExists { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.RoomInUse("Room alias already exists."), - } - } - } - - // If this is a direct message then we should invite the participants. - if len(r.Invite) > 0 { - // Build some stripped state for the invite. - var globalStrippedState []fclient.InviteV2StrippedState - for _, event := range builtEvents { - // Chosen events from the spec: - // https://spec.matrix.org/v1.3/client-server-api/#stripped-state - switch event.Type() { - case spec.MRoomCreate: - fallthrough - case spec.MRoomName: - fallthrough - case spec.MRoomAvatar: - fallthrough - case spec.MRoomTopic: - fallthrough - case spec.MRoomCanonicalAlias: - fallthrough - case spec.MRoomEncryption: - fallthrough - case spec.MRoomMember: - fallthrough - case spec.MRoomJoinRules: - ev := event.PDU - globalStrippedState = append( - globalStrippedState, - fclient.NewInviteV2StrippedState(ev), - ) - } - } - - // Process the invites. - var inviteEvent *types.HeaderedEvent - for _, invitee := range r.Invite { - // Build the invite event. - inviteEvent, err = buildMembershipEvent( - ctx, invitee, "", profileAPI, device, spec.Invite, - roomID, r.IsDirect, cfg, evTime, rsAPI, asAPI, - ) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed") - continue - } - inviteStrippedState := append( - globalStrippedState, - fclient.NewInviteV2StrippedState(inviteEvent.PDU), - ) - // Send the invite event to the roomserver. - event := inviteEvent - err = rsAPI.PerformInvite(ctx, &roomserverAPI.PerformInviteRequest{ - Event: event, - InviteRoomState: inviteStrippedState, - RoomVersion: event.Version(), - SendAsServer: string(userDomain), - }) - switch e := err.(type) { - case roomserverAPI.ErrInvalidID: - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.Unknown(e.Error()), - } - case roomserverAPI.ErrNotAllowed: - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: spec.Forbidden(e.Error()), - } - case nil: - default: - util.GetLogger(ctx).WithError(err).Error("PerformInvite failed") - sentry.CaptureException(err) - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - } - } - - if r.Visibility == spec.Public { - // expose this room in the published room list - if err = rsAPI.PerformPublish(ctx, &roomserverAPI.PerformPublishRequest{ - RoomID: roomID, - Visibility: spec.Public, - }); err != nil { - util.GetLogger(ctx).WithError(err).Error("failed to publish room") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } + roomAlias, createRes := rsAPI.PerformCreateRoom(ctx, *userID, *roomID, &req) + if createRes != nil { + return *createRes } response := createRoomResponse{ - RoomID: roomID, + RoomID: roomID.String(), RoomAlias: roomAlias, } diff --git a/clientapi/routing/joinroom_test.go b/clientapi/routing/joinroom_test.go index 4b67b09f0..0ddff8a95 100644 --- a/clientapi/routing/joinroom_test.go +++ b/clientapi/routing/joinroom_test.go @@ -11,6 +11,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/roomserver" @@ -63,7 +64,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { IsDirect: true, Topic: "testing", Visibility: "public", - Preset: presetPublicChat, + Preset: spec.PresetPublicChat, RoomAliasName: "alias", Invite: []string{bob.ID}, }, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now()) @@ -78,7 +79,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { IsDirect: true, Topic: "testing", Visibility: "public", - Preset: presetPublicChat, + Preset: spec.PresetPublicChat, Invite: []string{charlie.ID}, }, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now()) crRespWithGuestAccess, ok := resp.JSON.(createRoomResponse) diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 4f2a0e394..0fe0a4ade 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -16,12 +16,14 @@ package routing import ( "context" + "crypto/ed25519" "fmt" "net/http" "time" "github.com/getsentry/sentry-go" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" @@ -308,6 +310,41 @@ func sendInvite( }, nil } +func buildMembershipEventDirect( + ctx context.Context, + targetUserID, reason string, userDisplayName, userAvatarURL string, + sender string, senderDomain spec.ServerName, + membership, roomID string, isDirect bool, + keyID gomatrixserverlib.KeyID, privateKey ed25519.PrivateKey, evTime time.Time, + rsAPI roomserverAPI.ClientRoomserverAPI, +) (*types.HeaderedEvent, error) { + proto := gomatrixserverlib.ProtoEvent{ + Sender: sender, + RoomID: roomID, + Type: "m.room.member", + StateKey: &targetUserID, + } + + content := gomatrixserverlib.MemberContent{ + Membership: membership, + DisplayName: userDisplayName, + AvatarURL: userAvatarURL, + Reason: reason, + IsDirect: isDirect, + } + + if err := proto.SetContent(content); err != nil { + return nil, err + } + + identity := &fclient.SigningIdentity{ + ServerName: senderDomain, + KeyID: keyID, + PrivateKey: privateKey, + } + return eventutil.QueryAndBuildEvent(ctx, &proto, identity, evTime, rsAPI, nil) +} + func buildMembershipEvent( ctx context.Context, targetUserID, reason string, profileAPI userapi.ClientUserAPI, @@ -321,31 +358,13 @@ func buildMembershipEvent( return nil, err } - proto := gomatrixserverlib.ProtoEvent{ - Sender: device.UserID, - RoomID: roomID, - Type: "m.room.member", - StateKey: &targetUserID, - } - - content := gomatrixserverlib.MemberContent{ - Membership: membership, - DisplayName: profile.DisplayName, - AvatarURL: profile.AvatarURL, - Reason: reason, - IsDirect: isDirect, - } - - if err = proto.SetContent(content); err != nil { - return nil, err - } - identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain()) if err != nil { return nil, err } - return eventutil.QueryAndBuildEvent(ctx, &proto, cfg.Matrix, identity, evTime, rsAPI, nil) + return buildMembershipEventDirect(ctx, targetUserID, reason, profile.DisplayName, profile.AvatarURL, + device.UserID, device.UserDomain(), membership, roomID, isDirect, identity.KeyID, identity.PrivateKey, evTime, rsAPI) } // loadProfile lookups the profile of a given user from the database and returns diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 76129f0a8..2c9d0cbbe 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -387,7 +387,7 @@ func buildMembershipEvents( return nil, err } - event, err := eventutil.QueryAndBuildEvent(ctx, &proto, cfg.Matrix, identity, evTime, rsAPI, nil) + event, err := eventutil.QueryAndBuildEvent(ctx, &proto, identity, evTime, rsAPI, nil) if err != nil { return nil, err } diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go index ed70e5c5c..883126423 100644 --- a/clientapi/routing/redaction.go +++ b/clientapi/routing/redaction.go @@ -137,7 +137,7 @@ func SendRedaction( } var queryRes roomserverAPI.QueryLatestEventsAndStateResponse - e, err := eventutil.QueryAndBuildEvent(req.Context(), &proto, cfg.Matrix, identity, time.Now(), rsAPI, &queryRes) + e, err := eventutil.QueryAndBuildEvent(req.Context(), &proto, identity, time.Now(), rsAPI, &queryRes) if errors.Is(err, eventutil.ErrRoomNoExists{}) { return util.JSONResponse{ Code: http.StatusNotFound, diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index bc14642f8..1a2e25c9d 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -293,7 +293,7 @@ func generateSendEvent( } var queryRes api.QueryLatestEventsAndStateResponse - e, err := eventutil.QueryAndBuildEvent(ctx, &proto, cfg.Matrix, identity, evTime, rsAPI, &queryRes) + e, err := eventutil.QueryAndBuildEvent(ctx, &proto, identity, evTime, rsAPI, &queryRes) switch specificErr := err.(type) { case nil: case eventutil.ErrRoomNoExists: diff --git a/clientapi/routing/server_notices.go b/clientapi/routing/server_notices.go index ad50cc80b..06714ed1f 100644 --- a/clientapi/routing/server_notices.go +++ b/clientapi/routing/server_notices.go @@ -155,7 +155,7 @@ func SendServerNotice( Invite: []string{r.UserID}, Name: cfgNotices.RoomName, Visibility: "private", - Preset: presetPrivateChat, + Preset: spec.PresetPrivateChat, CreationContent: cc, RoomVersion: roomVersion, PowerLevelContentOverride: pl, diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index c296939d5..9f4f62e43 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -380,7 +380,7 @@ func emit3PIDInviteEvent( } queryRes := api.QueryLatestEventsAndStateResponse{} - event, err := eventutil.QueryAndBuildEvent(ctx, proto, cfg.Matrix, identity, evTime, rsAPI, &queryRes) + event, err := eventutil.QueryAndBuildEvent(ctx, proto, identity, evTime, rsAPI, &queryRes) if err != nil { return err } diff --git a/cmd/dendrite-upgrade-tests/tests.go b/cmd/dendrite-upgrade-tests/tests.go index 03438bd4d..692ab34ef 100644 --- a/cmd/dendrite-upgrade-tests/tests.go +++ b/cmd/dendrite-upgrade-tests/tests.go @@ -9,6 +9,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" ) const userPassword = "this_is_a_long_password" @@ -56,7 +57,7 @@ func runTests(baseURL string, v *semver.Version) error { // create DM room, join it and exchange messages createRoomResp, err := users[0].client.CreateRoom(&gomatrix.ReqCreateRoom{ - Preset: "trusted_private_chat", + Preset: spec.PresetTrustedPrivateChat, Invite: []string{users[1].userID}, IsDirect: true, }) @@ -98,7 +99,7 @@ func runTests(baseURL string, v *semver.Version) error { publicRoomID := "" createRoomResp, err = users[0].client.CreateRoom(&gomatrix.ReqCreateRoom{ RoomAliasName: "global", - Preset: "public_chat", + Preset: spec.PresetPublicChat, }) if err != nil { // this is okay and expected if the room already exists and the aliases clash // try to join it diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 4cbfc5e87..03d3309ae 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -121,7 +121,7 @@ func MakeJoin( queryRes := api.QueryLatestEventsAndStateResponse{ RoomVersion: roomVersion, } - event, err := eventutil.QueryAndBuildEvent(httpReq.Context(), proto, cfg.Matrix, identity, time.Now(), rsAPI, &queryRes) + event, err := eventutil.QueryAndBuildEvent(httpReq.Context(), proto, identity, time.Now(), rsAPI, &queryRes) switch e := err.(type) { case nil: case eventutil.ErrRoomNoExists: diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 3e576e09c..a767168d8 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -66,7 +66,7 @@ func MakeLeave( } queryRes := api.QueryLatestEventsAndStateResponse{} - event, err := eventutil.QueryAndBuildEvent(httpReq.Context(), proto, cfg.Matrix, identity, time.Now(), rsAPI, &queryRes) + event, err := eventutil.QueryAndBuildEvent(httpReq.Context(), proto, identity, time.Now(), rsAPI, &queryRes) switch e := err.(type) { case nil: case eventutil.ErrRoomNoExists: diff --git a/go.mod b/go.mod index 360ddf5b2..0e979de6f 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20230524095531-95ba6c68efb6 + github.com/matrix-org/gomatrixserverlib v0.0.0-20230531143710-c681a0658246 github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 github.com/mattn/go-sqlite3 v1.14.16 diff --git a/go.sum b/go.sum index 4a6054af8..8baa50e85 100644 --- a/go.sum +++ b/go.sum @@ -323,8 +323,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230524095531-95ba6c68efb6 h1:FQpdh/KGCCQJytz4GAdG6pbx3DJ1HNzdKFc/BCZ0hP0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230524095531-95ba6c68efb6/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230531143710-c681a0658246 h1:1sYXx7p9BIf0R7OIV/TZg3SCvNehEQPCKNqwV1ONfwU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230531143710-c681a0658246/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a h1:awrPDf9LEFySxTLKYBMCiObelNx/cBuv/wzllvCCH3A= github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a/go.mod h1:HchJX9oKMXaT2xYFs0Ha/6Zs06mxLU8k6F1ODnrGkeQ= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= diff --git a/internal/eventutil/events.go b/internal/eventutil/events.go index ca052c310..0f73db2d5 100644 --- a/internal/eventutil/events.go +++ b/internal/eventutil/events.go @@ -22,7 +22,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" @@ -51,7 +50,7 @@ func (e ErrRoomNoExists) Unwrap() error { // Returns an error if something else went wrong func QueryAndBuildEvent( ctx context.Context, - proto *gomatrixserverlib.ProtoEvent, cfg *config.Global, + proto *gomatrixserverlib.ProtoEvent, identity *fclient.SigningIdentity, evTime time.Time, rsAPI api.QueryLatestEventsAndStateAPI, queryRes *api.QueryLatestEventsAndStateResponse, ) (*types.HeaderedEvent, error) { @@ -64,14 +63,14 @@ func QueryAndBuildEvent( // This can pass through a ErrRoomNoExists to the caller return nil, err } - return BuildEvent(ctx, proto, cfg, identity, evTime, eventsNeeded, queryRes) + return BuildEvent(ctx, proto, identity, evTime, eventsNeeded, queryRes) } // BuildEvent builds a Matrix event from the builder and QueryLatestEventsAndStateResponse // provided. func BuildEvent( ctx context.Context, - proto *gomatrixserverlib.ProtoEvent, cfg *config.Global, + proto *gomatrixserverlib.ProtoEvent, identity *fclient.SigningIdentity, evTime time.Time, eventsNeeded *gomatrixserverlib.StateNeeded, queryRes *api.QueryLatestEventsAndStateResponse, ) (*types.HeaderedEvent, error) { diff --git a/roomserver/api/api.go b/roomserver/api/api.go index 213e16e5d..571aa40b3 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -5,6 +5,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" asAPI "github.com/matrix-org/dendrite/appservice/api" fsAPI "github.com/matrix-org/dendrite/federationapi/api" @@ -169,6 +170,7 @@ type ClientRoomserverAPI interface { GetRoomIDForAlias(ctx context.Context, req *GetRoomIDForAliasRequest, res *GetRoomIDForAliasResponse) error GetAliasesForRoomID(ctx context.Context, req *GetAliasesForRoomIDRequest, res *GetAliasesForRoomIDResponse) error + PerformCreateRoom(ctx context.Context, userID spec.UserID, roomID spec.RoomID, createRequest *PerformCreateRoomRequest) (string, *util.JSONResponse) // PerformRoomUpgrade upgrades a room to a newer version PerformRoomUpgrade(ctx context.Context, roomID, userID string, roomVersion gomatrixserverlib.RoomVersion) (newRoomID string, err error) PerformAdminEvacuateRoom(ctx context.Context, roomID string) (affected []string, err error) diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index c6e5f5a1c..8d9742c69 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -1,6 +1,10 @@ package api import ( + "crypto/ed25519" + "encoding/json" + "time" + "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/fclient" @@ -8,6 +12,26 @@ import ( "github.com/matrix-org/util" ) +type PerformCreateRoomRequest struct { + InvitedUsers []string + RoomName string + Visibility string + Topic string + StatePreset string + CreationContent json.RawMessage + InitialState []gomatrixserverlib.FledglingEvent + RoomAliasName string + RoomVersion gomatrixserverlib.RoomVersion + PowerLevelContentOverride json.RawMessage + IsDirect bool + + UserDisplayName string + UserAvatarURL string + KeyID gomatrixserverlib.KeyID + PrivateKey ed25519.PrivateKey + EventTime time.Time +} + type PerformJoinRequest struct { RoomIDOrAlias string `json:"room_id_or_alias"` UserID string `json:"user_id"` diff --git a/roomserver/internal/alias.go b/roomserver/internal/alias.go index 4d2de9a5a..52b90cf4e 100644 --- a/roomserver/internal/alias.go +++ b/roomserver/internal/alias.go @@ -208,7 +208,7 @@ func (r *RoomserverInternalAPI) RemoveRoomAlias( return err } - newEvent, err := eventutil.BuildEvent(ctx, proto, &r.Cfg.Global, identity, time.Now(), &eventsNeeded, stateRes) + newEvent, err := eventutil.BuildEvent(ctx, proto, identity, time.Now(), &eventsNeeded, stateRes) if err != nil { return err } diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index 81904c8b8..f61f89183 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -6,6 +6,7 @@ import ( "github.com/getsentry/sentry-go" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" "github.com/nats-io/nats.go" "github.com/sirupsen/logrus" @@ -40,6 +41,7 @@ type RoomserverInternalAPI struct { *perform.Forgetter *perform.Upgrader *perform.Admin + *perform.Creator ProcessContext *process.ProcessContext DB storage.Database Cfg *config.Dendrite @@ -191,6 +193,11 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio Queryer: r.Queryer, Leaver: r.Leaver, } + r.Creator = &perform.Creator{ + DB: r.DB, + Cfg: &r.Cfg.RoomServer, + RSAPI: r, + } if err := r.Inputer.Start(); err != nil { logrus.WithError(err).Panic("failed to start roomserver input API") @@ -206,6 +213,12 @@ func (r *RoomserverInternalAPI) SetAppserviceAPI(asAPI asAPI.AppServiceInternalA r.asAPI = asAPI } +func (r *RoomserverInternalAPI) PerformCreateRoom( + ctx context.Context, userID spec.UserID, roomID spec.RoomID, createRequest *api.PerformCreateRoomRequest, +) (string, *util.JSONResponse) { + return r.Creator.PerformCreateRoom(ctx, userID, roomID, createRequest) +} + func (r *RoomserverInternalAPI) PerformInvite( ctx context.Context, req *api.PerformInviteRequest, diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 02a1a2802..386083f6e 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -872,7 +872,7 @@ func (r *Inputer) kickGuests(ctx context.Context, event gomatrixserverlib.PDU, r return err } - event, err := eventutil.BuildEvent(ctx, fledglingEvent, r.Cfg.Matrix, r.SigningIdentity, time.Now(), &eventsNeeded, latestRes) + event, err := eventutil.BuildEvent(ctx, fledglingEvent, r.SigningIdentity, time.Now(), &eventsNeeded, latestRes) if err != nil { return err } diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go index 8d21b7829..575525e21 100644 --- a/roomserver/internal/perform/perform_admin.go +++ b/roomserver/internal/perform/perform_admin.go @@ -119,7 +119,7 @@ func (r *Admin) PerformAdminEvacuateRoom( continue } - event, err = eventutil.BuildEvent(ctx, fledglingEvent, r.Cfg.Matrix, identity, time.Now(), &eventsNeeded, latestRes) + event, err = eventutil.BuildEvent(ctx, fledglingEvent, identity, time.Now(), &eventsNeeded, latestRes) if err != nil { return nil, err } @@ -312,7 +312,7 @@ func (r *Admin) PerformAdminDownloadState( return err } - ev, err := eventutil.BuildEvent(ctx, proto, r.Cfg.Matrix, identity, time.Now(), &eventsNeeded, queryRes) + ev, err := eventutil.BuildEvent(ctx, proto, identity, time.Now(), &eventsNeeded, queryRes) if err != nil { return fmt.Errorf("eventutil.BuildEvent: %w", err) } diff --git a/roomserver/internal/perform/perform_create_room.go b/roomserver/internal/perform/perform_create_room.go new file mode 100644 index 000000000..0f9170087 --- /dev/null +++ b/roomserver/internal/perform/perform_create_room.go @@ -0,0 +1,498 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package perform + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/getsentry/sentry-go" + "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" +) + +const ( + historyVisibilityShared = "shared" +) + +type Creator struct { + DB storage.Database + Cfg *config.RoomServer + RSAPI api.RoomserverInternalAPI +} + +// PerformCreateRoom handles all the steps necessary to create a new room. +// nolint: gocyclo +func (c *Creator) PerformCreateRoom(ctx context.Context, userID spec.UserID, roomID spec.RoomID, createRequest *api.PerformCreateRoomRequest) (string, *util.JSONResponse) { + verImpl, err := gomatrixserverlib.GetRoomVersion(createRequest.RoomVersion) + if err != nil { + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("unknown room version"), + } + } + + createContent := map[string]interface{}{} + if len(createRequest.CreationContent) > 0 { + if err = json.Unmarshal(createRequest.CreationContent, &createContent); err != nil { + util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for creation_content failed") + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("invalid create content"), + } + } + } + createContent["creator"] = userID.String() + createContent["room_version"] = createRequest.RoomVersion + powerLevelContent := eventutil.InitialPowerLevelsContent(userID.String()) + joinRuleContent := gomatrixserverlib.JoinRuleContent{ + JoinRule: spec.Invite, + } + historyVisibilityContent := gomatrixserverlib.HistoryVisibilityContent{ + HistoryVisibility: historyVisibilityShared, + } + + if createRequest.PowerLevelContentOverride != nil { + // Merge powerLevelContentOverride fields by unmarshalling it atop the defaults + err = json.Unmarshal(createRequest.PowerLevelContentOverride, &powerLevelContent) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for power_level_content_override failed") + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("malformed power_level_content_override"), + } + } + } + + var guestsCanJoin bool + switch createRequest.StatePreset { + case spec.PresetPrivateChat: + joinRuleContent.JoinRule = spec.Invite + historyVisibilityContent.HistoryVisibility = historyVisibilityShared + guestsCanJoin = true + case spec.PresetTrustedPrivateChat: + joinRuleContent.JoinRule = spec.Invite + historyVisibilityContent.HistoryVisibility = historyVisibilityShared + for _, invitee := range createRequest.InvitedUsers { + powerLevelContent.Users[invitee] = 100 + } + guestsCanJoin = true + case spec.PresetPublicChat: + joinRuleContent.JoinRule = spec.Public + historyVisibilityContent.HistoryVisibility = historyVisibilityShared + } + + createEvent := gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomCreate, + Content: createContent, + } + powerLevelEvent := gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomPowerLevels, + Content: powerLevelContent, + } + joinRuleEvent := gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomJoinRules, + Content: joinRuleContent, + } + historyVisibilityEvent := gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomHistoryVisibility, + Content: historyVisibilityContent, + } + membershipEvent := gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomMember, + StateKey: userID.String(), + Content: gomatrixserverlib.MemberContent{ + Membership: spec.Join, + DisplayName: createRequest.UserDisplayName, + AvatarURL: createRequest.UserAvatarURL, + }, + } + + var nameEvent *gomatrixserverlib.FledglingEvent + var topicEvent *gomatrixserverlib.FledglingEvent + var guestAccessEvent *gomatrixserverlib.FledglingEvent + var aliasEvent *gomatrixserverlib.FledglingEvent + + if createRequest.RoomName != "" { + nameEvent = &gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomName, + Content: eventutil.NameContent{ + Name: createRequest.RoomName, + }, + } + } + + if createRequest.Topic != "" { + topicEvent = &gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomTopic, + Content: eventutil.TopicContent{ + Topic: createRequest.Topic, + }, + } + } + + if guestsCanJoin { + guestAccessEvent = &gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomGuestAccess, + Content: eventutil.GuestAccessContent{ + GuestAccess: "can_join", + }, + } + } + + var roomAlias string + if createRequest.RoomAliasName != "" { + roomAlias = fmt.Sprintf("#%s:%s", createRequest.RoomAliasName, userID.Domain()) + // check it's free + // TODO: This races but is better than nothing + hasAliasReq := api.GetRoomIDForAliasRequest{ + Alias: roomAlias, + IncludeAppservices: false, + } + + var aliasResp api.GetRoomIDForAliasResponse + err = c.RSAPI.GetRoomIDForAlias(ctx, &hasAliasReq, &aliasResp) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + if aliasResp.RoomID != "" { + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.RoomInUse("Room ID already exists."), + } + } + + aliasEvent = &gomatrixserverlib.FledglingEvent{ + Type: spec.MRoomCanonicalAlias, + Content: eventutil.CanonicalAlias{ + Alias: roomAlias, + }, + } + } + + var initialStateEvents []gomatrixserverlib.FledglingEvent + for i := range createRequest.InitialState { + if createRequest.InitialState[i].StateKey != "" { + initialStateEvents = append(initialStateEvents, createRequest.InitialState[i]) + continue + } + + switch createRequest.InitialState[i].Type { + case spec.MRoomCreate: + continue + + case spec.MRoomPowerLevels: + powerLevelEvent = createRequest.InitialState[i] + + case spec.MRoomJoinRules: + joinRuleEvent = createRequest.InitialState[i] + + case spec.MRoomHistoryVisibility: + historyVisibilityEvent = createRequest.InitialState[i] + + case spec.MRoomGuestAccess: + guestAccessEvent = &createRequest.InitialState[i] + + case spec.MRoomName: + nameEvent = &createRequest.InitialState[i] + + case spec.MRoomTopic: + topicEvent = &createRequest.InitialState[i] + + default: + initialStateEvents = append(initialStateEvents, createRequest.InitialState[i]) + } + } + + // send events into the room in order of: + // 1- m.room.create + // 2- room creator join member + // 3- m.room.power_levels + // 4- m.room.join_rules + // 5- m.room.history_visibility + // 6- m.room.canonical_alias (opt) + // 7- m.room.guest_access (opt) + // 8- other initial state items + // 9- m.room.name (opt) + // 10- m.room.topic (opt) + // 11- invite events (opt) - with is_direct flag if applicable TODO + // 12- 3pid invite events (opt) TODO + // This differs from Synapse slightly. Synapse would vary the ordering of 3-7 + // depending on if those events were in "initial_state" or not. This made it + // harder to reason about, hence sticking to a strict static ordering. + // TODO: Synapse has txn/token ID on each event. Do we need to do this here? + eventsToMake := []gomatrixserverlib.FledglingEvent{ + createEvent, membershipEvent, powerLevelEvent, joinRuleEvent, historyVisibilityEvent, + } + if guestAccessEvent != nil { + eventsToMake = append(eventsToMake, *guestAccessEvent) + } + eventsToMake = append(eventsToMake, initialStateEvents...) + if nameEvent != nil { + eventsToMake = append(eventsToMake, *nameEvent) + } + if topicEvent != nil { + eventsToMake = append(eventsToMake, *topicEvent) + } + if aliasEvent != nil { + // TODO: bit of a chicken and egg problem here as the alias doesn't exist and cannot until we have made the room. + // This means we might fail creating the alias but say the canonical alias is something that doesn't exist. + eventsToMake = append(eventsToMake, *aliasEvent) + } + + // TODO: invite events + // TODO: 3pid invite events + + var builtEvents []*types.HeaderedEvent + authEvents := gomatrixserverlib.NewAuthEvents(nil) + for i, e := range eventsToMake { + depth := i + 1 // depth starts at 1 + + builder := verImpl.NewEventBuilderFromProtoEvent(&gomatrixserverlib.ProtoEvent{ + Sender: userID.String(), + RoomID: roomID.String(), + Type: e.Type, + StateKey: &e.StateKey, + Depth: int64(depth), + }) + err = builder.SetContent(e.Content) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("builder.SetContent failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + if i > 0 { + builder.PrevEvents = []string{builtEvents[i-1].EventID()} + } + var ev gomatrixserverlib.PDU + if err = builder.AddAuthEvents(&authEvents); err != nil { + util.GetLogger(ctx).WithError(err).Error("AddAuthEvents failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + ev, err = builder.Build(createRequest.EventTime, userID.Domain(), createRequest.KeyID, createRequest.PrivateKey) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("buildEvent failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + if err = gomatrixserverlib.Allowed(ev, &authEvents); err != nil { + util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.Allowed failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + // Add the event to the list of auth events + builtEvents = append(builtEvents, &types.HeaderedEvent{PDU: ev}) + err = authEvents.AddEvent(ev) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("authEvents.AddEvent failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + } + + inputs := make([]api.InputRoomEvent, 0, len(builtEvents)) + for _, event := range builtEvents { + inputs = append(inputs, api.InputRoomEvent{ + Kind: api.KindNew, + Event: event, + Origin: userID.Domain(), + SendAsServer: api.DoNotSendToOtherServers, + }) + } + if err = api.SendInputRoomEvents(ctx, c.RSAPI, userID.Domain(), inputs, false); err != nil { + util.GetLogger(ctx).WithError(err).Error("roomserverAPI.SendInputRoomEvents failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + // TODO(#269): Reserve room alias while we create the room. This stops us + // from creating the room but still failing due to the alias having already + // been taken. + if roomAlias != "" { + aliasReq := api.SetRoomAliasRequest{ + Alias: roomAlias, + RoomID: roomID.String(), + UserID: userID.String(), + } + + var aliasResp api.SetRoomAliasResponse + err = c.RSAPI.SetRoomAlias(ctx, &aliasReq, &aliasResp) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("aliasAPI.SetRoomAlias failed") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + if aliasResp.AliasExists { + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.RoomInUse("Room alias already exists."), + } + } + } + + // If this is a direct message then we should invite the participants. + if len(createRequest.InvitedUsers) > 0 { + // Build some stripped state for the invite. + var globalStrippedState []fclient.InviteV2StrippedState + for _, event := range builtEvents { + // Chosen events from the spec: + // https://spec.matrix.org/v1.3/client-server-api/#stripped-state + switch event.Type() { + case spec.MRoomCreate: + fallthrough + case spec.MRoomName: + fallthrough + case spec.MRoomAvatar: + fallthrough + case spec.MRoomTopic: + fallthrough + case spec.MRoomCanonicalAlias: + fallthrough + case spec.MRoomEncryption: + fallthrough + case spec.MRoomMember: + fallthrough + case spec.MRoomJoinRules: + ev := event.PDU + globalStrippedState = append( + globalStrippedState, + fclient.NewInviteV2StrippedState(ev), + ) + } + } + + // Process the invites. + var inviteEvent *types.HeaderedEvent + for _, invitee := range createRequest.InvitedUsers { + proto := gomatrixserverlib.ProtoEvent{ + Sender: userID.String(), + RoomID: roomID.String(), + Type: "m.room.member", + StateKey: &invitee, + } + + content := gomatrixserverlib.MemberContent{ + Membership: spec.Invite, + DisplayName: createRequest.UserDisplayName, + AvatarURL: createRequest.UserAvatarURL, + Reason: "", + IsDirect: createRequest.IsDirect, + } + + if err = proto.SetContent(content); err != nil { + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + // Build the invite event. + identity := &fclient.SigningIdentity{ + ServerName: userID.Domain(), + KeyID: createRequest.KeyID, + PrivateKey: createRequest.PrivateKey, + } + inviteEvent, err = eventutil.QueryAndBuildEvent(ctx, &proto, identity, createRequest.EventTime, c.RSAPI, nil) + + if err != nil { + util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed") + continue + } + inviteStrippedState := append( + globalStrippedState, + fclient.NewInviteV2StrippedState(inviteEvent.PDU), + ) + // Send the invite event to the roomserver. + event := inviteEvent + err = c.RSAPI.PerformInvite(ctx, &api.PerformInviteRequest{ + Event: event, + InviteRoomState: inviteStrippedState, + RoomVersion: event.Version(), + SendAsServer: string(userID.Domain()), + }) + switch e := err.(type) { + case api.ErrInvalidID: + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.Unknown(e.Error()), + } + case api.ErrNotAllowed: + return "", &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden(e.Error()), + } + case nil: + default: + util.GetLogger(ctx).WithError(err).Error("PerformInvite failed") + sentry.CaptureException(err) + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + } + } + + if createRequest.Visibility == spec.Public { + // expose this room in the published room list + if err = c.RSAPI.PerformPublish(ctx, &api.PerformPublishRequest{ + RoomID: roomID.String(), + Visibility: spec.Public, + }); err != nil { + util.GetLogger(ctx).WithError(err).Error("failed to publish room") + return "", &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + } + + // TODO: visibility/presets/raw initial state + // TODO: Create room alias association + // Make sure this doesn't fall into an application service's namespace though! + + return roomAlias, nil +} diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go index 5f4ad1861..34bea5b6d 100644 --- a/roomserver/internal/perform/perform_join.go +++ b/roomserver/internal/perform/perform_join.go @@ -284,7 +284,7 @@ func (r *Joiner) performJoinRoomByID( if err != nil { return "", "", fmt.Errorf("error joining local room: %q", err) } - event, err := eventutil.QueryAndBuildEvent(ctx, &proto, r.Cfg.Matrix, identity, time.Now(), r.RSAPI, &buildRes) + event, err := eventutil.QueryAndBuildEvent(ctx, &proto, identity, time.Now(), r.RSAPI, &buildRes) switch err.(type) { case nil: diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go index e71b3e908..90102aeeb 100644 --- a/roomserver/internal/perform/perform_leave.go +++ b/roomserver/internal/perform/perform_leave.go @@ -183,7 +183,7 @@ func (r *Leaver) performLeaveRoomByID( if err != nil { return nil, fmt.Errorf("SigningIdentityFor: %w", err) } - event, err := eventutil.QueryAndBuildEvent(ctx, &proto, r.Cfg.Matrix, identity, time.Now(), r.RSAPI, &buildRes) + event, err := eventutil.QueryAndBuildEvent(ctx, &proto, identity, time.Now(), r.RSAPI, &buildRes) if err != nil { return nil, fmt.Errorf("eventutil.QueryAndBuildEvent: %w", err) } diff --git a/roomserver/internal/perform/perform_upgrade.go b/roomserver/internal/perform/perform_upgrade.go index 60085cb6d..ff4a6a1dc 100644 --- a/roomserver/internal/perform/perform_upgrade.go +++ b/roomserver/internal/perform/perform_upgrade.go @@ -35,13 +35,6 @@ type Upgrader struct { URSAPI api.RoomserverInternalAPI } -// fledglingEvent is a helper representation of an event used when creating many events in succession. -type fledglingEvent struct { - Type string `json:"type"` - StateKey string `json:"state_key"` - Content interface{} `json:"content"` -} - // PerformRoomUpgrade upgrades a room from one version to another func (r *Upgrader) PerformRoomUpgrade( ctx context.Context, @@ -154,7 +147,7 @@ func (r *Upgrader) restrictOldRoomPowerLevels(ctx context.Context, evTime time.T restrictedPowerLevelContent.EventsDefault = restrictedDefaultPowerLevel restrictedPowerLevelContent.Invite = restrictedDefaultPowerLevel - restrictedPowerLevelsHeadered, resErr := r.makeHeaderedEvent(ctx, evTime, userID, roomID, fledglingEvent{ + restrictedPowerLevelsHeadered, resErr := r.makeHeaderedEvent(ctx, evTime, userID, roomID, gomatrixserverlib.FledglingEvent{ Type: spec.MRoomPowerLevels, StateKey: "", Content: restrictedPowerLevelContent, @@ -216,7 +209,7 @@ func (r *Upgrader) clearOldCanonicalAliasEvent(ctx context.Context, oldRoom *api } } - emptyCanonicalAliasEvent, resErr := r.makeHeaderedEvent(ctx, evTime, userID, roomID, fledglingEvent{ + emptyCanonicalAliasEvent, resErr := r.makeHeaderedEvent(ctx, evTime, userID, roomID, gomatrixserverlib.FledglingEvent{ Type: spec.MRoomCanonicalAlias, Content: map[string]interface{}{}, }) @@ -298,7 +291,7 @@ func (r *Upgrader) userIsAuthorized(ctx context.Context, userID, roomID string, } // nolint:gocyclo -func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.QueryLatestEventsAndStateResponse, userID, roomID string, newVersion gomatrixserverlib.RoomVersion, tombstoneEvent *types.HeaderedEvent) ([]fledglingEvent, error) { +func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.QueryLatestEventsAndStateResponse, userID, roomID string, newVersion gomatrixserverlib.RoomVersion, tombstoneEvent *types.HeaderedEvent) ([]gomatrixserverlib.FledglingEvent, error) { state := make(map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent, len(oldRoom.StateEvents)) for _, event := range oldRoom.StateEvents { if event.StateKey() == nil { @@ -361,7 +354,7 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query EventID: tombstoneEvent.EventID(), RoomID: roomID, } - newCreateEvent := fledglingEvent{ + newCreateEvent := gomatrixserverlib.FledglingEvent{ Type: spec.MRoomCreate, StateKey: "", Content: newCreateContent, @@ -374,7 +367,7 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query newMembershipContent := map[string]interface{}{} _ = json.Unmarshal(oldMembershipEvent.Content(), &newMembershipContent) newMembershipContent["membership"] = spec.Join - newMembershipEvent := fledglingEvent{ + newMembershipEvent := gomatrixserverlib.FledglingEvent{ Type: spec.MRoomMember, StateKey: userID, Content: newMembershipContent, @@ -400,13 +393,13 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query "join_rule": spec.Invite, // sane default } _ = json.Unmarshal(oldJoinRulesEvent.Content(), &newJoinRulesContent) - newJoinRulesEvent := fledglingEvent{ + newJoinRulesEvent := gomatrixserverlib.FledglingEvent{ Type: spec.MRoomJoinRules, StateKey: "", Content: newJoinRulesContent, } - eventsToMake := make([]fledglingEvent, 0, len(state)) + eventsToMake := make([]gomatrixserverlib.FledglingEvent, 0, len(state)) eventsToMake = append( eventsToMake, newCreateEvent, newMembershipEvent, tempPowerLevelsEvent, newJoinRulesEvent, @@ -415,7 +408,7 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query // For some reason Sytest expects there to be a guest access event. // Create one if it doesn't exist. if _, ok := state[gomatrixserverlib.StateKeyTuple{EventType: spec.MRoomGuestAccess, StateKey: ""}]; !ok { - eventsToMake = append(eventsToMake, fledglingEvent{ + eventsToMake = append(eventsToMake, gomatrixserverlib.FledglingEvent{ Type: spec.MRoomGuestAccess, Content: map[string]string{ "guest_access": "forbidden", @@ -430,7 +423,7 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query // are already in `eventsToMake`. continue } - newEvent := fledglingEvent{ + newEvent := gomatrixserverlib.FledglingEvent{ Type: tuple.EventType, StateKey: tuple.StateKey, } @@ -444,7 +437,7 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query // If we sent a temporary power level event into the room before, // override that now by restoring the original power levels. if powerLevelsOverridden { - eventsToMake = append(eventsToMake, fledglingEvent{ + eventsToMake = append(eventsToMake, gomatrixserverlib.FledglingEvent{ Type: spec.MRoomPowerLevels, Content: powerLevelContent, }) @@ -452,7 +445,7 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query return eventsToMake, nil } -func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, userID string, userDomain spec.ServerName, newRoomID string, newVersion gomatrixserverlib.RoomVersion, eventsToMake []fledglingEvent) error { +func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, userID string, userDomain spec.ServerName, newRoomID string, newVersion gomatrixserverlib.RoomVersion, eventsToMake []gomatrixserverlib.FledglingEvent) error { var err error var builtEvents []*types.HeaderedEvent authEvents := gomatrixserverlib.NewAuthEvents(nil) @@ -527,14 +520,14 @@ func (r *Upgrader) makeTombstoneEvent( "body": "This room has been replaced", "replacement_room": newRoomID, } - event := fledglingEvent{ + event := gomatrixserverlib.FledglingEvent{ Type: "m.room.tombstone", Content: content, } return r.makeHeaderedEvent(ctx, evTime, userID, roomID, event) } -func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, userID, roomID string, event fledglingEvent) (*types.HeaderedEvent, error) { +func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, userID, roomID string, event gomatrixserverlib.FledglingEvent) (*types.HeaderedEvent, error) { proto := gomatrixserverlib.ProtoEvent{ Sender: userID, RoomID: roomID, @@ -555,7 +548,7 @@ func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, user return nil, fmt.Errorf("failed to get signing identity for %q: %w", senderDomain, err) } var queryRes api.QueryLatestEventsAndStateResponse - headeredEvent, err := eventutil.QueryAndBuildEvent(ctx, &proto, r.Cfg.Matrix, identity, evTime, r.URSAPI, &queryRes) + headeredEvent, err := eventutil.QueryAndBuildEvent(ctx, &proto, identity, evTime, r.URSAPI, &queryRes) switch e := err.(type) { case nil: case eventutil.ErrRoomNoExists: @@ -581,7 +574,7 @@ func (r *Upgrader) makeHeaderedEvent(ctx context.Context, evTime time.Time, user return headeredEvent, nil } -func createTemporaryPowerLevels(powerLevelContent *gomatrixserverlib.PowerLevelContent, userID string) (fledglingEvent, bool) { +func createTemporaryPowerLevels(powerLevelContent *gomatrixserverlib.PowerLevelContent, userID string) (gomatrixserverlib.FledglingEvent, bool) { // Work out what power level we need in order to be able to send events // of all types into the room. neededPowerLevel := powerLevelContent.StateDefault @@ -612,7 +605,7 @@ func createTemporaryPowerLevels(powerLevelContent *gomatrixserverlib.PowerLevelC } // Then return the temporary power levels event. - return fledglingEvent{ + return gomatrixserverlib.FledglingEvent{ Type: spec.MRoomPowerLevels, Content: tempPowerLevelContent, }, powerLevelsOverridden From ea6b368ad424a3d2e05135afb7fd0c0801b3609b Mon Sep 17 00:00:00 2001 From: devonh Date: Wed, 31 May 2023 16:33:49 +0000 Subject: [PATCH 5/5] Move Invite logic to GMSL (#3086) This is both the federation receiving & sending side logic (which were previously entangeld in a single function) --- clientapi/routing/createroom.go | 1 - federationapi/api/api.go | 8 +- federationapi/internal/perform.go | 47 ++- federationapi/routing/invite.go | 259 +++++++------- federationapi/routing/join.go | 21 +- federationapi/routing/routing.go | 20 +- go.mod | 2 +- go.sum | 4 +- roomserver/api/api.go | 5 + roomserver/api/perform.go | 11 +- roomserver/api/query.go | 20 ++ roomserver/internal/api.go | 29 +- roomserver/internal/helpers/auth.go | 38 +- .../internal/perform/perform_create_room.go | 6 +- roomserver/internal/perform/perform_invite.go | 330 +++++++----------- 15 files changed, 359 insertions(+), 442 deletions(-) diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index aaa305f06..799fc7976 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -135,7 +135,6 @@ func CreateRoom( // createRoom implements /createRoom func createRoom( ctx context.Context, - // TODO: remove dependency on createRoomRequest createRequest createRoomRequest, device *api.Device, cfg *config.ClientAPI, profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI, diff --git a/federationapi/api/api.go b/federationapi/api/api.go index b53ec3dd8..5b49e509e 100644 --- a/federationapi/api/api.go +++ b/federationapi/api/api.go @@ -62,7 +62,7 @@ type RoomserverFederationAPI interface { // Handle an instruction to make_leave & send_leave with a remote server. PerformLeave(ctx context.Context, request *PerformLeaveRequest, response *PerformLeaveResponse) error // Handle sending an invite to a remote server. - PerformInvite(ctx context.Context, request *PerformInviteRequest, response *PerformInviteResponse) error + SendInvite(ctx context.Context, event gomatrixserverlib.PDU, strippedState []gomatrixserverlib.InviteStrippedState) (gomatrixserverlib.PDU, error) // Handle an instruction to peek a room on a remote server. PerformOutboundPeek(ctx context.Context, request *PerformOutboundPeekRequest, response *PerformOutboundPeekResponse) error // Query the server names of the joined hosts in a room. @@ -190,9 +190,9 @@ type PerformLeaveResponse struct { } type PerformInviteRequest struct { - RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` - Event *rstypes.HeaderedEvent `json:"event"` - InviteRoomState []fclient.InviteV2StrippedState `json:"invite_room_state"` + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` + Event *rstypes.HeaderedEvent `json:"event"` + InviteRoomState []gomatrixserverlib.InviteStrippedState `json:"invite_room_state"` } type PerformInviteResponse struct { diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index 99943d86c..ed800d03a 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -503,60 +503,59 @@ func (r *FederationInternalAPI) PerformLeave( ) } -// PerformLeaveRequest implements api.FederationInternalAPI -func (r *FederationInternalAPI) PerformInvite( +// SendInvite implements api.FederationInternalAPI +func (r *FederationInternalAPI) SendInvite( ctx context.Context, - request *api.PerformInviteRequest, - response *api.PerformInviteResponse, -) (err error) { - _, origin, err := r.cfg.Matrix.SplitLocalID('@', request.Event.Sender()) + event gomatrixserverlib.PDU, + strippedState []gomatrixserverlib.InviteStrippedState, +) (gomatrixserverlib.PDU, error) { + _, origin, err := r.cfg.Matrix.SplitLocalID('@', event.Sender()) if err != nil { - return err + return nil, err } - if request.Event.StateKey() == nil { - return errors.New("invite must be a state event") + if event.StateKey() == nil { + return nil, errors.New("invite must be a state event") } - _, destination, err := gomatrixserverlib.SplitID('@', *request.Event.StateKey()) + _, destination, err := gomatrixserverlib.SplitID('@', *event.StateKey()) if err != nil { - return fmt.Errorf("gomatrixserverlib.SplitID: %w", err) + return nil, fmt.Errorf("gomatrixserverlib.SplitID: %w", err) } // TODO (devon): This should be allowed via a relay. Currently only transactions // can be sent to relays. Would need to extend relays to handle invites. if !r.shouldAttemptDirectFederation(destination) { - return fmt.Errorf("relay servers have no meaningful response for invite.") + return nil, fmt.Errorf("relay servers have no meaningful response for invite.") } logrus.WithFields(logrus.Fields{ - "event_id": request.Event.EventID(), - "user_id": *request.Event.StateKey(), - "room_id": request.Event.RoomID(), - "room_version": request.RoomVersion, + "event_id": event.EventID(), + "user_id": *event.StateKey(), + "room_id": event.RoomID(), + "room_version": event.Version(), "destination": destination, }).Info("Sending invite") - inviteReq, err := fclient.NewInviteV2Request(request.Event.PDU, request.InviteRoomState) + inviteReq, err := fclient.NewInviteV2Request(event, strippedState) if err != nil { - return fmt.Errorf("gomatrixserverlib.NewInviteV2Request: %w", err) + return nil, fmt.Errorf("gomatrixserverlib.NewInviteV2Request: %w", err) } inviteRes, err := r.federation.SendInviteV2(ctx, origin, destination, inviteReq) if err != nil { - return fmt.Errorf("r.federation.SendInviteV2: failed to send invite: %w", err) + return nil, fmt.Errorf("r.federation.SendInviteV2: failed to send invite: %w", err) } - verImpl, err := gomatrixserverlib.GetRoomVersion(request.RoomVersion) + verImpl, err := gomatrixserverlib.GetRoomVersion(event.Version()) if err != nil { - return err + return nil, err } inviteEvent, err := verImpl.NewEventFromUntrustedJSON(inviteRes.Event) if err != nil { - return fmt.Errorf("r.federation.SendInviteV2 failed to decode event response: %w", err) + return nil, fmt.Errorf("r.federation.SendInviteV2 failed to decode event response: %w", err) } - response.Event = &types.HeaderedEvent{PDU: inviteEvent} - return nil + return inviteEvent, nil } // PerformServersAlive implements api.FederationInternalAPI diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 993d40466..78a09d949 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -20,7 +20,6 @@ import ( "fmt" "net/http" - "github.com/getsentry/sentry-go" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" @@ -34,7 +33,7 @@ import ( func InviteV2( httpReq *http.Request, request *fclient.FederationRequest, - roomID string, + roomID spec.RoomID, eventID string, cfg *config.FederationAPI, rsAPI api.FederationRoomserverAPI, @@ -56,9 +55,55 @@ func InviteV2( JSON: spec.BadJSON(err.Error()), } case nil: - return processInvite( - httpReq.Context(), true, inviteReq.Event(), inviteReq.RoomVersion(), inviteReq.InviteRoomState(), roomID, eventID, cfg, rsAPI, keys, - ) + if inviteReq.Event().StateKey() == nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("The invite event has no state key"), + } + } + + invitedUser, userErr := spec.NewUserID(*inviteReq.Event().StateKey(), true) + if userErr != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("The user ID is invalid"), + } + } + if !cfg.Matrix.IsLocalServerName(invitedUser.Domain()) { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("The invited user domain does not belong to this server"), + } + } + + if inviteReq.Event().EventID() != eventID { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON("The event ID in the request path must match the event ID in the invite event JSON"), + } + } + + input := gomatrixserverlib.HandleInviteInput{ + RoomVersion: inviteReq.RoomVersion(), + RoomID: roomID, + InvitedUser: *invitedUser, + KeyID: cfg.Matrix.KeyID, + PrivateKey: cfg.Matrix.PrivateKey, + Verifier: keys, + RoomQuerier: rsAPI, + MembershipQuerier: &api.MembershipQuerier{Roomserver: rsAPI}, + StateQuerier: rsAPI.StateQuerier(), + InviteEvent: inviteReq.Event(), + StrippedState: inviteReq.InviteRoomState(), + } + event, jsonErr := handleInvite(httpReq.Context(), input, rsAPI) + if jsonErr != nil { + return *jsonErr + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: fclient.RespInviteV2{Event: event.JSON()}, + } default: return util.JSONResponse{ Code: http.StatusBadRequest, @@ -71,7 +116,7 @@ func InviteV2( func InviteV1( httpReq *http.Request, request *fclient.FederationRequest, - roomID string, + roomID spec.RoomID, eventID string, cfg *config.FederationAPI, rsAPI api.FederationRoomserverAPI, @@ -94,55 +139,11 @@ func InviteV1( JSON: spec.NotJSON("The request body could not be decoded into an invite v1 request. " + err.Error()), } } - var strippedState []fclient.InviteV2StrippedState - if err := json.Unmarshal(event.Unsigned(), &strippedState); err != nil { + var strippedState []gomatrixserverlib.InviteStrippedState + if jsonErr := json.Unmarshal(event.Unsigned(), &strippedState); jsonErr != nil { // just warn, they may not have added any. util.GetLogger(httpReq.Context()).Warnf("failed to extract stripped state from invite event") } - return processInvite( - httpReq.Context(), false, event, roomVer, strippedState, roomID, eventID, cfg, rsAPI, keys, - ) -} - -func processInvite( - ctx context.Context, - isInviteV2 bool, - event gomatrixserverlib.PDU, - roomVer gomatrixserverlib.RoomVersion, - strippedState []fclient.InviteV2StrippedState, - roomID string, - eventID string, - cfg *config.FederationAPI, - rsAPI api.FederationRoomserverAPI, - keys gomatrixserverlib.JSONVerifier, -) util.JSONResponse { - - // Check that we can accept invites for this room version. - verImpl, err := gomatrixserverlib.GetRoomVersion(roomVer) - if err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.UnsupportedRoomVersion( - fmt.Sprintf("Room version %q is not supported by this server.", roomVer), - ), - } - } - - // Check that the room ID is correct. - if event.RoomID() != roomID { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.BadJSON("The room ID in the request path must match the room ID in the invite event JSON"), - } - } - - // Check that the event ID is correct. - if event.EventID() != eventID { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.BadJSON("The event ID in the request path must match the event ID in the invite event JSON"), - } - } if event.StateKey() == nil { return util.JSONResponse{ @@ -151,105 +152,91 @@ func processInvite( } } - _, domain, err := cfg.Matrix.SplitLocalID('@', *event.StateKey()) + invitedUser, err := spec.NewUserID(*event.StateKey(), true) if err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: spec.InvalidParam(fmt.Sprintf("The user ID is invalid or domain %q does not belong to this server", domain)), + JSON: spec.InvalidParam("The user ID is invalid"), } } - - // Check that the event is signed by the server sending the request. - redacted, err := verImpl.RedactEventJSON(event.JSON()) - if err != nil { + if !cfg.Matrix.IsLocalServerName(invitedUser.Domain()) { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: spec.BadJSON("The event JSON could not be redacted"), + JSON: spec.InvalidParam("The invited user domain does not belong to this server"), } } - _, serverName, err := gomatrixserverlib.SplitID('@', event.Sender()) - if err != nil { + + if event.EventID() != eventID { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: spec.BadJSON("The event JSON contains an invalid sender"), - } - } - verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{ - ServerName: serverName, - Message: redacted, - AtTS: event.OriginServerTS(), - StrictValidityChecking: true, - }} - verifyResults, err := keys.VerifyJSONs(ctx, verifyRequests) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("keys.VerifyJSONs failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - if verifyResults[0].Error != nil { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: spec.Forbidden("The invite must be signed by the server it originated on"), + JSON: spec.BadJSON("The event ID in the request path must match the event ID in the invite event JSON"), } } - // Sign the event so that other servers will know that we have received the invite. - signedEvent := event.Sign( - string(domain), cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, - ) - - // Add the invite event to the roomserver. - inviteEvent := &types.HeaderedEvent{PDU: signedEvent} - request := &api.PerformInviteRequest{ - Event: inviteEvent, - InviteRoomState: strippedState, - RoomVersion: inviteEvent.Version(), - SendAsServer: string(api.DoNotSendToOtherServers), - TransactionID: nil, + input := gomatrixserverlib.HandleInviteInput{ + RoomVersion: roomVer, + RoomID: roomID, + InvitedUser: *invitedUser, + KeyID: cfg.Matrix.KeyID, + PrivateKey: cfg.Matrix.PrivateKey, + Verifier: keys, + RoomQuerier: rsAPI, + MembershipQuerier: &api.MembershipQuerier{Roomserver: rsAPI}, + StateQuerier: rsAPI.StateQuerier(), + InviteEvent: event, + StrippedState: strippedState, } - - if err = rsAPI.PerformInvite(ctx, request); err != nil { - util.GetLogger(ctx).WithError(err).Error("PerformInvite failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } + event, jsonErr := handleInvite(httpReq.Context(), input, rsAPI) + if jsonErr != nil { + return *jsonErr } - - switch e := err.(type) { - case api.ErrInvalidID: - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.Unknown(e.Error()), - } - case api.ErrNotAllowed: - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: spec.Forbidden(e.Error()), - } - case nil: - default: - util.GetLogger(ctx).WithError(err).Error("PerformInvite failed") - sentry.CaptureException(err) - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - - // Return the signed event to the originating server, it should then tell - // the other servers in the room that we have been invited. - if isInviteV2 { - return util.JSONResponse{ - Code: http.StatusOK, - JSON: fclient.RespInviteV2{Event: signedEvent.JSON()}, - } - } else { - return util.JSONResponse{ - Code: http.StatusOK, - JSON: fclient.RespInvite{Event: signedEvent.JSON()}, - } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: fclient.RespInvite{Event: event.JSON()}, } } + +func handleInvite(ctx context.Context, input gomatrixserverlib.HandleInviteInput, rsAPI api.FederationRoomserverAPI) (gomatrixserverlib.PDU, *util.JSONResponse) { + inviteEvent, err := gomatrixserverlib.HandleInvite(ctx, input) + switch e := err.(type) { + case nil: + case spec.InternalServerError: + util.GetLogger(ctx).WithError(err) + return nil, &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + case spec.MatrixError: + util.GetLogger(ctx).WithError(err) + code := http.StatusInternalServerError + switch e.ErrCode { + case spec.ErrorForbidden: + code = http.StatusForbidden + case spec.ErrorUnsupportedRoomVersion: + fallthrough // http.StatusBadRequest + case spec.ErrorBadJSON: + code = http.StatusBadRequest + } + + return nil, &util.JSONResponse{ + Code: code, + JSON: e, + } + default: + util.GetLogger(ctx).WithError(err) + return nil, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.Unknown("unknown error"), + } + } + + headeredInvite := &types.HeaderedEvent{PDU: inviteEvent} + if err = rsAPI.HandleInvite(ctx, headeredInvite); err != nil { + util.GetLogger(ctx).WithError(err).Error("HandleInvite failed") + return nil, &util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + return inviteEvent, nil +} diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 03d3309ae..c6f96375e 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -216,25 +216,6 @@ func MakeJoin( } } -type MembershipQuerier struct { - roomserver api.FederationRoomserverAPI -} - -func (mq *MembershipQuerier) CurrentMembership(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (string, error) { - req := api.QueryMembershipForUserRequest{ - RoomID: roomID.String(), - UserID: userID.String(), - } - res := api.QueryMembershipForUserResponse{} - err := mq.roomserver.QueryMembershipForUser(ctx, &req, &res) - - membership := "" - if err == nil { - membership = res.Membership - } - return membership, err -} - // SendJoin implements the /send_join API // The make-join send-join dance makes much more sense as a single // flow so the cyclomatic complexity is high: @@ -268,7 +249,7 @@ func SendJoin( KeyID: cfg.Matrix.KeyID, PrivateKey: cfg.Matrix.PrivateKey, Verifier: keys, - MembershipQuerier: &MembershipQuerier{roomserver: rsAPI}, + MembershipQuerier: &api.MembershipQuerier{Roomserver: rsAPI}, } response, joinErr := gomatrixserverlib.HandleSendJoin(input) switch e := joinErr.(type) { diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index fad06c1cf..8865022ff 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -152,8 +152,16 @@ func Setup( JSON: spec.Forbidden("Forbidden by server ACLs"), } } + + roomID, err := spec.NewRoomID(vars["roomID"]) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("Invalid RoomID"), + } + } return InviteV1( - httpReq, request, vars["roomID"], vars["eventID"], + httpReq, request, *roomID, vars["eventID"], cfg, rsAPI, keys, ) }, @@ -168,8 +176,16 @@ func Setup( JSON: spec.Forbidden("Forbidden by server ACLs"), } } + + roomID, err := spec.NewRoomID(vars["roomID"]) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("Invalid RoomID"), + } + } return InviteV2( - httpReq, request, vars["roomID"], vars["eventID"], + httpReq, request, *roomID, vars["eventID"], cfg, rsAPI, keys, ) }, diff --git a/go.mod b/go.mod index 0e979de6f..a20757bbc 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20230531143710-c681a0658246 + github.com/matrix-org/gomatrixserverlib v0.0.0-20230531155817-0e3adf17bee6 github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 github.com/mattn/go-sqlite3 v1.14.16 diff --git a/go.sum b/go.sum index 8baa50e85..a1946adaa 100644 --- a/go.sum +++ b/go.sum @@ -323,8 +323,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230531143710-c681a0658246 h1:1sYXx7p9BIf0R7OIV/TZg3SCvNehEQPCKNqwV1ONfwU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230531143710-c681a0658246/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230531155817-0e3adf17bee6 h1:Kh1TNvJDhWN5CdgtICNUC4G0wV2km51LGr46Dvl153A= +github.com/matrix-org/gomatrixserverlib v0.0.0-20230531155817-0e3adf17bee6/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a h1:awrPDf9LEFySxTLKYBMCiObelNx/cBuv/wzllvCCH3A= github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a/go.mod h1:HchJX9oKMXaT2xYFs0Ha/6Zs06mxLU8k6F1ODnrGkeQ= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= diff --git a/roomserver/api/api.go b/roomserver/api/api.go index 571aa40b3..7cb3379e0 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -225,6 +225,8 @@ type FederationRoomserverAPI interface { QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error QueryRestrictedJoinAllowed(ctx context.Context, req *QueryRestrictedJoinAllowedRequest, res *QueryRestrictedJoinAllowedResponse) error PerformInboundPeek(ctx context.Context, req *PerformInboundPeekRequest, res *PerformInboundPeekResponse) error + HandleInvite(ctx context.Context, event *types.HeaderedEvent) error + PerformInvite(ctx context.Context, req *PerformInviteRequest) error // Query a given amount (or less) of events prior to a given set of events. PerformBackfill(ctx context.Context, req *PerformBackfillRequest, res *PerformBackfillResponse) error @@ -234,6 +236,9 @@ type FederationRoomserverAPI interface { QueryRoomInfo(ctx context.Context, roomID spec.RoomID) (*types.RoomInfo, error) UserJoinedToRoom(ctx context.Context, roomID types.RoomNID, userID spec.UserID) (bool, error) LocallyJoinedUsers(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, roomNID types.RoomNID) ([]gomatrixserverlib.PDU, error) + + IsKnownRoom(ctx context.Context, roomID spec.RoomID) (bool, error) + StateQuerier() gomatrixserverlib.StateQuerier } type KeyserverRoomserverAPI interface { diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index 8d9742c69..6cbaf5b19 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -7,7 +7,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" ) @@ -52,11 +51,11 @@ type PerformLeaveResponse struct { } type PerformInviteRequest struct { - RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` - Event *types.HeaderedEvent `json:"event"` - InviteRoomState []fclient.InviteV2StrippedState `json:"invite_room_state"` - SendAsServer string `json:"send_as_server"` - TransactionID *TransactionID `json:"transaction_id"` + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` + Event *types.HeaderedEvent `json:"event"` + InviteRoomState []gomatrixserverlib.InviteStrippedState `json:"invite_room_state"` + SendAsServer string `json:"send_as_server"` + TransactionID *TransactionID `json:"transaction_id"` } type PerformPeekRequest struct { diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 1726bfe1f..b33698c82 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -17,6 +17,7 @@ package api import ( + "context" "encoding/json" "fmt" "strings" @@ -457,3 +458,22 @@ type QueryLeftUsersRequest struct { type QueryLeftUsersResponse struct { LeftUsers []string `json:"user_ids"` } + +type MembershipQuerier struct { + Roomserver FederationRoomserverAPI +} + +func (mq *MembershipQuerier) CurrentMembership(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (string, error) { + req := QueryMembershipForUserRequest{ + RoomID: roomID.String(), + UserID: userID.String(), + } + res := QueryMembershipForUserResponse{} + err := mq.Roomserver.QueryMembershipForUser(ctx, &req, &res) + + membership := "" + if err == nil { + membership = res.Membership + } + return membership, err +} diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index f61f89183..ee433f0d2 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -20,6 +20,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/internal/query" "github.com/matrix-org/dendrite/roomserver/producers" "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" @@ -132,6 +133,7 @@ func (r *RoomserverInternalAPI) SetFederationAPI(fsAPI fsAPI.RoomserverFederatio DB: r.DB, Cfg: &r.Cfg.RoomServer, FSAPI: r.fsAPI, + RSAPI: r, Inputer: r.Inputer, } r.Joiner = &perform.Joiner{ @@ -213,6 +215,24 @@ func (r *RoomserverInternalAPI) SetAppserviceAPI(asAPI asAPI.AppServiceInternalA r.asAPI = asAPI } +func (r *RoomserverInternalAPI) IsKnownRoom(ctx context.Context, roomID spec.RoomID) (bool, error) { + return r.Inviter.IsKnownRoom(ctx, roomID) +} + +func (r *RoomserverInternalAPI) StateQuerier() gomatrixserverlib.StateQuerier { + return r.Inviter.StateQuerier() +} + +func (r *RoomserverInternalAPI) HandleInvite( + ctx context.Context, inviteEvent *types.HeaderedEvent, +) error { + outputEvents, err := r.Inviter.ProcessInviteMembership(ctx, inviteEvent) + if err != nil { + return err + } + return r.OutputProducer.ProduceRoomEvents(inviteEvent.RoomID(), outputEvents) +} + func (r *RoomserverInternalAPI) PerformCreateRoom( ctx context.Context, userID spec.UserID, roomID spec.RoomID, createRequest *api.PerformCreateRoomRequest, ) (string, *util.JSONResponse) { @@ -223,14 +243,7 @@ func (r *RoomserverInternalAPI) PerformInvite( ctx context.Context, req *api.PerformInviteRequest, ) error { - outputEvents, err := r.Inviter.PerformInvite(ctx, req) - if err != nil { - return err - } - if len(outputEvents) == 0 { - return nil - } - return r.OutputProducer.ProduceRoomEvents(req.Event.RoomID(), outputEvents) + return r.Inviter.PerformInvite(ctx, req) } func (r *RoomserverInternalAPI) PerformLeave( diff --git a/roomserver/internal/helpers/auth.go b/roomserver/internal/helpers/auth.go index 24958091b..7ec0892e4 100644 --- a/roomserver/internal/helpers/auth.go +++ b/roomserver/internal/helpers/auth.go @@ -70,7 +70,7 @@ func CheckForSoftFail( ) // Load the actual auth events from the database. - authEvents, err := loadAuthEvents(ctx, db, roomInfo, stateNeeded, authStateEntries) + authEvents, err := loadAuthEvents(ctx, db, roomInfo.RoomVersion, stateNeeded, authStateEntries) if err != nil { return true, fmt.Errorf("loadAuthEvents: %w", err) } @@ -83,15 +83,14 @@ func CheckForSoftFail( return false, nil } -// CheckAuthEvents checks that the event passes authentication checks -// Returns the numeric IDs for the auth events. -func CheckAuthEvents( +// GetAuthEvents returns the numeric IDs for the auth events. +func GetAuthEvents( ctx context.Context, db storage.RoomDatabase, - roomInfo *types.RoomInfo, - event *types.HeaderedEvent, + roomVersion gomatrixserverlib.RoomVersion, + event gomatrixserverlib.PDU, authEventIDs []string, -) ([]types.EventNID, error) { +) (gomatrixserverlib.AuthEventProvider, error) { // Grab the numeric IDs for the supplied auth state events from the database. authStateEntries, err := db.StateEntriesForEventIDs(ctx, authEventIDs, true) if err != nil { @@ -100,25 +99,14 @@ func CheckAuthEvents( authStateEntries = types.DeduplicateStateEntries(authStateEntries) // Work out which of the state events we actually need. - stateNeeded := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.PDU{event.PDU}) + stateNeeded := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.PDU{event}) // Load the actual auth events from the database. - authEvents, err := loadAuthEvents(ctx, db, roomInfo, stateNeeded, authStateEntries) + authEvents, err := loadAuthEvents(ctx, db, roomVersion, stateNeeded, authStateEntries) if err != nil { return nil, fmt.Errorf("loadAuthEvents: %w", err) } - - // Check if the event is allowed. - if err = gomatrixserverlib.Allowed(event.PDU, &authEvents); err != nil { - return nil, err - } - - // Return the numeric IDs for the auth events. - result := make([]types.EventNID, len(authStateEntries)) - for i := range authStateEntries { - result[i] = authStateEntries[i].EventNID - } - return result, nil + return &authEvents, nil } type authEvents struct { @@ -196,7 +184,7 @@ func (ae *authEvents) lookupEvent(typeNID types.EventTypeNID, stateKey string) g func loadAuthEvents( ctx context.Context, db state.StateResolutionStorage, - roomInfo *types.RoomInfo, + roomVersion gomatrixserverlib.RoomVersion, needed gomatrixserverlib.StateNeeded, state []types.StateEntry, ) (result authEvents, err error) { @@ -220,11 +208,7 @@ func loadAuthEvents( } } - if roomInfo == nil { - err = types.ErrorInvalidRoomInfo - return - } - if result.events, err = db.Events(ctx, roomInfo.RoomVersion, eventNIDs); err != nil { + if result.events, err = db.Events(ctx, roomVersion, eventNIDs); err != nil { return } roomID := "" diff --git a/roomserver/internal/perform/perform_create_room.go b/roomserver/internal/perform/perform_create_room.go index 0f9170087..41194832d 100644 --- a/roomserver/internal/perform/perform_create_room.go +++ b/roomserver/internal/perform/perform_create_room.go @@ -376,7 +376,7 @@ func (c *Creator) PerformCreateRoom(ctx context.Context, userID spec.UserID, roo // If this is a direct message then we should invite the participants. if len(createRequest.InvitedUsers) > 0 { // Build some stripped state for the invite. - var globalStrippedState []fclient.InviteV2StrippedState + var globalStrippedState []gomatrixserverlib.InviteStrippedState for _, event := range builtEvents { // Chosen events from the spec: // https://spec.matrix.org/v1.3/client-server-api/#stripped-state @@ -399,7 +399,7 @@ func (c *Creator) PerformCreateRoom(ctx context.Context, userID spec.UserID, roo ev := event.PDU globalStrippedState = append( globalStrippedState, - fclient.NewInviteV2StrippedState(ev), + gomatrixserverlib.NewInviteStrippedState(ev), ) } } @@ -443,7 +443,7 @@ func (c *Creator) PerformCreateRoom(ctx context.Context, userID spec.UserID, roo } inviteStrippedState := append( globalStrippedState, - fclient.NewInviteV2StrippedState(inviteEvent.PDU), + gomatrixserverlib.NewInviteStrippedState(inviteEvent.PDU), ) // Send the invite event to the roomserver. event := inviteEvent diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go index a3fa2e011..1930b5ace 100644 --- a/roomserver/internal/perform/perform_invite.go +++ b/roomserver/internal/perform/perform_invite.go @@ -28,186 +28,149 @@ import ( "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" - log "github.com/sirupsen/logrus" ) +type QueryState struct { + storage.Database +} + +func (q *QueryState) GetAuthEvents(ctx context.Context, event gomatrixserverlib.PDU) (gomatrixserverlib.AuthEventProvider, error) { + return helpers.GetAuthEvents(ctx, q.Database, event.Version(), event, event.AuthEventIDs()) +} + +func (q *QueryState) GetState(ctx context.Context, roomID spec.RoomID, stateWanted []gomatrixserverlib.StateKeyTuple) ([]gomatrixserverlib.PDU, error) { + info, err := q.Database.RoomInfo(ctx, roomID.String()) + if err != nil { + return nil, fmt.Errorf("failed to load RoomInfo: %w", err) + } + if info != nil { + roomState := state.NewStateResolution(q.Database, info) + stateEntries, err := roomState.LoadStateAtSnapshotForStringTuples( + ctx, info.StateSnapshotNID(), stateWanted, + ) + if err != nil { + return nil, nil + } + stateNIDs := []types.EventNID{} + for _, stateNID := range stateEntries { + stateNIDs = append(stateNIDs, stateNID.EventNID) + } + stateEvents, err := q.Database.Events(ctx, info.RoomVersion, stateNIDs) + if err != nil { + return nil, fmt.Errorf("failed to obtain required events: %w", err) + } + + events := []gomatrixserverlib.PDU{} + for _, event := range stateEvents { + events = append(events, event.PDU) + } + return events, nil + } + + return nil, nil +} + type Inviter struct { DB storage.Database Cfg *config.RoomServer FSAPI federationAPI.RoomserverFederationAPI + RSAPI api.RoomserverInternalAPI Inputer *input.Inputer } +func (r *Inviter) IsKnownRoom(ctx context.Context, roomID spec.RoomID) (bool, error) { + info, err := r.DB.RoomInfo(ctx, roomID.String()) + if err != nil { + return false, fmt.Errorf("failed to load RoomInfo: %w", err) + } + return (info != nil && !info.IsStub()), nil +} + +func (r *Inviter) StateQuerier() gomatrixserverlib.StateQuerier { + return &QueryState{Database: r.DB} +} + +func (r *Inviter) ProcessInviteMembership( + ctx context.Context, inviteEvent *types.HeaderedEvent, +) ([]api.OutputEvent, error) { + var outputUpdates []api.OutputEvent + var updater *shared.MembershipUpdater + _, domain, err := gomatrixserverlib.SplitID('@', *inviteEvent.StateKey()) + if err != nil { + return nil, api.ErrInvalidID{Err: fmt.Errorf("the user ID %s is invalid", *inviteEvent.StateKey())} + } + isTargetLocal := r.Cfg.Matrix.IsLocalServerName(domain) + if updater, err = r.DB.MembershipUpdater(ctx, inviteEvent.RoomID(), *inviteEvent.StateKey(), isTargetLocal, inviteEvent.Version()); err != nil { + return nil, fmt.Errorf("r.DB.MembershipUpdater: %w", err) + } + outputUpdates, err = helpers.UpdateToInviteMembership(updater, &types.Event{ + EventNID: 0, + PDU: inviteEvent.PDU, + }, outputUpdates, inviteEvent.Version()) + if err != nil { + return nil, fmt.Errorf("updateToInviteMembership: %w", err) + } + if err = updater.Commit(); err != nil { + return nil, fmt.Errorf("updater.Commit: %w", err) + } + return outputUpdates, nil +} + // nolint:gocyclo func (r *Inviter) PerformInvite( ctx context.Context, req *api.PerformInviteRequest, -) ([]api.OutputEvent, error) { - var outputUpdates []api.OutputEvent +) error { event := req.Event + + sender, err := spec.NewUserID(event.Sender(), true) + if err != nil { + return spec.InvalidParam("The user ID is invalid") + } + if !r.Cfg.Matrix.IsLocalServerName(sender.Domain()) { + return api.ErrInvalidID{Err: fmt.Errorf("the invite must be from a local user")} + } + if event.StateKey() == nil { - return nil, fmt.Errorf("invite must be a state event") + return fmt.Errorf("invite must be a state event") } - _, senderDomain, err := gomatrixserverlib.SplitID('@', event.Sender()) + invitedUser, err := spec.NewUserID(*event.StateKey(), true) if err != nil { - return nil, fmt.Errorf("sender %q is invalid", event.Sender()) + return spec.InvalidParam("The user ID is invalid") } + isTargetLocal := r.Cfg.Matrix.IsLocalServerName(invitedUser.Domain()) - roomID := event.RoomID() - targetUserID := *event.StateKey() - info, err := r.DB.RoomInfo(ctx, roomID) + validRoomID, err := spec.NewRoomID(event.RoomID()) if err != nil { - return nil, fmt.Errorf("failed to load RoomInfo: %w", err) + return err } - _, domain, err := gomatrixserverlib.SplitID('@', targetUserID) + input := gomatrixserverlib.PerformInviteInput{ + RoomID: *validRoomID, + InviteEvent: event.PDU, + InvitedUser: *invitedUser, + IsTargetLocal: isTargetLocal, + StrippedState: req.InviteRoomState, + MembershipQuerier: &api.MembershipQuerier{Roomserver: r.RSAPI}, + StateQuerier: &QueryState{r.DB}, + } + inviteEvent, err := gomatrixserverlib.PerformInvite(ctx, input, r.FSAPI) if err != nil { - return nil, api.ErrInvalidID{Err: fmt.Errorf("the user ID %s is invalid", targetUserID)} - } - isTargetLocal := r.Cfg.Matrix.IsLocalServerName(domain) - isOriginLocal := r.Cfg.Matrix.IsLocalServerName(senderDomain) - if !isOriginLocal && !isTargetLocal { - return nil, api.ErrInvalidID{Err: fmt.Errorf("the invite must be either from or to a local user")} + switch e := err.(type) { + case spec.MatrixError: + if e.ErrCode == spec.ErrorForbidden { + return api.ErrNotAllowed{Err: fmt.Errorf("%s", e.Err)} + } + } + return err } - logger := util.GetLogger(ctx).WithFields(map[string]interface{}{ - "inviter": event.Sender(), - "invitee": *event.StateKey(), - "room_id": roomID, - "event_id": event.EventID(), - }) - logger.WithFields(log.Fields{ - "room_version": req.RoomVersion, - "room_info_exists": info != nil, - "target_local": isTargetLocal, - "origin_local": isOriginLocal, - }).Debug("processing invite event") - - inviteState := req.InviteRoomState - if len(inviteState) == 0 && info != nil { - var is []fclient.InviteV2StrippedState - if is, err = buildInviteStrippedState(ctx, r.DB, info, req); err == nil { - inviteState = is - } - } - if len(inviteState) == 0 { - if err = event.SetUnsignedField("invite_room_state", struct{}{}); err != nil { - return nil, fmt.Errorf("event.SetUnsignedField: %w", err) - } - } else { - if err = event.SetUnsignedField("invite_room_state", inviteState); err != nil { - return nil, fmt.Errorf("event.SetUnsignedField: %w", err) - } - } - - updateMembershipTableManually := func() ([]api.OutputEvent, error) { - var updater *shared.MembershipUpdater - if updater, err = r.DB.MembershipUpdater(ctx, roomID, targetUserID, isTargetLocal, req.RoomVersion); err != nil { - return nil, fmt.Errorf("r.DB.MembershipUpdater: %w", err) - } - outputUpdates, err = helpers.UpdateToInviteMembership(updater, &types.Event{ - EventNID: 0, - PDU: event.PDU, - }, outputUpdates, req.Event.Version()) - if err != nil { - return nil, fmt.Errorf("updateToInviteMembership: %w", err) - } - if err = updater.Commit(); err != nil { - return nil, fmt.Errorf("updater.Commit: %w", err) - } - logger.Debugf("updated membership to invite and sending invite OutputEvent") - return outputUpdates, nil - } - - if (info == nil || info.IsStub()) && !isOriginLocal && isTargetLocal { - // The invite came in over federation for a room that we don't know about - // yet. We need to handle this a bit differently to most invites because - // we don't know the room state, therefore the roomserver can't process - // an input event. Instead we will update the membership table with the - // new invite and generate an output event. - return updateMembershipTableManually() - } - - var isAlreadyJoined bool - if info != nil { - _, isAlreadyJoined, _, err = r.DB.GetMembership(ctx, info.RoomNID, *event.StateKey()) - if err != nil { - return nil, fmt.Errorf("r.DB.GetMembership: %w", err) - } - } - if isAlreadyJoined { - // If the user is joined to the room then that takes precedence over this - // invite event. It makes little sense to move a user that is already - // joined to the room into the invite state. - // This could plausibly happen if an invite request raced with a join - // request for a user. For example if a user was invited to a public - // room and they joined the room at the same time as the invite was sent. - // The other way this could plausibly happen is if an invite raced with - // a kick. For example if a user was kicked from a room in error and in - // response someone else in the room re-invited them then it is possible - // for the invite request to race with the leave event so that the - // target receives invite before it learns that it has been kicked. - // There are a few ways this could be plausibly handled in the roomserver. - // 1) Store the invite, but mark it as retired. That will result in the - // permanent rejection of that invite event. So even if the target - // user leaves the room and the invite is retransmitted it will be - // ignored. However a new invite with a new event ID would still be - // accepted. - // 2) Silently discard the invite event. This means that if the event - // was retransmitted at a later date after the target user had left - // the room we would accept the invite. However since we hadn't told - // the sending server that the invite had been discarded it would - // have no reason to attempt to retry. - // 3) Signal the sending server that the user is already joined to the - // room. - // For now we will implement option 2. Since in the abesence of a retry - // mechanism it will be equivalent to option 1, and we don't have a - // signalling mechanism to implement option 3. - logger.Debugf("user already joined") - return nil, api.ErrNotAllowed{Err: fmt.Errorf("user is already joined to room")} - } - - // If the invite originated remotely then we can't send an - // InputRoomEvent for the invite as it will never pass auth checks - // due to lacking room state, but we still need to tell the client - // about the invite so we can accept it, hence we return an output - // event to send to the Sync API. - if !isOriginLocal { - return updateMembershipTableManually() - } - - // The invite originated locally. Therefore we have a responsibility to - // try and see if the user is allowed to make this invite. We can't do - // this for invites coming in over federation - we have to take those on - // trust. - _, err = helpers.CheckAuthEvents(ctx, r.DB, info, event, event.AuthEventIDs()) - if err != nil { - logger.WithError(err).WithField("event_id", event.EventID()).WithField("auth_event_ids", event.AuthEventIDs()).Error( - "processInviteEvent.checkAuthEvents failed for event", - ) - return nil, api.ErrNotAllowed{Err: err} - } - - // If the invite originated from us and the target isn't local then we - // should try and send the invite over federation first. It might be - // that the remote user doesn't exist, in which case we can give up - // processing here. - if req.SendAsServer != api.DoNotSendToOtherServers && !isTargetLocal { - fsReq := &federationAPI.PerformInviteRequest{ - RoomVersion: req.RoomVersion, - Event: event, - InviteRoomState: inviteState, - } - fsRes := &federationAPI.PerformInviteResponse{} - if err = r.FSAPI.PerformInvite(ctx, fsReq, fsRes); err != nil { - logger.WithError(err).WithField("event_id", event.EventID()).Error("r.FSAPI.PerformInvite failed") - return nil, api.ErrNotAllowed{Err: err} - } - event = fsRes.Event - logger.Debugf("Federated PerformInvite success with event ID %s", event.EventID()) + // Use the returned event if there was one (due to federation), otherwise + // send the original invite event to the roomserver. + if inviteEvent == nil { + inviteEvent = event } // Send the invite event to the roomserver input stream. This will @@ -219,67 +182,18 @@ func (r *Inviter) PerformInvite( InputRoomEvents: []api.InputRoomEvent{ { Kind: api.KindNew, - Event: event, - Origin: senderDomain, + Event: &types.HeaderedEvent{PDU: inviteEvent}, + Origin: sender.Domain(), SendAsServer: req.SendAsServer, }, }, } inputRes := &api.InputRoomEventsResponse{} r.Inputer.InputRoomEvents(context.Background(), inputReq, inputRes) - if err = inputRes.Err(); err != nil { - logger.WithError(err).WithField("event_id", event.EventID()).Error("r.InputRoomEvents failed") - return nil, api.ErrNotAllowed{Err: err} + if err := inputRes.Err(); err != nil { + util.GetLogger(ctx).WithField("event_id", event.EventID()).Error("r.InputRoomEvents failed") + return api.ErrNotAllowed{Err: err} } - // Don't notify the sync api of this event in the same way as a federated invite so the invitee - // gets the invite, as the roomserver will do this when it processes the m.room.member invite. - return outputUpdates, nil -} - -func buildInviteStrippedState( - ctx context.Context, - db storage.Database, - info *types.RoomInfo, - input *api.PerformInviteRequest, -) ([]fclient.InviteV2StrippedState, error) { - stateWanted := []gomatrixserverlib.StateKeyTuple{} - // "If they are set on the room, at least the state for m.room.avatar, m.room.canonical_alias, m.room.join_rules, and m.room.name SHOULD be included." - // https://matrix.org/docs/spec/client_server/r0.6.0#m-room-member - for _, t := range []string{ - spec.MRoomName, spec.MRoomCanonicalAlias, - spec.MRoomJoinRules, spec.MRoomAvatar, - spec.MRoomEncryption, spec.MRoomCreate, - } { - stateWanted = append(stateWanted, gomatrixserverlib.StateKeyTuple{ - EventType: t, - StateKey: "", - }) - } - roomState := state.NewStateResolution(db, info) - stateEntries, err := roomState.LoadStateAtSnapshotForStringTuples( - ctx, info.StateSnapshotNID(), stateWanted, - ) - if err != nil { - return nil, err - } - stateNIDs := []types.EventNID{} - for _, stateNID := range stateEntries { - stateNIDs = append(stateNIDs, stateNID.EventNID) - } - if info == nil { - return nil, types.ErrorInvalidRoomInfo - } - stateEvents, err := db.Events(ctx, info.RoomVersion, stateNIDs) - if err != nil { - return nil, err - } - inviteState := []fclient.InviteV2StrippedState{ - fclient.NewInviteV2StrippedState(input.Event.PDU), - } - stateEvents = append(stateEvents, types.Event{PDU: input.Event.PDU}) - for _, event := range stateEvents { - inviteState = append(inviteState, fclient.NewInviteV2StrippedState(event.PDU)) - } - return inviteState, nil + return nil }