diff --git a/CHANGES.md b/CHANGES.md index 17fb75bed..095ab9c5b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,89 +1,161 @@ -# Dendrite 0.1.0 (2020-10-08) +# Changelog + +## Dendrite 0.2.1 (2020-10-22) + +### Fixes + +* Forward extremities are now calculated using only references from other extremities, rather than including outliers, which should fix cases where state can become corrupted ([#1556](https://github.com/matrix-org/dendrite/pull/1556)) +* Old state events will no longer be processed by the sync API as new, which should fix some cases where clients incorrectly believe they have joined or left rooms ([#1548](https://github.com/matrix-org/dendrite/pull/1548)) +* More SQLite database locking issues have been resolved in the latest events updater ([#1554](https://github.com/matrix-org/dendrite/pull/1554)) +* Internal HTTP API calls are now made using H2C (HTTP/2) in polylith mode, mitigating some potential head-of-line blocking issues ([#1541](https://github.com/matrix-org/dendrite/pull/1541)) +* Roomserver output events no longer incorrectly flag state rewrites ([#1557](https://github.com/matrix-org/dendrite/pull/1557)) +* Notification levels are now parsed correctly in power level events ([gomatrixserverlib#228](https://github.com/matrix-org/gomatrixserverlib/pull/228), contributed by [Pestdoktor](https://github.com/Pestdoktor)) +* Invalid UTF-8 is now correctly rejected when making federation requests ([gomatrixserverlib#229](https://github.com/matrix-org/gomatrixserverlib/pull/229), contributed by [Pestdoktor](https://github.com/Pestdoktor)) + +## Dendrite 0.2.0 (2020-10-20) + +### Important + +* This release makes breaking changes for polylith deployments, since they now use the multi-personality binary rather than separate binary files + * Users of polylith deployments should revise their setups to use the new binary - see the Features section below +* This release also makes breaking changes for Docker deployments, as are now publishing images to Docker Hub in separate repositories for monolith and polylith + * New repositories are as follows: [matrixdotorg/dendrite-monolith](https://hub.docker.com/repository/docker/matrixdotorg/dendrite-monolith) and [matrixdotorg/dendrite-polylith](https://hub.docker.com/repository/docker/matrixdotorg/dendrite-polylith) + * The new `latest` tag will be updated with the latest release, and new versioned tags, e.g. `v0.2.0`, will preserve specific release versions + * [Sample Compose configs](https://github.com/matrix-org/dendrite/tree/master/build/docker) have been updated - if you are running a Docker deployment, please review the changes + * Images for the client API proxy and federation API proxy are no longer provided as they are unsupported - please use [nginx](docs/nginx/) (or another reverse proxy) instead + +### Features + +* Dendrite polylith deployments now use a special multi-personality binary, rather than separate binaries + * This is cleaner, builds faster and simplifies deployment + * The first command line argument states the component to run, e.g. `./dendrite-polylith-multi roomserver` +* Database migrations are now run at startup +* Invalid UTF-8 in requests is now rejected (contributed by [Pestdoktor](https://github.com/Pestdoktor)) +* Fully read markers are now implemented in the client API (contributed by [Lesterpig](https://github.com/Lesterpig)) +* Missing auth events are now retrieved from other servers in the room, rather than just the event origin +* `m.room.create` events are now validated properly when processing a `/send_join` response +* The roomserver now implements `KindOld` for handling historic events without them becoming forward extremity candidates, i.e. for backfilled or missing events + +### Fixes + +* State resolution v2 performance has been improved dramatically when dealing with large state sets +* The roomserver no longer processes outlier events if they are already known +* A SQLite locking issue in the previous events updater has been fixed +* The client API `/state` endpoint now correctly returns state after the leave event, if the user has left the room +* The client API `/createRoom` endpoint now sends cumulative state to the roomserver for the initial room events +* The federation API `/send` endpoint now correctly requests the entire room state from the roomserver when needed +* Some internal HTTP API paths have been fixed in the user API (contributed by [S7evinK](https://github.com/S7evinK)) +* A race condition in the rate limiting code resulting in concurrent map writes has been fixed +* Each component now correctly starts a consumer/producer connection in monolith mode (when using Kafka) +* State resolution is no longer run for single trusted state snapshots that have been verified before +* A crash when rolling back the transaction in the latest events updater has been fixed +* Typing events are now ignored when the sender domain does not match the origin server +* Duplicate redaction entries no longer result in database errors +* Recursion has been removed from the code path for retrieving missing events +* `QueryMissingAuthPrevEvents` now returns events that have no associated state as if they are missing +* Signing key fetchers no longer ignore keys for the local domain, if retrieving a key that is not known in the local config +* Federation timeouts have been adjusted so we don't give up on remote requests so quickly +* `create-account` no longer relies on the device database (contributed by [ThatNerdyPikachu](https://github.com/ThatNerdyPikachu)) + +### Known issues + +* Old events can incorrectly appear in `/sync` as if they are new when retrieving missing events from federated servers, causing them to appear at the bottom of the timeline in clients + +## Dendrite 0.1.0 (2020-10-08) First versioned release of Dendrite. ## Client-Server API Features ### Account registration and management -- Registration: By password only. -- Login: By password only. No fallback. -- Logout: Yes. -- Change password: Yes. -- Link email/msisdn to account: No. -- Deactivate account: Yes. -- Check if username is available: Yes. -- Account data: Yes. -- OpenID: No. + +* Registration: By password only. +* Login: By password only. No fallback. +* Logout: Yes. +* Change password: Yes. +* Link email/msisdn to account: No. +* Deactivate account: Yes. +* Check if username is available: Yes. +* Account data: Yes. +* OpenID: No. ### Rooms -- Room creation: Yes, including presets. -- Joining rooms: Yes, including by alias or `?server_name=`. -- Event sending: Yes, including transaction IDs. -- Aliases: Yes. -- Published room directory: Yes. -- Kicking users: Yes. -- Banning users: Yes. -- Inviting users: Yes, but not third-party invites. -- Forgetting rooms: No. -- Room versions: All (v1 - v6) -- Tagging: Yes. + +* Room creation: Yes, including presets. +* Joining rooms: Yes, including by alias or `?server_name=`. +* Event sending: Yes, including transaction IDs. +* Aliases: Yes. +* Published room directory: Yes. +* Kicking users: Yes. +* Banning users: Yes. +* Inviting users: Yes, but not third-party invites. +* Forgetting rooms: No. +* Room versions: All (v1 * v6) +* Tagging: Yes. ### User management -- User directory: Basic support. -- Ignoring users: No. -- Groups/Communities: No. + +* User directory: Basic support. +* Ignoring users: No. +* Groups/Communities: No. ### Device management -- Creating devices: Yes. -- Deleting devices: Yes. -- Send-to-device messaging: Yes. + +* Creating devices: Yes. +* Deleting devices: Yes. +* Send-to-device messaging: Yes. ### Sync -- Filters: Timeline limit only. Rest unimplemented. -- Deprecated `/events` and `/initialSync`: No. + +* Filters: Timeline limit only. Rest unimplemented. +* Deprecated `/events` and `/initialSync`: No. ### Room events -- Typing: Yes. -- Receipts: No. -- Read Markers: No. -- Presence: No. -- Content repository (attachments): Yes. -- History visibility: No, defaults to `joined`. -- Push notifications: No. -- Event context: No. -- Reporting content: No. + +* Typing: Yes. +* Receipts: No. +* Read Markers: No. +* Presence: No. +* Content repository (attachments): Yes. +* History visibility: No, defaults to `joined`. +* Push notifications: No. +* Event context: No. +* Reporting content: No. ### End-to-End Encryption -- Uploading device keys: Yes. -- Downloading device keys: Yes. -- Claiming one-time keys: Yes. -- Querying key changes: Yes. -- Cross-Signing: No. + +* Uploading device keys: Yes. +* Downloading device keys: Yes. +* Claiming one-time keys: Yes. +* Querying key changes: Yes. +* Cross-Signing: No. ### Misc -- Server-side search: No. -- Guest access: Partial. -- Room previews: No, partial support for Peeking via MSC2753. -- Third-Party networks: No. -- Server notices: No. -- Policy lists: No. + +* Server-side search: No. +* Guest access: Partial. +* Room previews: No, partial support for Peeking via MSC2753. +* Third-Party networks: No. +* Server notices: No. +* Policy lists: No. ## Federation Features -- Querying keys (incl. notary): Yes. -- Server ACLs: Yes. -- Sending transactions: Yes. -- Joining rooms: Yes. -- Inviting to rooms: Yes, but not third-party invites. -- Leaving rooms: Yes. -- Content repository: Yes. -- Backfilling / get_missing_events: Yes. -- Retrieving state of the room (`/state` and `/state_ids`): Yes. -- Public rooms: Yes. -- Querying profile data: Yes. -- Device management: Yes. -- Send-to-Device messaging: Yes. -- Querying/Claiming E2E Keys: Yes. -- Typing: Yes. -- Presence: No. -- Receipts: No. -- OpenID: No. \ No newline at end of file + +* Querying keys (incl. notary): Yes. +* Server ACLs: Yes. +* Sending transactions: Yes. +* Joining rooms: Yes. +* Inviting to rooms: Yes, but not third-party invites. +* Leaving rooms: Yes. +* Content repository: Yes. +* Backfilling / get_missing_events: Yes. +* Retrieving state of the room (`/state` and `/state_ids`): Yes. +* Public rooms: Yes. +* Querying profile data: Yes. +* Device management: Yes. +* Send-to-Device messaging: Yes. +* Querying/Claiming E2E Keys: Yes. +* Typing: Yes. +* Presence: No. +* Receipts: No. +* OpenID: No. \ No newline at end of file diff --git a/README.md b/README.md index f27cb4029..ea61dac13 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ Then point your favourite Matrix client at `http://localhost:8008`. We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it -updates with CI. As of October 2020 we're at around 56% CS API coverage and 77% Federation coverage, though check +updates with CI. As of October 2020 we're at around 57% CS API coverage and 81% Federation coverage, though check CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse servers such as matrix.org reasonably well. There's a long list of features that are not implemented, notably: - Receipts diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index d8e07681f..5cab0530f 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.13.7-alpine3.11 AS builder +FROM docker.io/golang:1.15-alpine AS builder RUN apk --update --no-cache add bash build-base diff --git a/build/docker/Dockerfile.component b/build/docker/Dockerfile.component deleted file mode 100644 index 13634391a..000000000 --- a/build/docker/Dockerfile.component +++ /dev/null @@ -1,13 +0,0 @@ -FROM matrixdotorg/dendrite:latest AS base - -FROM alpine:latest - -ARG component=monolith -ENV entrypoint=${component} - -COPY --from=base /build/bin/${component} /usr/bin - -VOLUME /etc/dendrite -WORKDIR /etc/dendrite - -ENTRYPOINT /usr/bin/${entrypoint} $@ \ No newline at end of file diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith new file mode 100644 index 000000000..3e9d0cba4 --- /dev/null +++ b/build/docker/Dockerfile.monolith @@ -0,0 +1,13 @@ +FROM matrixdotorg/dendrite:latest AS base + +FROM alpine:latest + +COPY --from=base /build/bin/dendrite-monolith-server /usr/bin +COPY --from=base /build/bin/goose /usr/bin +COPY --from=base /build/bin/create-account /usr/bin +COPY --from=base /build/bin/generate-keys /usr/bin + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] \ No newline at end of file diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith new file mode 100644 index 000000000..dd4cbd38f --- /dev/null +++ b/build/docker/Dockerfile.polylith @@ -0,0 +1,13 @@ +FROM matrixdotorg/dendrite:latest AS base + +FROM alpine:latest + +COPY --from=base /build/bin/dendrite-polylith-multi /usr/bin +COPY --from=base /build/bin/goose /usr/bin +COPY --from=base /build/bin/create-account /usr/bin +COPY --from=base /build/bin/generate-keys /usr/bin + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] \ No newline at end of file diff --git a/build/docker/docker-compose.deps.yml b/build/docker/docker-compose.deps.yml index 74e478a8d..1a27ffac0 100644 --- a/build/docker/docker-compose.deps.yml +++ b/build/docker/docker-compose.deps.yml @@ -2,7 +2,7 @@ version: "3.4" services: postgres: hostname: postgres - image: postgres:9.5 + image: postgres:9.6 restart: always volumes: - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh diff --git a/build/docker/docker-compose.monolith.yml b/build/docker/docker-compose.monolith.yml index 336a43984..8fb798343 100644 --- a/build/docker/docker-compose.monolith.yml +++ b/build/docker/docker-compose.monolith.yml @@ -2,9 +2,8 @@ version: "3.4" services: monolith: hostname: monolith - image: matrixdotorg/dendrite:monolith + image: matrixdotorg/dendrite-monolith:latest command: [ - "--config=dendrite.yaml", "--tls-cert=server.crt", "--tls-key=server.key" ] diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index 8a4c50e06..f377e36fc 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -1,43 +1,18 @@ version: "3.4" services: - client_api_proxy: - hostname: client_api_proxy - image: matrixdotorg/dendrite:clientproxy - command: [ - "--bind-address=:8008", - "--client-api-server-url=http://client_api:8071", - "--sync-api-server-url=http://sync_api:8073", - "--media-api-server-url=http://media_api:8074" - ] - volumes: - - ./config:/etc/dendrite - networks: - - internal - depends_on: - - sync_api - - client_api - - media_api - ports: - - "8008:8008" - client_api: hostname: client_api - image: matrixdotorg/dendrite:clientapi - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: clientapi volumes: - ./config:/etc/dendrite - - room_server networks: - internal media_api: hostname: media_api - image: matrixdotorg/dendrite:mediaapi - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: mediaapi volumes: - ./config:/etc/dendrite networks: @@ -45,10 +20,8 @@ services: sync_api: hostname: sync_api - image: matrixdotorg/dendrite:syncapi - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: syncapi volumes: - ./config:/etc/dendrite networks: @@ -56,10 +29,8 @@ services: room_server: hostname: room_server - image: matrixdotorg/dendrite:roomserver - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: roomserver volumes: - ./config:/etc/dendrite networks: @@ -67,40 +38,17 @@ services: edu_server: hostname: edu_server - image: matrixdotorg/dendrite:eduserver - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: eduserver volumes: - ./config:/etc/dendrite networks: - internal - federation_api_proxy: - hostname: federation_api_proxy - image: matrixdotorg/dendrite:federationproxy - command: [ - "--bind-address=:8448", - "--federation-api-url=http://federation_api:8072", - "--media-api-server-url=http://media_api:8074" - ] - volumes: - - ./config:/etc/dendrite - depends_on: - - federation_api - - federation_sender - - media_api - networks: - - internal - ports: - - "8448:8448" - federation_api: hostname: federation_api - image: matrixdotorg/dendrite:federationapi - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: federationapi volumes: - ./config:/etc/dendrite networks: @@ -108,10 +56,8 @@ services: federation_sender: hostname: federation_sender - image: matrixdotorg/dendrite:federationsender - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: federationsender volumes: - ./config:/etc/dendrite networks: @@ -119,10 +65,8 @@ services: key_server: hostname: key_server - image: matrixdotorg/dendrite:keyserver - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: keyserver volumes: - ./config:/etc/dendrite networks: @@ -130,10 +74,8 @@ services: signing_key_server: hostname: signing_key_server - image: matrixdotorg/dendrite:signingkeyserver - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: signingkeyserver volumes: - ./config:/etc/dendrite networks: @@ -141,10 +83,8 @@ services: user_api: hostname: user_api - image: matrixdotorg/dendrite:userapi - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: userapi volumes: - ./config:/etc/dendrite networks: @@ -152,10 +92,8 @@ services: appservice_api: hostname: appservice_api - image: matrixdotorg/dendrite:appservice - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: appservice volumes: - ./config:/etc/dendrite networks: diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index d72bac214..f80f6bed2 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -2,20 +2,11 @@ cd $(git rev-parse --show-toplevel) -docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:latest . +TAG=${1:-latest} -docker build -t matrixdotorg/dendrite:monolith --build-arg component=dendrite-monolith-server -f build/docker/Dockerfile.component . +echo "Building tag '${TAG}'" -docker build -t matrixdotorg/dendrite:appservice --build-arg component=dendrite-appservice-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:clientapi --build-arg component=dendrite-client-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:clientproxy --build-arg component=client-api-proxy -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:eduserver --build-arg component=dendrite-edu-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationapi --build-arg component=dendrite-federation-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationsender --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationproxy --build-arg component=federation-api-proxy -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:keyserver --build-arg component=dendrite-key-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:signingkeyserver --build-arg component=dendrite-signing-key-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:userapi --build-arg component=dendrite-user-api-server -f build/docker/Dockerfile.component . +docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} . + +docker build -t matrixdotorg/dendrite-monolith:${TAG} -f build/docker/Dockerfile.monolith . +docker build -t matrixdotorg/dendrite-polylith:${TAG} -f build/docker/Dockerfile.polylith . \ No newline at end of file diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh index be9185464..496e80067 100755 --- a/build/docker/images-pull.sh +++ b/build/docker/images-pull.sh @@ -1,17 +1,8 @@ #!/bin/bash -docker pull matrixdotorg/dendrite:monolith +TAG=${1:-latest} -docker pull matrixdotorg/dendrite:appservice -docker pull matrixdotorg/dendrite:clientapi -docker pull matrixdotorg/dendrite:clientproxy -docker pull matrixdotorg/dendrite:eduserver -docker pull matrixdotorg/dendrite:federationapi -docker pull matrixdotorg/dendrite:federationsender -docker pull matrixdotorg/dendrite:federationproxy -docker pull matrixdotorg/dendrite:keyserver -docker pull matrixdotorg/dendrite:mediaapi -docker pull matrixdotorg/dendrite:roomserver -docker pull matrixdotorg/dendrite:syncapi -docker pull matrixdotorg/dendrite:signingkeyserver -docker pull matrixdotorg/dendrite:userapi +echo "Pulling tag '${TAG}'" + +docker pull matrixdotorg/dendrite-monolith:${TAG} +docker pull matrixdotorg/dendrite-polylith:${TAG} \ No newline at end of file diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index 64920171b..fd9b999ea 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -1,17 +1,8 @@ #!/bin/bash -docker push matrixdotorg/dendrite:monolith +TAG=${1:-latest} -docker push matrixdotorg/dendrite:appservice -docker push matrixdotorg/dendrite:clientapi -docker push matrixdotorg/dendrite:clientproxy -docker push matrixdotorg/dendrite:eduserver -docker push matrixdotorg/dendrite:federationapi -docker push matrixdotorg/dendrite:federationsender -docker push matrixdotorg/dendrite:federationproxy -docker push matrixdotorg/dendrite:keyserver -docker push matrixdotorg/dendrite:mediaapi -docker push matrixdotorg/dendrite:roomserver -docker push matrixdotorg/dendrite:syncapi -docker push matrixdotorg/dendrite:signingkeyserver -docker push matrixdotorg/dendrite:userapi +echo "Pushing tag '${TAG}'" + +docker push matrixdotorg/dendrite-monolith:${TAG} +docker push matrixdotorg/dendrite-polylith:${TAG} \ No newline at end of file diff --git a/build/docker/postgres/create_db.sh b/build/docker/postgres/create_db.sh index 97514467b..7495a3978 100755 --- a/build/docker/postgres/create_db.sh +++ b/build/docker/postgres/create_db.sh @@ -1,5 +1,5 @@ #!/bin/sh -for db in account device mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice e2ekey naffka; do +for db in account device mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice naffka; do createdb -U dendrite -O dendrite dendrite_$db done diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 9655339cd..cff3c9813 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -344,6 +344,7 @@ func createRoom( if err = roomserverAPI.SendEventWithState( req.Context(), rsAPI, + roomserverAPI.KindNew, &gomatrixserverlib.RespState{ StateEvents: accumulated, AuthEvents: accumulated, diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 88cb23647..fe0795577 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -76,6 +76,7 @@ func sendMembership(ctx context.Context, accountDB accounts.Database, device *us if err = roomserverAPI.SendEvents( ctx, rsAPI, + api.KindNew, []gomatrixserverlib.HeaderedEvent{event.Event.Headered(roomVer)}, cfg.Matrix.ServerName, nil, diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 60669a0c8..bbe35facd 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -170,7 +170,7 @@ func SetAvatarURL( return jsonerror.InternalServerError() } - if err := api.SendEvents(req.Context(), rsAPI, events, cfg.Matrix.ServerName, nil); err != nil { + if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, events, cfg.Matrix.ServerName, nil); err != nil { util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") return jsonerror.InternalServerError() } @@ -288,7 +288,7 @@ func SetDisplayName( return jsonerror.InternalServerError() } - if err := api.SendEvents(req.Context(), rsAPI, events, cfg.Matrix.ServerName, nil); err != nil { + if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, events, cfg.Matrix.ServerName, nil); err != nil { util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go index 9701685e0..266c0aff2 100644 --- a/clientapi/routing/redaction.go +++ b/clientapi/routing/redaction.go @@ -121,7 +121,7 @@ func SendRedaction( JSON: jsonerror.NotFound("Room does not exist"), } } - if err = roomserverAPI.SendEvents(context.Background(), rsAPI, []gomatrixserverlib.HeaderedEvent{*e}, cfg.Matrix.ServerName, nil); err != nil { + if err = roomserverAPI.SendEvents(context.Background(), rsAPI, api.KindNew, []gomatrixserverlib.HeaderedEvent{*e}, cfg.Matrix.ServerName, nil); err != nil { util.GetLogger(req.Context()).WithError(err).Errorf("failed to SendEvents") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 9744a5640..1303663ff 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -92,6 +92,7 @@ func SendEvent( // event ID in case of duplicate transaction is discarded if err := api.SendEvents( req.Context(), rsAPI, + api.KindNew, []gomatrixserverlib.HeaderedEvent{ e.Headered(verRes.RoomVersion), }, diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go index 2a424cbe8..f69b54bbc 100644 --- a/clientapi/routing/state.go +++ b/clientapi/routing/state.go @@ -141,7 +141,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a util.GetLogger(ctx).WithError(err).Error("Failed to QueryMembershipForUser") return jsonerror.InternalServerError() } - for _, ev := range stateRes.StateEvents { + for _, ev := range stateAfterRes.StateEvents { stateEvents = append( stateEvents, gomatrixserverlib.HeaderedToClientEvent(ev, gomatrixserverlib.FormatAll), diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index b9575a284..272d3407d 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -361,6 +361,7 @@ func emit3PIDInviteEvent( return api.SendEvents( ctx, rsAPI, + api.KindNew, []gomatrixserverlib.HeaderedEvent{ (*event).Headered(queryRes.RoomVersion), }, diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index a9bd92794..f6de2d0d4 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -22,7 +22,6 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/userapi/storage/accounts" - "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/matrix-org/gomatrixserverlib" ) @@ -39,7 +38,6 @@ var ( username = flag.String("username", "", "The user ID localpart to register e.g 'alice' in '@alice:localhost'.") password = flag.String("password", "", "Optional. The password to register with. If not specified, this account will be password-less.") serverNameStr = flag.String("servername", "localhost", "The Matrix server domain which will form the domain part of the user ID.") - accessToken = flag.String("token", "", "Optional. The desired access_token to have. If not specified, a random access_token will be made.") ) func main() { @@ -78,29 +76,5 @@ func main() { os.Exit(1) } - deviceDB, err := devices.NewDatabase(&config.DatabaseOptions{ - ConnectionString: config.DataSource(*database), - }, serverName) - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - - if *accessToken == "" { - t := "token_" + *username - accessToken = &t - } - - device, err := deviceDB.CreateDevice( - context.Background(), *username, nil, *accessToken, nil, "127.0.0.1", "", - ) - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - - fmt.Println("Created account:") - fmt.Printf("user_id = %s\n", device.UserID) - fmt.Printf("device_id = %s\n", device.ID) - fmt.Printf("access_token = %s\n", device.AccessToken) + fmt.Println("Created account") } diff --git a/cmd/dendrite-polylith-multi/main.go b/cmd/dendrite-polylith-multi/main.go new file mode 100644 index 000000000..0d6406c01 --- /dev/null +++ b/cmd/dendrite-polylith-multi/main.go @@ -0,0 +1,78 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "os" + "strings" + + "github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/setup" + "github.com/sirupsen/logrus" +) + +type entrypoint func(base *setup.BaseDendrite, cfg *config.Dendrite) + +func main() { + cfg := setup.ParseFlags(true) + + component := "" + if flag.NFlag() > 0 { + component = flag.Arg(0) // ./dendrite-polylith-multi --config=... clientapi + } else if len(os.Args) > 1 { + component = os.Args[1] // ./dendrite-polylith-multi clientapi + } + + components := map[string]entrypoint{ + "appservice": personalities.Appservice, + "clientapi": personalities.ClientAPI, + "eduserver": personalities.EDUServer, + "federationapi": personalities.FederationAPI, + "federationsender": personalities.FederationSender, + "keyserver": personalities.KeyServer, + "mediaapi": personalities.MediaAPI, + "roomserver": personalities.RoomServer, + "signingkeyserver": personalities.SigningKeyServer, + "syncapi": personalities.SyncAPI, + "userapi": personalities.UserAPI, + } + + start, ok := components[component] + if !ok { + if component == "" { + logrus.Errorf("No component specified") + logrus.Info("The first argument on the command line must be the name of the component to run") + } else { + logrus.Errorf("Unknown component %q specified", component) + } + + var list []string + for c := range components { + list = append(list, c) + } + logrus.Infof("Valid components: %s", strings.Join(list, ", ")) + + os.Exit(1) + } + + logrus.Infof("Starting %q component", component) + + base := setup.NewBaseDendrite(cfg, component, false) // TODO + defer base.Close() // nolint: errcheck + + start(base, cfg) +} diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-polylith-multi/personalities/appservice.go similarity index 83% rename from cmd/dendrite-appservice-server/main.go rename to cmd/dendrite-polylith-multi/personalities/appservice.go index 6adbdb17c..7fa87b115 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/appservice.go @@ -1,4 +1,4 @@ -// Copyright 2018 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "AppServiceAPI", true) - - defer base.Close() // nolint: errcheck +func Appservice(base *setup.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() rsAPI := base.RoomserverHTTPClient() diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/clientapi.go similarity index 87% rename from cmd/dendrite-client-api-server/main.go rename to cmd/dendrite-polylith-multi/personalities/clientapi.go index 0061de74f..09fc63ab3 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/clientapi.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,20 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( "github.com/matrix-org/dendrite/clientapi" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/internal/transactions" ) -func main() { - cfg := setup.ParseFlags(false) - - base := setup.NewBaseDendrite(cfg, "ClientAPI", true) - defer base.Close() // nolint: errcheck - +func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { accountDB := base.CreateAccountsDB() federation := base.CreateFederationClient() diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-polylith-multi/personalities/eduserver.go similarity index 77% rename from cmd/dendrite-edu-server/main.go rename to cmd/dendrite-polylith-multi/personalities/eduserver.go index 3a34b9a68..a5d2926f1 100644 --- a/cmd/dendrite-edu-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/eduserver.go @@ -1,3 +1,5 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -10,26 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( - _ "net/http/pprof" - "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" - "github.com/sirupsen/logrus" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "EDUServerAPI", true) - defer func() { - if err := base.Close(); err != nil { - logrus.WithError(err).Warn("BaseDendrite close failed") - } - }() - +func EDUServer(base *setup.BaseDendrite, cfg *config.Dendrite) { intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient()) eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI) diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/federationapi.go similarity index 86% rename from cmd/dendrite-federation-api-server/main.go rename to cmd/dendrite-polylith-multi/personalities/federationapi.go index 3ebb16f4b..a1bbeafad 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/federationapi.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "FederationAPI", true) - defer base.Close() // nolint: errcheck - +func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() federation := base.CreateFederationClient() serverKeyAPI := base.SigningKeyServerHTTPClient() diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-polylith-multi/personalities/federationsender.go similarity index 85% rename from cmd/dendrite-federation-sender-server/main.go rename to cmd/dendrite-polylith-multi/personalities/federationsender.go index 99b416c45..052523789 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/federationsender.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "FederationSender", true) - defer base.Close() // nolint: errcheck - +func FederationSender(base *setup.BaseDendrite, cfg *config.Dendrite) { federation := base.CreateFederationClient() serverKeyAPI := base.SigningKeyServerHTTPClient() diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-polylith-multi/personalities/keyserver.go similarity index 87% rename from cmd/dendrite-key-server/main.go rename to cmd/dendrite-polylith-multi/personalities/keyserver.go index ff5b22236..8c159ad06 100644 --- a/cmd/dendrite-key-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/keyserver.go @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/keyserver" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "KeyServer", true) - defer base.Close() // nolint: errcheck - +func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) { intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient()) intAPI.SetUserAPI(base.UserAPIClient()) diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/mediaapi.go similarity index 82% rename from cmd/dendrite-media-api-server/main.go rename to cmd/dendrite-polylith-multi/personalities/mediaapi.go index 2c2fe3b36..64e5bc312 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/mediaapi.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/mediaapi" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "MediaAPI", true) - defer base.Close() // nolint: errcheck - +func MediaAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() client := base.CreateClient() diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-polylith-multi/personalities/roomserver.go similarity index 84% rename from cmd/dendrite-room-server/main.go rename to cmd/dendrite-polylith-multi/personalities/roomserver.go index d3f145745..91027506d 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/roomserver.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/roomserver" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "RoomServerAPI", true) - defer base.Close() // nolint: errcheck - +func RoomServer(base *setup.BaseDendrite, cfg *config.Dendrite) { serverKeyAPI := base.SigningKeyServerHTTPClient() keyRing := serverKeyAPI.KeyRing() diff --git a/cmd/dendrite-signing-key-server/main.go b/cmd/dendrite-polylith-multi/personalities/signingkeyserver.go similarity index 86% rename from cmd/dendrite-signing-key-server/main.go rename to cmd/dendrite-polylith-multi/personalities/signingkeyserver.go index a4d48d361..a7bfff10b 100644 --- a/cmd/dendrite-signing-key-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/signingkeyserver.go @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/signingkeyserver" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "SigningKeyServer", true) - defer base.Close() // nolint: errcheck - +func SigningKeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) { federation := base.CreateFederationClient() intAPI := signingkeyserver.NewInternalAPI(&base.Cfg.SigningKeyServer, federation, base.Caches) diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/syncapi.go similarity index 84% rename from cmd/dendrite-sync-api-server/main.go rename to cmd/dendrite-polylith-multi/personalities/syncapi.go index 351dbc5f4..2d5c0b525 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/syncapi.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,19 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/syncapi" ) -func main() { - cfg := setup.ParseFlags(false) - - base := setup.NewBaseDendrite(cfg, "SyncAPI", true) - defer base.Close() // nolint: errcheck - +func SyncAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() federation := base.CreateFederationClient() diff --git a/cmd/dendrite-user-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/userapi.go similarity index 84% rename from cmd/dendrite-user-api-server/main.go rename to cmd/dendrite-polylith-multi/personalities/userapi.go index fb65fefbc..fe5e4fbd0 100644 --- a/cmd/dendrite-user-api-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/userapi.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/userapi" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "UserAPI", true) - defer base.Close() // nolint: errcheck - +func UserAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { accountDB := base.CreateAccountsDB() userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient()) diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go new file mode 100644 index 000000000..9fb14f056 --- /dev/null +++ b/cmd/resolve-state/main.go @@ -0,0 +1,132 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "strconv" + + "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/dendrite/roomserver/state" + "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" +) + +// This is a utility for inspecting state snapshots and running state resolution +// against real snapshots in an actual database. +// It takes one or more state snapshot NIDs as arguments, along with a room version +// to use for unmarshalling events, and will produce resolved output. +// +// Usage: ./resolve-state --roomversion=version snapshot [snapshot ...] +// e.g. ./resolve-state --roomversion=5 1254 1235 1282 + +var roomVersion = flag.String("roomversion", "5", "the room version to parse events as") + +// nolint:gocyclo +func main() { + ctx := context.Background() + cfg := setup.ParseFlags(true) + args := os.Args[1:] + + fmt.Println("Room version", *roomVersion) + + snapshotNIDs := []types.StateSnapshotNID{} + for _, arg := range args { + if i, err := strconv.Atoi(arg); err == nil { + snapshotNIDs = append(snapshotNIDs, types.StateSnapshotNID(i)) + } + } + + fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs") + + cache, err := caching.NewInMemoryLRUCache(true) + if err != nil { + panic(err) + } + + roomserverDB, err := storage.Open(&cfg.RoomServer.Database, cache) + if err != nil { + panic(err) + } + + blockNIDs, err := roomserverDB.StateBlockNIDs(ctx, snapshotNIDs) + if err != nil { + panic(err) + } + + var stateEntries []types.StateEntryList + for _, list := range blockNIDs { + entries, err2 := roomserverDB.StateEntries(ctx, list.StateBlockNIDs) + if err2 != nil { + panic(err2) + } + stateEntries = append(stateEntries, entries...) + } + + var eventNIDs []types.EventNID + for _, entry := range stateEntries { + for _, e := range entry.StateEntries { + eventNIDs = append(eventNIDs, e.EventNID) + } + } + + fmt.Println("Fetching", len(eventNIDs), "state events") + eventEntries, err := roomserverDB.Events(ctx, eventNIDs) + if err != nil { + panic(err) + } + + authEventIDMap := make(map[string]struct{}) + eventPtrs := make([]*gomatrixserverlib.Event, len(eventEntries)) + for i := range eventEntries { + eventPtrs[i] = &eventEntries[i].Event + for _, authEventID := range eventEntries[i].AuthEventIDs() { + authEventIDMap[authEventID] = struct{}{} + } + } + + authEventIDs := make([]string, 0, len(authEventIDMap)) + for authEventID := range authEventIDMap { + authEventIDs = append(authEventIDs, authEventID) + } + + fmt.Println("Fetching", len(authEventIDs), "auth events") + authEventEntries, err := roomserverDB.EventsFromIDs(ctx, authEventIDs) + if err != nil { + panic(err) + } + + authEventPtrs := make([]*gomatrixserverlib.Event, len(authEventEntries)) + for i := range authEventEntries { + authEventPtrs[i] = &authEventEntries[i].Event + } + + events := make([]gomatrixserverlib.Event, len(eventEntries)) + authEvents := make([]gomatrixserverlib.Event, len(authEventEntries)) + for i, ptr := range eventPtrs { + events[i] = *ptr + } + for i, ptr := range authEventPtrs { + authEvents[i] = *ptr + } + + fmt.Println("Resolving state") + resolved, err := state.ResolveConflictsAdhoc( + gomatrixserverlib.RoomVersion(*roomVersion), + events, + authEvents, + ) + if err != nil { + panic(err) + } + + fmt.Println("Resolved state contains", len(resolved), "events") + for _, event := range resolved { + fmt.Println() + fmt.Printf("* %s %s %q\n", event.EventID(), event.Type(), *event.StateKey()) + fmt.Printf(" %s\n", string(event.Content())) + } +} diff --git a/docs/INSTALL.md b/docs/INSTALL.md index be12d7b86..1cecd047c 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -24,7 +24,7 @@ use in production environments just yet! Dendrite requires: * Go 1.13 or higher -* Postgres 9.5 or higher (if using Postgres databases, not needed for SQLite) +* Postgres 9.6 or higher (if using Postgres databases, not needed for SQLite) If you want to run a polylith deployment, you also need: @@ -93,12 +93,12 @@ brew services start kafka ### SQLite database setup Dendrite can use the built-in SQLite database engine for small setups. -The SQLite databases do not need to be preconfigured - Dendrite will +The SQLite databases do not need to be pre-built - Dendrite will create them automatically at startup. ### Postgres database setup -Assuming that Postgres 9.5 (or later) is installed: +Assuming that Postgres 9.6 (or later) is installed: * Create role, choosing a new password when prompted: @@ -109,7 +109,7 @@ Assuming that Postgres 9.5 (or later) is installed: * Create the component databases: ```bash - for i in account device mediaapi syncapi roomserver signingkeyserver federationsender appservice e2ekey naffka; do + for i in mediaapi syncapi roomserver signingkeyserver federationsender appservice keyserver userapi_account userapi_device naffka; do sudo -u postgres createdb -O dendrite dendrite_$i done ``` @@ -129,14 +129,18 @@ for federation and the server signing key: ./bin/generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key ``` +If you have server keys from an older synapse instance, +[convert them](serverkeyformat.md#converting-synapse-keys) to Dendrite's PEM +format and configure them as `old_private_keys` in your config. + ### Configuration file Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Things that will need editing include *at least*: * The `server_name` entry to reflect the hostname of your Dendrite server * The `database` lines with an updated connection string based on your - desired setup, e.g. replacing `component` with the name of the component: - * For Postgres: `postgres://dendrite:password@localhost/component` + desired setup, e.g. replacing `database` with the name of the database: + * For Postgres: `postgres://dendrite:password@localhost/database` * For SQLite on disk: `file:component.db` or `file:///path/to/component.db` * Postgres and SQLite can be mixed and matched. * The `use_naffka` option if using Naffka in a monolith deployment @@ -147,6 +151,10 @@ then configuring `key_perspectives` (like `matrix.org` in the sample) can help to improve reliability considerably by allowing your homeserver to fetch public keys for dead homeservers from somewhere else. +**WARNING:** Dendrite supports running all components from the same database in +Postgres mode, but this is **NOT** a supported configuration with SQLite. When +using SQLite, all components **MUST** use their own database file. + ## Starting a monolith server It is possible to use Naffka as an in-process replacement to Kafka when using @@ -167,30 +175,17 @@ as shown below, it will also listen for HTTPS connections on port 8448. The following contains scripts which will run all the required processes in order to point a Matrix client at Dendrite. -### Client proxy +### nginx (or other reverse proxy) -This is what Matrix clients will talk to. If you use the script below, point -your client at `http://localhost:8008`. +This is what your clients and federated hosts will talk to. It must forward +requests onto the correct API server based on URL: -```bash -./bin/client-api-proxy \ ---bind-address ":8008" \ ---client-api-server-url "http://localhost:7771" \ ---sync-api-server-url "http://localhost:7773" \ ---media-api-server-url "http://localhost:7774" \ -``` +* `/_matrix/client` to the client API server +* `/_matrix/federation` to the federation API server +* `/_matrix/key` to the federation API server +* `/_matrix/media` to the media API server -### Federation proxy - -This is what Matrix servers will talk to. This is only required if you want -to support federation. - -```bash -./bin/federation-api-proxy \ ---bind-address ":8448" \ ---federation-api-url "http://localhost:7772" \ ---media-api-server-url "http://localhost:7774" \ -``` +See `docs/nginx/polylith-sample.conf` for a sample configuration. ### Client API server @@ -198,7 +193,7 @@ This is what implements CS API endpoints. Clients talk to this via the proxy in order to send messages, create and join rooms, etc. ```bash -./bin/dendrite-client-api-server --config=dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml clientapi ``` ### Sync server @@ -207,7 +202,7 @@ This is what implements `/sync` requests. Clients talk to this via the proxy in order to receive messages. ```bash -./bin/dendrite-sync-api-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml syncapi ``` ### Media server @@ -216,7 +211,7 @@ This implements `/media` requests. Clients talk to this via the proxy in order to upload and retrieve media. ```bash -./bin/dendrite-media-api-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml mediaapi ``` ### Federation API server @@ -226,7 +221,7 @@ order to send transactions. This is only required if you want to support federation. ```bash -./bin/dendrite-federation-api-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml federationapi ``` ### Internal components @@ -239,7 +234,7 @@ contacted by other components. This includes the following components. This is what implements the room DAG. Clients do not talk to this. ```bash -./bin/dendrite-room-server --config=dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml roomserver ``` #### Federation sender @@ -248,7 +243,7 @@ This sends events from our users to other servers. This is only required if you want to support federation. ```bash -./bin/dendrite-federation-sender-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml federationsender ``` #### Appservice server @@ -259,7 +254,7 @@ running locally. This is only required if you want to support running application services on your homeserver. ```bash -./bin/dendrite-appservice-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml appservice ``` #### Key server @@ -267,7 +262,7 @@ application services on your homeserver. This manages end-to-end encryption keys for users. ```bash -./bin/dendrite-key-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml keyserver ``` #### Signing key server @@ -275,7 +270,7 @@ This manages end-to-end encryption keys for users. This manages signing keys for servers. ```bash -./bin/dendrite-signing-key-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml signingkeyserver ``` #### EDU server @@ -283,7 +278,7 @@ This manages signing keys for servers. This manages processing EDUs such as typing, send-to-device events and presence. Clients do not talk to ```bash -./bin/dendrite-edu-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml eduserver ``` #### User server @@ -292,6 +287,6 @@ This manages user accounts, device access tokens and user account data, amongst other things. ```bash -./bin/dendrite-user-api-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml userapi ``` diff --git a/docs/serverkeyformat.md b/docs/serverkeyformat.md new file mode 100644 index 000000000..feda93454 --- /dev/null +++ b/docs/serverkeyformat.md @@ -0,0 +1,29 @@ +# Server Key Format + +Dendrite stores the server signing key in the PEM format with the following structure. + +``` +-----BEGIN MATRIX PRIVATE KEY----- +Key-ID: ed25519: + + +-----END MATRIX PRIVATE KEY----- +``` + +## Converting Synapse Keys + +If you have signing keys from a previous synapse server, you should ideally configure them as `old_private_keys` in your Dendrite config file. Synapse stores signing keys in the following format. + +``` +ed25519 +``` + +To convert this key to Dendrite's PEM format, use the following template. **It is important to include the equals sign, as the key data needs to be padded to 32 bytes.** + +``` +-----BEGIN MATRIX PRIVATE KEY----- +Key-ID: ed25519: + += +-----END MATRIX PRIVATE KEY----- +``` \ No newline at end of file diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index c637116f7..12f205366 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -290,6 +290,7 @@ func SendJoin( if !alreadyJoined { if err = api.SendEvents( httpReq.Context(), rsAPI, + api.KindNew, []gomatrixserverlib.HeaderedEvent{ event.Headered(stateAndAuthChainResponse.RoomVersion), }, diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index e16dfcc2e..fb81d9319 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -256,6 +256,7 @@ func SendLeave( // the room, so set SendAsServer to cfg.Matrix.ServerName if err = api.SendEvents( httpReq.Context(), rsAPI, + api.KindNew, []gomatrixserverlib.HeaderedEvent{ event.Headered(verRes.RoomVersion), }, diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 783fdc3b8..76dc3a2ee 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -403,6 +403,7 @@ func (t *txnReq) processEvent(ctx context.Context, e gomatrixserverlib.Event) er return api.SendEvents( context.Background(), t.rsAPI, + api.KindNew, []gomatrixserverlib.HeaderedEvent{ e.Headered(stateResp.RoomVersion), }, @@ -586,6 +587,7 @@ func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixser err = api.SendEventWithState( context.Background(), t.rsAPI, + api.KindOld, resolvedState, backwardsExtremity.Headered(roomVersion), t.haveEventIDs(), @@ -605,6 +607,7 @@ func (t *txnReq) processEventWithMissingState(ctx context.Context, e gomatrixser if err = api.SendEvents( context.Background(), t.rsAPI, + api.KindOld, append(headeredNewEvents, e.Headered(roomVersion)), api.DoNotSendToOtherServers, nil, diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index ec6cc1488..4db5273af 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -89,7 +89,7 @@ func CreateInvitesFrom3PIDInvites( } // Send all the events - if err := api.SendEvents(req.Context(), rsAPI, evs, cfg.Matrix.ServerName, nil); err != nil { + if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, evs, cfg.Matrix.ServerName, nil); err != nil { util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") return jsonerror.InternalServerError() } @@ -174,6 +174,7 @@ func ExchangeThirdPartyInvite( // Send the event to the roomserver if err = api.SendEvents( httpReq.Context(), rsAPI, + api.KindNew, []gomatrixserverlib.HeaderedEvent{ signedEvent.Event.Headered(verRes.RoomVersion), }, diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index efeb53fa6..ef945694c 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -87,6 +87,12 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { case api.OutputTypeNewRoomEvent: ev := &output.NewRoomEvent.Event + if output.NewRoomEvent.RewritesState { + if err := s.db.PurgeRoomState(context.TODO(), ev.RoomID()); err != nil { + return fmt.Errorf("s.db.PurgeRoom: %w", err) + } + } + if err := s.processMessage(*output.NewRoomEvent); err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index 254883e63..3904ab856 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -248,6 +248,7 @@ func (r *FederationSenderInternalAPI) performJoinUsingServer( // returned state to the roomserver to update our local view. if err = roomserverAPI.SendEventWithState( ctx, r.rsAPI, + roomserverAPI.KindNew, respState, event.Headered(respMakeJoin.RoomVersion), nil, diff --git a/federationsender/storage/interface.go b/federationsender/storage/interface.go index 734b368fe..a3f5073f9 100644 --- a/federationsender/storage/interface.go +++ b/federationsender/storage/interface.go @@ -32,6 +32,7 @@ type Database interface { GetAllJoinedHosts(ctx context.Context) ([]gomatrixserverlib.ServerName, error) // GetJoinedHostsForRooms returns the complete set of servers in the rooms given. GetJoinedHostsForRooms(ctx context.Context, roomIDs []string) ([]gomatrixserverlib.ServerName, error) + PurgeRoomState(ctx context.Context, roomID string) error StoreJSON(ctx context.Context, js string) (*shared.Receipt, error) diff --git a/federationsender/storage/postgres/joined_hosts_table.go b/federationsender/storage/postgres/joined_hosts_table.go index cc112b7f3..0bc9335dd 100644 --- a/federationsender/storage/postgres/joined_hosts_table.go +++ b/federationsender/storage/postgres/joined_hosts_table.go @@ -53,6 +53,9 @@ const insertJoinedHostsSQL = "" + const deleteJoinedHostsSQL = "" + "DELETE FROM federationsender_joined_hosts WHERE event_id = ANY($1)" +const deleteJoinedHostsForRoomSQL = "" + + "DELETE FROM federationsender_joined_hosts WHERE room_id = $1" + const selectJoinedHostsSQL = "" + "SELECT event_id, server_name FROM federationsender_joined_hosts" + " WHERE room_id = $1" @@ -67,6 +70,7 @@ type joinedHostsStatements struct { db *sql.DB insertJoinedHostsStmt *sql.Stmt deleteJoinedHostsStmt *sql.Stmt + deleteJoinedHostsForRoomStmt *sql.Stmt selectJoinedHostsStmt *sql.Stmt selectAllJoinedHostsStmt *sql.Stmt selectJoinedHostsForRoomsStmt *sql.Stmt @@ -86,6 +90,9 @@ func NewPostgresJoinedHostsTable(db *sql.DB) (s *joinedHostsStatements, err erro if s.deleteJoinedHostsStmt, err = s.db.Prepare(deleteJoinedHostsSQL); err != nil { return } + if s.deleteJoinedHostsForRoomStmt, err = s.db.Prepare(deleteJoinedHostsForRoomSQL); err != nil { + return + } if s.selectJoinedHostsStmt, err = s.db.Prepare(selectJoinedHostsSQL); err != nil { return } @@ -117,6 +124,14 @@ func (s *joinedHostsStatements) DeleteJoinedHosts( return err } +func (s *joinedHostsStatements) DeleteJoinedHostsForRoom( + ctx context.Context, txn *sql.Tx, roomID string, +) error { + stmt := sqlutil.TxStmt(txn, s.deleteJoinedHostsForRoomStmt) + _, err := stmt.ExecContext(ctx, roomID) + return err +} + func (s *joinedHostsStatements) SelectJoinedHostsWithTx( ctx context.Context, txn *sql.Tx, roomID string, ) ([]types.JoinedHost, error) { diff --git a/federationsender/storage/shared/storage.go b/federationsender/storage/shared/storage.go index 4c80c0792..d5731f31c 100644 --- a/federationsender/storage/shared/storage.go +++ b/federationsender/storage/shared/storage.go @@ -148,6 +148,20 @@ func (d *Database) StoreJSON( }, nil } +func (d *Database) PurgeRoomState( + ctx context.Context, roomID string, +) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + // If the event is a create event then we'll delete all of the existing + // data for the room. The only reason that a create event would be replayed + // to us in this way is if we're about to receive the entire room state. + if err := d.FederationSenderJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil { + return fmt.Errorf("d.FederationSenderJoinedHosts.DeleteJoinedHosts: %w", err) + } + return nil + }) +} + func (d *Database) AddServerToBlacklist(serverName gomatrixserverlib.ServerName) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { return d.FederationSenderBlacklist.InsertBlacklist(context.TODO(), txn, serverName) diff --git a/federationsender/storage/sqlite3/joined_hosts_table.go b/federationsender/storage/sqlite3/joined_hosts_table.go index 3bc45e7d5..1f906808d 100644 --- a/federationsender/storage/sqlite3/joined_hosts_table.go +++ b/federationsender/storage/sqlite3/joined_hosts_table.go @@ -53,6 +53,9 @@ const insertJoinedHostsSQL = "" + const deleteJoinedHostsSQL = "" + "DELETE FROM federationsender_joined_hosts WHERE event_id = $1" +const deleteJoinedHostsForRoomSQL = "" + + "DELETE FROM federationsender_joined_hosts WHERE room_id = $1" + const selectJoinedHostsSQL = "" + "SELECT event_id, server_name FROM federationsender_joined_hosts" + " WHERE room_id = $1" @@ -64,11 +67,12 @@ const selectJoinedHostsForRoomsSQL = "" + "SELECT DISTINCT server_name FROM federationsender_joined_hosts WHERE room_id IN ($1)" type joinedHostsStatements struct { - db *sql.DB - insertJoinedHostsStmt *sql.Stmt - deleteJoinedHostsStmt *sql.Stmt - selectJoinedHostsStmt *sql.Stmt - selectAllJoinedHostsStmt *sql.Stmt + db *sql.DB + insertJoinedHostsStmt *sql.Stmt + deleteJoinedHostsStmt *sql.Stmt + deleteJoinedHostsForRoomStmt *sql.Stmt + selectJoinedHostsStmt *sql.Stmt + selectAllJoinedHostsStmt *sql.Stmt // selectJoinedHostsForRoomsStmt *sql.Stmt - prepared at runtime due to variadic } @@ -86,6 +90,9 @@ func NewSQLiteJoinedHostsTable(db *sql.DB) (s *joinedHostsStatements, err error) if s.deleteJoinedHostsStmt, err = db.Prepare(deleteJoinedHostsSQL); err != nil { return } + if s.deleteJoinedHostsForRoomStmt, err = s.db.Prepare(deleteJoinedHostsForRoomSQL); err != nil { + return + } if s.selectJoinedHostsStmt, err = db.Prepare(selectJoinedHostsSQL); err != nil { return } @@ -118,6 +125,14 @@ func (s *joinedHostsStatements) DeleteJoinedHosts( return nil } +func (s *joinedHostsStatements) DeleteJoinedHostsForRoom( + ctx context.Context, txn *sql.Tx, roomID string, +) error { + stmt := sqlutil.TxStmt(txn, s.deleteJoinedHostsForRoomStmt) + _, err := stmt.ExecContext(ctx, roomID) + return err +} + func (s *joinedHostsStatements) SelectJoinedHostsWithTx( ctx context.Context, txn *sql.Tx, roomID string, ) ([]types.JoinedHost, error) { diff --git a/federationsender/storage/tables/interface.go b/federationsender/storage/tables/interface.go index c6f8a2d52..1167a212a 100644 --- a/federationsender/storage/tables/interface.go +++ b/federationsender/storage/tables/interface.go @@ -50,6 +50,7 @@ type FederationSenderQueueJSON interface { type FederationSenderJoinedHosts interface { InsertJoinedHosts(ctx context.Context, txn *sql.Tx, roomID, eventID string, serverName gomatrixserverlib.ServerName) error DeleteJoinedHosts(ctx context.Context, txn *sql.Tx, eventIDs []string) error + DeleteJoinedHostsForRoom(ctx context.Context, txn *sql.Tx, roomID string) error SelectJoinedHostsWithTx(ctx context.Context, txn *sql.Tx, roomID string) ([]types.JoinedHost, error) SelectJoinedHosts(ctx context.Context, roomID string) ([]types.JoinedHost, error) SelectAllJoinedHosts(ctx context.Context) ([]gomatrixserverlib.ServerName, error) diff --git a/go.mod b/go.mod index d3060fa0f..f785dd391 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8 + github.com/matrix-org/gomatrixserverlib v0.0.0-20201020162226-22169fe9cda7 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 @@ -40,6 +40,7 @@ require ( github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20201006093556-760d9a7fd5ee go.uber.org/atomic v1.6.0 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a + golang.org/x/net v0.0.0-20200528225125-3c3fba18258b gopkg.in/h2non/bimg.v1 v1.1.4 gopkg.in/yaml.v2 v2.3.0 ) diff --git a/go.sum b/go.sum index 377a2e093..7c24516d2 100644 --- a/go.sum +++ b/go.sum @@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8 h1:GF1PxbvImWDoz1DQZNMoaYtIqQXtyLAtmQOzwwmw1OI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201020162226-22169fe9cda7 h1:YPuewGCKaJh08NslYAhyGiLw2tg6ew9LtkW7Xr+4uTU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201020162226-22169fe9cda7/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/internal/setup/base.go b/internal/setup/base.go index 8bc4ae17a..4e1cee479 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -15,8 +15,10 @@ package setup import ( + "crypto/tls" "fmt" "io" + "net" "net/http" "net/url" "time" @@ -25,6 +27,8 @@ import ( "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/gomatrixserverlib" "github.com/prometheus/client_golang/prometheus/promhttp" + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/userapi/storage/accounts" @@ -107,7 +111,22 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo logrus.WithError(err).Warnf("Failed to create cache") } - apiClient := http.Client{Timeout: time.Minute * 10} + apiClient := http.Client{ + Timeout: time.Minute * 10, + Transport: &http2.Transport{ + AllowHTTP: true, + DialTLS: func(network, addr string, _ *tls.Config) (net.Conn, error) { + // Ordinarily HTTP/2 would expect TLS, but the remote listener is + // H2C-enabled (HTTP/2 without encryption). Overriding the DialTLS + // function with a plain Dial allows us to trick the HTTP client + // into establishing a HTTP/2 connection without TLS. + // TODO: Eventually we will want to look at authenticating and + // encrypting these internal HTTP APIs, at which point we will have + // to reconsider H2C and change all this anyway. + return net.Dial(network, addr) + }, + }, + } client := http.Client{Timeout: HTTPClientTimeout} if cfg.FederationSender.Proxy.Enabled { client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{ @@ -269,10 +288,17 @@ func (b *BaseDendrite) SetupAndServeHTTP( internalServ := externalServ if internalAddr != NoListener && externalAddr != internalAddr { + // H2C allows us to accept HTTP/2 connections without TLS + // encryption. Since we don't currently require any form of + // authentication or encryption on these internal HTTP APIs, + // H2C gives us all of the advantages of HTTP/2 (such as + // stream multiplexing and avoiding head-of-line blocking) + // without enabling TLS. + internalH2S := &http2.Server{} internalRouter = mux.NewRouter().SkipClean(true).UseEncodedPath() internalServ = &http.Server{ Addr: string(internalAddr), - Handler: internalRouter, + Handler: h2c.NewHandler(internalRouter, internalH2S), } } diff --git a/internal/version.go b/internal/version.go index a9e245d44..21f697086 100644 --- a/internal/version.go +++ b/internal/version.go @@ -16,8 +16,8 @@ var build string const ( VersionMajor = 0 - VersionMinor = 1 - VersionPatch = 0 + VersionMinor = 2 + VersionPatch = 1 VersionTag = "" // example: "rc1" ) diff --git a/roomserver/api/input.go b/roomserver/api/input.go index dd693203b..e1a8afa00 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -21,17 +21,25 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) +type Kind int + const ( // KindOutlier event fall outside the contiguous event graph. // We do not have the state for these events. // These events are state events used to authenticate other events. // They can become part of the contiguous event graph via backfill. - KindOutlier = 1 + KindOutlier Kind = iota + 1 // KindNew event extend the contiguous graph going forwards. // They usually don't need state, but may include state if the // there was a new event that references an event that we don't - // have a copy of. - KindNew = 2 + // have a copy of. New events will influence the fwd extremities + // of the room and output events will be generated as a result. + KindNew + // KindOld event extend the graph backwards, or fill gaps in + // history. They may or may not include state. They will not be + // considered for forward extremities, and output events will NOT + // be generated for them. + KindOld ) // DoNotSendToOtherServers tells us not to send the event to other matrix @@ -43,7 +51,7 @@ const DoNotSendToOtherServers = "" type InputRoomEvent struct { // Whether this event is new, backfilled or an outlier. // This controls how the event is processed. - Kind int `json:"kind"` + Kind Kind `json:"kind"` // The event JSON for the event to add. Event gomatrixserverlib.HeaderedEvent `json:"event"` // List of state event IDs that authenticate this event. diff --git a/roomserver/api/output.go b/roomserver/api/output.go index d57f3b04c..9cb814a47 100644 --- a/roomserver/api/output.go +++ b/roomserver/api/output.go @@ -24,6 +24,8 @@ type OutputType string const ( // OutputTypeNewRoomEvent indicates that the event is an OutputNewRoomEvent OutputTypeNewRoomEvent OutputType = "new_room_event" + // OutputTypeOldRoomEvent indicates that the event is an OutputOldRoomEvent + OutputTypeOldRoomEvent OutputType = "old_room_event" // OutputTypeNewInviteEvent indicates that the event is an OutputNewInviteEvent OutputTypeNewInviteEvent OutputType = "new_invite_event" // OutputTypeRetireInviteEvent indicates that the event is an OutputRetireInviteEvent @@ -58,6 +60,8 @@ type OutputEvent struct { Type OutputType `json:"type"` // The content of event with type OutputTypeNewRoomEvent NewRoomEvent *OutputNewRoomEvent `json:"new_room_event,omitempty"` + // The content of event with type OutputTypeOldRoomEvent + OldRoomEvent *OutputOldRoomEvent `json:"old_room_event,omitempty"` // The content of event with type OutputTypeNewInviteEvent NewInviteEvent *OutputNewInviteEvent `json:"new_invite_event,omitempty"` // The content of event with type OutputTypeRetireInviteEvent @@ -178,6 +182,20 @@ func (ore *OutputNewRoomEvent) AddsState() []gomatrixserverlib.HeaderedEvent { return append(ore.AddStateEvents, ore.Event) } +// An OutputOldRoomEvent is written when the roomserver receives an old event. +// This will typically happen as a result of getting either missing events +// or backfilling. Downstream components may wish to send these events to +// clients when it is advantageous to do so, but with the consideration that +// the event is likely a historic event. +// +// Old events do not update forward extremities or the current room state, +// therefore they must not be treated as if they do. Downstream components +// should build their current room state up from OutputNewRoomEvents only. +type OutputOldRoomEvent struct { + // The Event. + Event gomatrixserverlib.HeaderedEvent `json:"event"` +} + // An OutputNewInviteEvent is written whenever an invite becomes active. // Invite events can be received outside of an existing room so have to be // tracked separately from the room events themselves. diff --git a/roomserver/api/wrapper.go b/roomserver/api/wrapper.go index a38c00df7..9e8219103 100644 --- a/roomserver/api/wrapper.go +++ b/roomserver/api/wrapper.go @@ -24,13 +24,14 @@ import ( // SendEvents to the roomserver The events are written with KindNew. func SendEvents( - ctx context.Context, rsAPI RoomserverInternalAPI, events []gomatrixserverlib.HeaderedEvent, + ctx context.Context, rsAPI RoomserverInternalAPI, + kind Kind, events []gomatrixserverlib.HeaderedEvent, sendAsServer gomatrixserverlib.ServerName, txnID *TransactionID, ) error { ires := make([]InputRoomEvent, len(events)) for i, event := range events { ires[i] = InputRoomEvent{ - Kind: KindNew, + Kind: kind, Event: event, AuthEventIDs: event.AuthEventIDs(), SendAsServer: string(sendAsServer), @@ -40,12 +41,13 @@ func SendEvents( return SendInputRoomEvents(ctx, rsAPI, ires) } -// SendEventWithState writes an event with KindNew to the roomserver +// SendEventWithState writes an event with the specified kind to the roomserver // with the state at the event as KindOutlier before it. Will not send any event that is // marked as `true` in haveEventIDs func SendEventWithState( - ctx context.Context, rsAPI RoomserverInternalAPI, state *gomatrixserverlib.RespState, - event gomatrixserverlib.HeaderedEvent, haveEventIDs map[string]bool, + ctx context.Context, rsAPI RoomserverInternalAPI, kind Kind, + state *gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent, + haveEventIDs map[string]bool, ) error { outliers, err := state.Events() if err != nil { @@ -70,7 +72,7 @@ func SendEventWithState( } ires = append(ires, InputRoomEvent{ - Kind: KindNew, + Kind: kind, Event: event, AuthEventIDs: event.AuthEventIDs(), HasState: true, diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 113341591..c055289c9 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -17,6 +17,7 @@ package input import ( + "bytes" "context" "fmt" @@ -26,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) @@ -44,6 +46,28 @@ func (r *Inputer) processRoomEvent( headered := input.Event event := headered.Unwrap() + // if we have already got this event then do not process it again, if the input kind is an outlier. + // Outliers contain no extra information which may warrant a re-processing. + if input.Kind == api.KindOutlier { + evs, err2 := r.DB.EventsFromIDs(ctx, []string{event.EventID()}) + if err2 == nil && len(evs) == 1 { + // check hash matches if we're on early room versions where the event ID was a random string + idFormat, err2 := headered.RoomVersion.EventIDFormat() + if err2 == nil { + switch idFormat { + case gomatrixserverlib.EventIDFormatV1: + if bytes.Equal(event.EventReference().EventSHA256, evs[0].EventReference().EventSHA256) { + util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring") + return event.EventID(), nil + } + default: + util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring") + return event.EventID(), nil + } + } + } + } + // Check that the event passes authentication checks and work out // the numeric IDs for the auth events. isRejected := false @@ -119,7 +143,7 @@ func (r *Inputer) processRoomEvent( // We haven't calculated a state for this event yet. // Lets calculate one. err = r.calculateAndSetState(ctx, input, *roomInfo, &stateAtEvent, event, isRejected) - if err != nil { + if err != nil && input.Kind != api.KindOld { return "", fmt.Errorf("r.calculateAndSetState: %w", err) } } @@ -136,16 +160,31 @@ func (r *Inputer) processRoomEvent( return event.EventID(), rejectionErr } - if err = r.updateLatestEvents( - ctx, // context - roomInfo, // room info for the room being updated - stateAtEvent, // state at event (below) - event, // event - input.SendAsServer, // send as server - input.TransactionID, // transaction ID - input.HasState, // rewrites state? - ); err != nil { - return "", fmt.Errorf("r.updateLatestEvents: %w", err) + switch input.Kind { + case api.KindNew: + if err = r.updateLatestEvents( + ctx, // context + roomInfo, // room info for the room being updated + stateAtEvent, // state at event (below) + event, // event + input.SendAsServer, // send as server + input.TransactionID, // transaction ID + input.HasState, // rewrites state? + ); err != nil { + return "", fmt.Errorf("r.updateLatestEvents: %w", err) + } + case api.KindOld: + err = r.WriteOutputEvents(event.RoomID(), []api.OutputEvent{ + { + Type: api.OutputTypeOldRoomEvent, + OldRoomEvent: &api.OutputOldRoomEvent{ + Event: headered, + }, + }, + }) + if err != nil { + return "", fmt.Errorf("r.WriteOutputEvents (old): %w", err) + } } // processing this event resulted in an event (which may not be the one we're processing) @@ -163,7 +202,7 @@ func (r *Inputer) processRoomEvent( }, }) if err != nil { - return "", fmt.Errorf("r.WriteOutputEvents: %w", err) + return "", fmt.Errorf("r.WriteOutputEvents (redactions): %w", err) } } diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index ca5d214d7..2bf6b9f8a 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -17,7 +17,6 @@ package input import ( - "bytes" "context" "fmt" @@ -28,7 +27,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" - "github.com/sirupsen/logrus" ) // updateLatestEvents updates the list of latest events for this room in the database and writes the @@ -118,7 +116,6 @@ type latestEventsUpdater struct { func (u *latestEventsUpdater) doUpdateLatestEvents() error { u.lastEventIDSent = u.updater.LastEventIDSent() - u.oldStateNID = u.updater.CurrentStateSnapshotNID() // If we are doing a regular event update then we will get the // previous latest events to use as a part of the calculation. If @@ -127,7 +124,8 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { // then start with an empty set - none of the forward extremities // that we knew about before matter anymore. oldLatest := []types.StateAtEventAndReference{} - if !u.stateAtEvent.Overwrite { + if !u.rewritesState { + u.oldStateNID = u.updater.CurrentStateSnapshotNID() oldLatest = u.updater.LatestEvents() } @@ -141,31 +139,35 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { // Work out what the latest events are. This will include the new // event if it is not already referenced. - if err := u.calculateLatest( - oldLatest, + extremitiesChanged, err := u.calculateLatest( + oldLatest, &u.event, types.StateAtEventAndReference{ EventReference: u.event.EventReference(), StateAtEvent: u.stateAtEvent, }, - ); err != nil { + ) + if err != nil { return fmt.Errorf("u.calculateLatest: %w", err) } // Now that we know what the latest events are, it's time to get the // latest state. - if err := u.latestState(); err != nil { - return fmt.Errorf("u.latestState: %w", err) + var updates []api.OutputEvent + if extremitiesChanged || u.rewritesState { + if err = u.latestState(); err != nil { + return fmt.Errorf("u.latestState: %w", err) + } + + // If we need to generate any output events then here's where we do it. + // TODO: Move this! + if updates, err = u.api.updateMemberships(u.ctx, u.updater, u.removed, u.added); err != nil { + return fmt.Errorf("u.api.updateMemberships: %w", err) + } + } else { + u.newStateNID = u.oldStateNID } - // If we need to generate any output events then here's where we do it. - // TODO: Move this! - updates, err := u.api.updateMemberships(u.ctx, u.updater, u.removed, u.added) - if err != nil { - return fmt.Errorf("u.api.updateMemberships: %w", err) - } - - var update *api.OutputEvent - update, err = u.makeOutputNewRoomEvent() + update, err := u.makeOutputNewRoomEvent() if err != nil { return fmt.Errorf("u.makeOutputNewRoomEvent: %w", err) } @@ -234,18 +236,6 @@ func (u *latestEventsUpdater) latestState() error { if err != nil { return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err) } - if len(u.removed) > len(u.added) { - // This really shouldn't happen. - // TODO: What is ultimately the best way to handle this situation? - logrus.Errorf( - "Invalid state delta on event %q wants to remove %d state but only add %d state (between state snapshots %d and %d)", - u.event.EventID(), len(u.removed), len(u.added), u.oldStateNID, u.newStateNID, - ) - u.added = u.added[:0] - u.removed = u.removed[:0] - u.newStateNID = u.oldStateNID - return nil - } // Also work out the state before the event removes and the event // adds. @@ -259,56 +249,81 @@ func (u *latestEventsUpdater) latestState() error { return nil } +// calculateLatest works out the new set of forward extremities. Returns +// true if the new event is included in those extremites, false otherwise. func (u *latestEventsUpdater) calculateLatest( oldLatest []types.StateAtEventAndReference, - newEvent types.StateAtEventAndReference, -) error { - var newLatest []types.StateAtEventAndReference - - // First of all, let's see if any of the existing forward extremities - // now have entries in the previous events table. If they do then we - // will no longer include them as forward extremities. - for _, l := range oldLatest { - referenced, err := u.updater.IsReferenced(l.EventReference) - if err != nil { - logrus.WithError(err).Errorf("Failed to retrieve event reference for %q", l.EventID) - return fmt.Errorf("u.updater.IsReferenced (old): %w", err) - } else if !referenced { - newLatest = append(newLatest, l) - } + newEvent *gomatrixserverlib.Event, + newStateAndRef types.StateAtEventAndReference, +) (bool, error) { + // First of all, get a list of all of the events in our current + // set of forward extremities. + existingRefs := make(map[string]*types.StateAtEventAndReference) + existingNIDs := make([]types.EventNID, len(oldLatest)) + for i, old := range oldLatest { + existingRefs[old.EventID] = &oldLatest[i] + existingNIDs[i] = old.EventNID } - // Then check and see if our new event is already included in that set. - // This ordinarily won't happen but it covers the edge-case that we've - // already seen this event before and it's a forward extremity, so rather - // than adding a duplicate, we'll just return the set as complete. - for _, l := range newLatest { - if l.EventReference.EventID == newEvent.EventReference.EventID && bytes.Equal(l.EventReference.EventSHA256, newEvent.EventReference.EventSHA256) { - // We've already referenced this new event so we can just return - // the newly completed extremities at this point. - u.latest = newLatest - return nil - } - } - - // At this point we've processed the old extremities, and we've checked - // that our new event isn't already in that set. Therefore now we can - // check if our *new* event is a forward extremity, and if it is, add - // it in. - referenced, err := u.updater.IsReferenced(newEvent.EventReference) + // Look up the old extremity events. This allows us to find their + // prev events. + events, err := u.api.DB.Events(u.ctx, existingNIDs) if err != nil { - logrus.WithError(err).Errorf("Failed to retrieve event reference for %q", newEvent.EventReference.EventID) - return fmt.Errorf("u.updater.IsReferenced (new): %w", err) - } else if !referenced || len(newLatest) == 0 { - newLatest = append(newLatest, newEvent) + return false, fmt.Errorf("u.api.DB.Events: %w", err) + } + + // Make a list of all of the prev events as referenced by all of + // the current forward extremities. + existingPrevs := make(map[string]struct{}) + for _, old := range events { + for _, prevEventID := range old.PrevEventIDs() { + existingPrevs[prevEventID] = struct{}{} + } + } + + // If the "new" event is already referenced by a forward extremity + // then do nothing - it's not a candidate to be a new extremity if + // it has been referenced. + if _, ok := existingPrevs[newEvent.EventID()]; ok { + return false, nil + } + + // If the "new" event is already a forward extremity then stop, as + // nothing changes. + for _, event := range events { + if event.EventID() == newEvent.EventID() { + return false, nil + } + } + + // Include our new event in the extremities. + newLatest := []types.StateAtEventAndReference{newStateAndRef} + + // Then run through and see if the other extremities are still valid. + // If our new event references them then they are no longer good + // candidates. + for _, prevEventID := range newEvent.PrevEventIDs() { + delete(existingRefs, prevEventID) + } + + // Ensure that we don't add any candidate forward extremities from + // the old set that are, themselves, referenced by the old set of + // forward extremities. This shouldn't happen but guards against + // the possibility anyway. + for prevEventID := range existingPrevs { + delete(existingRefs, prevEventID) + } + + // Then re-add any old extremities that are still valid after all. + for _, old := range existingRefs { + newLatest = append(newLatest, *old) } u.latest = newLatest - return nil + return true, nil } func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) { - latestEventIDs := make([]string, len(u.latest)) for i := range u.latest { latestEventIDs[i] = u.latest[i].EventID @@ -326,7 +341,6 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) if err != nil { return nil, err } - for _, entry := range u.added { ore.AddsStateEventIDs = append(ore.AddsStateEventIDs, eventIDMap[entry.EventNID]) } @@ -339,21 +353,17 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) for _, entry := range u.stateBeforeEventAdds { ore.StateBeforeAddsEventIDs = append(ore.StateBeforeAddsEventIDs, eventIDMap[entry.EventNID]) } + ore.SendAsServer = u.sendAsServer // include extra state events if they were added as nearly every downstream component will care about it // and we'd rather not have them all hit QueryEventsByID at the same time! if len(ore.AddsStateEventIDs) > 0 { - ore.AddStateEvents, err = u.extraEventsForIDs(u.roomInfo.RoomVersion, ore.AddsStateEventIDs) - if err != nil { + var err error + if ore.AddStateEvents, err = u.extraEventsForIDs(u.roomInfo.RoomVersion, ore.AddsStateEventIDs); err != nil { return nil, fmt.Errorf("failed to load add_state_events from db: %w", err) } } - // State is rewritten if the input room event HasState and we actually produced a delta on state events. - // Without this check, /get_missing_events which produce events with associated (but not complete) state - // will incorrectly purge the room and set it to no state. TODO: This is likely flakey, as if /gme produced - // a state conflict res which just so happens to include 2+ events we might purge the room state downstream. - ore.RewritesState = len(ore.AddsStateEventIDs) > 1 return &api.OutputEvent{ Type: api.OutputTypeNewRoomEvent, diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index 1b692a098..41cbd2637 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -191,7 +191,7 @@ func mustSendEvents(t *testing.T, ver gomatrixserverlib.RoomVersion, events []js t.Helper() rsAPI, dp := mustCreateRoomserverAPI(t) hevents := mustLoadRawEvents(t, ver, events) - if err := api.SendEvents(ctx, rsAPI, hevents, testOrigin, nil); err != nil { + if err := api.SendEvents(ctx, rsAPI, api.KindNew, hevents, testOrigin, nil); err != nil { t.Errorf("failed to SendEvents: %s", err) } return rsAPI, dp, hevents @@ -337,7 +337,7 @@ func TestOutputRewritesState(t *testing.T) { deleteDatabase() rsAPI, producer := mustCreateRoomserverAPI(t) defer deleteDatabase() - err := api.SendEvents(context.Background(), rsAPI, originalEvents, testOrigin, nil) + err := api.SendEvents(context.Background(), rsAPI, api.KindNew, originalEvents, testOrigin, nil) if err != nil { t.Fatalf("failed to send original events: %s", err) } @@ -379,7 +379,7 @@ func TestOutputRewritesState(t *testing.T) { if len(producer.producedMessages) != 1 { t.Fatalf("Rewritten events got output, want only 1 got %d", len(producer.producedMessages)) } - outputEvent := producer.producedMessages[0] + outputEvent := producer.producedMessages[len(producer.producedMessages)-1] if !outputEvent.NewRoomEvent.RewritesState { t.Errorf("RewritesState flag not set on output event") } diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 2944f71c1..d23f14c84 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -526,13 +526,7 @@ func (v StateResolution) CalculateAndStoreStateBeforeEvent( isRejected bool, ) (types.StateSnapshotNID, error) { // Load the state at the prev events. - prevEventRefs := event.PrevEvents() - prevEventIDs := make([]string, len(prevEventRefs)) - for i := range prevEventRefs { - prevEventIDs[i] = prevEventRefs[i].EventID - } - - prevStates, err := v.db.StateAtEventIDs(ctx, prevEventIDs) + prevStates, err := v.db.StateAtEventIDs(ctx, event.PrevEventIDs()) if err != nil { return 0, err } diff --git a/roomserver/storage/shared/latest_events_updater.go b/roomserver/storage/shared/latest_events_updater.go index b316f639d..8825dc464 100644 --- a/roomserver/storage/shared/latest_events_updater.go +++ b/roomserver/storage/shared/latest_events_updater.go @@ -70,16 +70,14 @@ func (u *LatestEventsUpdater) CurrentStateSnapshotNID() types.StateSnapshotNID { return u.currentStateSnapshotNID } -// StorePreviousEvents implements types.RoomRecentEventsUpdater +// StorePreviousEvents implements types.RoomRecentEventsUpdater - This must be called from a Writer func (u *LatestEventsUpdater) StorePreviousEvents(eventNID types.EventNID, previousEventReferences []gomatrixserverlib.EventReference) error { - return u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error { - for _, ref := range previousEventReferences { - if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { - return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err) - } + for _, ref := range previousEventReferences { + if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, u.txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { + return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err) } - return nil - }) + } + return nil } // IsReferenced implements types.RoomRecentEventsUpdater diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index f2be8b3cf..aec15ab22 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -27,23 +27,24 @@ import ( const redactionsArePermanent = true type Database struct { - DB *sql.DB - Cache caching.RoomServerCaches - Writer sqlutil.Writer - EventsTable tables.Events - EventJSONTable tables.EventJSON - EventTypesTable tables.EventTypes - EventStateKeysTable tables.EventStateKeys - RoomsTable tables.Rooms - TransactionsTable tables.Transactions - StateSnapshotTable tables.StateSnapshot - StateBlockTable tables.StateBlock - RoomAliasesTable tables.RoomAliases - PrevEventsTable tables.PreviousEvents - InvitesTable tables.Invites - MembershipTable tables.Membership - PublishedTable tables.Published - RedactionsTable tables.Redactions + DB *sql.DB + Cache caching.RoomServerCaches + Writer sqlutil.Writer + EventsTable tables.Events + EventJSONTable tables.EventJSON + EventTypesTable tables.EventTypes + EventStateKeysTable tables.EventStateKeys + RoomsTable tables.Rooms + TransactionsTable tables.Transactions + StateSnapshotTable tables.StateSnapshot + StateBlockTable tables.StateBlock + RoomAliasesTable tables.RoomAliases + PrevEventsTable tables.PreviousEvents + InvitesTable tables.Invites + MembershipTable tables.Membership + PublishedTable tables.Published + RedactionsTable tables.Redactions + GetLatestEventsForUpdateFn func(ctx context.Context, roomInfo types.RoomInfo) (*LatestEventsUpdater, error) } func (d *Database) SupportsConcurrentRoomInputs() bool { @@ -372,6 +373,9 @@ func (d *Database) MembershipUpdater( func (d *Database) GetLatestEventsForUpdate( ctx context.Context, roomInfo types.RoomInfo, ) (*LatestEventsUpdater, error) { + if d.GetLatestEventsForUpdateFn != nil { + return d.GetLatestEventsForUpdateFn(ctx, roomInfo) + } txn, err := d.DB.Begin() if err != nil { return nil, err @@ -492,15 +496,32 @@ func (d *Database) StoreEvent( if roomInfo == nil && len(prevEvents) > 0 { return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("expected room %q to exist", event.RoomID()) } + // Create an updater - NB: on sqlite this WILL create a txn as we are directly calling the shared DB form of + // GetLatestEventsForUpdate - not via the SQLiteDatabase form which has `nil` txns. This + // function only does SELECTs though so the created txn (at this point) is just a read txn like + // any other so this is fine. If we ever update GetLatestEventsForUpdate or NewLatestEventsUpdater + // to do writes however then this will need to go inside `Writer.Do`. updater, err = d.GetLatestEventsForUpdate(ctx, *roomInfo) if err != nil { return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("NewLatestEventsUpdater: %w", err) } - if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil { - return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("updater.StorePreviousEvents: %w", err) + // Ensure that we atomically store prev events AND commit them. If we don't wrap StorePreviousEvents + // and EndTransaction in a writer then it's possible for a new write txn to be made between the two + // function calls which will then fail with 'database is locked'. This new write txn would HAVE to be + // something like SetRoomAlias/RemoveRoomAlias as normal input events are already done sequentially due to + // SupportsConcurrentRoomInputs() == false on sqlite, though this does not apply to setting room aliases + // as they don't go via InputRoomEvents + err = d.Writer.Do(d.DB, updater.txn, func(txn *sql.Tx) error { + if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil { + return fmt.Errorf("updater.StorePreviousEvents: %w", err) + } + succeeded := true + err = sqlutil.EndTransaction(updater, &succeeded) + return err + }) + if err != nil { + return 0, types.StateAtEvent{}, nil, "", err } - succeeded := true - err = sqlutil.EndTransaction(updater, &succeeded) } return roomNID, types.StateAtEvent{ diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 4a74bf736..6d9b860f5 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -120,23 +120,24 @@ func Open(dbProperties *config.DatabaseOptions, cache caching.RoomServerCaches) return nil, err } d.Database = shared.Database{ - DB: d.db, - Cache: cache, - Writer: d.writer, - EventsTable: d.events, - EventTypesTable: d.eventTypes, - EventStateKeysTable: d.eventStateKeys, - EventJSONTable: d.eventJSON, - RoomsTable: d.rooms, - TransactionsTable: d.transactions, - StateBlockTable: stateBlock, - StateSnapshotTable: stateSnapshot, - PrevEventsTable: d.prevEvents, - RoomAliasesTable: roomAliases, - InvitesTable: d.invites, - MembershipTable: d.membership, - PublishedTable: published, - RedactionsTable: redactions, + DB: d.db, + Cache: cache, + Writer: d.writer, + EventsTable: d.events, + EventTypesTable: d.eventTypes, + EventStateKeysTable: d.eventStateKeys, + EventJSONTable: d.eventJSON, + RoomsTable: d.rooms, + TransactionsTable: d.transactions, + StateBlockTable: stateBlock, + StateSnapshotTable: stateSnapshot, + PrevEventsTable: d.prevEvents, + RoomAliasesTable: roomAliases, + InvitesTable: d.invites, + MembershipTable: d.membership, + PublishedTable: published, + RedactionsTable: redactions, + GetLatestEventsForUpdateFn: d.GetLatestEventsForUpdate, } return &d, nil } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index ca48c8300..ac1128c11 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -97,6 +97,8 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { } } return s.onNewRoomEvent(context.TODO(), *output.NewRoomEvent) + case api.OutputTypeOldRoomEvent: + return s.onOldRoomEvent(context.TODO(), *output.OldRoomEvent) case api.OutputTypeNewInviteEvent: return s.onNewInviteEvent(context.TODO(), *output.NewInviteEvent) case api.OutputTypeRetireInviteEvent: @@ -147,7 +149,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( } if msg.RewritesState { - if err = s.db.PurgeRoom(ctx, ev.RoomID()); err != nil { + if err = s.db.PurgeRoomState(ctx, ev.RoomID()); err != nil { return fmt.Errorf("s.db.PurgeRoom: %w", err) } } @@ -168,7 +170,46 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( log.ErrorKey: err, "add": msg.AddsStateEventIDs, "del": msg.RemovesStateEventIDs, - }).Panicf("roomserver output log: write event failure") + }).Panicf("roomserver output log: write new event failure") + return nil + } + + if pduPos, err = s.notifyJoinedPeeks(ctx, &ev, pduPos); err != nil { + logrus.WithError(err).Errorf("Failed to notifyJoinedPeeks for PDU pos %d", pduPos) + return err + } + + s.notifier.OnNewEvent(&ev, "", nil, types.NewStreamToken(pduPos, 0, nil)) + + return nil +} + +func (s *OutputRoomEventConsumer) onOldRoomEvent( + ctx context.Context, msg api.OutputOldRoomEvent, +) error { + ev := msg.Event + + // TODO: The state key check when excluding from sync is designed + // to stop us from lying to clients with old state, whilst still + // allowing normal timeline events through. This is an absolute + // hack but until we have some better strategy for dealing with + // old events in the sync API, this should at least prevent us + // from confusing clients into thinking they've joined/left rooms. + pduPos, err := s.db.WriteEvent( + ctx, + &ev, + []gomatrixserverlib.HeaderedEvent{}, + []string{}, // adds no state + []string{}, // removes no state + nil, // no transaction + ev.StateKey() != nil, // exclude from sync? + ) + if err != nil { + // panic rather than continue with an inconsistent database + log.WithFields(log.Fields{ + "event": string(ev.JSON()), + log.ErrorKey: err, + }).Panicf("roomserver output log: write old event failure") return nil } diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index ce7f1c152..e12a1166e 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -43,9 +43,9 @@ type Database interface { // Returns an error if there was a problem inserting this event. WriteEvent(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent, addStateEvents []gomatrixserverlib.HeaderedEvent, addStateEventIDs []string, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool) (types.StreamPosition, error) - // PurgeRoom completely purges room state from the sync API. This is done when + // PurgeRoomState completely purges room state from the sync API. This is done when // receiving an output event that completely resets the state. - PurgeRoom(ctx context.Context, roomID string) error + PurgeRoomState(ctx context.Context, roomID string) error // GetStateEvent returns the Matrix state event of a given type for a given room with a given state key // If no event could be found, returns nil // If there was an issue during the retrieval, returns an error diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index b9f21913e..a7c07f943 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -276,7 +276,7 @@ func (d *Database) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, e return nil } -func (d *Database) PurgeRoom( +func (d *Database) PurgeRoomState( ctx context.Context, roomID string, ) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { @@ -286,15 +286,6 @@ func (d *Database) PurgeRoom( if err := d.CurrentRoomState.DeleteRoomStateForRoom(ctx, txn, roomID); err != nil { return fmt.Errorf("d.CurrentRoomState.DeleteRoomStateForRoom: %w", err) } - if err := d.OutputEvents.DeleteEventsForRoom(ctx, txn, roomID); err != nil { - return fmt.Errorf("d.Events.DeleteEventsForRoom: %w", err) - } - if err := d.Topology.DeleteTopologyForRoom(ctx, txn, roomID); err != nil { - return fmt.Errorf("d.Topology.DeleteTopologyForRoom: %w", err) - } - if err := d.BackwardExtremities.DeleteBackwardExtremitiesForRoom(ctx, txn, roomID); err != nil { - return fmt.Errorf("d.BackwardExtremities.DeleteBackwardExtremitiesForRoom: %w", err) - } return nil }) } diff --git a/sytest-blacklist b/sytest-blacklist index ff7fdf7e0..f493f94fe 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -57,4 +57,7 @@ The only membership state included in a gapped incremental sync is for senders i # Blacklisted out of flakiness after #1479 Invited user can reject local invite after originator leaves Invited user can reject invite for empty room -If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes \ No newline at end of file +If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes + +# Blacklisted due to flakiness +A prev_batch token from incremental sync can be used in the v1 messages API \ No newline at end of file diff --git a/sytest-whitelist b/sytest-whitelist index 2ba0a88b2..1a12b591b 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -481,6 +481,8 @@ m.room.history_visibility == "joined" allows/forbids appropriately for Guest use m.room.history_visibility == "joined" allows/forbids appropriately for Real users POST rejects invalid utf-8 in JSON Users cannot kick users who have already left a room -A prev_batch token from incremental sync can be used in the v1 messages API Event with an invalid signature in the send_join response should not cause room join to fail Inbound federation rejects typing notifications from wrong remote +Should not be able to take over the room by pretending there is no PL event +Can get rooms/{roomId}/state for a departed room (SPEC-216) +Users cannot set notifications powerlevel higher than their own diff --git a/userapi/inthttp/server.go b/userapi/inthttp/server.go index e24aad3a9..81e936e58 100644 --- a/userapi/inthttp/server.go +++ b/userapi/inthttp/server.go @@ -39,7 +39,7 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - internalAPIMux.Handle(PerformAccountCreationPath, + internalAPIMux.Handle(PerformPasswordUpdatePath, httputil.MakeInternalAPI("performPasswordUpdate", func(req *http.Request) util.JSONResponse { request := api.PerformPasswordUpdateRequest{} response := api.PerformPasswordUpdateResponse{} @@ -169,7 +169,7 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - internalAPIMux.Handle(QueryDeviceInfosPath, + internalAPIMux.Handle(QuerySearchProfilesPath, httputil.MakeInternalAPI("querySearchProfiles", func(req *http.Request) util.JSONResponse { request := api.QuerySearchProfilesRequest{} response := api.QuerySearchProfilesResponse{} @@ -182,4 +182,17 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(InputAccountDataPath, + httputil.MakeInternalAPI("inputAccountDataPath", func(req *http.Request) util.JSONResponse { + request := api.InputAccountDataRequest{} + response := api.InputAccountDataResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.InputAccountData(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) }