mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2026-01-11 08:03:09 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into s7evink/docker

Browse source
This commit is contained in:
Neil Alexander 2022-09-27 13:15:32 +01:00 committed by GitHub
parent 75efbf37aa f18bce93cc
commit 958d145f61
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
561 changed files with 23215 additions and 15882 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/PULL_REQUEST_TEMPLATE.md vendored
View file

@ -2,7 +2,7 @@
<!-- Please read docs/CONTRIBUTING.md before submitting your pull request -->
* [ ] I have added added tests for PR _or_ I have justified why this PR doesn't need tests.
* [ ] I have added tests for PR _or_ I have justified why this PR doesn't need tests.
* [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off)
Signed-off-by: `Your Name <your@email.example.org>`

70
.github/workflows/dendrite.yml vendored
View file

@ -7,6 +7,7 @@ on:
pull_request:
release:
types: [published]
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@ -17,13 +18,14 @@ jobs:
name: WASM build test
timeout-minutes: 5
runs-on: ubuntu-latest
if: ${{ false }} # disable for now
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Install Go
uses: actions/setup-go@v2
uses: actions/setup-go@v3
with:
go-version: 1.16
go-version: 1.18
- uses: actions/cache@v2
with:
@ -65,8 +67,12 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: 1.18
- name: golangci-lint
uses: golangci/golangci-lint-action@v2
uses: golangci/golangci-lint-action@v3
# run go test with different go versions
test:
@ -96,11 +102,11 @@ jobs:
strategy:
fail-fast: false
matrix:
go: ["1.16", "1.17", "1.18"]
go: ["1.18", "1.19"]
steps:
- uses: actions/checkout@v3
- name: Setup go
uses: actions/setup-go@v2
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- uses: actions/cache@v3
@ -111,7 +117,7 @@ jobs:
key: ${{ runner.os }}-go${{ matrix.go }}-test-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go${{ matrix.go }}-test-
- run: go test -p 1 ./...
- run: go test ./...
env:
POSTGRES_HOST: localhost
POSTGRES_USER: postgres
@ -126,13 +132,13 @@ jobs:
strategy:
fail-fast: false
matrix:
go: ["1.16", "1.17", "1.18"]
go: ["1.18", "1.19"]
goos: ["linux"]
goarch: ["amd64", "386"]
steps:
- uses: actions/checkout@v3
- name: Setup go
uses: actions/setup-go@v2
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- name: Install dependencies x86
@ -150,6 +156,7 @@ jobs:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: 1
CGO_CFLAGS: -fno-stack-protector
run: go build -trimpath -v -o "bin/" ./cmd/...
# build for Windows 64-bit
@ -159,13 +166,13 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
go: ["1.16", "1.17", "1.18"]
go: ["1.18", "1.19"]
goos: ["windows"]
goarch: ["amd64"]
steps:
- uses: actions/checkout@v3
- name: Setup Go ${{ matrix.go }}
uses: actions/setup-go@v2
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- name: Install dependencies
@ -206,9 +213,9 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Setup go
uses: actions/setup-go@v2
uses: actions/setup-go@v3
with:
go-version: "1.16"
go-version: "1.18"
- uses: actions/cache@v3
with:
path: |
@ -222,6 +229,31 @@ jobs:
- name: Test upgrade
run: ./dendrite-upgrade-tests --head .
# run database upgrade tests, skipping over one version
upgrade_test_direct:
name: Upgrade tests from HEAD-2
timeout-minutes: 20
needs: initial-tests-done
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: "1.18"
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-upgrade
- name: Build upgrade-tests
run: go build ./cmd/dendrite-upgrade-tests
- name: Test upgrade
run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
# run Sytest in different variations
sytest:
timeout-minutes: 20
@ -250,6 +282,7 @@ jobs:
env:
POSTGRES: ${{ matrix.postgres && 1}}
API: ${{ matrix.api && 1 }}
SYTEST_BRANCH: ${{ github.head_ref }}
steps:
- uses: actions/checkout@v2
- name: Run Sytest
@ -343,6 +376,8 @@ jobs:
# Build initial Dendrite image
- run: docker build -t complement-dendrite -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
working-directory: dendrite
env:
DOCKER_BUILDKIT: 1
# Run Complement
- run: |
@ -357,7 +392,14 @@ jobs:
integration-tests-done:
name: Integration tests passed
needs: [initial-tests-done, upgrade_test, sytest, complement]
needs:
[
initial-tests-done,
upgrade_test,
upgrade_test_direct,
sytest,
complement,
]
runs-on: ubuntu-latest
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
steps:

60
.github/workflows/docker.yml vendored
View file

@ -161,3 +161,63 @@ jobs:
${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
demo-pinecone:
name: Pinecone demo image
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Get release tag
if: github.event_name == 'release' # Only for GitHub releases
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build main pinecone demo image
if: github.ref_name == 'main'
id: docker_build_demo_pinecone
uses: docker/build-push-action@v2
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
file: ./build/docker/Dockerfile.demo-pinecone
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }}
- name: Build release pinecone demo image
if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_demo_pinecone_release
uses: docker/build-push-action@v2
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
file: ./build/docker/Dockerfile.demo-pinecone
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:latest
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ env.RELEASE_VERSION }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ env.RELEASE_VERSION }}

7
.gitignore vendored
View file

@ -41,6 +41,10 @@ _testmain.go
*.test
*.prof
*.wasm
*.aar
*.jar
*.framework
*.xcframework
# Generated keys
*.pem
@ -65,4 +69,7 @@ test/wasm/node_modules
# Ignore complement folder when running locally
complement/
# Stuff from GitHub Pages
docs/_site
media_store/

333
CHANGES.md
View file

@ -1,5 +1,338 @@
# Changelog
## Dendrite 0.9.9 (2022-09-22)
### Features
* Dendrite will now try to keep HTTP connections open to remote federated servers for a few minutes after a request and attempt to reuse those connections where possible
* This should reduce the amount of time spent on TLS handshakes and often speed up requests to remote servers
* This new behaviour can be disabled with the `federation_api.disable_http_keepalives` option if needed
* A number of dependencies have been updated
### Fixes
* A bug where the roomserver did not correctly propagate rewritten room state to downstream components (like the federation API and sync API) has been fixed, which could cause issues when performing a federated join to a previously left room
* Event auth now correctly parses the `join_authorised_via_users_server` field in the membership event content
* Database migrations should no longer produce unique constraint errors at Dendrite startup
* The `origin` of device list updates should now be populated correctly
* Send-to-device messages will no longer be dropped if we fail to publish them to specific devices
* The roomserver query to find state after events will now always resolve state if there are multiple prev events
* The roomserver will now return no memberships if querying history visibility for an event which has no state snapshot
* The device list updater will now mark a device list as stale if a requesting device ID is not known
* Transactions sent to appservices should no longer have accidental duplicated transaction IDs (contributed by [tak-hntlabs](https://github.com/tak-hntlabs))
## Dendrite 0.9.8 (2022-09-12)
### Important
* This is a **security release** to fix a vulnerability where missing events retrieved from other servers did not have their signatures verified in all cases, affecting all versions of Dendrite before 0.9.8. Upgrading to this version is highly recommended. For more information, [see here](https://github.com/matrix-org/dendrite/security/advisories/GHSA-pfw4-xjgm-267c).
### Features
* The built-in NATS Server has been updated to the final 2.9.0 release version
### Fixes
* Dendrite will now correctly verify the signatures of events retrieved using `/get_missing_events`
## Dendrite 0.9.7 (2022-09-09)
### Features
* Initial supporting code to enable full-text search has been merged (although not ready for use yet)
* Newly created rooms now have higher default power levels for enabling encryption, setting server ACLs or sending tombstone events
* Incoming signing key updates over federation are now queued in JetStream for processing, so that they cannot be dropped accidentally
### Fixes
* A race condition between the roomserver output events being generated, forward extremities being updated and room info being updated has been fixed
* Appservices will no longer receive invite events which they are not interested in, which caused heavy load in some cases or excessive request sizes in others
* A bug in state resolution v2 where events could incorrectly be classified as control events has been fixed
* A bug in state resolution v2 where some specific events with unexpected non-empty state keys are dropped has been fixed
* A bug in state resolution v2 when fetching auth events vs partial state has been fixed
* Stale device lists should now be handled correctly for all user IDs, which may help with E2EE reliability
* A number of database writer issues have been fixed in the user API and sync API, which should help to reduce `database is locked` errors with SQLite databases
* Database migrations should now be detected more reliably to prevent unexpected errors at startup
* A number of minor database transaction issues have been fixed, particularly for assigning NIDs in the roomserver, cleaning up device keys and cleaning up notifications
* The database query for finding shared users in the sync API has been optimised, using significantly less CPU time as a result
## Dendrite 0.9.6 (2022-09-01)
### Features
* The appservice API has been refactored for improved performance and stability
* The appservice database has been deprecated, as the roomserver output stream is now used as the data source instead
* The `generate-config` tool has been updated to support additional scenarios, i.e. for CI configuration generation and generating both monolith and polylith skeleton config files
### Fixes
* The username length check has been fixed on new account creation
* The length of the `type`, `sender`, `state_key` and `room_id` fields in events are now verified by number of codepoints rather than bytes, fixing the "Cat Overflow" bug
* UTF-16 surrogate handling in the canonical JSON implementation has been fixed
* A race condition when starting the keyserver has been fixed
* A race condition when configuring HTTP servers and routing at startup has been fixed
* A bug where the incorrect limit was used for lazy-loading memberships has been fixed
* The number of push notifications will now be sent to the push gateway
* A missing index causing slow performance on the sync API send-to-device table has been added (contributed by [PiotrKozimor](https://github.com/PiotrKozimor))
* Event auth will now correctly check for the existence of the `"creator"` field in create events
## Dendrite 0.9.5 (2022-08-25)
### Fixes
* The roomserver will now correctly unreject previously rejected events if necessary when reprocessing
* The handling of event soft-failure has been improved on the roomserver input by no longer applying rejection rules and still calculating state before the event if possible
* The federation `/state` and `/state_ids` endpoints should now return the correct error code when the state isn't known instead of returning a HTTP 500
* The federation `/event` should now return outlier events correctly instead of returning a HTTP 500
* A bug in the federation backoff allowing zero intervals has been corrected
* The `create-account` utility will no longer error if the homeserver URL ends in a trailing slash
* A regression in `/sync` introduced in 0.9.4 should be fixed
## Dendrite 0.9.4 (2022-08-19)
### Fixes
* A bug in the roomserver around handling rejected outliers has been fixed
* Backfilled events will now use the correct history visibility where possible
* The device list updater backoff has been fixed, which should reduce the number of outbound HTTP requests and `Failed to query device keys for some users` log entries for dead servers
* The `/sync` endpoint will no longer incorrectly return room entries for retired invites which could cause some rooms to show up in the client "Historical" section
* The `/createRoom` endpoint will now correctly populate `is_direct` in invite membership events, which may help clients to classify direct messages correctly
* The `create-account` tool will now log an error if the shared secret is not set in the Dendrite config
* A couple of minor bugs have been fixed in the membership lazy-loading
* Queued EDUs in the federation API are now cached properly
## Dendrite 0.9.3 (2022-08-15)
### Important
* This is a **security release** to fix a vulnerability within event auth, affecting all versions of Dendrite before 0.9.3. Upgrading to this version is highly recommended. For more information, [see here](https://github.com/matrix-org/gomatrixserverlib/security/advisories/GHSA-grvv-h2f9-7v9c).
### Fixes
* Dendrite will now correctly parse the `"events_default"` power level value for event auth.
## Dendrite 0.9.2 (2022-08-12)
### Features
* Dendrite now supports history visibility on the `/sync`, `/messages` and `/context` endpoints
* It should now be possible to view the history of a room in more cases (as opposed to limiting scrollback to the join event or defaulting to the restrictive `"join"` visibility rule as before)
* The default room version for newly created rooms is now room version 9
* New admin endpoint `/_dendrite/admin/resetPassword/{userID}` has been added, which replaces the `-reset-password` flag in `create-account`
* The `create-account` binary now uses shared secret registration over HTTP to create new accounts, which fixes a number of problems with account data and push rules not being configured correctly for new accounts
* The internal HTTP APIs for polylith deployments have been refactored for correctness and consistency
* The federation API will now automatically clean up some EDUs that have failed to send within a certain period of time
* The `/hierarchy` endpoint will now return potentially joinable rooms (contributed by [texuf](https://github.com/texuf))
* The user directory will now show or hide users correctly
### Fixes
* Send-to-device messages should no longer be incorrectly duplicated in `/sync`
* The federation sender will no longer create unnecessary destination queues as a result of a logic error
* A bug where database migrations may not execute properly when upgrading from older versions has been fixed
* A crash when failing to update user account data has been fixed
* A race condition when generating notification counts has been fixed
* A race condition when setting up NATS has been fixed (contributed by [brianathere](https://github.com/brianathere))
* Stale cache data for membership lazy-loading is now correctly invalidated when doing a complete sync
* Data races within user-interactive authentication have been fixed (contributed by [tak-hntlabs](https://github.com/tak-hntlabs))
## Dendrite 0.9.1 (2022-08-03)
### Fixes
* Upgrades a dependency which caused issues building Dendrite with Go 1.19
* The roomserver will no longer give up prematurely after failing to call `/state_ids`
* Removes the faulty room info cache, which caused of a number of race conditions and occasional bugs (including when creating and joining rooms)
* The media endpoint now sets the `Cache-Control` header correctly to prevent web-based clients from hitting media endpoints excessively
* The sync API will now advance the PDU stream position correctly in all cases (contributed by [sergekh2](https://github.com/sergekh2))
* The sync API will now delete the correct range of send-to-device messages when advancing the stream position
* The device list `changed` key in the `/sync` response should now return the correct users
* A data race when looking up missing state has been fixed
* The `/send_join` API is now applying stronger validation to the received membership event
## Dendrite 0.9.0 (2022-08-01)
### Features
* Dendrite now uses Ristretto for managing in-memory caches
* Should improve cache utilisation considerably over time by more intelligently selecting and managing cache entries compared to the previous LRU-based cache
* Defaults to a 1GB cache size if not configured otherwise
* The estimated cache size in memory and maximum age can now be configured with new [configuration options](https://github.com/matrix-org/dendrite/blob/e94ef84aaba30e12baf7f524c4e7a36d2fdeb189/dendrite-sample.monolith.yaml#L44-L61) to prevent unbounded cache growth
* Added support for serving the `/.well-known/matrix/client` hint directly from Dendrite
* Configurable with the new [configuration option](https://github.com/matrix-org/dendrite/blob/e94ef84aaba30e12baf7f524c4e7a36d2fdeb189/dendrite-sample.monolith.yaml#L67-L69)
* Refactored membership updater, which should eliminate some bugs caused by the membership table getting out of sync with the room state
* The User API is now responsible for sending account data updates to other components, which may fix some races and duplicate account data events
* Optimised database query for checking whether a remote server is allowed to request an event over federation without using anywhere near as much CPU time (PostgreSQL only)
* Database migrations have been refactored to eliminate some problems that were present with `goose` and upgrading from older Dendrite versions
* Media fetching will now use the `/v3` endpoints for downloading media from remote homeservers
* HTTP 404 and HTTP 405 errors from the client-facing APIs should now be returned with CORS headers so that web-based clients do not produce incorrect access control warnings for unknown endpoints
* Some preparation work for full history visibility support
### Fixes
* Fixes a crash that could occur during event redaction
* The `/members` endpoint will no longer incorrectly return HTTP 500 as a result of some invite events
* Send-to-device messages should now be ordered more reliably and the last position in the stream updated correctly
* Parsing of appservice configuration files is now less strict (contributed by [Kab1r](https://github.com/Kab1r))
* The sync API should now identify shared users correctly when waking up for E2EE key changes
* The federation `/state` endpoint will now return a HTTP 403 when the state before an event isn't known instead of a HTTP 500
* Presence timestamps should now be calculated with the correct precision
* A race condition in the roomserver's room info has been fixed
* A race condition in the sync API has been fixed
## Dendrite 0.8.9 (2022-07-01)
### Features
* Incoming device list updates over federation are now queued in JetStream for processing so that they will no longer block incoming federation transactions and should never end up dropped, which will hopefully help E2EE reliability
* The `/context` endpoint now returns `"start"` and `"end"` parameters to allow pagination from a context call
* The `/messages` endpoint will no longer return `"end"` when there are no more messages remaining
* Deactivated user accounts will now leave all rooms automatically
* New admin endpoint `/_dendrite/admin/evacuateUser/{userID}` has been added for forcing a local user to leave all joined rooms
* Dendrite will now automatically attempt to raise the file descriptor limit at startup if it is too low
### Fixes
* A rare crash when retrieving remote device lists has been fixed
* Fixes a bug where events were not redacted properly over federation
* The `/invite` endpoints will now return an error instead of silently proceeding if the user ID is obviously malformed
## Dendrite 0.8.8 (2022-06-09)
### Features
* The performance of state resolution has been increased significantly for larger rooms
* A number of changes have been made to rate limiting:
* Logged in users will now be rate-limited on a per-session basis rather than by remote IP
* Rate limiting no longer applies to admin or appservice users
* It is now possible to configure additional users that are exempt from rate limiting using the `exempt_user_ids` option in the `rate_limiting` section of the Dendrite config
* Setting state is now idempotent via the client API state endpoints
### Fixes
* Room upgrades now properly propagate tombstone events to remote servers
* Room upgrades will no longer send tombstone events if creating the upgraded room fails
* A crash has been fixed when evaluating restricted room joins
## Dendrite 0.8.7 (2022-06-01)
### Features
* Support added for room version 10
### Fixes
* A number of state handling bugs have been fixed, which previously resulted in missing state events, unexpected state deletions, reverted memberships and unexpectedly rejected/soft-failed events in some specific cases
* Fixed destination queue performance issues as a result of missing indexes, which speeds up outbound federation considerably
* A bug which could cause the `/register` endpoint to return HTTP 500 has been fixed
## Dendrite 0.8.6 (2022-05-26)
### Features
* Room versions 8 and 9 are now marked as stable
* Dendrite can now assist remote users to join restricted rooms via `/make_join` and `/send_join`
### Fixes
* The sync API no longer returns immediately on `/sync` requests unnecessarily if it can be avoided
* A race condition has been fixed in the sync API when updating presence via `/sync`
* A race condition has been fixed sending E2EE keys to remote servers over federation when joining rooms
* The `trusted_private_chat` preset should now grant power level 100 to all participant users, which should improve the user experience of direct messages
* Invited users are now authed correctly in restricted rooms
* The `join_authorised_by_users_server` key is now correctly stripped in restricted rooms when updating the membership event
* Appservices should now receive invite events correctly
* Device list updates should no longer contain optional fields with `null` values
* The `/deactivate` endpoint has been fixed to no longer confuse Element with incorrect completed flows
## Dendrite 0.8.5 (2022-05-13)
### Features
* New living documentation available at <https://matrix-org.github.io/dendrite/>, including new installation instructions
* The built-in NATS Server has been updated to version 2.8.2
### Fixes
* Monolith deployments will no longer panic at startup if given a config file that does not include the `internal_api` and `external_api` options
* State resolution v2 now correctly identifies other events related to power events, which should fix some event auth issues
* The latest events updater will no longer implicitly trust the new forward extremities when calculating the current room state, which may help to avoid some state resets
* The one-time key count is now correctly returned in `/sync` even if the request otherwise timed out, which should reduce the chance that unnecessary one-time keys will be uploaded by clients
* The `create-account` tool should now work properly when the database is configured using the global connection pool
## Dendrite 0.8.4 (2022-05-10)
### Fixes
* Fixes a regression introduced in the previous version where appservices, push and phone-home statistics would not work over plain HTTP
* Adds missing indexes to the sync API output events table, which should significantly improve `/sync` performance and reduce database CPU usage
* Building Dendrite with the `bimg` thumbnailer should now work again (contributed by [database64128](https://github.com/database64128))
## Dendrite 0.8.3 (2022-05-09)
### Features
* Open registration is now harder to enable, which should reduce the chance that Dendrite servers will be used to conduct spam or abuse attacks
* Dendrite will only enable open registration if you pass the `--really-enable-open-registration` command line flag at startup
* If open registration is enabled but this command line flag is not passed, Dendrite will fail to start up
* Dendrite now supports phone-home statistic reporting
* These statistics include things like the number of registered and active users, some configuration options and platform/environment details, to help us to understand how Dendrite is used
* This is not enabled by default — it must be enabled in the `global.report_stats` section of the config file
* Monolith installations can now be configured with a single global database connection pool (in `global.database` in the config) rather than having to configure each component separately
* This also means that you no longer need to balance connection counts between different components, as they will share the same larger pool
* Specific components can override the global database settings by specifying their own `database` block
* To use only the global pool, you must configure `global.database` and then remove the `database` block from all of the component sections of the config file
* A new admin API endpoint `/_dendrite/admin/evacuateRoom/{roomID}` has been added, allowing server admins to forcefully part all local users from a given room
* The sync notifier now only loads members for the relevant rooms, which should reduce CPU usage and load on the database
* A number of component interfaces have been refactored for cleanliness and developer ease
* Event auth errors in the log should now be much more useful, including the reason for the event failures
* The forward extremity calculation in the roomserver has been simplified
* A new index has been added to the one-time keys table in the keyserver which should speed up key count lookups
### Fixes
* Dendrite will no longer process events for rooms where there are no local users joined, which should help to reduce CPU and RAM usage
* A bug has been fixed in event auth when changing the user levels in `m.room.power_levels` events
* Usernames should no longer be duplicated when no room name is set
* Device display names should now be correctly propagated over federation
* A panic when uploading cross-signing signatures has been fixed
* Presence is now correctly limited in `/sync` based on the filters
* The presence stream position returned by `/sync` will now be correct if no presence events were returned
* The media `/config` endpoint will no longer return a maximum upload size field if it is configured to be unlimited in the Dendrite config
* The server notices room will no longer produce "User is already joined to the room" errors
* Consumer errors will no longer flood the logs during a graceful shutdown
* Sync API and federation API consumers will no longer unnecessarily query added state events matching the one in the output event
* The Sync API will no longer unnecessarily track invites for remote users
## Dendrite 0.8.2 (2022-04-27)
### Features
* Lazy-loading has been added to the `/sync` endpoint, which should speed up syncs considerably
* Filtering has been added to the `/messages` endpoint
* The room summary now contains "heroes" (up to 5 users in the room) for clients to display when no room name is set
* The existing lazy-loading caches will now be used by `/messages` and `/context` so that member events will not be sent to clients more times than necessary
* The account data stream now uses the provided filters
* The built-in NATS Server has been updated to version 2.8.0
* The `/state` and `/state_ids` endpoints will now return `M_NOT_FOUND` for rejected events
* Repeated calls to the `/redact` endpoint will now be idempotent when a transaction ID is given
* Dendrite should now be able to run as a Windows service under Service Control Manager
### Fixes
* Fictitious presence updates will no longer be created for users which have not sent us presence updates, which should speed up complete syncs considerably
* Uploading cross-signing device signatures should now be more reliable, fixing a number of bugs with cross-signing
* All account data should now be sent properly on a complete sync, which should eliminate problems with client settings or key backups appearing to be missing
* Account data will now be limited correctly on incremental syncs, returning the stream position of the most recent update rather than the latest stream position
* Account data will not be sent for parted rooms, which should reduce the number of left/forgotten rooms reappearing in clients as empty rooms
* The TURN username hash has been fixed which should help to resolve some problems when using TURN for voice calls (contributed by [fcwoknhenuxdfiyv](https://github.com/fcwoknhenuxdfiyv))
* Push rules can no longer be modified using the account data endpoints
* Querying account availability should now work properly in polylith deployments
* A number of bugs with sync filters have been fixed
* A default sync filter will now be used if the request contains a filter ID that does not exist
* The `pushkey_ts` field is now using seconds instead of milliseconds
* A race condition when gracefully shutting down has been fixed, so JetStream should no longer cause the process to exit before other Dendrite components are finished shutting down
## Dendrite 0.8.1 (2022-04-07)
### Fixes

88
README.md
View file

@ -1,4 +1,5 @@
# Dendrite
[![Build status](https://github.com/matrix-org/dendrite/actions/workflows/dendrite.yml/badge.svg?event=push)](https://github.com/matrix-org/dendrite/actions/workflows/dendrite.yml) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org)
Dendrite is a second-generation Matrix homeserver written in Go.
@ -6,26 +7,24 @@ It intends to provide an **efficient**, **reliable** and **scalable** alternativ
- Efficient: A small memory footprint with better baseline performance than an out-of-the-box Synapse.
- Reliable: Implements the Matrix specification as written, using the
[same test suite](https://github.com/matrix-org/sytest) as Synapse as well as
a [brand new Go test suite](https://github.com/matrix-org/complement).
[same test suite](https://github.com/matrix-org/sytest) as Synapse as well as
a [brand new Go test suite](https://github.com/matrix-org/complement).
- Scalable: can run on multiple machines and eventually scale to massive homeserver deployments.
As of October 2020, Dendrite has now entered **beta** which means:
Dendrite is **beta** software, which means:
- Dendrite is ready for early adopters. We recommend running in Monolith mode with a PostgreSQL database.
- Dendrite has periodic semver releases. We intend to release new versions as we land significant features.
- Dendrite has periodic releases. We intend to release new versions as we fix bugs and land significant features.
- Dendrite supports database schema upgrades between releases. This means you should never lose your messages when upgrading Dendrite.
- Breaking changes will not occur on minor releases. This means you can safely upgrade Dendrite without modifying your database or config file.
This does not mean:
- Dendrite is bug-free. It has not yet been battle-tested in the real world and so will be error prone initially.
- All of the CS/Federation APIs are implemented. We are tracking progress via a script called 'Are We Synapse Yet?'. In particular,
presence and push notifications are entirely missing from Dendrite. See [CHANGES.md](CHANGES.md) for updates.
- Dendrite is ready for massive homeserver deployments. You cannot shard each microservice, only run each one on a different machine.
- Dendrite is feature-complete. There may be client or federation APIs that are not implemented.
- Dendrite is ready for massive homeserver deployments. There is no sharding of microservices (although it is possible to run them on separate machines) and there is no high-availability/clustering support.
Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices.
In the future, we will be able to scale up to gigantic servers (equivalent to matrix.org) via polylith mode.
In the future, we will be able to scale up to gigantic servers (equivalent to `matrix.org`) via polylith mode.
If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or join us in:
@ -35,7 +34,10 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j
## Requirements
To build Dendrite, you will need Go 1.16 or later.
See the [Planning your Installation](https://matrix-org.github.io/dendrite/installation/planning) page for
more information on requirements.
To build Dendrite, you will need Go 1.18 or later.
For a usable federating Dendrite deployment, you will also need:
@ -52,7 +54,7 @@ The [Federation Tester](https://federationtester.matrix.org) can be used to veri
## Get started
If you wish to build a fully-federating Dendrite instance, see [INSTALL.md](docs/INSTALL.md). For running in Docker, see [build/docker](build/docker).
If you wish to build a fully-federating Dendrite instance, see [the Installation documentation](https://matrix-org.github.io/dendrite/installation). For running in Docker, see [build/docker](build/docker).
The following instructions are enough to get Dendrite started as a non-federating test deployment using self-signed certificates and SQLite databases:
@ -70,10 +72,14 @@ $ ./bin/generate-keys --tls-cert server.crt --tls-key server.key
# Copy and modify the config file - you'll need to set a server name and paths to the keys
# at the very least, along with setting up the database connection strings.
$ cp dendrite-config.yaml dendrite.yaml
$ cp dendrite-sample.monolith.yaml dendrite.yaml
# Build and run the server:
$ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml
# Create an user account (add -admin for an admin user).
# Specify the localpart only, e.g. 'alice' for '@alice:domain.com'
$ ./bin/create-account --config dendrite.yaml --url http://localhost:8008 --username alice
```
Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`.
@ -82,19 +88,18 @@ Then point your favourite Matrix client at `http://localhost:8008` or `https://l
We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver
test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
updates with CI. As of April 2022 we're at around 83% CS API coverage and 95% Federation coverage, though check
updates with CI. As of August 2022 we're at around 90% CS API coverage and 95% Federation coverage, though check
CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
servers such as matrix.org reasonably well, although there are still some missing features (like Search).
We are prioritising features that will benefit single-user homeservers first (e.g Receipts, E2E) rather
than features that massive deployments may be interested in (User Directory, OpenID, Guests, Admin APIs, AS API).
than features that massive deployments may be interested in (OpenID, Guests, Admin APIs, AS API).
This means Dendrite supports amongst others:
- Core room functionality (creating rooms, invites, auth rules)
- Full support for room versions 1 to 7
- Experimental support for room versions 8 to 9
- Room versions 1 to 10 supported
- Backfilling locally and via federation
- Accounts, Profiles and Devices
- Accounts, profiles and devices
- Published room lists
- Typing
- Media APIs
@ -115,53 +120,8 @@ We would be grateful for any help on issues marked as
all have related Sytests which need to pass in order for the issue to be closed. Once you've written your
code, you can quickly run Sytest to ensure that the test names are now passing.
For example, if the test `Local device key changes get to remote servers` was marked as failing, find the
test file (e.g via `grep` or via the
[CI log output](https://buildkite.com/matrix-dot-org/dendrite/builds/2826#39cff5de-e032-4ad0-ad26-f819e6919c42)
it's `tests/50federation/40devicelists.pl` ) then to run Sytest:
```
docker run --rm --name sytest
-v "/Users/kegan/github/sytest:/sytest"
-v "/Users/kegan/github/dendrite:/src"
-v "/Users/kegan/logs:/logs"
-v "/Users/kegan/go/:/gopath"
-e "POSTGRES=1" -e "DENDRITE_TRACE_HTTP=1"
matrixdotorg/sytest-dendrite:latest tests/50federation/40devicelists.pl
```
See [sytest.md](docs/sytest.md) for the full description of these flags.
You can try running sytest outside of docker for faster runs, but the dependencies can be temperamental
and we recommend using docker where possible.
```
cd sytest
export PERL5LIB=$HOME/lib/perl5
export PERL_MB_OPT=--install_base=$HOME
export PERL_MM_OPT=INSTALL_BASE=$HOME
./install-deps.pl
./run-tests.pl -I Dendrite::Monolith -d $PATH_TO_DENDRITE_BINARIES
```
Sometimes Sytest is testing the wrong thing or is flakey, so it will need to be patched.
Ask on `#dendrite-dev:matrix.org` if you think this is the case for you and we'll be happy to help.
If you're new to the project, see [CONTRIBUTING.md](docs/CONTRIBUTING.md) to get up to speed then
If you're new to the project, see our
[Contributing page](https://matrix-org.github.io/dendrite/development/contributing) to get up to speed, then
look for [Good First Issues](https://github.com/matrix-org/dendrite/labels/good%20first%20issue). If you're
familiar with the project, look for [Help Wanted](https://github.com/matrix-org/dendrite/labels/help-wanted)
issues.
## Hardware requirements
Dendrite in Monolith + SQLite works in a range of environments including iOS and in-browser via WASM.
For small homeserver installations joined on ~10s rooms on matrix.org with ~100s of users in those rooms, including some
encrypted rooms:
- Memory: uses around 100MB of RAM, with peaks at around 200MB.
- Disk space: After a few months of usage, the database grew to around 2GB (in Monolith mode).
- CPU: Brief spikes when processing events, typically idles at 1% CPU.
This means Dendrite should comfortably work on things like Raspberry Pis.

10
appservice/README.md
View file

@ -1,10 +0,0 @@
# Application Service
This component interfaces with external [Application
Services](https://matrix.org/docs/spec/application_service/unstable.html).
This includes any HTTP endpoints that application services call, as well as talking
to any HTTP endpoints that application services provide themselves.
## Consumers
This component consumes and filters events from the Roomserver Kafka stream, passing on any necessary events to subscribing application services.

38
appservice/api/query.go
View file

@ -26,6 +26,23 @@ import (
"github.com/matrix-org/gomatrixserverlib"
)
// AppServiceInternalAPI is used to query user and room alias data from application
// services
type AppServiceInternalAPI interface {
// Check whether a room alias exists within any application service namespaces
RoomAliasExists(
ctx context.Context,
req *RoomAliasExistsRequest,
resp *RoomAliasExistsResponse,
) error
// Check whether a user ID exists within any application service namespaces
UserIDExists(
ctx context.Context,
req *UserIDExistsRequest,
resp *UserIDExistsResponse,
) error
}
// RoomAliasExistsRequest is a request to an application service
// about whether a room alias exists
type RoomAliasExistsRequest struct {
@ -60,31 +77,14 @@ type UserIDExistsResponse struct {
UserIDExists bool `json:"exists"`
}
// AppServiceQueryAPI is used to query user and room alias data from application
// services
type AppServiceQueryAPI interface {
// Check whether a room alias exists within any application service namespaces
RoomAliasExists(
ctx context.Context,
req *RoomAliasExistsRequest,
resp *RoomAliasExistsResponse,
) error
// Check whether a user ID exists within any application service namespaces
UserIDExists(
ctx context.Context,
req *UserIDExistsRequest,
resp *UserIDExistsResponse,
) error
}
// RetrieveUserProfile is a wrapper that queries both the local database and
// application services for a given user's profile
// TODO: Remove this, it's called from federationapi and clientapi but is a pure function
func RetrieveUserProfile(
ctx context.Context,
userID string,
asAPI AppServiceQueryAPI,
profileAPI userapi.UserProfileAPI,
asAPI AppServiceInternalAPI,
profileAPI userapi.ClientUserAPI,
) (*authtypes.Profile, error) {
localpart, _, err := gomatrixserverlib.SplitID('@', userID)
if err != nil {

63
appservice/appservice.go
View file

@ -18,7 +18,6 @@ import (
"context"
"crypto/tls"
"net/http"
"sync"
"time"
"github.com/gorilla/mux"
@ -28,18 +27,14 @@ import (
"github.com/matrix-org/dendrite/appservice/consumers"
"github.com/matrix-org/dendrite/appservice/inthttp"
"github.com/matrix-org/dendrite/appservice/query"
"github.com/matrix-org/dendrite/appservice/storage"
"github.com/matrix-org/dendrite/appservice/types"
"github.com/matrix-org/dendrite/appservice/workers"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/jetstream"
userapi "github.com/matrix-org/dendrite/userapi/api"
)
// AddInternalRoutes registers HTTP handlers for internal API calls
func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceQueryAPI) {
func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInternalAPI) {
inthttp.AddRoutes(queryAPI, router)
}
@ -49,7 +44,7 @@ func NewInternalAPI(
base *base.BaseDendrite,
userAPI userapi.UserInternalAPI,
rsAPI roomserverAPI.RoomserverInternalAPI,
) appserviceAPI.AppServiceQueryAPI {
) appserviceAPI.AppServiceInternalAPI {
client := &http.Client{
Timeout: time.Second * 30,
Transport: &http.Transport{
@ -57,59 +52,43 @@ func NewInternalAPI(
TLSClientConfig: &tls.Config{
InsecureSkipVerify: base.Cfg.AppServiceAPI.DisableTLSValidation,
},
Proxy: http.ProxyFromEnvironment,
},
}
js, _ := jetstream.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
// Create appserivce query API with an HTTP client that will be used for all
// outbound and inbound requests (inbound only for the internal API)
appserviceQueryAPI := &query.AppServiceQueryAPI{
HTTPClient: client,
Cfg: &base.Cfg.AppServiceAPI,
}
// Create a connection to the appservice postgres DB
appserviceDB, err := storage.NewDatabase(&base.Cfg.AppServiceAPI.Database)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to appservice db")
if len(base.Cfg.Derived.ApplicationServices) == 0 {
return appserviceQueryAPI
}
// Wrap application services in a type that relates the application service and
// a sync.Cond object that can be used to notify workers when there are new
// events to be sent out.
workerStates := make([]types.ApplicationServiceWorkerState, len(base.Cfg.Derived.ApplicationServices))
for i, appservice := range base.Cfg.Derived.ApplicationServices {
m := sync.Mutex{}
ws := types.ApplicationServiceWorkerState{
AppService: appservice,
Cond: sync.NewCond(&m),
}
workerStates[i] = ws
for _, appservice := range base.Cfg.Derived.ApplicationServices {
// Create bot account for this AS if it doesn't already exist
if err = generateAppServiceAccount(userAPI, appservice); err != nil {
if err := generateAppServiceAccount(userAPI, appservice); err != nil {
logrus.WithFields(logrus.Fields{
"appservice": appservice.ID,
}).WithError(err).Panicf("failed to generate bot account for appservice")
}
}
// Create appserivce query API with an HTTP client that will be used for all
// outbound and inbound requests (inbound only for the internal API)
appserviceQueryAPI := &query.AppServiceQueryAPI{
HTTPClient: client,
Cfg: base.Cfg,
}
// Only consume if we actually have ASes to track, else we'll just chew cycles needlessly.
// We can't add ASes at runtime so this is safe to do.
if len(workerStates) > 0 {
consumer := consumers.NewOutputRoomEventConsumer(
base.ProcessContext, base.Cfg, js, appserviceDB,
rsAPI, workerStates,
)
if err := consumer.Start(); err != nil {
logrus.WithError(err).Panicf("failed to start appservice roomserver consumer")
}
js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
consumer := consumers.NewOutputRoomEventConsumer(
base.ProcessContext, &base.Cfg.AppServiceAPI,
client, js, rsAPI,
)
if err := consumer.Start(); err != nil {
logrus.WithError(err).Panicf("failed to start appservice roomserver consumer")
}
// Create application service transaction workers
if err := workers.SetupTransactionWorkers(client, appserviceDB, workerStates); err != nil {
logrus.WithError(err).Panicf("failed to start app service transaction workers")
}
return appserviceQueryAPI
}
@ -117,7 +96,7 @@ func NewInternalAPI(
// `sender_localpart` field of each application service if it doesn't
// exist already
func generateAppServiceAccount(
userAPI userapi.UserInternalAPI,
userAPI userapi.AppserviceUserAPI,
as config.ApplicationService,
) error {
var accRes userapi.PerformAccountCreationResponse

344
appservice/consumers/roomserver.go
View file

@ -15,184 +15,224 @@
package consumers
import (
"bytes"
"context"
"encoding/json"
"fmt"
"math"
"net/http"
"net/url"
"strconv"
"time"
"github.com/matrix-org/gomatrixserverlib"
"github.com/nats-io/nats.go"
"github.com/matrix-org/dendrite/appservice/storage"
"github.com/matrix-org/dendrite/appservice/types"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/jetstream"
"github.com/matrix-org/dendrite/setup/process"
"github.com/matrix-org/gomatrixserverlib"
"github.com/nats-io/nats.go"
log "github.com/sirupsen/logrus"
)
// OutputRoomEventConsumer consumes events that originated in the room server.
type OutputRoomEventConsumer struct {
ctx context.Context
jetstream nats.JetStreamContext
durable string
topic string
asDB storage.Database
rsAPI api.RoomserverInternalAPI
serverName string
workerStates []types.ApplicationServiceWorkerState
ctx context.Context
cfg *config.AppServiceAPI
client *http.Client
jetstream nats.JetStreamContext
topic string
rsAPI api.AppserviceRoomserverAPI
}
type appserviceState struct {
*config.ApplicationService
backoff int
}
// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call
// Start() to begin consuming from room servers.
func NewOutputRoomEventConsumer(
process *process.ProcessContext,
cfg *config.Dendrite,
cfg *config.AppServiceAPI,
client *http.Client,
js nats.JetStreamContext,
appserviceDB storage.Database,
rsAPI api.RoomserverInternalAPI,
workerStates []types.ApplicationServiceWorkerState,
rsAPI api.AppserviceRoomserverAPI,
) *OutputRoomEventConsumer {
return &OutputRoomEventConsumer{
ctx: process.Context(),
jetstream: js,
durable: cfg.Global.JetStream.Durable("AppserviceRoomserverConsumer"),
topic: cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent),
asDB: appserviceDB,
rsAPI: rsAPI,
serverName: string(cfg.Global.ServerName),
workerStates: workerStates,
ctx: process.Context(),
cfg: cfg,
client: client,
jetstream: js,
topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent),
rsAPI: rsAPI,
}
}
// Start consuming from room servers
func (s *OutputRoomEventConsumer) Start() error {
return jetstream.JetStreamConsumer(
s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
nats.DeliverAll(), nats.ManualAck(),
)
for _, as := range s.cfg.Derived.ApplicationServices {
appsvc := as
state := &appserviceState{
ApplicationService: &appsvc,
}
token := jetstream.Tokenise(as.ID)
if err := jetstream.JetStreamConsumer(
s.ctx, s.jetstream, s.topic,
s.cfg.Matrix.JetStream.Durable("Appservice_"+token),
50, // maximum number of events to send in a single transaction
func(ctx context.Context, msgs []*nats.Msg) bool {
return s.onMessage(ctx, state, msgs)
},
nats.DeliverNew(), nats.ManualAck(),
); err != nil {
return fmt.Errorf("failed to create %q consumer: %w", token, err)
}
}
return nil
}
// onMessage is called when the appservice component receives a new event from
// the room server output log.
func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
// Parse out the event JSON
var output api.OutputEvent
if err := json.Unmarshal(msg.Data, &output); err != nil {
// If the message was invalid, log it and move on to the next message in the stream
log.WithError(err).Errorf("roomserver output log: message parse failure")
return true
}
if output.Type != api.OutputTypeNewRoomEvent || output.NewRoomEvent == nil {
return true
}
newEventID := output.NewRoomEvent.Event.EventID()
events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(output.NewRoomEvent.AddsStateEventIDs))
events = append(events, output.NewRoomEvent.Event)
if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
eventsReq := &api.QueryEventsByIDRequest{
EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
func (s *OutputRoomEventConsumer) onMessage(
ctx context.Context, state *appserviceState, msgs []*nats.Msg,
) bool {
log.WithField("appservice", state.ID).Tracef("Appservice worker received %d message(s) from roomserver", len(msgs))
events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(msgs))
for _, msg := range msgs {
// Parse out the event JSON
var output api.OutputEvent
if err := json.Unmarshal(msg.Data, &output); err != nil {
// If the message was invalid, log it and move on to the next message in the stream
log.WithField("appservice", state.ID).WithError(err).Errorf("Appservice failed to parse message, ignoring")
continue
}
eventsRes := &api.QueryEventsByIDResponse{}
for _, eventID := range output.NewRoomEvent.AddsStateEventIDs {
if eventID != newEventID {
eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
switch output.Type {
case api.OutputTypeNewRoomEvent:
if output.NewRoomEvent == nil || !s.appserviceIsInterestedInEvent(ctx, output.NewRoomEvent.Event, state.ApplicationService) {
continue
}
}
if len(eventsReq.EventIDs) > 0 {
if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
return false
}
events = append(events, eventsRes.Events...)
}
}
// Send event to any relevant application services
if err := s.filterRoomserverEvents(context.TODO(), events); err != nil {
log.WithError(err).Errorf("roomserver output log: filter error")
return true
}
return true
}
// filterRoomserverEvents takes in events and decides whether any of them need
// to be passed on to an external application service. It does this by checking
// each namespace of each registered application service, and if there is a
// match, adds the event to the queue for events to be sent to a particular
// application service.
func (s *OutputRoomEventConsumer) filterRoomserverEvents(
ctx context.Context,
events []*gomatrixserverlib.HeaderedEvent,
) error {
for _, ws := range s.workerStates {
for _, event := range events {
// Check if this event is interesting to this application service
if s.appserviceIsInterestedInEvent(ctx, event, ws.AppService) {
// Queue this event to be sent off to the application service
if err := s.asDB.StoreEvent(ctx, ws.AppService.ID, event); err != nil {
log.WithError(err).Warn("failed to insert incoming event into appservices database")
return err
} else {
// Tell our worker to send out new messages by updating remaining message
// count and waking them up with a broadcast
ws.NotifyNewEvents()
events = append(events, output.NewRoomEvent.Event)
if len(output.NewRoomEvent.AddsStateEventIDs) > 0 {
newEventID := output.NewRoomEvent.Event.EventID()
eventsReq := &api.QueryEventsByIDRequest{
EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)),
}
eventsRes := &api.QueryEventsByIDResponse{}
for _, eventID := range output.NewRoomEvent.AddsStateEventIDs {
if eventID != newEventID {
eventsReq.EventIDs = append(eventsReq.EventIDs, eventID)
}
}
if len(eventsReq.EventIDs) > 0 {
if err := s.rsAPI.QueryEventsByID(s.ctx, eventsReq, eventsRes); err != nil {
log.WithError(err).Errorf("s.rsAPI.QueryEventsByID failed")
return false
}
events = append(events, eventsRes.Events...)
}
}
case api.OutputTypeNewInviteEvent:
if output.NewInviteEvent == nil || !s.appserviceIsInterestedInEvent(ctx, output.NewInviteEvent.Event, state.ApplicationService) {
continue
}
events = append(events, output.NewInviteEvent.Event)
default:
continue
}
}
// If there are no events selected for sending then we should
// ack the messages so that we don't get sent them again in the
// future.
if len(events) == 0 {
return true
}
txnID := ""
// Try to get the message metadata, if we're able to, use the timestamp as the txnID
metadata, err := msgs[0].Metadata()
if err == nil {
txnID = strconv.Itoa(int(metadata.Timestamp.UnixNano()))
}
// Send event to any relevant application services. If we hit
// an error here, return false, so that we negatively ack.
log.WithField("appservice", state.ID).Debugf("Appservice worker sending %d events(s) from roomserver", len(events))
return s.sendEvents(ctx, state, events, txnID) == nil
}
// sendEvents passes events to the appservice by using the transactions
// endpoint. It will block for the backoff period if necessary.
func (s *OutputRoomEventConsumer) sendEvents(
ctx context.Context, state *appserviceState,
events []*gomatrixserverlib.HeaderedEvent,
txnID string,
) error {
// Create the transaction body.
transaction, err := json.Marshal(
gomatrixserverlib.ApplicationServiceTransaction{
Events: gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatAll),
},
)
if err != nil {
return err
}
// If txnID is not defined, generate one from the events.
if txnID == "" {
txnID = fmt.Sprintf("%d_%d", events[0].Event.OriginServerTS(), len(transaction))
}
// Send the transaction to the appservice.
// https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid
address := fmt.Sprintf("%s/transactions/%s?access_token=%s", state.URL, txnID, url.QueryEscape(state.HSToken))
req, err := http.NewRequestWithContext(ctx, "PUT", address, bytes.NewBuffer(transaction))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := s.client.Do(req)
if err != nil {
return state.backoffAndPause(err)
}
// If the response was fine then we can clear any backoffs in place and
// report that everything was OK. Otherwise, back off for a while.
switch resp.StatusCode {
case http.StatusOK:
state.backoff = 0
default:
return state.backoffAndPause(fmt.Errorf("received HTTP status code %d from appservice", resp.StatusCode))
}
return nil
}
// appserviceJoinedAtEvent returns a boolean depending on whether a given
// appservice has membership at the time a given event was created.
func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice config.ApplicationService) bool {
// TODO: This is only checking the current room state, not the state at
// the event in question. Pretty sure this is what Synapse does too, but
// until we have a lighter way of checking the state before the event that
// doesn't involve state res, then this is probably OK.
membershipReq := &api.QueryMembershipsForRoomRequest{
RoomID: event.RoomID(),
JoinedOnly: true,
// backoff pauses the calling goroutine for a 2^some backoff exponent seconds
func (s *appserviceState) backoffAndPause(err error) error {
if s.backoff < 6 {
s.backoff++
}
membershipRes := &api.QueryMembershipsForRoomResponse{}
// XXX: This could potentially race if the state for the event is not known yet
// e.g. the event came over federation but we do not have the full state persisted.
if err := s.rsAPI.QueryMembershipsForRoom(ctx, membershipReq, membershipRes); err == nil {
for _, ev := range membershipRes.JoinEvents {
var membership gomatrixserverlib.MemberContent
if err = json.Unmarshal(ev.Content, &membership); err != nil || ev.StateKey == nil {
continue
}
if appservice.IsInterestedInUserID(*ev.StateKey) {
return true
}
}
} else {
log.WithFields(log.Fields{
"room_id": event.RoomID(),
}).WithError(err).Errorf("Unable to get membership for room")
}
return false
duration := time.Second * time.Duration(math.Pow(2, float64(s.backoff)))
log.WithField("appservice", s.ID).WithError(err).Errorf("Unable to send transaction to appservice, backing off for %s", duration.String())
time.Sleep(duration)
return err
}
// appserviceIsInterestedInEvent returns a boolean depending on whether a given
// event falls within one of a given application service's namespaces.
//
// TODO: This should be cached, see https://github.com/matrix-org/dendrite/issues/1682
func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice config.ApplicationService) bool {
// No reason to queue events if they'll never be sent to the application
// service
if appservice.URL == "" {
func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice *config.ApplicationService) bool {
switch {
case appservice.URL == "":
return false
}
// Check Room ID and Sender of the event
if appservice.IsInterestedInUserID(event.Sender()) ||
appservice.IsInterestedInRoomID(event.RoomID()) {
case appservice.IsInterestedInUserID(event.Sender()):
return true
case appservice.IsInterestedInRoomID(event.RoomID()):
return true
}
@ -213,10 +253,54 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont
}
} else {
log.WithFields(log.Fields{
"room_id": event.RoomID(),
"appservice": appservice.ID,
"room_id": event.RoomID(),
}).WithError(err).Errorf("Unable to get aliases for room")
}
// Check if any of the members in the room match the appservice
return s.appserviceJoinedAtEvent(ctx, event, appservice)
}
// appserviceJoinedAtEvent returns a boolean depending on whether a given
// appservice has membership at the time a given event was created.
func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, appservice *config.ApplicationService) bool {
// TODO: This is only checking the current room state, not the state at
// the event in question. Pretty sure this is what Synapse does too, but
// until we have a lighter way of checking the state before the event that
// doesn't involve state res, then this is probably OK.
membershipReq := &api.QueryMembershipsForRoomRequest{
RoomID: event.RoomID(),
JoinedOnly: true,
}
membershipRes := &api.QueryMembershipsForRoomResponse{}
// XXX: This could potentially race if the state for the event is not known yet
// e.g. the event came over federation but we do not have the full state persisted.
if err := s.rsAPI.QueryMembershipsForRoom(ctx, membershipReq, membershipRes); err == nil {
for _, ev := range membershipRes.JoinEvents {
switch {
case ev.StateKey == nil:
continue
case ev.Type != gomatrixserverlib.MRoomMember:
continue
}
var membership gomatrixserverlib.MemberContent
err = json.Unmarshal(ev.Content, &membership)
switch {
case err != nil:
continue
case membership.Membership == gomatrixserverlib.Join:
if appservice.IsInterestedInUserID(*ev.StateKey) {
return true
}
}
}
} else {
log.WithFields(log.Fields{
"appservice": appservice.ID,
"room_id": event.RoomID(),
}).WithError(err).Errorf("Unable to get membership for room")
}
return false
}

21
appservice/inthttp/client.go
View file

@ -7,7 +7,6 @@ import (
"github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/opentracing/opentracing-go"
)
// HTTP paths for the internal HTTP APIs
@ -29,7 +28,7 @@ type httpAppServiceQueryAPI struct {
func NewAppserviceClient(
appserviceURL string,
httpClient *http.Client,
) (api.AppServiceQueryAPI, error) {
) (api.AppServiceInternalAPI, error) {
if httpClient == nil {
return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is <nil>")
}
@ -42,11 +41,10 @@ func (h *httpAppServiceQueryAPI) RoomAliasExists(
request *api.RoomAliasExistsRequest,
response *api.RoomAliasExistsResponse,
) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "appserviceRoomAliasExists")
defer span.Finish()
apiURL := h.appserviceURL + AppServiceRoomAliasExistsPath
return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
return httputil.CallInternalRPCAPI(
"RoomAliasExists", h.appserviceURL+AppServiceRoomAliasExistsPath,
h.httpClient, ctx, request, response,
)
}
// UserIDExists implements AppServiceQueryAPI
@ -55,9 +53,8 @@ func (h *httpAppServiceQueryAPI) UserIDExists(
request *api.UserIDExistsRequest,
response *api.UserIDExistsResponse,
) error {
span, ctx := opentracing.StartSpanFromContext(ctx, "appserviceUserIDExists")
defer span.Finish()
apiURL := h.appserviceURL + AppServiceUserIDExistsPath
return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
return httputil.CallInternalRPCAPI(
"UserIDExists", h.appserviceURL+AppServiceUserIDExistsPath,
h.httpClient, ctx, request, response,
)
}

31
appservice/inthttp/server.go
View file

@ -1,43 +1,20 @@
package inthttp
import (
"encoding/json"
"net/http"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/util"
)
// AddRoutes adds the AppServiceQueryAPI handlers to the http.ServeMux.
func AddRoutes(a api.AppServiceQueryAPI, internalAPIMux *mux.Router) {
func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) {
internalAPIMux.Handle(
AppServiceRoomAliasExistsPath,
httputil.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse {
var request api.RoomAliasExistsRequest
var response api.RoomAliasExistsResponse
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
return util.ErrorResponse(err)
}
if err := a.RoomAliasExists(req.Context(), &request, &response); err != nil {
return util.ErrorResponse(err)
}
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
httputil.MakeInternalRPCAPI("AppserviceRoomAliasExists", a.RoomAliasExists),
)
internalAPIMux.Handle(
AppServiceUserIDExistsPath,
httputil.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse {
var request api.UserIDExistsRequest
var response api.UserIDExistsResponse
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
return util.ErrorResponse(err)
}
if err := a.UserIDExists(req.Context(), &request, &response); err != nil {
return util.ErrorResponse(err)
}
return util.JSONResponse{Code: http.StatusOK, JSON: &response}
}),
httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists),
)
}

2
appservice/query/query.go
View file

@ -33,7 +33,7 @@ const userIDExistsPath = "/users/"
// AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI
type AppServiceQueryAPI struct {
HTTPClient *http.Client
Cfg *config.Dendrite
Cfg *config.AppServiceAPI
}
// RoomAliasExists performs a request to '/room/{roomAlias}' on all known

30
appservice/storage/interface.go
View file

@ -1,30 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storage
import (
"context"
"github.com/matrix-org/gomatrixserverlib"
)
type Database interface {
StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.HeaderedEvent) error
GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error)
CountEventsWithAppServiceID(ctx context.Context, appServiceID string) (int, error)
UpdateTxnIDForEvents(ctx context.Context, appserviceID string, maxID, txnID int) error
RemoveEventsBeforeAndIncludingID(ctx context.Context, appserviceID string, eventTableID int) error
GetLatestTxnID(ctx context.Context) (int, error)
}

256
appservice/storage/postgres/appservice_events_table.go
View file

@ -1,256 +0,0 @@
// Copyright 2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgres
import (
"context"
"database/sql"
"encoding/json"
"time"
"github.com/matrix-org/gomatrixserverlib"
log "github.com/sirupsen/logrus"
)
const appserviceEventsSchema = `
-- Stores events to be sent to application services
CREATE TABLE IF NOT EXISTS appservice_events (
-- An auto-incrementing id unique to each event in the table
id BIGSERIAL NOT NULL PRIMARY KEY,
-- The ID of the application service the event will be sent to
as_id TEXT NOT NULL,
-- JSON representation of the event
headered_event_json TEXT NOT NULL,
-- The ID of the transaction that this event is a part of
txn_id BIGINT NOT NULL
);
CREATE INDEX IF NOT EXISTS appservice_events_as_id ON appservice_events(as_id);
`
const selectEventsByApplicationServiceIDSQL = "" +
"SELECT id, headered_event_json, txn_id " +
"FROM appservice_events WHERE as_id = $1 ORDER BY txn_id DESC, id ASC"
const countEventsByApplicationServiceIDSQL = "" +
"SELECT COUNT(id) FROM appservice_events WHERE as_id = $1"
const insertEventSQL = "" +
"INSERT INTO appservice_events(as_id, headered_event_json, txn_id) " +
"VALUES ($1, $2, $3)"
const updateTxnIDForEventsSQL = "" +
"UPDATE appservice_events SET txn_id = $1 WHERE as_id = $2 AND id <= $3"
const deleteEventsBeforeAndIncludingIDSQL = "" +
"DELETE FROM appservice_events WHERE as_id = $1 AND id <= $2"
const (
// A transaction ID number that no transaction should ever have. Used for
// checking again the default value.
invalidTxnID = -2
)
type eventsStatements struct {
selectEventsByApplicationServiceIDStmt *sql.Stmt
countEventsByApplicationServiceIDStmt *sql.Stmt
insertEventStmt *sql.Stmt
updateTxnIDForEventsStmt *sql.Stmt
deleteEventsBeforeAndIncludingIDStmt *sql.Stmt
}
func (s *eventsStatements) prepare(db *sql.DB) (err error) {
_, err = db.Exec(appserviceEventsSchema)
if err != nil {
return
}
if s.selectEventsByApplicationServiceIDStmt, err = db.Prepare(selectEventsByApplicationServiceIDSQL); err != nil {
return
}
if s.countEventsByApplicationServiceIDStmt, err = db.Prepare(countEventsByApplicationServiceIDSQL); err != nil {
return
}
if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil {
return
}
if s.updateTxnIDForEventsStmt, err = db.Prepare(updateTxnIDForEventsSQL); err != nil {
return
}
if s.deleteEventsBeforeAndIncludingIDStmt, err = db.Prepare(deleteEventsBeforeAndIncludingIDSQL); err != nil {
return
}
return
}
// selectEventsByApplicationServiceID takes in an application service ID and
// returns a slice of events that need to be sent to that application service,
// as well as an int later used to remove these same events from the database
// once successfully sent to an application service.
func (s *eventsStatements) selectEventsByApplicationServiceID(
ctx context.Context,
applicationServiceID string,
limit int,
) (
txnID, maxID int,
events []gomatrixserverlib.HeaderedEvent,
eventsRemaining bool,
err error,
) {
defer func() {
if err != nil {
log.WithFields(log.Fields{
"appservice": applicationServiceID,
}).WithError(err).Fatalf("appservice unable to select new events to send")
}
}()
// Retrieve events from the database. Unsuccessfully sent events first
eventRows, err := s.selectEventsByApplicationServiceIDStmt.QueryContext(ctx, applicationServiceID)
if err != nil {
return
}
defer checkNamedErr(eventRows.Close, &err)
events, maxID, txnID, eventsRemaining, err = retrieveEvents(eventRows, limit)
if err != nil {
return
}
return
}
// checkNamedErr calls fn and overwrite err if it was nil and fn returned non-nil
func checkNamedErr(fn func() error, err *error) {
if e := fn(); e != nil && *err == nil {
*err = e
}
}
func retrieveEvents(eventRows *sql.Rows, limit int) (events []gomatrixserverlib.HeaderedEvent, maxID, txnID int, eventsRemaining bool, err error) {
// Get current time for use in calculating event age
nowMilli := time.Now().UnixNano() / int64(time.Millisecond)
// Iterate through each row and store event contents
// If txn_id changes dramatically, we've switched from collecting old events to
// new ones. Send back those events first.
lastTxnID := invalidTxnID
for eventsProcessed := 0; eventRows.Next(); {
var event gomatrixserverlib.HeaderedEvent
var eventJSON []byte
var id int
err = eventRows.Scan(
&id,
&eventJSON,
&txnID,
)
if err != nil {
return nil, 0, 0, false, err
}
// Unmarshal eventJSON
if err = json.Unmarshal(eventJSON, &event); err != nil {
return nil, 0, 0, false, err
}
// If txnID has changed on this event from the previous event, then we've
// reached the end of a transaction's events. Return only those events.
if lastTxnID > invalidTxnID && lastTxnID != txnID {
return events, maxID, lastTxnID, true, nil
}
lastTxnID = txnID
// Limit events that aren't part of an old transaction
if txnID == -1 {
// Return if we've hit the limit
if eventsProcessed++; eventsProcessed > limit {
return events, maxID, lastTxnID, true, nil
}
}
if id > maxID {
maxID = id
}
// Portion of the event that is unsigned due to rapid change
// TODO: Consider removing age as not many app services use it
if err = event.SetUnsignedField("age", nowMilli-int64(event.OriginServerTS())); err != nil {
return nil, 0, 0, false, err
}
events = append(events, event)
}
return
}
// countEventsByApplicationServiceID inserts an event mapped to its corresponding application service
// IDs into the db.
func (s *eventsStatements) countEventsByApplicationServiceID(
ctx context.Context,
appServiceID string,
) (int, error) {
var count int
err := s.countEventsByApplicationServiceIDStmt.QueryRowContext(ctx, appServiceID).Scan(&count)
if err != nil && err != sql.ErrNoRows {
return 0, err
}
return count, nil
}
// insertEvent inserts an event mapped to its corresponding application service
// IDs into the db.
func (s *eventsStatements) insertEvent(
ctx context.Context,
appServiceID string,
event *gomatrixserverlib.HeaderedEvent,
) (err error) {
// Convert event to JSON before inserting
eventJSON, err := json.Marshal(event)
if err != nil {
return err
}
_, err = s.insertEventStmt.ExecContext(
ctx,
appServiceID,
eventJSON,
-1, // No transaction ID yet
)
return
}
// updateTxnIDForEvents sets the transactionID for a collection of events. Done
// before sending them to an AppService. Referenced before sending to make sure
// we aren't constructing multiple transactions with the same events.
func (s *eventsStatements) updateTxnIDForEvents(
ctx context.Context,
appserviceID string,
maxID, txnID int,
) (err error) {
_, err = s.updateTxnIDForEventsStmt.ExecContext(ctx, txnID, appserviceID, maxID)
return
}
// deleteEventsBeforeAndIncludingID removes events matching given IDs from the database.
func (s *eventsStatements) deleteEventsBeforeAndIncludingID(
ctx context.Context,
appserviceID string,
eventTableID int,
) (err error) {
_, err = s.deleteEventsBeforeAndIncludingIDStmt.ExecContext(ctx, appserviceID, eventTableID)
return
}

115
appservice/storage/postgres/storage.go
View file

@ -1,115 +0,0 @@
// Copyright 2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgres
import (
"context"
"database/sql"
// Import postgres database driver
_ "github.com/lib/pq"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
)
// Database stores events intended to be later sent to application services
type Database struct {
events eventsStatements
txnID txnStatements
db *sql.DB
writer sqlutil.Writer
}
// NewDatabase opens a new database
func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) {
var result Database
var err error
if result.db, err = sqlutil.Open(dbProperties); err != nil {
return nil, err
}
result.writer = sqlutil.NewDummyWriter()
if err = result.prepare(); err != nil {
return nil, err
}
return &result, nil
}
func (d *Database) prepare() error {
if err := d.events.prepare(d.db); err != nil {
return err
}
return d.txnID.prepare(d.db)
}
// StoreEvent takes in a gomatrixserverlib.HeaderedEvent and stores it in the database
// for a transaction worker to pull and later send to an application service.
func (d *Database) StoreEvent(
ctx context.Context,
appServiceID string,
event *gomatrixserverlib.HeaderedEvent,
) error {
return d.events.insertEvent(ctx, appServiceID, event)
}
// GetEventsWithAppServiceID returns a slice of events and their IDs intended to
// be sent to an application service given its ID.
func (d *Database) GetEventsWithAppServiceID(
ctx context.Context,
appServiceID string,
limit int,
) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error) {
return d.events.selectEventsByApplicationServiceID(ctx, appServiceID, limit)
}
// CountEventsWithAppServiceID returns the number of events destined for an
// application service given its ID.
func (d *Database) CountEventsWithAppServiceID(
ctx context.Context,
appServiceID string,
) (int, error) {
return d.events.countEventsByApplicationServiceID(ctx, appServiceID)
}
// UpdateTxnIDForEvents takes in an application service ID and a
// and stores them in the DB, unless the pair already exists, in
// which case it updates them.
func (d *Database) UpdateTxnIDForEvents(
ctx context.Context,
appserviceID string,
maxID, txnID int,
) error {
return d.events.updateTxnIDForEvents(ctx, appserviceID, maxID, txnID)
}
// RemoveEventsBeforeAndIncludingID removes all events from the database that
// are less than or equal to a given maximum ID. IDs here are implemented as a
// serial, thus this should always delete events in chronological order.
func (d *Database) RemoveEventsBeforeAndIncludingID(
ctx context.Context,
appserviceID string,
eventTableID int,
) error {
return d.events.deleteEventsBeforeAndIncludingID(ctx, appserviceID, eventTableID)
}
// GetLatestTxnID returns the latest available transaction id
func (d *Database) GetLatestTxnID(
ctx context.Context,
) (int, error) {
return d.txnID.selectTxnID(ctx)
}

53
appservice/storage/postgres/txn_id_counter_table.go
View file

@ -1,53 +0,0 @@
// Copyright 2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package postgres
import (
"context"
"database/sql"
)
const txnIDSchema = `
-- Keeps a count of the current transaction ID
CREATE SEQUENCE IF NOT EXISTS txn_id_counter START 1;
`
const selectTxnIDSQL = "SELECT nextval('txn_id_counter')"
type txnStatements struct {
selectTxnIDStmt *sql.Stmt
}
func (s *txnStatements) prepare(db *sql.DB) (err error) {
_, err = db.Exec(txnIDSchema)
if err != nil {
return
}
if s.selectTxnIDStmt, err = db.Prepare(selectTxnIDSQL); err != nil {
return
}
return
}
// selectTxnID selects the latest ascending transaction ID
func (s *txnStatements) selectTxnID(
ctx context.Context,
) (txnID int, err error) {
err = s.selectTxnIDStmt.QueryRowContext(ctx).Scan(&txnID)
return
}

267
appservice/storage/sqlite3/appservice_events_table.go
View file

@ -1,267 +0,0 @@
// Copyright 2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sqlite3
import (
"context"
"database/sql"
"encoding/json"
"time"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/gomatrixserverlib"
log "github.com/sirupsen/logrus"
)
const appserviceEventsSchema = `
-- Stores events to be sent to application services
CREATE TABLE IF NOT EXISTS appservice_events (
-- An auto-incrementing id unique to each event in the table
id INTEGER PRIMARY KEY AUTOINCREMENT,
-- The ID of the application service the event will be sent to
as_id TEXT NOT NULL,
-- JSON representation of the event
headered_event_json TEXT NOT NULL,
-- The ID of the transaction that this event is a part of
txn_id INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS appservice_events_as_id ON appservice_events(as_id);
`
const selectEventsByApplicationServiceIDSQL = "" +
"SELECT id, headered_event_json, txn_id " +
"FROM appservice_events WHERE as_id = $1 ORDER BY txn_id DESC, id ASC"
const countEventsByApplicationServiceIDSQL = "" +
"SELECT COUNT(id) FROM appservice_events WHERE as_id = $1"
const insertEventSQL = "" +
"INSERT INTO appservice_events(as_id, headered_event_json, txn_id) " +
"VALUES ($1, $2, $3)"
const updateTxnIDForEventsSQL = "" +
"UPDATE appservice_events SET txn_id = $1 WHERE as_id = $2 AND id <= $3"
const deleteEventsBeforeAndIncludingIDSQL = "" +
"DELETE FROM appservice_events WHERE as_id = $1 AND id <= $2"
const (
// A transaction ID number that no transaction should ever have. Used for
// checking again the default value.
invalidTxnID = -2
)
type eventsStatements struct {
db *sql.DB
writer sqlutil.Writer
selectEventsByApplicationServiceIDStmt *sql.Stmt
countEventsByApplicationServiceIDStmt *sql.Stmt
insertEventStmt *sql.Stmt
updateTxnIDForEventsStmt *sql.Stmt
deleteEventsBeforeAndIncludingIDStmt *sql.Stmt
}
func (s *eventsStatements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) {
s.db = db
s.writer = writer
_, err = db.Exec(appserviceEventsSchema)
if err != nil {
return
}
if s.selectEventsByApplicationServiceIDStmt, err = db.Prepare(selectEventsByApplicationServiceIDSQL); err != nil {
return
}
if s.countEventsByApplicationServiceIDStmt, err = db.Prepare(countEventsByApplicationServiceIDSQL); err != nil {
return
}
if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil {
return
}
if s.updateTxnIDForEventsStmt, err = db.Prepare(updateTxnIDForEventsSQL); err != nil {
return
}
if s.deleteEventsBeforeAndIncludingIDStmt, err = db.Prepare(deleteEventsBeforeAndIncludingIDSQL); err != nil {
return
}
return
}
// selectEventsByApplicationServiceID takes in an application service ID and
// returns a slice of events that need to be sent to that application service,
// as well as an int later used to remove these same events from the database
// once successfully sent to an application service.
func (s *eventsStatements) selectEventsByApplicationServiceID(
ctx context.Context,
applicationServiceID string,
limit int,
) (
txnID, maxID int,
events []gomatrixserverlib.HeaderedEvent,
eventsRemaining bool,
err error,
) {
defer func() {
if err != nil {
log.WithFields(log.Fields{
"appservice": applicationServiceID,
}).WithError(err).Fatalf("appservice unable to select new events to send")
}
}()
// Retrieve events from the database. Unsuccessfully sent events first
eventRows, err := s.selectEventsByApplicationServiceIDStmt.QueryContext(ctx, applicationServiceID)
if err != nil {
return
}
defer checkNamedErr(eventRows.Close, &err)
events, maxID, txnID, eventsRemaining, err = retrieveEvents(eventRows, limit)
if err != nil {
return
}
return
}
// checkNamedErr calls fn and overwrite err if it was nil and fn returned non-nil
func checkNamedErr(fn func() error, err *error) {
if e := fn(); e != nil && *err == nil {
*err = e
}
}
func retrieveEvents(eventRows *sql.Rows, limit int) (events []gomatrixserverlib.HeaderedEvent, maxID, txnID int, eventsRemaining bool, err error) {
// Get current time for use in calculating event age
nowMilli := time.Now().UnixNano() / int64(time.Millisecond)
// Iterate through each row and store event contents
// If txn_id changes dramatically, we've switched from collecting old events to
// new ones. Send back those events first.
lastTxnID := invalidTxnID
for eventsProcessed := 0; eventRows.Next(); {
var event gomatrixserverlib.HeaderedEvent
var eventJSON []byte
var id int
err = eventRows.Scan(
&id,
&eventJSON,
&txnID,
)
if err != nil {
return nil, 0, 0, false, err
}
// Unmarshal eventJSON
if err = json.Unmarshal(eventJSON, &event); err != nil {
return nil, 0, 0, false, err
}
// If txnID has changed on this event from the previous event, then we've
// reached the end of a transaction's events. Return only those events.
if lastTxnID > invalidTxnID && lastTxnID != txnID {
return events, maxID, lastTxnID, true, nil
}
lastTxnID = txnID
// Limit events that aren't part of an old transaction
if txnID == -1 {
// Return if we've hit the limit
if eventsProcessed++; eventsProcessed > limit {
return events, maxID, lastTxnID, true, nil
}
}
if id > maxID {
maxID = id
}
// Portion of the event that is unsigned due to rapid change
// TODO: Consider removing age as not many app services use it
if err = event.SetUnsignedField("age", nowMilli-int64(event.OriginServerTS())); err != nil {
return nil, 0, 0, false, err
}
events = append(events, event)
}
return
}
// countEventsByApplicationServiceID inserts an event mapped to its corresponding application service
// IDs into the db.
func (s *eventsStatements) countEventsByApplicationServiceID(
ctx context.Context,
appServiceID string,
) (int, error) {
var count int
err := s.countEventsByApplicationServiceIDStmt.QueryRowContext(ctx, appServiceID).Scan(&count)
if err != nil && err != sql.ErrNoRows {
return 0, err
}
return count, nil
}
// insertEvent inserts an event mapped to its corresponding application service
// IDs into the db.
func (s *eventsStatements) insertEvent(
ctx context.Context,
appServiceID string,
event *gomatrixserverlib.HeaderedEvent,
) (err error) {
// Convert event to JSON before inserting
eventJSON, err := json.Marshal(event)
if err != nil {
return err
}
return s.writer.Do(s.db, nil, func(txn *sql.Tx) error {
_, err := s.insertEventStmt.ExecContext(
ctx,
appServiceID,
eventJSON,
-1, // No transaction ID yet
)
return err
})
}
// updateTxnIDForEvents sets the transactionID for a collection of events. Done
// before sending them to an AppService. Referenced before sending to make sure
// we aren't constructing multiple transactions with the same events.
func (s *eventsStatements) updateTxnIDForEvents(
ctx context.Context,
appserviceID string,
maxID, txnID int,
) (err error) {
return s.writer.Do(s.db, nil, func(txn *sql.Tx) error {
_, err := s.updateTxnIDForEventsStmt.ExecContext(ctx, txnID, appserviceID, maxID)
return err
})
}
// deleteEventsBeforeAndIncludingID removes events matching given IDs from the database.
func (s *eventsStatements) deleteEventsBeforeAndIncludingID(
ctx context.Context,
appserviceID string,
eventTableID int,
) (err error) {
return s.writer.Do(s.db, nil, func(txn *sql.Tx) error {
_, err := s.deleteEventsBeforeAndIncludingIDStmt.ExecContext(ctx, appserviceID, eventTableID)
return err
})
}

114
appservice/storage/sqlite3/storage.go
View file

@ -1,114 +0,0 @@
// Copyright 2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sqlite3
import (
"context"
"database/sql"
// Import SQLite database driver
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
)
// Database stores events intended to be later sent to application services
type Database struct {
events eventsStatements
txnID txnStatements
db *sql.DB
writer sqlutil.Writer
}
// NewDatabase opens a new database
func NewDatabase(dbProperties *config.DatabaseOptions) (*Database, error) {
var result Database
var err error
if result.db, err = sqlutil.Open(dbProperties); err != nil {
return nil, err
}
result.writer = sqlutil.NewExclusiveWriter()
if err = result.prepare(); err != nil {
return nil, err
}
return &result, nil
}
func (d *Database) prepare() error {
if err := d.events.prepare(d.db, d.writer); err != nil {
return err
}
return d.txnID.prepare(d.db, d.writer)
}
// StoreEvent takes in a gomatrixserverlib.HeaderedEvent and stores it in the database
// for a transaction worker to pull and later send to an application service.
func (d *Database) StoreEvent(
ctx context.Context,
appServiceID string,
event *gomatrixserverlib.HeaderedEvent,
) error {
return d.events.insertEvent(ctx, appServiceID, event)
}
// GetEventsWithAppServiceID returns a slice of events and their IDs intended to
// be sent to an application service given its ID.
func (d *Database) GetEventsWithAppServiceID(
ctx context.Context,
appServiceID string,
limit int,
) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error) {
return d.events.selectEventsByApplicationServiceID(ctx, appServiceID, limit)
}
// CountEventsWithAppServiceID returns the number of events destined for an
// application service given its ID.
func (d *Database) CountEventsWithAppServiceID(
ctx context.Context,
appServiceID string,
) (int, error) {
return d.events.countEventsByApplicationServiceID(ctx, appServiceID)
}
// UpdateTxnIDForEvents takes in an application service ID and a
// and stores them in the DB, unless the pair already exists, in
// which case it updates them.
func (d *Database) UpdateTxnIDForEvents(
ctx context.Context,
appserviceID string,
maxID, txnID int,
) error {
return d.events.updateTxnIDForEvents(ctx, appserviceID, maxID, txnID)
}
// RemoveEventsBeforeAndIncludingID removes all events from the database that
// are less than or equal to a given maximum ID. IDs here are implemented as a
// serial, thus this should always delete events in chronological order.
func (d *Database) RemoveEventsBeforeAndIncludingID(
ctx context.Context,
appserviceID string,
eventTableID int,
) error {
return d.events.deleteEventsBeforeAndIncludingID(ctx, appserviceID, eventTableID)
}
// GetLatestTxnID returns the latest available transaction id
func (d *Database) GetLatestTxnID(
ctx context.Context,
) (int, error) {
return d.txnID.selectTxnID(ctx)
}

82
appservice/storage/sqlite3/txn_id_counter_table.go
View file

@ -1,82 +0,0 @@
// Copyright 2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sqlite3
import (
"context"
"database/sql"
"github.com/matrix-org/dendrite/internal/sqlutil"
)
const txnIDSchema = `
-- Keeps a count of the current transaction ID
CREATE TABLE IF NOT EXISTS appservice_counters (
name TEXT PRIMARY KEY NOT NULL,
last_id INTEGER DEFAULT 1
);
INSERT OR IGNORE INTO appservice_counters (name, last_id) VALUES('txn_id', 1);
`
const selectTxnIDSQL = `
SELECT last_id FROM appservice_counters WHERE name='txn_id'
`
const updateTxnIDSQL = `
UPDATE appservice_counters SET last_id=last_id+1 WHERE name='txn_id'
`
type txnStatements struct {
db *sql.DB
writer sqlutil.Writer
selectTxnIDStmt *sql.Stmt
updateTxnIDStmt *sql.Stmt
}
func (s *txnStatements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) {
s.db = db
s.writer = writer
_, err = db.Exec(txnIDSchema)
if err != nil {
return
}
if s.selectTxnIDStmt, err = db.Prepare(selectTxnIDSQL); err != nil {
return
}
if s.updateTxnIDStmt, err = db.Prepare(updateTxnIDSQL); err != nil {
return
}
return
}
// selectTxnID selects the latest ascending transaction ID
func (s *txnStatements) selectTxnID(
ctx context.Context,
) (txnID int, err error) {
err = s.writer.Do(s.db, nil, func(txn *sql.Tx) error {
err := s.selectTxnIDStmt.QueryRowContext(ctx).Scan(&txnID)
if err != nil {
return err
}
_, err = s.updateTxnIDStmt.ExecContext(ctx)
return err
})
return
}

39
appservice/storage/storage.go
View file

@ -1,39 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !wasm
// +build !wasm
package storage
import (
"fmt"
"github.com/matrix-org/dendrite/appservice/storage/postgres"
"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
"github.com/matrix-org/dendrite/setup/config"
)
// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)
// and sets DB connection parameters
func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) {
switch {
case dbProperties.ConnectionString.IsSQLite():
return sqlite3.NewDatabase(dbProperties)
case dbProperties.ConnectionString.IsPostgres():
return postgres.NewDatabase(dbProperties)
default:
return nil, fmt.Errorf("unexpected database type")
}
}

64
appservice/types/types.go
View file

@ -1,64 +0,0 @@
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
import (
"sync"
"github.com/matrix-org/dendrite/setup/config"
)
const (
// AppServiceDeviceID is the AS dummy device ID
AppServiceDeviceID = "AS_Device"
)
// ApplicationServiceWorkerState is a type that couples an application service,
// a lockable condition as well as some other state variables, allowing the
// roomserver to notify appservice workers when there are events ready to send
// externally to application services.
type ApplicationServiceWorkerState struct {
AppService config.ApplicationService
Cond *sync.Cond
// Events ready to be sent
EventsReady bool
// Backoff exponent (2^x secs). Max 6, aka 64s.
Backoff int
}
// NotifyNewEvents wakes up all waiting goroutines, notifying that events remain
// in the event queue for this application service worker.
func (a *ApplicationServiceWorkerState) NotifyNewEvents() {
a.Cond.L.Lock()
a.EventsReady = true
a.Cond.Broadcast()
a.Cond.L.Unlock()
}
// FinishEventProcessing marks all events of this worker as being sent to the
// application service.
func (a *ApplicationServiceWorkerState) FinishEventProcessing() {
a.Cond.L.Lock()
a.EventsReady = false
a.Cond.L.Unlock()
}
// WaitForNewEvents causes the calling goroutine to wait on the worker state's
// condition for a broadcast or similar wakeup, if there are no events ready.
func (a *ApplicationServiceWorkerState) WaitForNewEvents() {
a.Cond.L.Lock()
if !a.EventsReady {
a.Cond.Wait()
}
a.Cond.L.Unlock()
}

236
appservice/workers/transaction_scheduler.go
View file

@ -1,236 +0,0 @@
// Copyright 2018 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package workers
import (
"bytes"
"context"
"encoding/json"
"fmt"
"math"
"net/http"
"net/url"
"time"
"github.com/matrix-org/dendrite/appservice/storage"
"github.com/matrix-org/dendrite/appservice/types"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
log "github.com/sirupsen/logrus"
)
var (
// Maximum size of events sent in each transaction.
transactionBatchSize = 50
)
// SetupTransactionWorkers spawns a separate goroutine for each application
// service. Each of these "workers" handle taking all events intended for their
// app service, batch them up into a single transaction (up to a max transaction
// size), then send that off to the AS's /transactions/{txnID} endpoint. It also
// handles exponentially backing off in case the AS isn't currently available.
func SetupTransactionWorkers(
client *http.Client,
appserviceDB storage.Database,
workerStates []types.ApplicationServiceWorkerState,
) error {
// Create a worker that handles transmitting events to a single homeserver
for _, workerState := range workerStates {
// Don't create a worker if this AS doesn't want to receive events
if workerState.AppService.URL != "" {
go worker(client, appserviceDB, workerState)
}
}
return nil
}
// worker is a goroutine that sends any queued events to the application service
// it is given.
func worker(client *http.Client, db storage.Database, ws types.ApplicationServiceWorkerState) {
log.WithFields(log.Fields{
"appservice": ws.AppService.ID,
}).Info("Starting application service")
ctx := context.Background()
// Initial check for any leftover events to send from last time
eventCount, err := db.CountEventsWithAppServiceID(ctx, ws.AppService.ID)
if err != nil {
log.WithFields(log.Fields{
"appservice": ws.AppService.ID,
}).WithError(err).Fatal("appservice worker unable to read queued events from DB")
return
}
if eventCount > 0 {
ws.NotifyNewEvents()
}
// Loop forever and keep waiting for more events to send
for {
// Wait for more events if we've sent all the events in the database
ws.WaitForNewEvents()
// Batch events up into a transaction
transactionJSON, txnID, maxEventID, eventsRemaining, err := createTransaction(ctx, db, ws.AppService.ID)
if err != nil {
log.WithFields(log.Fields{
"appservice": ws.AppService.ID,
}).WithError(err).Fatal("appservice worker unable to create transaction")
return
}
// Send the events off to the application service
// Backoff if the application service does not respond
err = send(client, ws.AppService, txnID, transactionJSON)
if err != nil {
log.WithFields(log.Fields{
"appservice": ws.AppService.ID,
}).WithError(err).Error("unable to send event")
// Backoff
backoff(&ws, err)
continue
}
// We sent successfully, hooray!
ws.Backoff = 0
// Transactions have a maximum event size, so there may still be some events
// left over to send. Keep sending until none are left
if !eventsRemaining {
ws.FinishEventProcessing()
}
// Remove sent events from the DB
err = db.RemoveEventsBeforeAndIncludingID(ctx, ws.AppService.ID, maxEventID)
if err != nil {
log.WithFields(log.Fields{
"appservice": ws.AppService.ID,
}).WithError(err).Fatal("unable to remove appservice events from the database")
return
}
}
}
// backoff pauses the calling goroutine for a 2^some backoff exponent seconds
func backoff(ws *types.ApplicationServiceWorkerState, err error) {
// Calculate how long to backoff for
backoffDuration := time.Duration(math.Pow(2, float64(ws.Backoff)))
backoffSeconds := time.Second * backoffDuration
log.WithFields(log.Fields{
"appservice": ws.AppService.ID,
}).WithError(err).Warnf("unable to send transactions successfully, backing off for %ds",
backoffDuration)
ws.Backoff++
if ws.Backoff > 6 {
ws.Backoff = 6
}
// Backoff
time.Sleep(backoffSeconds)
}
// createTransaction takes in a slice of AS events, stores them in an AS
// transaction, and JSON-encodes the results.
func createTransaction(
ctx context.Context,
db storage.Database,
appserviceID string,
) (
transactionJSON []byte,
txnID, maxID int,
eventsRemaining bool,
err error,
) {
// Retrieve the latest events from the DB (will return old events if they weren't successfully sent)
txnID, maxID, events, eventsRemaining, err := db.GetEventsWithAppServiceID(ctx, appserviceID, transactionBatchSize)
if err != nil {
log.WithFields(log.Fields{
"appservice": appserviceID,
}).WithError(err).Fatalf("appservice worker unable to read queued events from DB")
return
}
// Check if these events do not already have a transaction ID
if txnID == -1 {
// If not, grab next available ID from the DB
txnID, err = db.GetLatestTxnID(ctx)
if err != nil {
return nil, 0, 0, false, err
}
// Mark new events with current transactionID
if err = db.UpdateTxnIDForEvents(ctx, appserviceID, maxID, txnID); err != nil {
return nil, 0, 0, false, err
}
}
var ev []*gomatrixserverlib.HeaderedEvent
for i := range events {
ev = append(ev, &events[i])
}
// Create a transaction and store the events inside
transaction := gomatrixserverlib.ApplicationServiceTransaction{
Events: gomatrixserverlib.HeaderedToClientEvents(ev, gomatrixserverlib.FormatAll),
}
transactionJSON, err = json.Marshal(transaction)
if err != nil {
return
}
return
}
// send sends events to an application service. Returns an error if an OK was not
// received back from the application service or the request timed out.
func send(
client *http.Client,
appservice config.ApplicationService,
txnID int,
transaction []byte,
) (err error) {
// PUT a transaction to our AS
// https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid
address := fmt.Sprintf("%s/transactions/%d?access_token=%s", appservice.URL, txnID, url.QueryEscape(appservice.HSToken))
req, err := http.NewRequest("PUT", address, bytes.NewBuffer(transaction))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
return err
}
defer checkNamedErr(resp.Body.Close, &err)
// Check the AS received the events correctly
if resp.StatusCode != http.StatusOK {
// TODO: Handle non-200 error codes from application services
return fmt.Errorf("non-OK status code %d returned from AS", resp.StatusCode)
}
return nil
}
// checkNamedErr calls fn and overwrite err if it was nil and fn returned non-nil
func checkNamedErr(fn func() error, err *error) {
if e := fn(); e != nil && *err == nil {
*err = e
}
}

24
are-we-synapse-yet.list
View file

@ -212,11 +212,12 @@ plv Users cannot set kick powerlevel higher than their own (2 subtests)
plv Users cannot set redact powerlevel higher than their own (2 subtests)
v1s Check that event streams started after a client joined a room work (SYT-1)
v1s Event stream catches up fully after many messages
xxx POST /rooms/:room_id/redact/:event_id as power user redacts message
xxx POST /rooms/:room_id/redact/:event_id as original message sender redacts message
xxx POST /rooms/:room_id/redact/:event_id as random user does not redact message
xxx POST /redact disallows redaction of event in different room
xxx PUT /rooms/:room_id/redact/:event_id/:txn_id as power user redacts message
xxx PUT /rooms/:room_id/redact/:event_id/:txn_id as original message sender redacts message
xxx PUT /rooms/:room_id/redact/:event_id/:txn_id as random user does not redact message
xxx PUT /redact disallows redaction of event in different room
xxx Redaction of a redaction redacts the redaction reason
xxx PUT /rooms/:room_id/redact/:event_id/:txn_id is idempotent
v1s A departed room is still included in /initialSync (SPEC-216)
v1s Can get rooms/{roomId}/initialSync for a departed room (SPEC-216)
rst Can get rooms/{roomId}/state for a departed room (SPEC-216)
@ -921,3 +922,18 @@ msc We can't peek into rooms with invited history_visibility
msc We can't peek into rooms with joined history_visibility
msc Local users can peek by room alias
msc Peeked rooms only turn up in the sync for the device who peeked them
ban 'ban' event respects room powerlevel (2 subtests)
inv Test that we can be reinvited to a room we created (11 subtests)
fiv Rejecting invite over federation doesn't break incremental /sync
pre Presence can be set from sync
fst /state returns M_NOT_FOUND for an outlier
fst /state_ids returns M_NOT_FOUND for an outlier
fst /state returns M_NOT_FOUND for a rejected message event
fst /state_ids returns M_NOT_FOUND for a rejected message event
fst /state returns M_NOT_FOUND for a rejected state event
fst /state_ids returns M_NOT_FOUND for a rejected state event
fst Room state after a rejected message event is the same as before
fst Room state after a rejected state event is the same as before
fpb Federation publicRoom Name/topic keys are correct
fed New federated private chats get full presence information (SYN-115) (10 subtests)
dvk Rejects invalid device keys

2
build.sh
View file

@ -21,4 +21,4 @@ mkdir -p bin
CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/...
CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs-pinecone
# CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs-pinecone

25
build/docker/Dockerfile.demo-pinecone Normal file
View file

@ -0,0 +1,25 @@
FROM docker.io/golang:1.19-alpine AS base
RUN apk --update --no-cache add bash build-base
WORKDIR /build
COPY . /build
RUN mkdir -p bin
RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-pinecone
RUN go build -trimpath -o bin/ ./cmd/create-account
RUN go build -trimpath -o bin/ ./cmd/generate-keys
FROM alpine:latest
LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)"
LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
LABEL org.opencontainers.image.licenses="Apache-2.0"
COPY --from=base /build/bin/* /usr/bin/
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"]

5
build/docker/Dockerfile.monolith
View file

@ -1,4 +1,4 @@
FROM docker.io/golang:1.17-alpine AS base
FROM docker.io/golang:1.19-alpine AS base
RUN apk --update --no-cache add bash build-base
@ -8,7 +8,6 @@ COPY . /build
RUN mkdir -p bin
RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server
RUN go build -trimpath -o bin/ ./cmd/goose
RUN go build -trimpath -o bin/ ./cmd/create-account
RUN go build -trimpath -o bin/ ./cmd/generate-keys
@ -23,4 +22,4 @@ COPY --from=base /build/bin/* /usr/bin/
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]

5
build/docker/Dockerfile.polylith
View file

@ -1,4 +1,4 @@
FROM docker.io/golang:1.17-alpine AS base
FROM docker.io/golang:1.19-alpine AS base
RUN apk --update --no-cache add bash build-base
@ -8,7 +8,6 @@ COPY . /build
RUN mkdir -p bin
RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi
RUN go build -trimpath -o bin/ ./cmd/goose
RUN go build -trimpath -o bin/ ./cmd/create-account
RUN go build -trimpath -o bin/ ./cmd/generate-keys
@ -23,4 +22,4 @@ COPY --from=base /build/bin/* /usr/bin/
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]
ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]

7
build/docker/README.md
View file

@ -27,8 +27,7 @@ There are three sample `docker-compose` files:
The `docker-compose` files refer to the `/etc/dendrite` volume as where the
runtime config should come from. The mounted folder must contain:
- `dendrite.yaml` configuration file (from the [Docker config folder](https://github.com/matrix-org/dendrite/tree/master/build/docker/config)
sample in the `build/docker/config` folder of this repository.)
- `dendrite.yaml` configuration file (based on one of the sample config files)
- `matrix_key.pem` server key, as generated using `cmd/generate-keys`
- `server.crt` certificate file
- `server.key` private key file for the above certificate
@ -49,7 +48,7 @@ The key files will now exist in your current working directory, and can be mount
## Starting Dendrite as a monolith deployment
Create your config based on the [`dendrite.yaml`](https://github.com/matrix-org/dendrite/tree/master/build/docker/config) configuration file in the `build/docker/config` folder of this repository.
Create your config based on the [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml) sample configuration file.
Then start the deployment:
@ -59,7 +58,7 @@ docker-compose -f docker-compose.monolith.yml up
## Starting Dendrite as a polylith deployment
Create your config based on the [`dendrite-config.yaml`](https://github.com/matrix-org/dendrite/tree/master/build/docker/config) configuration file in the `build/docker/config` folder of this repository.
Create your config based on the [`dendrite-sample.polylith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) sample configuration file.
Then start the deployment:

348
build/docker/config/dendrite.yaml
View file

@ -1,348 +0,0 @@
# This is the Dendrite configuration file.
#
# The configuration is split up into sections - each Dendrite component has a
# configuration section, in addition to the "global" section which applies to
# all components.
#
# At a minimum, to get started, you will need to update the settings in the
# "global" section for your deployment, and you will need to check that the
# database "connection_string" line in each component section is correct.
#
# Each component with a "database" section can accept the following formats
# for "connection_string":
# SQLite: file:filename.db
# file:///path/to/filename.db
# PostgreSQL: postgresql://user:pass@hostname/database?params=...
#
# SQLite is embedded into Dendrite and therefore no further prerequisites are
# needed for the database when using SQLite mode. However, performance with
# PostgreSQL is significantly better and recommended for multi-user deployments.
# SQLite is typically around 20-30% slower than PostgreSQL when tested with a
# small number of users and likely will perform worse still with a higher volume
# of users.
#
# The "max_open_conns" and "max_idle_conns" settings configure the maximum
# number of open/idle database connections. The value 0 will use the database
# engine default, and a negative value will use unlimited connections. The
# "conn_max_lifetime" option controls the maximum length of time a database
# connection can be idle in seconds - a negative value is unlimited.
# The version of the configuration file.
version: 2
# Global Matrix configuration. This configuration applies to all components.
global:
# The domain name of this homeserver.
server_name: example.com
# The path to the signing private key file, used to sign requests and events.
private_key: matrix_key.pem
# The paths and expiry timestamps (as a UNIX timestamp in millisecond precision)
# to old signing private keys that were formerly in use on this domain. These
# keys will not be used for federation request or event signing, but will be
# provided to any other homeserver that asks when trying to verify old events.
# old_private_keys:
# - private_key: old_matrix_key.pem
# expired_at: 1601024554498
# How long a remote server can cache our server signing key before requesting it
# again. Increasing this number will reduce the number of requests made by other
# servers for our key but increases the period that a compromised key will be
# considered valid by other homeservers.
key_validity_period: 168h0m0s
# The server name to delegate server-server communications to, with optional port
# e.g. localhost:443
well_known_server_name: ""
# Lists of domains that the server will trust as identity servers to verify third
# party identifiers such as phone numbers and email addresses.
trusted_third_party_id_servers:
- matrix.org
- vector.im
# Disables federation. Dendrite will not be able to make any outbound HTTP requests
# to other servers and the federation API will not be exposed.
disable_federation: false
# Configures the handling of presence events.
presence:
# Whether inbound presence events are allowed, e.g. receiving presence events from other servers
enable_inbound: false
# Whether outbound presence events are allowed, e.g. sending presence events to other servers
enable_outbound: false
# Configuration for NATS JetStream
jetstream:
# A list of NATS Server addresses to connect to. If none are specified, an
# internal NATS server will be started automatically when running Dendrite
# in monolith mode. It is required to specify the address of at least one
# NATS Server node if running in polylith mode.
addresses:
- jetstream:4222
# Keep all NATS streams in memory, rather than persisting it to the storage
# path below. This option is present primarily for integration testing and
# should not be used on a real world Dendrite deployment.
in_memory: false
# Persistent directory to store JetStream streams in. This directory
# should be preserved across Dendrite restarts.
storage_path: ./
# The prefix to use for stream names for this homeserver - really only
# useful if running more than one Dendrite on the same NATS deployment.
topic_prefix: Dendrite
# Configuration for Prometheus metric collection.
metrics:
# Whether or not Prometheus metrics are enabled.
enabled: false
# HTTP basic authentication to protect access to monitoring.
basic_auth:
username: metrics
password: metrics
# DNS cache options. The DNS cache may reduce the load on DNS servers
# if there is no local caching resolver available for use.
dns_cache:
# Whether or not the DNS cache is enabled.
enabled: false
# Maximum number of entries to hold in the DNS cache, and
# for how long those items should be considered valid in seconds.
cache_size: 256
cache_lifetime: 300
# Configuration for the Appservice API.
app_service_api:
internal_api:
listen: http://0.0.0.0:7777
connect: http://appservice_api:7777
database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_appservice?sslmode=disable
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
# Appservice configuration files to load into this homeserver.
config_files: []
# Configuration for the Client API.
client_api:
internal_api:
listen: http://0.0.0.0:7771
connect: http://client_api:7771
external_api:
listen: http://0.0.0.0:8071
# Prevents new users from being able to register on this homeserver, except when
# using the registration shared secret below.
registration_disabled: false
# If set, allows registration by anyone who knows the shared secret, regardless of
# whether registration is otherwise disabled.
registration_shared_secret: ""
# Whether to require reCAPTCHA for registration.
enable_registration_captcha: false
# Settings for ReCAPTCHA.
recaptcha_public_key: ""
recaptcha_private_key: ""
recaptcha_bypass_secret: ""
recaptcha_siteverify_api: ""
# TURN server information that this homeserver should send to clients.
turn:
turn_user_lifetime: ""
turn_uris: []
turn_shared_secret: ""
turn_username: ""
turn_password: ""
# Settings for rate-limited endpoints. Rate limiting will kick in after the
# threshold number of "slots" have been taken by requests from a specific
# host. Each "slot" will be released after the cooloff time in milliseconds.
rate_limiting:
enabled: true
threshold: 5
cooloff_ms: 500
# Configuration for the Federation API.
federation_api:
internal_api:
listen: http://0.0.0.0:7772
connect: http://federation_api:7772
external_api:
listen: http://0.0.0.0:8072
database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_federationapi?sslmode=disable
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
# How many times we will try to resend a failed transaction to a specific server. The
# backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc.
send_max_retries: 16
# Disable the validation of TLS certificates of remote federated homeservers. Do not
# enable this option in production as it presents a security risk!
disable_tls_validation: false
# Use the following proxy server for outbound federation traffic.
proxy_outbound:
enabled: false
protocol: http
host: localhost
port: 8080
# Perspective keyservers to use as a backup when direct key fetches fail. This may
# be required to satisfy key requests for servers that are no longer online when
# joining some rooms.
key_perspectives:
- server_name: matrix.org
keys:
- key_id: ed25519:auto
public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw
- key_id: ed25519:a_RXGa
public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ
# This option will control whether Dendrite will prefer to look up keys directly
# or whether it should try perspective servers first, using direct fetches as a
# last resort.
prefer_direct_fetch: false
# Configuration for the Key Server (for end-to-end encryption).
key_server:
internal_api:
listen: http://0.0.0.0:7779
connect: http://key_server:7779
database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_keyserver?sslmode=disable
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
# Configuration for the Media API.
media_api:
internal_api:
listen: http://0.0.0.0:7774
connect: http://media_api:7774
external_api:
listen: http://0.0.0.0:8074
database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_mediaapi?sslmode=disable
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
# Storage path for uploaded media. May be relative or absolute.
base_path: /var/dendrite/media
# The maximum allowed file size (in bytes) for media uploads to this homeserver
# (0 = unlimited).
max_file_size_bytes: 10485760
# Whether to dynamically generate thumbnails if needed.
dynamic_thumbnails: false
# The maximum number of simultaneous thumbnail generators to run.
max_thumbnail_generators: 10
# A list of thumbnail sizes to be generated for media content.
thumbnail_sizes:
- width: 32
height: 32
method: crop
- width: 96
height: 96
method: crop
- width: 640
height: 480
method: scale
# Configuration for experimental MSC's
mscs:
# A list of enabled MSC's
# Currently valid values are:
# - msc2836 (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836)
# - msc2946 (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946)
mscs: []
database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_mscs?sslmode=disable
max_open_conns: 5
max_idle_conns: 2
conn_max_lifetime: -1
# Configuration for the Room Server.
room_server:
internal_api:
listen: http://0.0.0.0:7770
connect: http://room_server:7770
database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_roomserver?sslmode=disable
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
# Configuration for the Sync API.
sync_api:
internal_api:
listen: http://0.0.0.0:7773
connect: http://sync_api:7773
external_api:
listen: http://0.0.0.0:8073
database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_syncapi?sslmode=disable
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
# Configuration for the User API.
user_api:
internal_api:
listen: http://0.0.0.0:7781
connect: http://user_api:7781
account_database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_userapi_accounts?sslmode=disable
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
# Configuration for the Push Server API.
push_server:
internal_api:
listen: http://localhost:7782
connect: http://localhost:7782
database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_pushserver?sslmode=disable
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
# Configuration for Opentracing.
# See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on
# how this works and how to set it up.
tracing:
enabled: false
jaeger:
serviceName: ""
disabled: false
rpc_metrics: false
tags: []
sampler: null
reporter: null
headers: null
baggage_restrictions: null
throttler: null
# Logging configuration, in addition to the standard logging that is sent to
# stdout by Dendrite.
logging:
- type: file
level: info
params:
path: /var/log/dendrite

2
build/docker/images-build.sh
View file

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
cd $(git rev-parse --show-toplevel)

2
build/docker/images-pull.sh
View file

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
TAG=${1:-latest}

2
build/docker/images-push.sh
View file

@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash
TAG=${1:-latest}

91
build/gobind-pinecone/monolith.go
View file

@ -22,10 +22,10 @@ import (
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"time"
@ -45,6 +45,7 @@ import (
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/process"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/userapi"
userapiAPI "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
@ -204,28 +205,46 @@ func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, e
// nolint:gocyclo
func (m *DendriteMonolith) Start() {
var err error
var sk ed25519.PrivateKey
var pk ed25519.PublicKey
keyfile := fmt.Sprintf("%s/p2p.key", m.StorageDirectory)
if _, err = os.Stat(keyfile); os.IsNotExist(err) {
if pk, sk, err = ed25519.GenerateKey(nil); err != nil {
panic(err)
keyfile := filepath.Join(m.StorageDirectory, "p2p.pem")
if _, err := os.Stat(keyfile); os.IsNotExist(err) {
oldkeyfile := filepath.Join(m.StorageDirectory, "p2p.key")
if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
if err = test.NewMatrixKey(keyfile); err != nil {
panic("failed to generate a new PEM key: " + err.Error())
}
if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
panic("failed to load PEM key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
} else {
if sk, err = os.ReadFile(oldkeyfile); err != nil {
panic("failed to read the old private key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
if err = test.SaveMatrixKey(keyfile, sk); err != nil {
panic("failed to convert the private key to PEM format: " + err.Error())
}
}
if err = ioutil.WriteFile(keyfile, sk, 0644); err != nil {
panic(err)
}
} else if err == nil {
if sk, err = ioutil.ReadFile(keyfile); err != nil {
panic(err)
} else {
if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
panic("failed to load PEM key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
pk = sk.Public().(ed25519.PublicKey)
}
m.listener, err = net.Listen("tcp", ":65432")
pk = sk.Public().(ed25519.PublicKey)
var err error
m.listener, err = net.Listen("tcp", "localhost:65432")
if err != nil {
panic(err)
}
@ -236,14 +255,17 @@ func (m *DendriteMonolith) Start() {
m.logger.SetOutput(BindLogger{})
logrus.SetOutput(BindLogger{})
m.PineconeRouter = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false)
m.PineconeRouter = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk)
m.PineconeQUIC = pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), m.PineconeRouter, []string{"matrix"})
m.PineconeMulticast = pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), m.PineconeRouter)
m.PineconeManager = pineconeConnections.NewConnectionManager(m.PineconeRouter)
m.PineconeManager = pineconeConnections.NewConnectionManager(m.PineconeRouter, nil)
prefix := hex.EncodeToString(pk)
cfg := &config.Dendrite{}
cfg.Defaults(true)
cfg.Defaults(config.DefaultOpts{
Generate: true,
Monolithic: true,
})
cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
cfg.Global.PrivateKey = sk
cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
@ -255,18 +277,18 @@ func (m *DendriteMonolith) Start() {
cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-roomserver.db", m.StorageDirectory, prefix))
cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-keyserver.db", m.StorageDirectory, prefix))
cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-federationsender.db", m.StorageDirectory, prefix))
cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-appservice.db", m.StorageDirectory, prefix))
cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory))
cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory))
cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
if err := cfg.Derive(); err != nil {
cfg.ClientAPI.RegistrationDisabled = false
cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
if err = cfg.Derive(); err != nil {
panic(err)
}
base := base.NewBaseDendrite(cfg, "Monolith")
defer base.Close() // nolint: errcheck
accountDB := base.CreateAccountsDB()
federation := conn.CreateFederationClient(base, m.PineconeQUIC)
serverKeyAPI := &signing.YggdrasilKeys{}
@ -279,7 +301,7 @@ func (m *DendriteMonolith) Start() {
)
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
m.userAPI = userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
m.userAPI = userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
keyAPI.SetUserAPI(m.userAPI)
asAPI := appservice.NewInternalAPI(base, m.userAPI, rsAPI)
@ -293,7 +315,6 @@ func (m *DendriteMonolith) Start() {
monolith := setup.Monolith{
Config: base.Cfg,
AccountDB: accountDB,
Client: conn.CreateClient(base, m.PineconeQUIC),
FedClient: federation,
KeyRing: keyRing,
@ -306,15 +327,7 @@ func (m *DendriteMonolith) Start() {
ExtPublicRoomsProvider: roomProvider,
ExtUserDirectoryProvider: userProvider,
}
monolith.AddAllPublicRoutes(
base.ProcessContext,
base.PublicClientAPIMux,
base.PublicFederationAPIMux,
base.PublicKeyAPIMux,
base.PublicWellKnownAPIMux,
base.PublicMediaAPIMux,
base.SynapseAdminMux,
)
monolith.AddAllPublicRoutes(base)
httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
@ -350,11 +363,23 @@ func (m *DendriteMonolith) Start() {
go func() {
m.logger.Info("Listening on ", cfg.Global.ServerName)
m.logger.Fatal(m.httpServer.Serve(m.PineconeQUIC.Protocol("matrix")))
switch m.httpServer.Serve(m.PineconeQUIC.Protocol("matrix")) {
case net.ErrClosed, http.ErrServerClosed:
m.logger.Info("Stopped listening on ", cfg.Global.ServerName)
default:
m.logger.Fatal(err)
}
}()
go func() {
logrus.Info("Listening on ", m.listener.Addr())
logrus.Fatal(http.Serve(m.listener, httpRouter))
switch http.Serve(m.listener, httpRouter) {
case net.ErrClosed, http.ErrServerClosed:
m.logger.Info("Stopped listening on ", cfg.Global.ServerName)
default:
m.logger.Fatal(err)
}
}()
}

76
build/gobind-yggdrasil/monolith.go
View file

@ -2,10 +2,14 @@ package gobind
import (
"context"
"crypto/ed25519"
"crypto/tls"
"encoding/hex"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"time"
"github.com/gorilla/mux"
@ -22,6 +26,7 @@ import (
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/process"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/userapi"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
@ -63,28 +68,70 @@ func (m *DendriteMonolith) DisconnectMulticastPeers() {
}
func (m *DendriteMonolith) Start() {
var pk ed25519.PublicKey
var sk ed25519.PrivateKey
m.logger = logrus.Logger{
Out: BindLogger{},
}
m.logger.SetOutput(BindLogger{})
logrus.SetOutput(BindLogger{})
keyfile := filepath.Join(m.StorageDirectory, "p2p.pem")
if _, err := os.Stat(keyfile); os.IsNotExist(err) {
oldkeyfile := filepath.Join(m.StorageDirectory, "p2p.key")
if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
if err = test.NewMatrixKey(keyfile); err != nil {
panic("failed to generate a new PEM key: " + err.Error())
}
if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
panic("failed to load PEM key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
} else {
if sk, err = os.ReadFile(oldkeyfile); err != nil {
panic("failed to read the old private key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
if err := test.SaveMatrixKey(keyfile, sk); err != nil {
panic("failed to convert the private key to PEM format: " + err.Error())
}
}
} else {
var err error
if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
panic("failed to load PEM key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
}
pk = sk.Public().(ed25519.PublicKey)
var err error
m.listener, err = net.Listen("tcp", "localhost:65432")
if err != nil {
panic(err)
}
ygg, err := yggconn.Setup("dendrite", m.StorageDirectory, "")
ygg, err := yggconn.Setup(sk, "dendrite", m.StorageDirectory, "", "")
if err != nil {
panic(err)
}
m.YggdrasilNode = ygg
cfg := &config.Dendrite{}
cfg.Defaults(true)
cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
cfg.Global.PrivateKey = ygg.PrivateKey()
cfg.Defaults(config.DefaultOpts{
Generate: true,
Monolithic: true,
})
cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
cfg.Global.PrivateKey = sk
cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", m.StorageDirectory))
cfg.Global.JetStream.InMemory = true
@ -94,9 +141,10 @@ func (m *DendriteMonolith) Start() {
cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-roomserver.db", m.StorageDirectory))
cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-keyserver.db", m.StorageDirectory))
cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-federationsender.db", m.StorageDirectory))
cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-appservice.db", m.StorageDirectory))
cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory))
cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory))
cfg.ClientAPI.RegistrationDisabled = false
cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
if err = cfg.Derive(); err != nil {
panic(err)
}
@ -105,7 +153,6 @@ func (m *DendriteMonolith) Start() {
m.processContext = base.ProcessContext
defer base.Close() // nolint: errcheck
accountDB := base.CreateAccountsDB()
federation := ygg.CreateFederationClient(base)
serverKeyAPI := &signing.YggdrasilKeys{}
@ -118,7 +165,7 @@ func (m *DendriteMonolith) Start() {
)
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
keyAPI.SetUserAPI(userAPI)
asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
@ -130,7 +177,6 @@ func (m *DendriteMonolith) Start() {
monolith := setup.Monolith{
Config: base.Cfg,
AccountDB: accountDB,
Client: ygg.CreateClient(base),
FedClient: federation,
KeyRing: keyRing,
@ -144,15 +190,7 @@ func (m *DendriteMonolith) Start() {
ygg, fsAPI, federation,
),
}
monolith.AddAllPublicRoutes(
base.ProcessContext,
base.PublicClientAPIMux,
base.PublicFederationAPIMux,
base.PublicKeyAPIMux,
base.PublicWellKnownAPIMux,
base.PublicMediaAPIMux,
base.SynapseAdminMux,
)
monolith.AddAllPublicRoutes(base)
httpRouter := mux.NewRouter()
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
@ -178,11 +216,11 @@ func (m *DendriteMonolith) Start() {
go func() {
m.logger.Info("Listening on ", ygg.DerivedServerName())
m.logger.Fatal(m.httpServer.Serve(ygg))
m.logger.Error(m.httpServer.Serve(ygg))
}()
go func() {
logrus.Info("Listening on ", m.listener.Addr())
logrus.Fatal(http.Serve(m.listener, httpRouter))
logrus.Error(http.Serve(m.listener, httpRouter))
}()
go func() {
logrus.Info("Sending wake-up message to known nodes")

26
build/scripts/Complement.Dockerfile
View file

@ -1,4 +1,6 @@
FROM golang:1.16-stretch as build
#syntax=docker/dockerfile:1.2
FROM golang:1.18-stretch as build
RUN apt-get update && apt-get install -y sqlite3
WORKDIR /build
@ -8,14 +10,12 @@ RUN mkdir /dendrite
# Utilise Docker caching when downloading dependencies, this stops us needlessly
# downloading dependencies every time.
COPY go.mod .
COPY go.sum .
RUN go mod download
COPY . .
RUN go build -o /dendrite ./cmd/dendrite-monolith-server
RUN go build -o /dendrite ./cmd/generate-keys
RUN go build -o /dendrite ./cmd/generate-config
RUN --mount=target=. \
--mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
go build -o /dendrite ./cmd/generate-config && \
go build -o /dendrite ./cmd/generate-keys && \
go build -o /dendrite ./cmd/dendrite-monolith-server
WORKDIR /dendrite
RUN ./generate-keys --private-key matrix_key.pem
@ -26,7 +26,7 @@ EXPOSE 8008 8448
# At runtime, generate TLS cert based on the CA now mounted at /ca
# At runtime, replace the SERVER_NAME with what we are told
CMD ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
CMD ./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}

47
build/scripts/ComplementLocal.Dockerfile
View file

@ -1,3 +1,5 @@
#syntax=docker/dockerfile:1.2
# A local development Complement dockerfile, to be used with host mounts
# /cache -> Contains the entire dendrite code at Dockerfile build time. Builds binaries but only keeps the generate-* ones. Pre-compilation saves time.
# /dendrite -> Host-mounted sources
@ -6,48 +8,45 @@
#
# Use these mounts to make use of this dockerfile:
# COMPLEMENT_HOST_MOUNTS='/your/local/dendrite:/dendrite:ro;/your/go/path:/go:ro'
FROM golang:1.16-stretch
FROM golang:1.18-stretch
RUN apt-get update && apt-get install -y sqlite3
WORKDIR /runtime
ENV SERVER_NAME=localhost
EXPOSE 8008 8448
WORKDIR /runtime
# This script compiles Dendrite for us.
RUN echo '\
#!/bin/bash -eux \n\
if test -f "/runtime/dendrite-monolith-server"; then \n\
#!/bin/bash -eux \n\
if test -f "/runtime/dendrite-monolith-server"; then \n\
echo "Skipping compilation; binaries exist" \n\
exit 0 \n\
fi \n\
cd /dendrite \n\
go build -v -o /runtime /dendrite/cmd/dendrite-monolith-server \n\
' > compile.sh && chmod +x compile.sh
fi \n\
cd /dendrite \n\
go build -v -o /runtime /dendrite/cmd/dendrite-monolith-server \n\
' > compile.sh && chmod +x compile.sh
# This script runs Dendrite for us. Must be run in the /runtime directory.
RUN echo '\
#!/bin/bash -eu \n\
./generate-keys --private-key matrix_key.pem \n\
./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\
cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\
./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
' > run.sh && chmod +x run.sh
#!/bin/bash -eu \n\
./generate-keys --private-key matrix_key.pem \n\
./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\
cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\
exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
' > run.sh && chmod +x run.sh
WORKDIR /cache
# Pre-download deps; we don't need to do this if the GOPATH is mounted.
COPY go.mod .
COPY go.sum .
RUN go mod download
# Build the monolith in /cache - we won't actually use this but will rely on build artifacts to speed
# up the real compilation. Build the generate-* binaries in the true /runtime locations.
# If the generate-* source is changed, this dockerfile needs re-running.
COPY . .
RUN go build ./cmd/dendrite-monolith-server && go build -o /runtime ./cmd/generate-keys && go build -o /runtime ./cmd/generate-config
RUN --mount=target=. \
--mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
go build -o /runtime ./cmd/generate-config && \
go build -o /runtime ./cmd/generate-keys
WORKDIR /runtime
CMD /runtime/compile.sh && /runtime/run.sh
CMD /runtime/compile.sh && exec /runtime/run.sh

51
build/scripts/ComplementPostgres.Dockerfile
View file

@ -1,4 +1,6 @@
FROM golang:1.16-stretch as build
#syntax=docker/dockerfile:1.2
FROM golang:1.18-stretch as build
RUN apt-get update && apt-get install -y postgresql
WORKDIR /build
@ -9,16 +11,16 @@ RUN sed -i "s%127.0.0.1/32 md5%127.0.0.1/32 trust%g" /etc/
# This entry script starts postgres, waits for it to be up then starts dendrite
RUN echo '\
#!/bin/bash -eu \n\
pg_lsclusters \n\
pg_ctlcluster 9.6 main start \n\
\n\
until pg_isready \n\
do \n\
echo "Waiting for postgres"; \n\
sleep 1; \n\
done \n\
' > run_postgres.sh && chmod +x run_postgres.sh
#!/bin/bash -eu \n\
pg_lsclusters \n\
pg_ctlcluster 9.6 main start \n\
\n\
until pg_isready \n\
do \n\
echo "Waiting for postgres"; \n\
sleep 1; \n\
done \n\
' > run_postgres.sh && chmod +x run_postgres.sh
# we will dump the binaries and config file to this location to ensure any local untracked files
# that come from the COPY . . file don't contaminate the build
@ -26,14 +28,12 @@ RUN mkdir /dendrite
# Utilise Docker caching when downloading dependencies, this stops us needlessly
# downloading dependencies every time.
COPY go.mod .
COPY go.sum .
RUN go mod download
COPY . .
RUN go build -o /dendrite ./cmd/dendrite-monolith-server
RUN go build -o /dendrite ./cmd/generate-keys
RUN go build -o /dendrite ./cmd/generate-config
RUN --mount=target=. \
--mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
go build -o /dendrite ./cmd/generate-config && \
go build -o /dendrite ./cmd/generate-keys && \
go build -o /dendrite ./cmd/dendrite-monolith-server
WORKDIR /dendrite
RUN ./generate-keys --private-key matrix_key.pem
@ -45,10 +45,9 @@ EXPOSE 8008 8448
# At runtime, generate TLS cert based on the CA now mounted at /ca
# At runtime, replace the SERVER_NAME with what we are told
CMD /build/run_postgres.sh && ./generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
# Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password, bump max_conns
sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml && \
sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml && \
cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
./dendrite-monolith-server --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
CMD /build/run_postgres.sh && ./generate-keys --keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
./generate-config -server $SERVER_NAME --ci --db postgresql://postgres@localhost/postgres?sslmode=disable > dendrite.yaml && \
# Bump max_open_conns up here in the global database config
sed -i 's/max_open_conns:.*$/max_open_conns: 1990/g' dendrite.yaml && \
cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}

2
build/scripts/build-test-lint.sh
View file

@ -13,4 +13,4 @@ go build ./cmd/...
./build/scripts/find-lint.sh
echo "Testing..."
go test -v ./...
go test --race -v ./...

4
build/scripts/find-lint.sh
View file

@ -25,7 +25,7 @@ echo "Installing golangci-lint..."
# Make a backup of go.{mod,sum} first
cp go.mod go.mod.bak && cp go.sum go.sum.bak
go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.41.1
go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.45.2
# Run linting
echo "Looking for lint..."
@ -33,7 +33,7 @@ echo "Looking for lint..."
# Capture exit code to ensure go.{mod,sum} is restored before exiting
exit_code=0
PATH="$PATH:${GOPATH:-~/go}/bin" golangci-lint run $args || exit_code=1
PATH="$PATH:$(go env GOPATH)/bin" golangci-lint run $args || exit_code=1
# Restore go.{mod,sum}
mv go.mod.bak go.mod && mv go.sum.bak go.sum

2
clientapi/auth/auth.go
View file

@ -51,7 +51,7 @@ type AccountDatabase interface {
// Note: For an AS user, AS dummy device is returned.
// On failure returns an JSON error response which can be sent to the client.
func VerifyUserFromRequest(
req *http.Request, userAPI api.UserInternalAPI,
req *http.Request, userAPI api.QueryAcccessTokenAPI,
) (*api.Device, *util.JSONResponse) {
// Try to find the Application Service user
token, err := ExtractAccessToken(req)

5
clientapi/auth/login.go
View file

@ -18,7 +18,6 @@ import (
"context"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
@ -33,8 +32,8 @@ import (
// called after authorization has completed, with the result of the authorization.
// If the final return value is non-nil, an error occurred and the cleanup function
// is nil.
func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserAccountAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) {
reqBytes, err := ioutil.ReadAll(r)
func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserLoginAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) {
reqBytes, err := io.ReadAll(r)
if err != nil {
err := &util.JSONResponse{
Code: http.StatusBadRequest,

5
clientapi/auth/login_test.go
View file

@ -160,7 +160,6 @@ func TestBadLoginFromJSONReader(t *testing.T) {
type fakeUserInternalAPI struct {
UserInternalAPIForLogin
uapi.UserAccountAPI
DeletedTokens []string
}
@ -179,6 +178,10 @@ func (ua *fakeUserInternalAPI) PerformLoginTokenDeletion(ctx context.Context, re
return nil
}
func (ua *fakeUserInternalAPI) PerformLoginTokenCreation(ctx context.Context, req *uapi.PerformLoginTokenCreationRequest, res *uapi.PerformLoginTokenCreationResponse) error {
return nil
}
func (*fakeUserInternalAPI) QueryLoginToken(ctx context.Context, req *uapi.QueryLoginTokenRequest, res *uapi.QueryLoginTokenResponse) error {
if req.Token == "invalidtoken" {
return nil

41
clientapi/auth/user_interactive.go
View file

@ -18,6 +18,7 @@ import (
"context"
"encoding/json"
"net/http"
"sync"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/setup/config"
@ -102,21 +103,20 @@ type userInteractiveFlow struct {
// the user already has a valid access token, but we want to double-check
// that it isn't stolen by re-authenticating them.
type UserInteractive struct {
Completed []string
Flows []userInteractiveFlow
sync.RWMutex
Flows []userInteractiveFlow
// Map of login type to implementation
Types map[string]Type
// Map of session ID to completed login types, will need to be extended in future
Sessions map[string][]string
}
func NewUserInteractive(userAccountAPI api.UserAccountAPI, cfg *config.ClientAPI) *UserInteractive {
func NewUserInteractive(userAccountAPI api.UserLoginAPI, cfg *config.ClientAPI) *UserInteractive {
typePassword := &LoginTypePassword{
GetAccountByPassword: userAccountAPI.QueryAccountByPassword,
Config: cfg,
}
return &UserInteractive{
Completed: []string{},
Flows: []userInteractiveFlow{
{
Stages: []string{typePassword.Name()},
@ -130,6 +130,8 @@ func NewUserInteractive(userAccountAPI api.UserAccountAPI, cfg *config.ClientAPI
}
func (u *UserInteractive) IsSingleStageFlow(authType string) bool {
u.RLock()
defer u.RUnlock()
for _, f := range u.Flows {
if len(f.Stages) == 1 && f.Stages[0] == authType {
return true
@ -139,9 +141,10 @@ func (u *UserInteractive) IsSingleStageFlow(authType string) bool {
}
func (u *UserInteractive) AddCompletedStage(sessionID, authType string) {
u.Lock()
// TODO: Handle multi-stage flows
u.Completed = append(u.Completed, authType)
delete(u.Sessions, sessionID)
u.Unlock()
}
type Challenge struct {
@ -153,12 +156,17 @@ type Challenge struct {
}
// Challenge returns an HTTP 401 with the supported flows for authenticating
func (u *UserInteractive) Challenge(sessionID string) *util.JSONResponse {
func (u *UserInteractive) challenge(sessionID string) *util.JSONResponse {
u.RLock()
completed := u.Sessions[sessionID]
flows := u.Flows
u.RUnlock()
return &util.JSONResponse{
Code: 401,
JSON: Challenge{
Completed: u.Completed,
Flows: u.Flows,
Completed: completed,
Flows: flows,
Session: sessionID,
Params: make(map[string]interface{}),
},
@ -173,8 +181,10 @@ func (u *UserInteractive) NewSession() *util.JSONResponse {
res := jsonerror.InternalServerError()
return &res
}
u.Lock()
u.Sessions[sessionID] = []string{}
return u.Challenge(sessionID)
u.Unlock()
return u.challenge(sessionID)
}
// ResponseWithChallenge mixes together a JSON body (e.g an error with errcode/message) with the
@ -187,7 +197,7 @@ func (u *UserInteractive) ResponseWithChallenge(sessionID string, response inter
return &ise
}
_ = json.Unmarshal(b, &mixedObjects)
challenge := u.Challenge(sessionID)
challenge := u.challenge(sessionID)
b, err = json.Marshal(challenge.JSON)
if err != nil {
ise := jsonerror.InternalServerError()
@ -216,7 +226,11 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *
// extract the type so we know which login type to use
authType := gjson.GetBytes(bodyBytes, "auth.type").Str
u.RLock()
loginType, ok := u.Types[authType]
u.RUnlock()
if !ok {
return nil, &util.JSONResponse{
Code: http.StatusBadRequest,
@ -226,7 +240,12 @@ func (u *UserInteractive) Verify(ctx context.Context, bodyBytes []byte, device *
// retrieve the session
sessionID := gjson.GetBytes(bodyBytes, "auth.session").Str
if _, ok = u.Sessions[sessionID]; !ok {
u.RLock()
_, ok = u.Sessions[sessionID]
u.RUnlock()
if !ok {
// if the login type is part of a single stage flow then allow them to omit the session ID
if !u.IsSingleStageFlow(authType) {
return nil, &util.JSONResponse{

39
clientapi/auth/user_interactive_test.go
View file

@ -24,9 +24,7 @@ var (
}
)
type fakeAccountDatabase struct {
api.UserAccountAPI
}
type fakeAccountDatabase struct{}
func (d *fakeAccountDatabase) PerformPasswordUpdate(ctx context.Context, req *api.PerformPasswordUpdateRequest, res *api.PerformPasswordUpdateResponse) error {
return nil
@ -189,3 +187,38 @@ func TestUserInteractivePasswordBadLogin(t *testing.T) {
}
}
}
func TestUserInteractive_AddCompletedStage(t *testing.T) {
tests := []struct {
name string
sessionID string
}{
{
name: "first user",
sessionID: util.RandomString(8),
},
{
name: "second user",
sessionID: util.RandomString(8),
},
{
name: "third user",
sessionID: util.RandomString(8),
},
}
u := setup()
ctx := context.Background()
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, resp := u.Verify(ctx, []byte("{}"), nil)
challenge, ok := resp.JSON.(Challenge)
if !ok {
t.Fatalf("expected a Challenge, got %T", resp.JSON)
}
if len(challenge.Completed) > 0 {
t.Fatalf("expected 0 completed stages, got %d", len(challenge.Completed))
}
u.AddCompletedStage(tt.sessionID, "")
})
}
}

33
clientapi/clientapi.go
View file

@ -15,7 +15,6 @@
package clientapi
import (
"github.com/gorilla/mux"
appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/clientapi/api"
"github.com/matrix-org/dendrite/clientapi/producers"
@ -24,35 +23,31 @@ import (
"github.com/matrix-org/dendrite/internal/transactions"
keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/jetstream"
"github.com/matrix-org/dendrite/setup/process"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
)
// AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component.
func AddPublicRoutes(
process *process.ProcessContext,
router *mux.Router,
synapseAdminRouter *mux.Router,
cfg *config.ClientAPI,
base *base.BaseDendrite,
federation *gomatrixserverlib.FederationClient,
rsAPI roomserverAPI.RoomserverInternalAPI,
asAPI appserviceAPI.AppServiceQueryAPI,
rsAPI roomserverAPI.ClientRoomserverAPI,
asAPI appserviceAPI.AppServiceInternalAPI,
transactionsCache *transactions.Cache,
fsAPI federationAPI.FederationInternalAPI,
userAPI userapi.UserInternalAPI,
userDirectoryProvider userapi.UserDirectoryProvider,
keyAPI keyserverAPI.KeyInternalAPI,
fsAPI federationAPI.ClientFederationAPI,
userAPI userapi.ClientUserAPI,
userDirectoryProvider userapi.QuerySearchProfilesAPI,
keyAPI keyserverAPI.ClientKeyAPI,
extRoomsProvider api.ExtraPublicRoomsProvider,
mscCfg *config.MSCs,
) {
js, natsClient := jetstream.Prepare(process, &cfg.Matrix.JetStream)
cfg := &base.Cfg.ClientAPI
mscCfg := &base.Cfg.MSCs
js, natsClient := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
syncProducer := &producers.SyncAPIProducer{
JetStream: js,
TopicClientData: cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData),
TopicReceiptEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
TopicTypingEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
@ -62,7 +57,11 @@ func AddPublicRoutes(
}
routing.Setup(
router, synapseAdminRouter, cfg, rsAPI, asAPI,
base.PublicClientAPIMux,
base.PublicWellKnownAPIMux,
base.SynapseAdminMux,
base.DendriteAdminMux,
cfg, rsAPI, asAPI,
userAPI, userDirectoryProvider, federation,
syncProducer, transactionsCache, fsAPI, keyAPI,
extRoomsProvider, mscCfg, natsClient,

6
clientapi/httputil/httputil.go
View file

@ -16,7 +16,7 @@ package httputil
import (
"encoding/json"
"io/ioutil"
"io"
"net/http"
"unicode/utf8"
@ -29,9 +29,9 @@ import (
func UnmarshalJSONRequest(req *http.Request, iface interface{}) *util.JSONResponse {
// encoding/json allows invalid utf-8, matrix does not
// https://matrix.org/docs/spec/client_server/r0.6.1#api-standards
body, err := ioutil.ReadAll(req.Body)
body, err := io.ReadAll(req.Body)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("ioutil.ReadAll failed")
util.GetLogger(req.Context()).WithError(err).Error("io.ReadAll failed")
resp := jsonerror.InternalServerError()
return &resp
}

20
clientapi/jsonerror/jsonerror.go
View file

@ -15,11 +15,13 @@
package jsonerror
import (
"context"
"fmt"
"net/http"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/sirupsen/logrus"
)
// MatrixError represents the "standard error response" in Matrix.
@ -154,6 +156,12 @@ func MissingParam(msg string) *MatrixError {
return &MatrixError{"M_MISSING_PARAM", msg}
}
// UnableToAuthoriseJoin is an error that is returned when a server can't
// determine whether to allow a restricted join or not.
func UnableToAuthoriseJoin(msg string) *MatrixError {
return &MatrixError{"M_UNABLE_TO_AUTHORISE_JOIN", msg}
}
// LeaveServerNoticeError is an error returned when trying to reject an invite
// for a server notice room.
func LeaveServerNoticeError() *MatrixError {
@ -207,3 +215,15 @@ func NotTrusted(serverName string) *MatrixError {
Err: fmt.Sprintf("Untrusted server '%s'", serverName),
}
}
// InternalAPIError is returned when Dendrite failed to reach an internal API.
func InternalAPIError(ctx context.Context, err error) util.JSONResponse {
logrus.WithContext(ctx).WithError(err).Error("Error reaching an internal API")
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: &MatrixError{
ErrCode: "M_INTERNAL_SERVER_ERROR",
Err: "Dendrite encountered an error reaching an internal API.",
},
}
}

69
clientapi/producers/syncapi.go
View file

@ -17,58 +17,28 @@ package producers
import (
"context"
"encoding/json"
"fmt"
"strconv"
"time"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/setup/jetstream"
"github.com/matrix-org/dendrite/syncapi/types"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/nats-io/nats.go"
log "github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/setup/jetstream"
"github.com/matrix-org/dendrite/syncapi/types"
userapi "github.com/matrix-org/dendrite/userapi/api"
)
// SyncAPIProducer produces events for the sync API server to consume
type SyncAPIProducer struct {
TopicClientData string
TopicReceiptEvent string
TopicSendToDeviceEvent string
TopicTypingEvent string
TopicPresenceEvent string
JetStream nats.JetStreamContext
ServerName gomatrixserverlib.ServerName
UserAPI userapi.UserInternalAPI
}
// SendData sends account data to the sync API server
func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string, readMarker *eventutil.ReadMarkerJSON, ignoredUsers *types.IgnoredUsers) error {
m := &nats.Msg{
Subject: p.TopicClientData,
Header: nats.Header{},
}
m.Header.Set(jetstream.UserID, userID)
data := eventutil.AccountData{
RoomID: roomID,
Type: dataType,
ReadMarker: readMarker,
IgnoredUsers: ignoredUsers,
}
var err error
m.Data, err = json.Marshal(data)
if err != nil {
return err
}
log.WithFields(log.Fields{
"user_id": userID,
"room_id": roomID,
"data_type": dataType,
}).Tracef("Producing to topic '%s'", p.TopicClientData)
_, err = p.JetStream.PublishMsg(m)
return err
UserAPI userapi.ClientUserAPI
}
func (p *SyncAPIProducer) SendReceipt(
@ -83,7 +53,7 @@ func (p *SyncAPIProducer) SendReceipt(
m.Header.Set(jetstream.RoomID, roomID)
m.Header.Set(jetstream.EventID, eventID)
m.Header.Set("type", receiptType)
m.Header.Set("timestamp", strconv.Itoa(int(timestamp)))
m.Header.Set("timestamp", fmt.Sprintf("%d", timestamp))
log.WithFields(log.Fields{}).Tracef("Producing to topic '%s'", p.TopicReceiptEvent)
_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
@ -92,7 +62,7 @@ func (p *SyncAPIProducer) SendReceipt(
func (p *SyncAPIProducer) SendToDevice(
ctx context.Context, sender, userID, deviceID, eventType string,
message interface{},
message json.RawMessage,
) error {
devices := []string{}
_, domain, err := gomatrixserverlib.SplitID('@', userID)
@ -120,24 +90,19 @@ func (p *SyncAPIProducer) SendToDevice(
devices = append(devices, deviceID)
}
js, err := json.Marshal(message)
if err != nil {
return err
}
log.WithFields(log.Fields{
"user_id": userID,
"num_devices": len(devices),
"type": eventType,
}).Tracef("Producing to topic '%s'", p.TopicSendToDeviceEvent)
for _, device := range devices {
for i, device := range devices {
ote := &types.OutputSendToDeviceEvent{
UserID: userID,
DeviceID: device,
SendToDeviceEvent: gomatrixserverlib.SendToDeviceEvent{
Sender: sender,
Type: eventType,
Content: js,
Content: message,
},
}
@ -146,15 +111,17 @@ func (p *SyncAPIProducer) SendToDevice(
log.WithError(err).Error("sendToDevice failed json.Marshal")
return err
}
m := &nats.Msg{
Subject: p.TopicSendToDeviceEvent,
Data: eventJSON,
Header: nats.Header{},
}
m := nats.NewMsg(p.TopicSendToDeviceEvent)
m.Data = eventJSON
m.Header.Set("sender", sender)
m.Header.Set(jetstream.UserID, userID)
if _, err = p.JetStream.PublishMsg(m, nats.Context(ctx)); err != nil {
log.WithError(err).Error("sendToDevice failed t.Producer.SendMessage")
if i < len(devices)-1 {
log.WithError(err).Warn("sendToDevice failed to PublishMsg, trying further devices")
continue
}
log.WithError(err).Error("sendToDevice failed to PublishMsg for all devices")
return err
}
}

30
clientapi/routing/account_data.go
View file

@ -17,7 +17,7 @@ package routing
import (
"encoding/json"
"fmt"
"io/ioutil"
"io"
"net/http"
"github.com/matrix-org/dendrite/clientapi/httputil"
@ -25,7 +25,6 @@ import (
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/dendrite/internal/eventutil"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/syncapi/types"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/util"
@ -33,7 +32,7 @@ import (
// GetAccountData implements GET /user/{userId}/[rooms/{roomid}/]account_data/{type}
func GetAccountData(
req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
userID string, roomID string, dataType string,
) util.JSONResponse {
if userID != device.UserID {
@ -76,7 +75,7 @@ func GetAccountData(
// SaveAccountData implements PUT /user/{userId}/[rooms/{roomId}/]account_data/{type}
func SaveAccountData(
req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
userID string, roomID string, dataType string, syncProducer *producers.SyncAPIProducer,
) util.JSONResponse {
if userID != device.UserID {
@ -102,9 +101,9 @@ func SaveAccountData(
}
}
body, err := ioutil.ReadAll(req.Body)
body, err := io.ReadAll(req.Body)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("ioutil.ReadAll failed")
util.GetLogger(req.Context()).WithError(err).Error("io.ReadAll failed")
return jsonerror.InternalServerError()
}
@ -127,18 +126,6 @@ func SaveAccountData(
return util.ErrorResponse(err)
}
var ignoredUsers *types.IgnoredUsers
if dataType == "m.ignored_user_list" {
ignoredUsers = &types.IgnoredUsers{}
_ = json.Unmarshal(body, ignoredUsers)
}
// TODO: user API should do this since it's account data
if err := syncProducer.SendData(userID, roomID, dataType, nil, ignoredUsers); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed")
return jsonerror.InternalServerError()
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
@ -152,7 +139,7 @@ type fullyReadEvent struct {
// SaveReadMarker implements POST /rooms/{roomId}/read_markers
func SaveReadMarker(
req *http.Request,
userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI,
userAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
syncProducer *producers.SyncAPIProducer, device *api.Device, roomID string,
) util.JSONResponse {
// Verify that the user is a member of this room
@ -191,11 +178,6 @@ func SaveReadMarker(
return util.ErrorResponse(err)
}
if err := syncProducer.SendData(device.UserID, roomID, "m.fully_read", &r, nil); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed")
return jsonerror.InternalServerError()
}
// Handle the read receipt that may be included in the read marker
if r.Read != "" {
return SetReceipt(req, syncProducer, device, roomID, "m.read", r.Read)

140
clientapi/routing/admin.go Normal file
View file

@ -0,0 +1,140 @@
package routing
import (
"encoding/json"
"net/http"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/httputil"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
roomID, ok := vars["roomID"]
if !ok {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.MissingArgument("Expecting room ID."),
}
}
res := &roomserverAPI.PerformAdminEvacuateRoomResponse{}
if err := rsAPI.PerformAdminEvacuateRoom(
req.Context(),
&roomserverAPI.PerformAdminEvacuateRoomRequest{
RoomID: roomID,
},
res,
); err != nil {
return util.ErrorResponse(err)
}
if err := res.Error; err != nil {
return err.JSONResponse()
}
return util.JSONResponse{
Code: 200,
JSON: map[string]interface{}{
"affected": res.Affected,
},
}
}
func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
userID, ok := vars["userID"]
if !ok {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.MissingArgument("Expecting user ID."),
}
}
_, domain, err := gomatrixserverlib.SplitID('@', userID)
if err != nil {
return util.MessageResponse(http.StatusBadRequest, err.Error())
}
if domain != cfg.Matrix.ServerName {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.MissingArgument("User ID must belong to this server."),
}
}
res := &roomserverAPI.PerformAdminEvacuateUserResponse{}
if err := rsAPI.PerformAdminEvacuateUser(
req.Context(),
&roomserverAPI.PerformAdminEvacuateUserRequest{
UserID: userID,
},
res,
); err != nil {
return jsonerror.InternalAPIError(req.Context(), err)
}
if err := res.Error; err != nil {
return err.JSONResponse()
}
return util.JSONResponse{
Code: 200,
JSON: map[string]interface{}{
"affected": res.Affected,
},
}
}
func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
localpart, ok := vars["localpart"]
if !ok {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.MissingArgument("Expecting user localpart."),
}
}
request := struct {
Password string `json:"password"`
}{}
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.Unknown("Failed to decode request body: " + err.Error()),
}
}
if request.Password == "" {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.MissingArgument("Expecting non-empty password."),
}
}
updateReq := &userapi.PerformPasswordUpdateRequest{
Localpart: localpart,
Password: request.Password,
LogoutDevices: true,
}
updateRes := &userapi.PerformPasswordUpdateResponse{}
if err := userAPI.PerformPasswordUpdate(req.Context(), updateReq, updateRes); err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.Unknown("Failed to perform password update: " + err.Error()),
}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct {
Updated bool `json:"password_updated"`
}{
Updated: updateRes.PasswordUpdated,
},
}
}

2
clientapi/routing/admin_whois.go
View file

@ -44,7 +44,7 @@ type connectionInfo struct {
// GetAdminWhois implements GET /admin/whois/{userId}
func GetAdminWhois(
req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
userID string,
) util.JSONResponse {
allowed := device.AccountType == api.AccountTypeAdmin || userID == device.UserID

4
clientapi/routing/aliases.go
View file

@ -28,7 +28,7 @@ import (
// GetAliases implements GET /_matrix/client/r0/rooms/{roomId}/aliases
func GetAliases(
req *http.Request, rsAPI api.RoomserverInternalAPI, device *userapi.Device, roomID string,
req *http.Request, rsAPI api.ClientRoomserverAPI, device *userapi.Device, roomID string,
) util.JSONResponse {
stateTuple := gomatrixserverlib.StateKeyTuple{
EventType: gomatrixserverlib.MRoomHistoryVisibility,
@ -44,7 +44,7 @@ func GetAliases(
return util.ErrorResponse(fmt.Errorf("rsAPI.QueryCurrentState: %w", err))
}
visibility := "invite"
visibility := gomatrixserverlib.HistoryVisibilityInvited
if historyVisEvent, ok := stateRes.StateEvents[stateTuple]; ok {
var err error
visibility, err = historyVisEvent.HistoryVisibility()

2
clientapi/routing/capabilities.go
View file

@ -26,7 +26,7 @@ import (
// GetCapabilities returns information about the server's supported feature set
// and other relevant capabilities to an authenticated user.
func GetCapabilities(
req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI,
req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI,
) util.JSONResponse {
roomVersionsQueryReq := roomserverAPI.QueryRoomVersionCapabilitiesRequest{}
roomVersionsQueryRes := roomserverAPI.QueryRoomVersionCapabilitiesResponse{}

55
clientapi/routing/createroom.go
View file

@ -49,6 +49,7 @@ type createRoomRequest struct {
GuestCanJoin bool `json:"guest_can_join"`
RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"`
PowerLevelContentOverride json.RawMessage `json:"power_level_content_override"`
IsDirect bool `json:"is_direct"`
}
const (
@ -137,8 +138,8 @@ type fledglingEvent struct {
func CreateRoom(
req *http.Request, device *api.Device,
cfg *config.ClientAPI,
profileAPI api.UserProfileAPI, rsAPI roomserverAPI.RoomserverInternalAPI,
asAPI appserviceAPI.AppServiceQueryAPI,
profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
asAPI appserviceAPI.AppServiceInternalAPI,
) util.JSONResponse {
var r createRoomRequest
resErr := httputil.UnmarshalJSONRequest(req, &r)
@ -164,8 +165,8 @@ func createRoom(
ctx context.Context,
r createRoomRequest, device *api.Device,
cfg *config.ClientAPI,
profileAPI api.UserProfileAPI, rsAPI roomserverAPI.RoomserverInternalAPI,
asAPI appserviceAPI.AppServiceQueryAPI,
profileAPI api.ClientUserAPI, rsAPI roomserverAPI.ClientRoomserverAPI,
asAPI appserviceAPI.AppServiceInternalAPI,
evTime time.Time,
) util.JSONResponse {
// TODO (#267): Check room ID doesn't clash with an existing one, and we
@ -245,7 +246,9 @@ func createRoom(
case presetTrustedPrivateChat:
joinRuleContent.JoinRule = gomatrixserverlib.Invite
historyVisibilityContent.HistoryVisibility = historyVisibilityShared
// TODO If trusted_private_chat, all invitees are given the same power level as the room creator.
for _, invitee := range r.Invite {
powerLevelContent.Users[invitee] = 100
}
case presetPublicChat:
joinRuleContent.JoinRule = gomatrixserverlib.Public
historyVisibilityContent.HistoryVisibility = historyVisibilityShared
@ -497,9 +500,17 @@ func createRoom(
// Build some stripped state for the invite.
var globalStrippedState []gomatrixserverlib.InviteV2StrippedState
for _, event := range builtEvents {
// Chosen events from the spec:
// https://spec.matrix.org/v1.3/client-server-api/#stripped-state
switch event.Type() {
case gomatrixserverlib.MRoomCreate:
fallthrough
case gomatrixserverlib.MRoomName:
fallthrough
case gomatrixserverlib.MRoomAvatar:
fallthrough
case gomatrixserverlib.MRoomTopic:
fallthrough
case gomatrixserverlib.MRoomCanonicalAlias:
fallthrough
case gomatrixserverlib.MRoomEncryption:
@ -520,7 +531,7 @@ func createRoom(
// Build the invite event.
inviteEvent, err := buildMembershipEvent(
ctx, invitee, "", profileAPI, device, gomatrixserverlib.Invite,
roomID, true, cfg, evTime, rsAPI, asAPI,
roomID, r.IsDirect, cfg, evTime, rsAPI, asAPI,
)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
@ -531,35 +542,35 @@ func createRoom(
gomatrixserverlib.NewInviteV2StrippedState(inviteEvent.Event),
)
// Send the invite event to the roomserver.
err = roomserverAPI.SendInvite(
ctx,
rsAPI,
inviteEvent.Headered(roomVersion),
inviteStrippedState, // invite room state
cfg.Matrix.ServerName, // send as server
nil, // transaction ID
)
switch e := err.(type) {
case *roomserverAPI.PerformError:
return e.JSONResponse()
case nil:
default:
util.GetLogger(ctx).WithError(err).Error("roomserverAPI.SendInvite failed")
var inviteRes roomserverAPI.PerformInviteResponse
event := inviteEvent.Headered(roomVersion)
if err := rsAPI.PerformInvite(ctx, &roomserverAPI.PerformInviteRequest{
Event: event,
InviteRoomState: inviteStrippedState,
RoomVersion: event.RoomVersion,
SendAsServer: string(cfg.Matrix.ServerName),
}, &inviteRes); err != nil {
util.GetLogger(ctx).WithError(err).Error("PerformInvite failed")
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: jsonerror.InternalServerError(),
}
}
if inviteRes.Error != nil {
return inviteRes.Error.JSONResponse()
}
}
}
if r.Visibility == "public" {
// expose this room in the published room list
var pubRes roomserverAPI.PerformPublishResponse
rsAPI.PerformPublish(ctx, &roomserverAPI.PerformPublishRequest{
if err := rsAPI.PerformPublish(ctx, &roomserverAPI.PerformPublishRequest{
RoomID: roomID,
Visibility: "public",
}, &pubRes)
}, &pubRes); err != nil {
return jsonerror.InternalAPIError(ctx, err)
}
if pubRes.Error != nil {
// treat as non-fatal since the room is already made by this point
util.GetLogger(ctx).WithError(pubRes.Error).Error("failed to visibility:public")

6
clientapi/routing/deactivate.go
View file

@ -1,7 +1,7 @@
package routing
import (
"io/ioutil"
"io"
"net/http"
"github.com/matrix-org/dendrite/clientapi/auth"
@ -15,12 +15,12 @@ import (
func Deactivate(
req *http.Request,
userInteractiveAuth *auth.UserInteractive,
accountAPI api.UserAccountAPI,
accountAPI api.ClientUserAPI,
deviceAPI *api.Device,
) util.JSONResponse {
ctx := req.Context()
defer req.Body.Close() // nolint:errcheck
bodyBytes, err := ioutil.ReadAll(req.Body)
bodyBytes, err := io.ReadAll(req.Body)
if err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,

14
clientapi/routing/device.go
View file

@ -15,7 +15,7 @@
package routing
import (
"io/ioutil"
"io"
"net"
"net/http"
@ -50,7 +50,7 @@ type devicesDeleteJSON struct {
// GetDeviceByID handles /devices/{deviceID}
func GetDeviceByID(
req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
deviceID string,
) util.JSONResponse {
var queryRes api.QueryDevicesResponse
@ -88,7 +88,7 @@ func GetDeviceByID(
// GetDevicesByLocalpart handles /devices
func GetDevicesByLocalpart(
req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
) util.JSONResponse {
var queryRes api.QueryDevicesResponse
err := userAPI.QueryDevices(req.Context(), &api.QueryDevicesRequest{
@ -118,7 +118,7 @@ func GetDevicesByLocalpart(
// UpdateDeviceByID handles PUT on /devices/{deviceID}
func UpdateDeviceByID(
req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
deviceID string,
) util.JSONResponse {
@ -161,7 +161,7 @@ func UpdateDeviceByID(
// DeleteDeviceById handles DELETE requests to /devices/{deviceId}
func DeleteDeviceById(
req *http.Request, userInteractiveAuth *auth.UserInteractive, userAPI api.UserInternalAPI, device *api.Device,
req *http.Request, userInteractiveAuth *auth.UserInteractive, userAPI api.ClientUserAPI, device *api.Device,
deviceID string,
) util.JSONResponse {
var (
@ -175,7 +175,7 @@ func DeleteDeviceById(
}()
ctx := req.Context()
defer req.Body.Close() // nolint:errcheck
bodyBytes, err := ioutil.ReadAll(req.Body)
bodyBytes, err := io.ReadAll(req.Body)
if err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
@ -242,7 +242,7 @@ func DeleteDeviceById(
// DeleteDevices handles POST requests to /delete_devices
func DeleteDevices(
req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
) util.JSONResponse {
ctx := req.Context()
payload := devicesDeleteJSON{}

18
clientapi/routing/directory.go
View file

@ -46,8 +46,8 @@ func DirectoryRoom(
roomAlias string,
federation *gomatrixserverlib.FederationClient,
cfg *config.ClientAPI,
rsAPI roomserverAPI.RoomserverInternalAPI,
fedSenderAPI federationAPI.FederationInternalAPI,
rsAPI roomserverAPI.ClientRoomserverAPI,
fedSenderAPI federationAPI.ClientFederationAPI,
) util.JSONResponse {
_, domain, err := gomatrixserverlib.SplitID('#', roomAlias)
if err != nil {
@ -117,7 +117,7 @@ func SetLocalAlias(
device *userapi.Device,
alias string,
cfg *config.ClientAPI,
rsAPI roomserverAPI.RoomserverInternalAPI,
rsAPI roomserverAPI.ClientRoomserverAPI,
) util.JSONResponse {
_, domain, err := gomatrixserverlib.SplitID('#', alias)
if err != nil {
@ -199,7 +199,7 @@ func RemoveLocalAlias(
req *http.Request,
device *userapi.Device,
alias string,
rsAPI roomserverAPI.RoomserverInternalAPI,
rsAPI roomserverAPI.ClientRoomserverAPI,
) util.JSONResponse {
queryReq := roomserverAPI.RemoveRoomAliasRequest{
Alias: alias,
@ -237,7 +237,7 @@ type roomVisibility struct {
// GetVisibility implements GET /directory/list/room/{roomID}
func GetVisibility(
req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI,
req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI,
roomID string,
) util.JSONResponse {
var res roomserverAPI.QueryPublishedRoomsResponse
@ -265,7 +265,7 @@ func GetVisibility(
// SetVisibility implements PUT /directory/list/room/{roomID}
// TODO: Allow admin users to edit the room visibility
func SetVisibility(
req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, dev *userapi.Device,
req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, dev *userapi.Device,
roomID string,
) util.JSONResponse {
resErr := checkMemberInRoom(req.Context(), rsAPI, dev.UserID, roomID)
@ -302,10 +302,12 @@ func SetVisibility(
}
var publishRes roomserverAPI.PerformPublishResponse
rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
if err := rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
RoomID: roomID,
Visibility: v.Visibility,
}, &publishRes)
}, &publishRes); err != nil {
return jsonerror.InternalAPIError(req.Context(), err)
}
if publishRes.Error != nil {
util.GetLogger(req.Context()).WithError(publishRes.Error).Error("PerformPublish failed")
return publishRes.Error.JSONResponse()

23
clientapi/routing/directory_public.go
View file

@ -23,13 +23,14 @@ import (
"strings"
"sync"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/matrix-org/dendrite/clientapi/api"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
var (
@ -50,7 +51,7 @@ type filter struct {
// GetPostPublicRooms implements GET and POST /publicRooms
func GetPostPublicRooms(
req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI,
req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI,
extRoomsProvider api.ExtraPublicRoomsProvider,
federation *gomatrixserverlib.FederationClient,
cfg *config.ClientAPI,
@ -91,7 +92,7 @@ func GetPostPublicRooms(
}
func publicRooms(
ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.RoomserverInternalAPI, extRoomsProvider api.ExtraPublicRoomsProvider,
ctx context.Context, request PublicRoomReq, rsAPI roomserverAPI.ClientRoomserverAPI, extRoomsProvider api.ExtraPublicRoomsProvider,
) (*gomatrixserverlib.RespPublicRooms, error) {
response := gomatrixserverlib.RespPublicRooms{
@ -196,14 +197,14 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO
// sliceInto returns a subslice of `slice` which honours the since/limit values given.
//
// 0 1 2 3 4 5 6 index
// [A, B, C, D, E, F, G] slice
// 0 1 2 3 4 5 6 index
// [A, B, C, D, E, F, G] slice
//
// limit=3 => A,B,C (prev='', next='3')
// limit=3&since=3 => D,E,F (prev='0', next='6')
// limit=3&since=6 => G (prev='3', next='')
// limit=3 => A,B,C (prev='', next='3')
// limit=3&since=3 => D,E,F (prev='0', next='6')
// limit=3&since=6 => G (prev='3', next='')
//
// A value of '-1' for prev/next indicates no position.
// A value of '-1' for prev/next indicates no position.
func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int16) (subset []gomatrixserverlib.PublicRoom, prev, next int) {
prev = -1
next = -1
@ -229,7 +230,7 @@ func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int16) (
}
func refreshPublicRoomCache(
ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI, extRoomsProvider api.ExtraPublicRoomsProvider,
ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, extRoomsProvider api.ExtraPublicRoomsProvider,
) []gomatrixserverlib.PublicRoom {
cacheMu.Lock()
defer cacheMu.Unlock()

5
clientapi/routing/getevent.go
View file

@ -31,7 +31,6 @@ type getEventRequest struct {
roomID string
eventID string
cfg *config.ClientAPI
federation *gomatrixserverlib.FederationClient
requestedEvent *gomatrixserverlib.Event
}
@ -43,8 +42,7 @@ func GetEvent(
roomID string,
eventID string,
cfg *config.ClientAPI,
rsAPI api.RoomserverInternalAPI,
federation *gomatrixserverlib.FederationClient,
rsAPI api.ClientRoomserverAPI,
) util.JSONResponse {
eventsReq := api.QueryEventsByIDRequest{
EventIDs: []string{eventID},
@ -72,7 +70,6 @@ func GetEvent(
roomID: roomID,
eventID: eventID,
cfg: cfg,
federation: federation,
requestedEvent: requestedEvent,
}

9
clientapi/routing/joinroom.go
View file

@ -29,8 +29,8 @@ import (
func JoinRoomByIDOrAlias(
req *http.Request,
device *api.Device,
rsAPI roomserverAPI.RoomserverInternalAPI,
profileAPI api.UserProfileAPI,
rsAPI roomserverAPI.ClientRoomserverAPI,
profileAPI api.ClientUserAPI,
roomIDOrAlias string,
) util.JSONResponse {
// Prepare to ask the roomserver to perform the room join.
@ -81,8 +81,9 @@ func JoinRoomByIDOrAlias(
done := make(chan util.JSONResponse, 1)
go func() {
defer close(done)
rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes)
if joinRes.Error != nil {
if err := rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes); err != nil {
done <- jsonerror.InternalAPIError(req.Context(), err)
} else if joinRes.Error != nil {
done <- joinRes.Error.JSONResponse()
} else {
done <- util.JSONResponse{

24
clientapi/routing/key_backup.go
View file

@ -55,7 +55,7 @@ type keyBackupSessionResponse struct {
// Create a new key backup. Request must contain a `keyBackupVersion`. Returns a `keyBackupVersionCreateResponse`.
// Implements POST /_matrix/client/r0/room_keys/version
func CreateKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device) util.JSONResponse {
func CreateKeyBackupVersion(req *http.Request, userAPI userapi.ClientUserAPI, device *userapi.Device) util.JSONResponse {
var kb keyBackupVersion
resErr := httputil.UnmarshalJSONRequest(req, &kb)
if resErr != nil {
@ -89,12 +89,14 @@ func CreateKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI,
// KeyBackupVersion returns the key backup version specified. If `version` is empty, the latest `keyBackupVersionResponse` is returned.
// Implements GET /_matrix/client/r0/room_keys/version and GET /_matrix/client/r0/room_keys/version/{version}
func KeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string) util.JSONResponse {
func KeyBackupVersion(req *http.Request, userAPI userapi.ClientUserAPI, device *userapi.Device, version string) util.JSONResponse {
var queryResp userapi.QueryKeyBackupResponse
userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{
if err := userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{
UserID: device.UserID,
Version: version,
}, &queryResp)
}, &queryResp); err != nil {
return jsonerror.InternalAPIError(req.Context(), err)
}
if queryResp.Error != "" {
return util.ErrorResponse(fmt.Errorf("QueryKeyBackup: %s", queryResp.Error))
}
@ -118,7 +120,7 @@ func KeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device
// Modify the auth data of a key backup. Version must not be empty. Request must contain a `keyBackupVersion`
// Implements PUT /_matrix/client/r0/room_keys/version/{version}
func ModifyKeyBackupVersionAuthData(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string) util.JSONResponse {
func ModifyKeyBackupVersionAuthData(req *http.Request, userAPI userapi.ClientUserAPI, device *userapi.Device, version string) util.JSONResponse {
var kb keyBackupVersion
resErr := httputil.UnmarshalJSONRequest(req, &kb)
if resErr != nil {
@ -159,7 +161,7 @@ func ModifyKeyBackupVersionAuthData(req *http.Request, userAPI userapi.UserInter
// Delete a version of key backup. Version must not be empty. If the key backup was previously deleted, will return 200 OK.
// Implements DELETE /_matrix/client/r0/room_keys/version/{version}
func DeleteKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string) util.JSONResponse {
func DeleteKeyBackupVersion(req *http.Request, userAPI userapi.ClientUserAPI, device *userapi.Device, version string) util.JSONResponse {
var performKeyBackupResp userapi.PerformKeyBackupResponse
if err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
UserID: device.UserID,
@ -194,7 +196,7 @@ func DeleteKeyBackupVersion(req *http.Request, userAPI userapi.UserInternalAPI,
// Upload a bunch of session keys for a given `version`.
func UploadBackupKeys(
req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version string, keys *keyBackupSessionRequest,
req *http.Request, userAPI userapi.ClientUserAPI, device *userapi.Device, version string, keys *keyBackupSessionRequest,
) util.JSONResponse {
var performKeyBackupResp userapi.PerformKeyBackupResponse
if err := userAPI.PerformKeyBackup(req.Context(), &userapi.PerformKeyBackupRequest{
@ -230,16 +232,18 @@ func UploadBackupKeys(
// Get keys from a given backup version. Response returned varies depending on if roomID and sessionID are set.
func GetBackupKeys(
req *http.Request, userAPI userapi.UserInternalAPI, device *userapi.Device, version, roomID, sessionID string,
req *http.Request, userAPI userapi.ClientUserAPI, device *userapi.Device, version, roomID, sessionID string,
) util.JSONResponse {
var queryResp userapi.QueryKeyBackupResponse
userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{
if err := userAPI.QueryKeyBackup(req.Context(), &userapi.QueryKeyBackupRequest{
UserID: device.UserID,
Version: version,
ReturnKeys: true,
KeysForRoomID: roomID,
KeysForSessionID: sessionID,
}, &queryResp)
}, &queryResp); err != nil {
return jsonerror.InternalAPIError(req.Context(), err)
}
if queryResp.Error != "" {
return util.ErrorResponse(fmt.Errorf("QueryKeyBackup: %s", queryResp.Error))
}

14
clientapi/routing/key_crosssigning.go
View file

@ -34,8 +34,8 @@ type crossSigningRequest struct {
func UploadCrossSigningDeviceKeys(
req *http.Request, userInteractiveAuth *auth.UserInteractive,
keyserverAPI api.KeyInternalAPI, device *userapi.Device,
accountAPI userapi.UserAccountAPI, cfg *config.ClientAPI,
keyserverAPI api.ClientKeyAPI, device *userapi.Device,
accountAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
) util.JSONResponse {
uploadReq := &crossSigningRequest{}
uploadRes := &api.PerformUploadDeviceKeysResponse{}
@ -72,7 +72,9 @@ func UploadCrossSigningDeviceKeys(
sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypePassword)
uploadReq.UserID = device.UserID
keyserverAPI.PerformUploadDeviceKeys(req.Context(), &uploadReq.PerformUploadDeviceKeysRequest, uploadRes)
if err := keyserverAPI.PerformUploadDeviceKeys(req.Context(), &uploadReq.PerformUploadDeviceKeysRequest, uploadRes); err != nil {
return jsonerror.InternalAPIError(req.Context(), err)
}
if err := uploadRes.Error; err != nil {
switch {
@ -105,7 +107,7 @@ func UploadCrossSigningDeviceKeys(
}
}
func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.KeyInternalAPI, device *userapi.Device) util.JSONResponse {
func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse {
uploadReq := &api.PerformUploadDeviceSignaturesRequest{}
uploadRes := &api.PerformUploadDeviceSignaturesResponse{}
@ -114,7 +116,9 @@ func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.KeyI
}
uploadReq.UserID = device.UserID
keyserverAPI.PerformUploadDeviceSignatures(req.Context(), uploadReq, uploadRes)
if err := keyserverAPI.PerformUploadDeviceSignatures(req.Context(), uploadReq, uploadRes); err != nil {
return jsonerror.InternalAPIError(req.Context(), err)
}
if err := uploadRes.Error; err != nil {
switch {

22
clientapi/routing/keys.go
View file

@ -31,7 +31,7 @@ type uploadKeysRequest struct {
OneTimeKeys map[string]json.RawMessage `json:"one_time_keys"`
}
func UploadKeys(req *http.Request, keyAPI api.KeyInternalAPI, device *userapi.Device) util.JSONResponse {
func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse {
var r uploadKeysRequest
resErr := httputil.UnmarshalJSONRequest(req, &r)
if resErr != nil {
@ -62,7 +62,9 @@ func UploadKeys(req *http.Request, keyAPI api.KeyInternalAPI, device *userapi.De
}
var uploadRes api.PerformUploadKeysResponse
keyAPI.PerformUploadKeys(req.Context(), uploadReq, &uploadRes)
if err := keyAPI.PerformUploadKeys(req.Context(), uploadReq, &uploadRes); err != nil {
return util.ErrorResponse(err)
}
if uploadRes.Error != nil {
util.GetLogger(req.Context()).WithError(uploadRes.Error).Error("Failed to PerformUploadKeys")
return jsonerror.InternalServerError()
@ -100,19 +102,21 @@ func (r *queryKeysRequest) GetTimeout() time.Duration {
return time.Duration(r.Timeout) * time.Millisecond
}
func QueryKeys(req *http.Request, keyAPI api.KeyInternalAPI, device *userapi.Device) util.JSONResponse {
func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse {
var r queryKeysRequest
resErr := httputil.UnmarshalJSONRequest(req, &r)
if resErr != nil {
return *resErr
}
queryRes := api.QueryKeysResponse{}
keyAPI.QueryKeys(req.Context(), &api.QueryKeysRequest{
if err := keyAPI.QueryKeys(req.Context(), &api.QueryKeysRequest{
UserID: device.UserID,
UserToDevices: r.DeviceKeys,
Timeout: r.GetTimeout(),
// TODO: Token?
}, &queryRes)
}, &queryRes); err != nil {
return util.ErrorResponse(err)
}
return util.JSONResponse{
Code: 200,
JSON: map[string]interface{}{
@ -138,17 +142,19 @@ func (r *claimKeysRequest) GetTimeout() time.Duration {
return time.Duration(r.TimeoutMS) * time.Millisecond
}
func ClaimKeys(req *http.Request, keyAPI api.KeyInternalAPI) util.JSONResponse {
func ClaimKeys(req *http.Request, keyAPI api.ClientKeyAPI) util.JSONResponse {
var r claimKeysRequest
resErr := httputil.UnmarshalJSONRequest(req, &r)
if resErr != nil {
return *resErr
}
claimRes := api.PerformClaimKeysResponse{}
keyAPI.PerformClaimKeys(req.Context(), &api.PerformClaimKeysRequest{
if err := keyAPI.PerformClaimKeys(req.Context(), &api.PerformClaimKeysRequest{
OneTimeKeys: r.OneTimeKeys,
Timeout: r.GetTimeout(),
}, &claimRes)
}, &claimRes); err != nil {
return jsonerror.InternalAPIError(req.Context(), err)
}
if claimRes.Error != nil {
util.GetLogger(req.Context()).WithError(claimRes.Error).Error("failed to PerformClaimKeys")
return jsonerror.InternalServerError()

2
clientapi/routing/leaveroom.go
View file

@ -26,7 +26,7 @@ import (
func LeaveRoomByID(
req *http.Request,
device *api.Device,
rsAPI roomserverAPI.RoomserverInternalAPI,
rsAPI roomserverAPI.ClientRoomserverAPI,
roomID string,
) util.JSONResponse {
// Prepare to ask the roomserver to perform the room join.

4
clientapi/routing/login.go
View file

@ -53,7 +53,7 @@ func passwordLogin() flows {
// Login implements GET and POST /login
func Login(
req *http.Request, userAPI userapi.UserInternalAPI,
req *http.Request, userAPI userapi.ClientUserAPI,
cfg *config.ClientAPI,
) util.JSONResponse {
if req.Method == http.MethodGet {
@ -79,7 +79,7 @@ func Login(
}
func completeAuth(
ctx context.Context, serverName gomatrixserverlib.ServerName, userAPI userapi.UserInternalAPI, login *auth.Login,
ctx context.Context, serverName gomatrixserverlib.ServerName, userAPI userapi.ClientUserAPI, login *auth.Login,
ipAddr, userAgent string,
) util.JSONResponse {
token, err := auth.GenerateAccessToken()

4
clientapi/routing/logout.go
View file

@ -24,7 +24,7 @@ import (
// Logout handles POST /logout
func Logout(
req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
) util.JSONResponse {
var performRes api.PerformDeviceDeletionResponse
err := userAPI.PerformDeviceDeletion(req.Context(), &api.PerformDeviceDeletionRequest{
@ -44,7 +44,7 @@ func Logout(
// LogoutAll handles POST /logout/all
func LogoutAll(
req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
) util.JSONResponse {
var performRes api.PerformDeviceDeletionResponse
err := userAPI.PerformDeviceDeletion(req.Context(), &api.PerformDeviceDeletionRequest{

90
clientapi/routing/membership.go
View file

@ -27,6 +27,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/threepid"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/roomserver/api"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
@ -38,9 +39,9 @@ import (
var errMissingUserID = errors.New("'user_id' must be supplied")
func SendBan(
req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device,
req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
roomID string, cfg *config.ClientAPI,
rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
) util.JSONResponse {
body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
if reqErr != nil {
@ -80,10 +81,10 @@ func SendBan(
return sendMembership(req.Context(), profileAPI, device, roomID, "ban", body.Reason, cfg, body.UserID, evTime, roomVer, rsAPI, asAPI)
}
func sendMembership(ctx context.Context, profileAPI userapi.UserProfileAPI, device *userapi.Device,
func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, device *userapi.Device,
roomID, membership, reason string, cfg *config.ClientAPI, targetUserID string, evTime time.Time,
roomVer gomatrixserverlib.RoomVersion,
rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI) util.JSONResponse {
rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI) util.JSONResponse {
event, err := buildMembershipEvent(
ctx, targetUserID, reason, profileAPI, device, membership,
@ -124,9 +125,9 @@ func sendMembership(ctx context.Context, profileAPI userapi.UserProfileAPI, devi
}
func SendKick(
req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device,
req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
roomID string, cfg *config.ClientAPI,
rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
) util.JSONResponse {
body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
if reqErr != nil {
@ -164,9 +165,9 @@ func SendKick(
}
func SendUnban(
req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device,
req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
roomID string, cfg *config.ClientAPI,
rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
) util.JSONResponse {
body, evTime, roomVer, reqErr := extractRequestData(req, roomID, rsAPI)
if reqErr != nil {
@ -187,6 +188,12 @@ func SendUnban(
if err != nil {
return util.ErrorResponse(err)
}
if !queryRes.RoomExists {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("room does not exist"),
}
}
// unban is only valid if the user is currently banned
if queryRes.Membership != "ban" {
return util.JSONResponse{
@ -199,9 +206,9 @@ func SendUnban(
}
func SendInvite(
req *http.Request, profileAPI userapi.UserProfileAPI, device *userapi.Device,
req *http.Request, profileAPI userapi.ClientUserAPI, device *userapi.Device,
roomID string, cfg *config.ClientAPI,
rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
) util.JSONResponse {
body, evTime, _, reqErr := extractRequestData(req, roomID, rsAPI)
if reqErr != nil {
@ -233,12 +240,12 @@ func SendInvite(
// sendInvite sends an invitation to a user. Returns a JSONResponse and an error
func sendInvite(
ctx context.Context,
profileAPI userapi.UserProfileAPI,
profileAPI userapi.ClientUserAPI,
device *userapi.Device,
roomID, userID, reason string,
cfg *config.ClientAPI,
rsAPI roomserverAPI.RoomserverInternalAPI,
asAPI appserviceAPI.AppServiceQueryAPI, evTime time.Time,
rsAPI roomserverAPI.ClientRoomserverAPI,
asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time,
) (util.JSONResponse, error) {
event, err := buildMembershipEvent(
ctx, userID, reason, profileAPI, device, "invite",
@ -259,37 +266,36 @@ func sendInvite(
return jsonerror.InternalServerError(), err
}
err = roomserverAPI.SendInvite(
ctx, rsAPI,
event,
nil, // ask the roomserver to draw up invite room state for us
cfg.Matrix.ServerName,
nil,
)
switch e := err.(type) {
case *roomserverAPI.PerformError:
return e.JSONResponse(), err
case nil:
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
}, nil
default:
util.GetLogger(ctx).WithError(err).Error("roomserverAPI.SendInvite failed")
var inviteRes api.PerformInviteResponse
if err := rsAPI.PerformInvite(ctx, &api.PerformInviteRequest{
Event: event,
InviteRoomState: nil, // ask the roomserver to draw up invite room state for us
RoomVersion: event.RoomVersion,
SendAsServer: string(cfg.Matrix.ServerName),
}, &inviteRes); err != nil {
util.GetLogger(ctx).WithError(err).Error("PerformInvite failed")
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: jsonerror.InternalServerError(),
}, err
}
if inviteRes.Error != nil {
return inviteRes.Error.JSONResponse(), inviteRes.Error
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
}, nil
}
func buildMembershipEvent(
ctx context.Context,
targetUserID, reason string, profileAPI userapi.UserProfileAPI,
targetUserID, reason string, profileAPI userapi.ClientUserAPI,
device *userapi.Device,
membership, roomID string, isDirect bool,
cfg *config.ClientAPI, evTime time.Time,
rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI,
rsAPI roomserverAPI.ClientRoomserverAPI, asAPI appserviceAPI.AppServiceInternalAPI,
) (*gomatrixserverlib.HeaderedEvent, error) {
profile, err := loadProfile(ctx, targetUserID, cfg, profileAPI, asAPI)
if err != nil {
@ -326,8 +332,8 @@ func loadProfile(
ctx context.Context,
userID string,
cfg *config.ClientAPI,
profileAPI userapi.UserProfileAPI,
asAPI appserviceAPI.AppServiceQueryAPI,
profileAPI userapi.ClientUserAPI,
asAPI appserviceAPI.AppServiceInternalAPI,
) (*authtypes.Profile, error) {
_, serverName, err := gomatrixserverlib.SplitID('@', userID)
if err != nil {
@ -344,7 +350,7 @@ func loadProfile(
return profile, err
}
func extractRequestData(req *http.Request, roomID string, rsAPI roomserverAPI.RoomserverInternalAPI) (
func extractRequestData(req *http.Request, roomID string, rsAPI roomserverAPI.ClientRoomserverAPI) (
body *threepid.MembershipRequest, evTime time.Time, roomVer gomatrixserverlib.RoomVersion, resErr *util.JSONResponse,
) {
verReq := roomserverAPI.QueryRoomVersionForRoomRequest{RoomID: roomID}
@ -379,8 +385,8 @@ func checkAndProcessThreepid(
device *userapi.Device,
body *threepid.MembershipRequest,
cfg *config.ClientAPI,
rsAPI roomserverAPI.RoomserverInternalAPI,
profileAPI userapi.UserProfileAPI,
rsAPI roomserverAPI.ClientRoomserverAPI,
profileAPI userapi.ClientUserAPI,
roomID string,
evTime time.Time,
) (inviteStored bool, errRes *util.JSONResponse) {
@ -418,7 +424,7 @@ func checkAndProcessThreepid(
return
}
func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI, userID, roomID string) *util.JSONResponse {
func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, userID, roomID string) *util.JSONResponse {
tuple := gomatrixserverlib.StateKeyTuple{
EventType: gomatrixserverlib.MRoomMember,
StateKey: userID,
@ -457,7 +463,7 @@ func checkMemberInRoom(ctx context.Context, rsAPI roomserverAPI.RoomserverIntern
func SendForget(
req *http.Request, device *userapi.Device,
roomID string, rsAPI roomserverAPI.RoomserverInternalAPI,
roomID string, rsAPI roomserverAPI.ClientRoomserverAPI,
) util.JSONResponse {
ctx := req.Context()
logger := util.GetLogger(ctx).WithField("roomID", roomID).WithField("userID", device.UserID)
@ -471,6 +477,12 @@ func SendForget(
logger.WithError(err).Error("QueryMembershipForUser: could not query membership for user")
return jsonerror.InternalServerError()
}
if !membershipRes.RoomExists {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("room does not exist"),
}
}
if membershipRes.IsInRoom {
return util.JSONResponse{
Code: http.StatusBadRequest,

4
clientapi/routing/memberships.go
View file

@ -55,7 +55,7 @@ type databaseJoinedMember struct {
func GetMemberships(
req *http.Request, device *userapi.Device, roomID string, joinedOnly bool,
_ *config.ClientAPI,
rsAPI api.RoomserverInternalAPI,
rsAPI api.ClientRoomserverAPI,
) util.JSONResponse {
queryReq := api.QueryMembershipsForRoomRequest{
JoinedOnly: joinedOnly,
@ -100,7 +100,7 @@ func GetMemberships(
func GetJoinedRooms(
req *http.Request,
device *userapi.Device,
rsAPI api.RoomserverInternalAPI,
rsAPI api.ClientRoomserverAPI,
) util.JSONResponse {
var res api.QueryRoomsForUserResponse
err := rsAPI.QueryRoomsForUser(req.Context(), &api.QueryRoomsForUserRequest{

2
clientapi/routing/notification.go
View file

@ -27,7 +27,7 @@ import (
// GetNotifications handles /_matrix/client/r0/notifications
func GetNotifications(
req *http.Request, device *userapi.Device,
userAPI userapi.UserInternalAPI,
userAPI userapi.ClientUserAPI,
) util.JSONResponse {
var limit int64
if limitStr := req.URL.Query().Get("limit"); limitStr != "" {

2
clientapi/routing/openid.go
View file

@ -34,7 +34,7 @@ type openIDTokenResponse struct {
// can supply to an OpenID Relying Party to verify their identity
func CreateOpenIDToken(
req *http.Request,
userAPI api.UserInternalAPI,
userAPI api.ClientUserAPI,
device *api.Device,
userID string,
cfg *config.ClientAPI,

2
clientapi/routing/password.go
View file

@ -28,7 +28,7 @@ type newPasswordAuth struct {
func Password(
req *http.Request,
userAPI api.UserInternalAPI,
userAPI api.ClientUserAPI,
device *api.Device,
cfg *config.ClientAPI,
) util.JSONResponse {

13
clientapi/routing/peekroom.go
View file

@ -17,6 +17,7 @@ package routing
import (
"net/http"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
@ -26,7 +27,7 @@ import (
func PeekRoomByIDOrAlias(
req *http.Request,
device *api.Device,
rsAPI roomserverAPI.RoomserverInternalAPI,
rsAPI roomserverAPI.ClientRoomserverAPI,
roomIDOrAlias string,
) util.JSONResponse {
// if this is a remote roomIDOrAlias, we have to ask the roomserver (or federation sender?) to
@ -54,7 +55,9 @@ func PeekRoomByIDOrAlias(
}
// Ask the roomserver to perform the peek.
rsAPI.PerformPeek(req.Context(), &peekReq, &peekRes)
if err := rsAPI.PerformPeek(req.Context(), &peekReq, &peekRes); err != nil {
return util.ErrorResponse(err)
}
if peekRes.Error != nil {
return peekRes.Error.JSONResponse()
}
@ -79,7 +82,7 @@ func PeekRoomByIDOrAlias(
func UnpeekRoomByID(
req *http.Request,
device *api.Device,
rsAPI roomserverAPI.RoomserverInternalAPI,
rsAPI roomserverAPI.ClientRoomserverAPI,
roomID string,
) util.JSONResponse {
unpeekReq := roomserverAPI.PerformUnpeekRequest{
@ -89,7 +92,9 @@ func UnpeekRoomByID(
}
unpeekRes := roomserverAPI.PerformUnpeekResponse{}
rsAPI.PerformUnpeek(req.Context(), &unpeekReq, &unpeekRes)
if err := rsAPI.PerformUnpeek(req.Context(), &unpeekReq, &unpeekRes); err != nil {
return jsonerror.InternalAPIError(req.Context(), err)
}
if unpeekRes.Error != nil {
return unpeekRes.Error.JSONResponse()
}

26
clientapi/routing/profile.go
View file

@ -35,9 +35,9 @@ import (
// GetProfile implements GET /profile/{userID}
func GetProfile(
req *http.Request, profileAPI userapi.UserProfileAPI, cfg *config.ClientAPI,
req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
userID string,
asAPI appserviceAPI.AppServiceQueryAPI,
asAPI appserviceAPI.AppServiceInternalAPI,
federation *gomatrixserverlib.FederationClient,
) util.JSONResponse {
profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation)
@ -64,8 +64,8 @@ func GetProfile(
// GetAvatarURL implements GET /profile/{userID}/avatar_url
func GetAvatarURL(
req *http.Request, profileAPI userapi.UserProfileAPI, cfg *config.ClientAPI,
userID string, asAPI appserviceAPI.AppServiceQueryAPI,
req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
userID string, asAPI appserviceAPI.AppServiceInternalAPI,
federation *gomatrixserverlib.FederationClient,
) util.JSONResponse {
profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation)
@ -91,8 +91,8 @@ func GetAvatarURL(
// SetAvatarURL implements PUT /profile/{userID}/avatar_url
func SetAvatarURL(
req *http.Request, profileAPI userapi.UserProfileAPI,
device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI,
req *http.Request, profileAPI userapi.ClientUserAPI,
device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.ClientRoomserverAPI,
) util.JSONResponse {
if userID != device.UserID {
return util.JSONResponse{
@ -193,8 +193,8 @@ func SetAvatarURL(
// GetDisplayName implements GET /profile/{userID}/displayname
func GetDisplayName(
req *http.Request, profileAPI userapi.UserProfileAPI, cfg *config.ClientAPI,
userID string, asAPI appserviceAPI.AppServiceQueryAPI,
req *http.Request, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
userID string, asAPI appserviceAPI.AppServiceInternalAPI,
federation *gomatrixserverlib.FederationClient,
) util.JSONResponse {
profile, err := getProfile(req.Context(), profileAPI, cfg, userID, asAPI, federation)
@ -220,8 +220,8 @@ func GetDisplayName(
// SetDisplayName implements PUT /profile/{userID}/displayname
func SetDisplayName(
req *http.Request, profileAPI userapi.UserProfileAPI,
device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.RoomserverInternalAPI,
req *http.Request, profileAPI userapi.ClientUserAPI,
device *userapi.Device, userID string, cfg *config.ClientAPI, rsAPI api.ClientRoomserverAPI,
) util.JSONResponse {
if userID != device.UserID {
return util.JSONResponse{
@ -325,9 +325,9 @@ func SetDisplayName(
// Returns an error when something goes wrong or specifically
// eventutil.ErrProfileNoExists when the profile doesn't exist.
func getProfile(
ctx context.Context, profileAPI userapi.UserProfileAPI, cfg *config.ClientAPI,
ctx context.Context, profileAPI userapi.ClientUserAPI, cfg *config.ClientAPI,
userID string,
asAPI appserviceAPI.AppServiceQueryAPI,
asAPI appserviceAPI.AppServiceInternalAPI,
federation *gomatrixserverlib.FederationClient,
) (*authtypes.Profile, error) {
localpart, domain, err := gomatrixserverlib.SplitID('@', userID)
@ -366,7 +366,7 @@ func buildMembershipEvents(
ctx context.Context,
roomIDs []string,
newProfile authtypes.Profile, userID string, cfg *config.ClientAPI,
evTime time.Time, rsAPI api.RoomserverInternalAPI,
evTime time.Time, rsAPI api.ClientRoomserverAPI,
) ([]*gomatrixserverlib.HeaderedEvent, error) {
evs := []*gomatrixserverlib.HeaderedEvent{}

4
clientapi/routing/pusher.go
View file

@ -28,7 +28,7 @@ import (
// GetPushers handles /_matrix/client/r0/pushers
func GetPushers(
req *http.Request, device *userapi.Device,
userAPI userapi.UserInternalAPI,
userAPI userapi.ClientUserAPI,
) util.JSONResponse {
var queryRes userapi.QueryPushersResponse
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
@ -57,7 +57,7 @@ func GetPushers(
// The behaviour of this endpoint varies depending on the values in the JSON body.
func SetPusher(
req *http.Request, device *userapi.Device,
userAPI userapi.UserInternalAPI,
userAPI userapi.ClientUserAPI,
) util.JSONResponse {
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {

20
clientapi/routing/pushrules.go
View file

@ -30,7 +30,7 @@ func errorResponse(ctx context.Context, err error, msg string, args ...interface
return jsonerror.InternalServerError()
}
func GetAllPushRules(ctx context.Context, device *userapi.Device, userAPI userapi.UserInternalAPI) util.JSONResponse {
func GetAllPushRules(ctx context.Context, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
if err != nil {
return errorResponse(ctx, err, "queryPushRulesJSON failed")
@ -41,7 +41,7 @@ func GetAllPushRules(ctx context.Context, device *userapi.Device, userAPI userap
}
}
func GetPushRulesByScope(ctx context.Context, scope string, device *userapi.Device, userAPI userapi.UserInternalAPI) util.JSONResponse {
func GetPushRulesByScope(ctx context.Context, scope string, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
if err != nil {
return errorResponse(ctx, err, "queryPushRulesJSON failed")
@ -56,7 +56,7 @@ func GetPushRulesByScope(ctx context.Context, scope string, device *userapi.Devi
}
}
func GetPushRulesByKind(ctx context.Context, scope, kind string, device *userapi.Device, userAPI userapi.UserInternalAPI) util.JSONResponse {
func GetPushRulesByKind(ctx context.Context, scope, kind string, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
if err != nil {
return errorResponse(ctx, err, "queryPushRules failed")
@ -75,7 +75,7 @@ func GetPushRulesByKind(ctx context.Context, scope, kind string, device *userapi
}
}
func GetPushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, device *userapi.Device, userAPI userapi.UserInternalAPI) util.JSONResponse {
func GetPushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
if err != nil {
return errorResponse(ctx, err, "queryPushRules failed")
@ -98,7 +98,7 @@ func GetPushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, device
}
}
func PutPushRuleByRuleID(ctx context.Context, scope, kind, ruleID, afterRuleID, beforeRuleID string, body io.Reader, device *userapi.Device, userAPI userapi.UserInternalAPI) util.JSONResponse {
func PutPushRuleByRuleID(ctx context.Context, scope, kind, ruleID, afterRuleID, beforeRuleID string, body io.Reader, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
var newRule pushrules.Rule
if err := json.NewDecoder(body).Decode(&newRule); err != nil {
return errorResponse(ctx, err, "JSON Decode failed")
@ -160,7 +160,7 @@ func PutPushRuleByRuleID(ctx context.Context, scope, kind, ruleID, afterRuleID,
return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}}
}
func DeletePushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, device *userapi.Device, userAPI userapi.UserInternalAPI) util.JSONResponse {
func DeletePushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
ruleSets, err := queryPushRules(ctx, device.UserID, userAPI)
if err != nil {
return errorResponse(ctx, err, "queryPushRules failed")
@ -187,7 +187,7 @@ func DeletePushRuleByRuleID(ctx context.Context, scope, kind, ruleID string, dev
return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}}
}
func GetPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr string, device *userapi.Device, userAPI userapi.UserInternalAPI) util.JSONResponse {
func GetPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr string, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
attrGet, err := pushRuleAttrGetter(attr)
if err != nil {
return errorResponse(ctx, err, "pushRuleAttrGetter failed")
@ -216,7 +216,7 @@ func GetPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr stri
}
}
func PutPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr string, body io.Reader, device *userapi.Device, userAPI userapi.UserInternalAPI) util.JSONResponse {
func PutPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr string, body io.Reader, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
var newPartialRule pushrules.Rule
if err := json.NewDecoder(body).Decode(&newPartialRule); err != nil {
return util.JSONResponse{
@ -266,7 +266,7 @@ func PutPushRuleAttrByRuleID(ctx context.Context, scope, kind, ruleID, attr stri
return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}}
}
func queryPushRules(ctx context.Context, userID string, userAPI userapi.UserInternalAPI) (*pushrules.AccountRuleSets, error) {
func queryPushRules(ctx context.Context, userID string, userAPI userapi.ClientUserAPI) (*pushrules.AccountRuleSets, error) {
var res userapi.QueryPushRulesResponse
if err := userAPI.QueryPushRules(ctx, &userapi.QueryPushRulesRequest{UserID: userID}, &res); err != nil {
util.GetLogger(ctx).WithError(err).Error("userAPI.QueryPushRules failed")
@ -275,7 +275,7 @@ func queryPushRules(ctx context.Context, userID string, userAPI userapi.UserInte
return res.RuleSets, nil
}
func putPushRules(ctx context.Context, userID string, ruleSets *pushrules.AccountRuleSets, userAPI userapi.UserInternalAPI) error {
func putPushRules(ctx context.Context, userID string, ruleSets *pushrules.AccountRuleSets, userAPI userapi.ClientUserAPI) error {
req := userapi.PerformPushRulesPutRequest{
UserID: userID,
RuleSets: ruleSets,

22
clientapi/routing/redaction.go
View file

@ -22,6 +22,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/internal/transactions"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
@ -39,13 +40,22 @@ type redactionResponse struct {
func SendRedaction(
req *http.Request, device *userapi.Device, roomID, eventID string, cfg *config.ClientAPI,
rsAPI roomserverAPI.RoomserverInternalAPI,
rsAPI roomserverAPI.ClientRoomserverAPI,
txnID *string,
txnCache *transactions.Cache,
) util.JSONResponse {
resErr := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
if resErr != nil {
return *resErr
}
if txnID != nil {
// Try to fetch response from transactionsCache
if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok {
return *res
}
}
ev := roomserverAPI.GetEvent(req.Context(), rsAPI, eventID)
if ev == nil {
return util.JSONResponse{
@ -124,10 +134,18 @@ func SendRedaction(
util.GetLogger(req.Context()).WithError(err).Errorf("failed to SendEvents")
return jsonerror.InternalServerError()
}
return util.JSONResponse{
res := util.JSONResponse{
Code: 200,
JSON: redactionResponse{
EventID: e.EventID(),
},
}
// Add response to transactionsCache
if txnID != nil {
txnCache.AddTransaction(device.AccessToken, *txnID, &res)
}
return res
}

107
clientapi/routing/register.go
View file

@ -19,7 +19,7 @@ import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"io"
"net/http"
"net/url"
"regexp"
@ -29,9 +29,10 @@ import (
"sync"
"time"
"github.com/tidwall/gjson"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/setup/config"
"github.com/tidwall/gjson"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/gomatrixserverlib/tokens"
@ -68,9 +69,10 @@ const (
// It shouldn't be passed by value because it contains a mutex.
type sessionsDict struct {
sync.RWMutex
sessions map[string][]authtypes.LoginType
params map[string]registerRequest
timer map[string]*time.Timer
sessions map[string][]authtypes.LoginType
sessionCompletedResult map[string]registerResponse
params map[string]registerRequest
timer map[string]*time.Timer
// deleteSessionToDeviceID protects requests to DELETE /devices/{deviceID} from being abused.
// If a UIA session is started by trying to delete device1, and then UIA is completed by deleting device2,
// the delete request will fail for device2 since the UIA was initiated by trying to delete device1.
@ -115,6 +117,7 @@ func (d *sessionsDict) deleteSession(sessionID string) {
delete(d.params, sessionID)
delete(d.sessions, sessionID)
delete(d.deleteSessionToDeviceID, sessionID)
delete(d.sessionCompletedResult, sessionID)
// stop the timer, e.g. because the registration was completed
if t, ok := d.timer[sessionID]; ok {
if !t.Stop() {
@ -130,6 +133,7 @@ func (d *sessionsDict) deleteSession(sessionID string) {
func newSessionsDict() *sessionsDict {
return &sessionsDict{
sessions: make(map[string][]authtypes.LoginType),
sessionCompletedResult: make(map[string]registerResponse),
params: make(map[string]registerRequest),
timer: make(map[string]*time.Timer),
deleteSessionToDeviceID: make(map[string]string),
@ -173,6 +177,19 @@ func (d *sessionsDict) addDeviceToDelete(sessionID, deviceID string) {
d.deleteSessionToDeviceID[sessionID] = deviceID
}
func (d *sessionsDict) addCompletedRegistration(sessionID string, response registerResponse) {
d.Lock()
defer d.Unlock()
d.sessionCompletedResult[sessionID] = response
}
func (d *sessionsDict) getCompletedRegistration(sessionID string) (registerResponse, bool) {
d.RLock()
defer d.RUnlock()
result, ok := d.sessionCompletedResult[sessionID]
return result, ok
}
func (d *sessionsDict) getDeviceToDelete(sessionID string) (string, bool) {
d.RLock()
defer d.RUnlock()
@ -259,19 +276,19 @@ type recaptchaResponse struct {
}
// validateUsername returns an error response if the username is invalid
func validateUsername(username string) *util.JSONResponse {
func validateUsername(localpart string, domain gomatrixserverlib.ServerName) *util.JSONResponse {
// https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
if len(username) > maxUsernameLength {
if id := fmt.Sprintf("@%s:%s", localpart, domain); len(id) > maxUsernameLength {
return &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON(fmt.Sprintf("'username' >%d characters", maxUsernameLength)),
JSON: jsonerror.BadJSON(fmt.Sprintf("%q exceeds the maximum length of %d characters", id, maxUsernameLength)),
}
} else if !validUsernameRegex.MatchString(username) {
} else if !validUsernameRegex.MatchString(localpart) {
return &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./='"),
}
} else if username[0] == '_' { // Regex checks its not a zero length string
} else if localpart[0] == '_' { // Regex checks its not a zero length string
return &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.InvalidUsername("Username cannot start with a '_'"),
@ -281,13 +298,13 @@ func validateUsername(username string) *util.JSONResponse {
}
// validateApplicationServiceUsername returns an error response if the username is invalid for an application service
func validateApplicationServiceUsername(username string) *util.JSONResponse {
if len(username) > maxUsernameLength {
func validateApplicationServiceUsername(localpart string, domain gomatrixserverlib.ServerName) *util.JSONResponse {
if id := fmt.Sprintf("@%s:%s", localpart, domain); len(id) > maxUsernameLength {
return &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON(fmt.Sprintf("'username' >%d characters", maxUsernameLength)),
JSON: jsonerror.BadJSON(fmt.Sprintf("%q exceeds the maximum length of %d characters", id, maxUsernameLength)),
}
} else if !validUsernameRegex.MatchString(username) {
} else if !validUsernameRegex.MatchString(localpart) {
return &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./='"),
@ -354,7 +371,7 @@ func validateRecaptcha(
// Grab the body of the response from the captcha server
var r recaptchaResponse
body, err := ioutil.ReadAll(resp.Body)
body, err := io.ReadAll(resp.Body)
if err != nil {
return &util.JSONResponse{
Code: http.StatusGatewayTimeout,
@ -506,7 +523,7 @@ func validateApplicationService(
}
// Check username application service is trying to register is valid
if err := validateApplicationServiceUsername(username); err != nil {
if err := validateApplicationServiceUsername(username, cfg.Matrix.ServerName); err != nil {
return "", err
}
@ -518,11 +535,11 @@ func validateApplicationService(
// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
func Register(
req *http.Request,
userAPI userapi.UserRegisterAPI,
userAPI userapi.ClientUserAPI,
cfg *config.ClientAPI,
) util.JSONResponse {
defer req.Body.Close() // nolint: errcheck
reqBody, err := ioutil.ReadAll(req.Body)
reqBody, err := io.ReadAll(req.Body)
if err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
@ -544,6 +561,14 @@ func Register(
r.DeviceID = data.DeviceID
r.InitialDisplayName = data.InitialDisplayName
r.InhibitLogin = data.InhibitLogin
// Check if the user already registered using this session, if so, return that result
if response, ok := sessions.getCompletedRegistration(sessionID); ok {
return util.JSONResponse{
Code: http.StatusOK,
JSON: response,
}
}
}
if resErr := httputil.UnmarshalJSON(reqBody, &r); resErr != nil {
return *resErr
@ -579,7 +604,7 @@ func Register(
case r.Type == authtypes.LoginTypeApplicationService && accessTokenErr == nil:
// Spec-compliant case (the access_token is specified and the login type
// is correctly set, so it's an appservice registration)
if resErr := validateApplicationServiceUsername(r.Username); resErr != nil {
if resErr := validateApplicationServiceUsername(r.Username, cfg.Matrix.ServerName); resErr != nil {
return *resErr
}
case accessTokenErr == nil:
@ -592,7 +617,7 @@ func Register(
default:
// Spec-compliant case (neither the access_token nor the login type are
// specified, so it's a normal user registration)
if resErr := validateUsername(r.Username); resErr != nil {
if resErr := validateUsername(r.Username, cfg.Matrix.ServerName); resErr != nil {
return *resErr
}
}
@ -614,7 +639,7 @@ func handleGuestRegistration(
req *http.Request,
r registerRequest,
cfg *config.ClientAPI,
userAPI userapi.UserRegisterAPI,
userAPI userapi.ClientUserAPI,
) util.JSONResponse {
if cfg.RegistrationDisabled || cfg.GuestsDisabled {
return util.JSONResponse{
@ -679,7 +704,7 @@ func handleRegistrationFlow(
r registerRequest,
sessionID string,
cfg *config.ClientAPI,
userAPI userapi.UserRegisterAPI,
userAPI userapi.ClientUserAPI,
accessToken string,
accessTokenErr error,
) util.JSONResponse {
@ -768,7 +793,7 @@ func handleApplicationServiceRegistration(
req *http.Request,
r registerRequest,
cfg *config.ClientAPI,
userAPI userapi.UserRegisterAPI,
userAPI userapi.ClientUserAPI,
) util.JSONResponse {
// Check if we previously had issues extracting the access token from the
// request.
@ -806,7 +831,7 @@ func checkAndCompleteFlow(
r registerRequest,
sessionID string,
cfg *config.ClientAPI,
userAPI userapi.UserRegisterAPI,
userAPI userapi.ClientUserAPI,
) util.JSONResponse {
if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
// This flow was completed, registration can continue
@ -833,19 +858,12 @@ func checkAndCompleteFlow(
// not all
func completeRegistration(
ctx context.Context,
userAPI userapi.UserRegisterAPI,
userAPI userapi.ClientUserAPI,
username, password, appserviceID, ipAddr, userAgent, sessionID string,
inhibitLogin eventutil.WeakBoolean,
displayName, deviceID *string,
accType userapi.AccountType,
) util.JSONResponse {
var registrationOK bool
defer func() {
if registrationOK {
sessions.deleteSession(sessionID)
}
}()
if username == "" {
return util.JSONResponse{
Code: http.StatusBadRequest,
@ -886,7 +904,6 @@ func completeRegistration(
// Check whether inhibit_login option is set. If so, don't create an access
// token or a device for this user
if inhibitLogin {
registrationOK = true
return util.JSONResponse{
Code: http.StatusOK,
JSON: registerResponse{
@ -920,15 +937,17 @@ func completeRegistration(
}
}
registrationOK = true
result := registerResponse{
UserID: devRes.Device.UserID,
AccessToken: devRes.Device.AccessToken,
HomeServer: accRes.Account.ServerName,
DeviceID: devRes.Device.ID,
}
sessions.addCompletedRegistration(sessionID, result)
return util.JSONResponse{
Code: http.StatusOK,
JSON: registerResponse{
UserID: devRes.Device.UserID,
AccessToken: devRes.Device.AccessToken,
HomeServer: accRes.Account.ServerName,
DeviceID: devRes.Device.ID,
},
JSON: result,
}
}
@ -992,14 +1011,14 @@ type availableResponse struct {
func RegisterAvailable(
req *http.Request,
cfg *config.ClientAPI,
registerAPI userapi.UserRegisterAPI,
registerAPI userapi.ClientUserAPI,
) util.JSONResponse {
username := req.URL.Query().Get("username")
// Squash username to all lowercase letters
username = strings.ToLower(username)
if err := validateUsername(username); err != nil {
if err := validateUsername(username, cfg.Matrix.ServerName); err != nil {
return *err
}
@ -1040,7 +1059,7 @@ func RegisterAvailable(
}
}
func handleSharedSecretRegistration(userAPI userapi.UserInternalAPI, sr *SharedSecretRegistration, req *http.Request) util.JSONResponse {
func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.ClientUserAPI, sr *SharedSecretRegistration, req *http.Request) util.JSONResponse {
ssrr, err := NewSharedSecretRegistrationRequest(req.Body)
if err != nil {
return util.JSONResponse{
@ -1061,7 +1080,7 @@ func handleSharedSecretRegistration(userAPI userapi.UserInternalAPI, sr *SharedS
// downcase capitals
ssrr.User = strings.ToLower(ssrr.User)
if resErr := validateUsername(ssrr.User); resErr != nil {
if resErr := validateUsername(ssrr.User, cfg.Matrix.ServerName); resErr != nil {
return *resErr
}
if resErr := validatePassword(ssrr.Password); resErr != nil {

4
clientapi/routing/register_secret_test.go
View file

@ -2,7 +2,7 @@ package routing
import (
"bytes"
"io/ioutil"
"io"
"testing"
"github.com/patrickmn/go-cache"
@ -13,7 +13,7 @@ func TestSharedSecretRegister(t *testing.T) {
jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice"}`)
sharedSecret := "dendritetest"
req, err := NewSharedSecretRegistrationRequest(ioutil.NopCloser(bytes.NewBuffer(jsonStr)))
req, err := NewSharedSecretRegistrationRequest(io.NopCloser(bytes.NewBuffer(jsonStr)))
if err != nil {
t.Fatalf("failed to read request: %s", err)
}

5
clientapi/routing/register_test.go
View file

@ -181,7 +181,10 @@ func TestValidationOfApplicationServices(t *testing.T) {
// Set up a config
fakeConfig := &config.Dendrite{}
fakeConfig.Defaults(true)
fakeConfig.Defaults(config.DefaultOpts{
Generate: true,
Monolithic: true,
})
fakeConfig.Global.ServerName = "localhost"
fakeConfig.ClientAPI.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService}

21
clientapi/routing/room_tagging.go
View file

@ -18,8 +18,6 @@ import (
"encoding/json"
"net/http"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/producers"
@ -31,7 +29,7 @@ import (
// GetTags implements GET /_matrix/client/r0/user/{userID}/rooms/{roomID}/tags
func GetTags(
req *http.Request,
userAPI api.UserInternalAPI,
userAPI api.ClientUserAPI,
device *api.Device,
userID string,
roomID string,
@ -62,7 +60,7 @@ func GetTags(
// the tag to the "map" and saving the new "map" to the DB
func PutTag(
req *http.Request,
userAPI api.UserInternalAPI,
userAPI api.ClientUserAPI,
device *api.Device,
userID string,
roomID string,
@ -98,10 +96,6 @@ func PutTag(
return jsonerror.InternalServerError()
}
if err = syncProducer.SendData(userID, roomID, "m.tag", nil, nil); err != nil {
logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
@ -113,7 +107,7 @@ func PutTag(
// the "map" and then saving the new "map" in the DB
func DeleteTag(
req *http.Request,
userAPI api.UserInternalAPI,
userAPI api.ClientUserAPI,
device *api.Device,
userID string,
roomID string,
@ -150,11 +144,6 @@ func DeleteTag(
return jsonerror.InternalServerError()
}
// TODO: user API should do this since it's account data
if err := syncProducer.SendData(userID, roomID, "m.tag", nil, nil); err != nil {
logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
@ -167,7 +156,7 @@ func obtainSavedTags(
req *http.Request,
userID string,
roomID string,
userAPI api.UserInternalAPI,
userAPI api.ClientUserAPI,
) (tags gomatrix.TagContent, err error) {
dataReq := api.QueryAccountDataRequest{
UserID: userID,
@ -194,7 +183,7 @@ func saveTagData(
req *http.Request,
userID string,
roomID string,
userAPI api.UserInternalAPI,
userAPI api.ClientUserAPI,
Tag gomatrix.TagContent,
) error {
newTagData, err := json.Marshal(Tag)

121
clientapi/routing/routing.go
View file

@ -48,16 +48,17 @@ import (
// applied:
// nolint: gocyclo
func Setup(
publicAPIMux, synapseAdminRouter *mux.Router, cfg *config.ClientAPI,
rsAPI roomserverAPI.RoomserverInternalAPI,
asAPI appserviceAPI.AppServiceQueryAPI,
userAPI userapi.UserInternalAPI,
userDirectoryProvider userapi.UserDirectoryProvider,
publicAPIMux, wkMux, synapseAdminRouter, dendriteAdminRouter *mux.Router,
cfg *config.ClientAPI,
rsAPI roomserverAPI.ClientRoomserverAPI,
asAPI appserviceAPI.AppServiceInternalAPI,
userAPI userapi.ClientUserAPI,
userDirectoryProvider userapi.QuerySearchProfilesAPI,
federation *gomatrixserverlib.FederationClient,
syncProducer *producers.SyncAPIProducer,
transactionsCache *transactions.Cache,
federationSender federationAPI.FederationInternalAPI,
keyAPI keyserverAPI.KeyInternalAPI,
federationSender federationAPI.ClientFederationAPI,
keyAPI keyserverAPI.ClientKeyAPI,
extRoomsProvider api.ExtraPublicRoomsProvider,
mscCfg *config.MSCs, natsClient *nats.Conn,
) {
@ -73,6 +74,26 @@ func Setup(
unstableFeatures["org.matrix."+msc] = true
}
if cfg.Matrix.WellKnownClientName != "" {
logrus.Infof("Setting m.homeserver base_url as %s at /.well-known/matrix/client", cfg.Matrix.WellKnownClientName)
wkMux.Handle("/client", httputil.MakeExternalAPI("wellknown", func(r *http.Request) util.JSONResponse {
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct {
HomeserverName struct {
BaseUrl string `json:"base_url"`
} `json:"m.homeserver"`
}{
HomeserverName: struct {
BaseUrl string `json:"base_url"`
}{
BaseUrl: cfg.Matrix.WellKnownClientName,
},
},
}
})).Methods(http.MethodGet, http.MethodOptions)
}
publicAPIMux.Handle("/versions",
httputil.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse {
return util.JSONResponse{
@ -88,6 +109,9 @@ func Setup(
"r0.4.0",
"r0.5.0",
"r0.6.1",
"v1.0",
"v1.1",
"v1.2",
}, UnstableFeatures: unstableFeatures},
}
}),
@ -109,7 +133,7 @@ func Setup(
}
}
if req.Method == http.MethodPost {
return handleSharedSecretRegistration(userAPI, sr, req)
return handleSharedSecretRegistration(cfg, userAPI, sr, req)
}
return util.JSONResponse{
Code: http.StatusMethodNotAllowed,
@ -119,6 +143,24 @@ func Setup(
).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
}
dendriteAdminRouter.Handle("/admin/evacuateRoom/{roomID}",
httputil.MakeAdminAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return AdminEvacuateRoom(req, cfg, device, rsAPI)
}),
).Methods(http.MethodGet, http.MethodOptions)
dendriteAdminRouter.Handle("/admin/evacuateUser/{userID}",
httputil.MakeAdminAPI("admin_evacuate_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return AdminEvacuateUser(req, cfg, device, rsAPI)
}),
).Methods(http.MethodGet, http.MethodOptions)
dendriteAdminRouter.Handle("/admin/resetPassword/{localpart}",
httputil.MakeAdminAPI("admin_reset_password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return AdminResetPassword(req, cfg, device, userAPI)
}),
).Methods(http.MethodPost, http.MethodOptions)
// server notifications
if cfg.Matrix.ServerNotices.Enabled {
logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
@ -130,7 +172,7 @@ func Setup(
synapseAdminRouter.Handle("/admin/v1/send_server_notice/{txnID}",
httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
// not specced, but ensure we're rate limiting requests to this endpoint
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -150,7 +192,7 @@ func Setup(
synapseAdminRouter.Handle("/admin/v1/send_server_notice",
httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
// not specced, but ensure we're rate limiting requests to this endpoint
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
return SendServerNotice(
@ -180,7 +222,7 @@ func Setup(
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/join/{roomIDOrAlias}",
httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -196,7 +238,7 @@ func Setup(
if mscCfg.Enabled("msc2753") {
v3mux.Handle("/peek/{roomIDOrAlias}",
httputil.MakeAuthAPI(gomatrixserverlib.Peek, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -216,7 +258,7 @@ func Setup(
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/join",
httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -230,7 +272,7 @@ func Setup(
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/leave",
httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -264,7 +306,7 @@ func Setup(
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/invite",
httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -318,7 +360,7 @@ func Setup(
if err != nil {
return util.ErrorResponse(err)
}
return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation)
return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI)
}),
).Methods(http.MethodGet, http.MethodOptions)
@ -382,14 +424,14 @@ func Setup(
).Methods(http.MethodPut, http.MethodOptions)
v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, nil); r != nil {
return *r
}
return Register(req, userAPI, cfg)
})).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, nil); r != nil {
return *r
}
return RegisterAvailable(req, cfg, userAPI)
@ -463,7 +505,7 @@ func Setup(
v3mux.Handle("/rooms/{roomID}/typing/{userID}",
httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -479,7 +521,7 @@ func Setup(
if err != nil {
return util.ErrorResponse(err)
}
return SendRedaction(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI)
return SendRedaction(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, nil, nil)
}),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/redact/{eventID}/{txnId}",
@ -488,7 +530,8 @@ func Setup(
if err != nil {
return util.ErrorResponse(err)
}
return SendRedaction(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI)
txnID := vars["txnId"]
return SendRedaction(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, &txnID, transactionsCache)
}),
).Methods(http.MethodPut, http.MethodOptions)
@ -519,7 +562,7 @@ func Setup(
v3mux.Handle("/account/whoami",
httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
return Whoami(req, device)
@ -528,7 +571,7 @@ func Setup(
v3mux.Handle("/account/password",
httputil.MakeAuthAPI("password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
return Password(req, userAPI, device, cfg)
@ -537,7 +580,7 @@ func Setup(
v3mux.Handle("/account/deactivate",
httputil.MakeAuthAPI("deactivate", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
return Deactivate(req, userInteractiveAuth, userAPI, device)
@ -548,7 +591,7 @@ func Setup(
v3mux.Handle("/login",
httputil.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, nil); r != nil {
return *r
}
return Login(req, userAPI, cfg)
@ -656,7 +699,7 @@ func Setup(
v3mux.Handle("/pushrules/{scope}/{kind}/{ruleID}",
httputil.MakeAuthAPI("push_rules", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -722,7 +765,7 @@ func Setup(
v3mux.Handle("/profile/{userID}/avatar_url",
httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -747,7 +790,7 @@ func Setup(
v3mux.Handle("/profile/{userID}/displayname",
httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -786,7 +829,7 @@ func Setup(
v3mux.Handle("/voip/turnServer",
httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
return RequestTurnServer(req, device, cfg)
@ -865,7 +908,7 @@ func Setup(
v3mux.Handle("/user/{userID}/openid/request_token",
httputil.MakeAuthAPI("openid_request_token", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -878,7 +921,7 @@ func Setup(
v3mux.Handle("/user_directory/search",
httputil.MakeAuthAPI("userdirectory_search", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
postContent := struct {
@ -889,15 +932,15 @@ func Setup(
if resErr := clientutil.UnmarshalJSONRequest(req, &postContent); resErr != nil {
return *resErr
}
return *SearchUserDirectory(
return SearchUserDirectory(
req.Context(),
device,
userAPI,
rsAPI,
userDirectoryProvider,
cfg.Matrix.ServerName,
postContent.SearchString,
postContent.Limit,
federation,
cfg.Matrix.ServerName,
)
}),
).Methods(http.MethodPost, http.MethodOptions)
@ -924,7 +967,7 @@ func Setup(
v3mux.Handle("/rooms/{roomID}/read_markers",
httputil.MakeAuthAPI("rooms_read_markers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -937,7 +980,7 @@ func Setup(
v3mux.Handle("/rooms/{roomID}/forget",
httputil.MakeAuthAPI("rooms_forget", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -1014,7 +1057,7 @@ func Setup(
v3mux.Handle("/pushers/set",
httputil.MakeAuthAPI("set_pushers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
return SetPusher(req, device, userAPI)
@ -1072,7 +1115,7 @@ func Setup(
v3mux.Handle("/capabilities",
httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
return GetCapabilities(req, rsAPI)
@ -1288,7 +1331,7 @@ func Setup(
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}",
httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req); r != nil {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))

61
clientapi/routing/sendevent.go
View file

@ -19,6 +19,7 @@ import (
"encoding/json"
"fmt"
"net/http"
"reflect"
"sync"
"time"
@ -62,15 +63,16 @@ var sendEventDuration = prometheus.NewHistogramVec(
)
// SendEvent implements:
// /rooms/{roomID}/send/{eventType}
// /rooms/{roomID}/send/{eventType}/{txnID}
// /rooms/{roomID}/state/{eventType}/{stateKey}
//
// /rooms/{roomID}/send/{eventType}
// /rooms/{roomID}/send/{eventType}/{txnID}
// /rooms/{roomID}/state/{eventType}/{stateKey}
func SendEvent(
req *http.Request,
device *userapi.Device,
roomID, eventType string, txnID, stateKey *string,
cfg *config.ClientAPI,
rsAPI api.RoomserverInternalAPI,
rsAPI api.ClientRoomserverAPI,
txnCache *transactions.Cache,
) util.JSONResponse {
verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
@ -96,14 +98,28 @@ func SendEvent(
mutex.(*sync.Mutex).Lock()
defer mutex.(*sync.Mutex).Unlock()
startedGeneratingEvent := time.Now()
var r map[string]interface{} // must be a JSON object
resErr := httputil.UnmarshalJSONRequest(req, &r)
if resErr != nil {
return *resErr
}
if stateKey != nil {
// If the existing/new state content are equal, return the existing event_id, making the request idempotent.
if resp := stateEqual(req.Context(), rsAPI, eventType, *stateKey, roomID, r); resp != nil {
return *resp
}
}
startedGeneratingEvent := time.Now()
// If we're sending a membership update, make sure to strip the authorised
// via key if it is present, otherwise other servers won't be able to auth
// the event if the room is set to the "restricted" join rule.
if eventType == gomatrixserverlib.MRoomMember {
delete(r, "join_authorised_via_users_server")
}
evTime, err := httputil.ParseTSParam(req)
if err != nil {
return util.JSONResponse{
@ -201,13 +217,44 @@ func SendEvent(
return res
}
// stateEqual compares the new and the existing state event content. If they are equal, returns a *util.JSONResponse
// with the existing event_id, making this an idempotent request.
func stateEqual(ctx context.Context, rsAPI api.ClientRoomserverAPI, eventType, stateKey, roomID string, newContent map[string]interface{}) *util.JSONResponse {
stateRes := api.QueryCurrentStateResponse{}
tuple := gomatrixserverlib.StateKeyTuple{
EventType: eventType,
StateKey: stateKey,
}
err := rsAPI.QueryCurrentState(ctx, &api.QueryCurrentStateRequest{
RoomID: roomID,
StateTuples: []gomatrixserverlib.StateKeyTuple{tuple},
}, &stateRes)
if err != nil {
return nil
}
if existingEvent, ok := stateRes.StateEvents[tuple]; ok {
var existingContent map[string]interface{}
if err = json.Unmarshal(existingEvent.Content(), &existingContent); err != nil {
return nil
}
if reflect.DeepEqual(existingContent, newContent) {
return &util.JSONResponse{
Code: http.StatusOK,
JSON: sendEventResponse{existingEvent.EventID()},
}
}
}
return nil
}
func generateSendEvent(
ctx context.Context,
r map[string]interface{},
device *userapi.Device,
roomID, eventType string, stateKey *string,
cfg *config.ClientAPI,
rsAPI api.RoomserverInternalAPI,
rsAPI api.ClientRoomserverAPI,
evTime time.Time,
) (*gomatrixserverlib.Event, *util.JSONResponse) {
// parse the incoming http request

2
clientapi/routing/sendtyping.go
View file

@ -32,7 +32,7 @@ type typingContentJSON struct {
// sends the typing events to client API typingProducer
func SendTyping(
req *http.Request, device *userapi.Device, roomID string,
userID string, rsAPI roomserverAPI.RoomserverInternalAPI,
userID string, rsAPI roomserverAPI.ClientRoomserverAPI,
syncProducer *producers.SyncAPIProducer,
) util.JSONResponse {
if device.UserID != userID {

57
clientapi/routing/server_notices.go
View file

@ -21,6 +21,7 @@ import (
"net/http"
"time"
"github.com/matrix-org/dendrite/roomserver/version"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/gomatrixserverlib/tokens"
@ -55,9 +56,9 @@ func SendServerNotice(
req *http.Request,
cfgNotices *config.ServerNotices,
cfgClient *config.ClientAPI,
userAPI userapi.UserInternalAPI,
rsAPI api.RoomserverInternalAPI,
asAPI appserviceAPI.AppServiceQueryAPI,
userAPI userapi.ClientUserAPI,
rsAPI api.ClientRoomserverAPI,
asAPI appserviceAPI.AppServiceInternalAPI,
device *userapi.Device,
senderDevice *userapi.Device,
txnID *string,
@ -95,29 +96,16 @@ func SendServerNotice(
// get rooms for specified user
allUserRooms := []string{}
userRooms := api.QueryRoomsForUserResponse{}
if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
UserID: r.UserID,
WantMembership: "join",
}, &userRooms); err != nil {
return util.ErrorResponse(err)
// Get rooms the user is either joined, invited or has left.
for _, membership := range []string{"join", "invite", "leave"} {
if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
UserID: r.UserID,
WantMembership: membership,
}, &userRooms); err != nil {
return util.ErrorResponse(err)
}
allUserRooms = append(allUserRooms, userRooms.RoomIDs...)
}
allUserRooms = append(allUserRooms, userRooms.RoomIDs...)
// get invites for specified user
if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
UserID: r.UserID,
WantMembership: "invite",
}, &userRooms); err != nil {
return util.ErrorResponse(err)
}
allUserRooms = append(allUserRooms, userRooms.RoomIDs...)
// get left rooms for specified user
if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
UserID: r.UserID,
WantMembership: "leave",
}, &userRooms); err != nil {
return util.ErrorResponse(err)
}
allUserRooms = append(allUserRooms, userRooms.RoomIDs...)
// get rooms of the sender
senderUserID := fmt.Sprintf("@%s:%s", cfgNotices.LocalPart, cfgClient.Matrix.ServerName)
@ -145,7 +133,7 @@ func SendServerNotice(
var (
roomID string
roomVersion = gomatrixserverlib.RoomVersionV6
roomVersion = version.DefaultRoomVersion()
)
// create a new room for the user
@ -194,14 +182,21 @@ func SendServerNotice(
// if we didn't get a createRoomResponse, we probably received an error, so return that.
return roomRes
}
} else {
// we've found a room in common, check the membership
roomID = commonRooms[0]
// re-invite the user
res, err := sendInvite(ctx, userAPI, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, asAPI, time.Now())
membershipRes := api.QueryMembershipForUserResponse{}
err := rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{UserID: r.UserID, RoomID: roomID}, &membershipRes)
if err != nil {
return res
util.GetLogger(ctx).WithError(err).Error("unable to query membership for user")
return jsonerror.InternalServerError()
}
if !membershipRes.IsInRoom {
// re-invite the user
res, err := sendInvite(ctx, userAPI, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, asAPI, time.Now())
if err != nil {
return res
}
}
}
@ -281,7 +276,7 @@ func (r sendServerNoticeRequest) valid() (ok bool) {
// It returns an userapi.Device, which is used for building the event
func getSenderDevice(
ctx context.Context,
userAPI userapi.UserInternalAPI,
userAPI userapi.ClientUserAPI,
cfg *config.ClientAPI,
) (*userapi.Device, error) {
var accRes userapi.PerformAccountCreationResponse

10
clientapi/routing/state.go
View file

@ -41,7 +41,7 @@ type stateEventInStateResp struct {
// TODO: Check if the user is in the room. If not, check if the room's history
// is publicly visible. Current behaviour is returning an empty array if the
// user cannot see the room's history.
func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI api.RoomserverInternalAPI, roomID string) util.JSONResponse {
func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI api.ClientRoomserverAPI, roomID string) util.JSONResponse {
var worldReadable bool
var wantLatestState bool
@ -56,6 +56,12 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed")
return jsonerror.InternalServerError()
}
if !stateRes.RoomExists {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("room does not exist"),
}
}
// Look at the room state and see if we have a history visibility event
// that marks the room as world-readable. If we don't then we assume that
@ -162,7 +168,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
// is then (by default) we return the content, otherwise a 404.
// If eventFormat=true, sends the whole event else just the content.
func OnIncomingStateTypeRequest(
ctx context.Context, device *userapi.Device, rsAPI api.RoomserverInternalAPI,
ctx context.Context, device *userapi.Device, rsAPI api.ClientRoomserverAPI,
roomID, evType, stateKey string, eventFormat bool,
) util.JSONResponse {
var worldReadable bool

13
clientapi/routing/threepid.go
View file

@ -38,9 +38,10 @@ type threePIDsResponse struct {
}
// RequestEmailToken implements:
// POST /account/3pid/email/requestToken
// POST /register/email/requestToken
func RequestEmailToken(req *http.Request, threePIDAPI api.UserThreePIDAPI, cfg *config.ClientAPI) util.JSONResponse {
//
// POST /account/3pid/email/requestToken
// POST /register/email/requestToken
func RequestEmailToken(req *http.Request, threePIDAPI api.ClientUserAPI, cfg *config.ClientAPI) util.JSONResponse {
var body threepid.EmailAssociationRequest
if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
return *reqErr
@ -90,7 +91,7 @@ func RequestEmailToken(req *http.Request, threePIDAPI api.UserThreePIDAPI, cfg *
// CheckAndSave3PIDAssociation implements POST /account/3pid
func CheckAndSave3PIDAssociation(
req *http.Request, threePIDAPI api.UserThreePIDAPI, device *api.Device,
req *http.Request, threePIDAPI api.ClientUserAPI, device *api.Device,
cfg *config.ClientAPI,
) util.JSONResponse {
var body threepid.EmailAssociationCheckRequest
@ -158,7 +159,7 @@ func CheckAndSave3PIDAssociation(
// GetAssociated3PIDs implements GET /account/3pid
func GetAssociated3PIDs(
req *http.Request, threepidAPI api.UserThreePIDAPI, device *api.Device,
req *http.Request, threepidAPI api.ClientUserAPI, device *api.Device,
) util.JSONResponse {
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {
@ -182,7 +183,7 @@ func GetAssociated3PIDs(
}
// Forget3PID implements POST /account/3pid/delete
func Forget3PID(req *http.Request, threepidAPI api.UserThreePIDAPI) util.JSONResponse {
func Forget3PID(req *http.Request, threepidAPI api.ClientUserAPI) util.JSONResponse {
var body authtypes.ThreePID
if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
return *reqErr

10
clientapi/routing/upgrade_room.go
View file

@ -40,9 +40,9 @@ type upgradeRoomResponse struct {
func UpgradeRoom(
req *http.Request, device *userapi.Device,
cfg *config.ClientAPI,
roomID string, profileAPI userapi.UserProfileAPI,
rsAPI roomserverAPI.RoomserverInternalAPI,
asAPI appserviceAPI.AppServiceQueryAPI,
roomID string, profileAPI userapi.ClientUserAPI,
rsAPI roomserverAPI.ClientRoomserverAPI,
asAPI appserviceAPI.AppServiceInternalAPI,
) util.JSONResponse {
var r upgradeRoomRequest
if rErr := httputil.UnmarshalJSONRequest(req, &r); rErr != nil {
@ -64,7 +64,9 @@ func UpgradeRoom(
}
upgradeResp := roomserverAPI.PerformRoomUpgradeResponse{}
rsAPI.PerformRoomUpgrade(req.Context(), &upgradeReq, &upgradeResp)
if err := rsAPI.PerformRoomUpgrade(req.Context(), &upgradeReq, &upgradeResp); err != nil {
return jsonerror.InternalAPIError(req.Context(), err)
}
if upgradeResp.Error != nil {
if upgradeResp.Error.Code == roomserverAPI.PerformErrorNoRoom {

118
clientapi/routing/userdirectory.go
View file

@ -18,10 +18,13 @@ import (
"context"
"database/sql"
"fmt"
"net/http"
"strings"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/roomserver/api"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
@ -34,13 +37,13 @@ type UserDirectoryResponse struct {
func SearchUserDirectory(
ctx context.Context,
device *userapi.Device,
userAPI userapi.UserInternalAPI,
rsAPI api.RoomserverInternalAPI,
provider userapi.UserDirectoryProvider,
serverName gomatrixserverlib.ServerName,
rsAPI api.ClientRoomserverAPI,
provider userapi.QuerySearchProfilesAPI,
searchString string,
limit int,
) *util.JSONResponse {
federation *gomatrixserverlib.FederationClient,
localServerName gomatrixserverlib.ServerName,
) util.JSONResponse {
if limit < 10 {
limit = 10
}
@ -51,61 +54,74 @@ func SearchUserDirectory(
Limited: false,
}
// First start searching local users.
userReq := &userapi.QuerySearchProfilesRequest{
SearchString: searchString,
Limit: limit,
// Get users we share a room with
knownUsersReq := &api.QueryKnownUsersRequest{
UserID: device.UserID,
Limit: limit,
}
userRes := &userapi.QuerySearchProfilesResponse{}
if err := provider.QuerySearchProfiles(ctx, userReq, userRes); err != nil {
errRes := util.ErrorResponse(fmt.Errorf("userAPI.QuerySearchProfiles: %w", err))
return &errRes
knownUsersRes := &api.QueryKnownUsersResponse{}
if err := rsAPI.QueryKnownUsers(ctx, knownUsersReq, knownUsersRes); err != nil && err != sql.ErrNoRows {
return util.ErrorResponse(fmt.Errorf("rsAPI.QueryKnownUsers: %w", err))
}
for _, user := range userRes.Profiles {
knownUsersLoop:
for _, profile := range knownUsersRes.Users {
if len(results) == limit {
response.Limited = true
break
}
var userID string
if user.ServerName != "" {
userID = fmt.Sprintf("@%s:%s", user.Localpart, user.ServerName)
userID := profile.UserID
// get the full profile of the local user
localpart, serverName, _ := gomatrixserverlib.SplitID('@', userID)
if serverName == localServerName {
userReq := &userapi.QuerySearchProfilesRequest{
SearchString: localpart,
Limit: limit,
}
userRes := &userapi.QuerySearchProfilesResponse{}
if err := provider.QuerySearchProfiles(ctx, userReq, userRes); err != nil {
return util.ErrorResponse(fmt.Errorf("userAPI.QuerySearchProfiles: %w", err))
}
for _, p := range userRes.Profiles {
if strings.Contains(p.DisplayName, searchString) ||
strings.Contains(p.Localpart, searchString) {
profile.DisplayName = p.DisplayName
profile.AvatarURL = p.AvatarURL
results[userID] = profile
if len(results) == limit {
response.Limited = true
break knownUsersLoop
}
}
}
} else {
userID = fmt.Sprintf("@%s:%s", user.Localpart, serverName)
}
if _, ok := results[userID]; !ok {
results[userID] = authtypes.FullyQualifiedProfile{
UserID: userID,
DisplayName: user.DisplayName,
AvatarURL: user.AvatarURL,
// If the username already contains the search string, don't bother hitting federation.
// This will result in missing avatars and displaynames, but saves the federation roundtrip.
if strings.Contains(localpart, searchString) {
results[userID] = profile
if len(results) == limit {
response.Limited = true
break knownUsersLoop
}
continue
}
}
}
// Then, if we have enough room left in the response,
// start searching for known users from joined rooms.
if len(results) <= limit {
stateReq := &api.QueryKnownUsersRequest{
UserID: device.UserID,
SearchString: searchString,
Limit: limit - len(results),
}
stateRes := &api.QueryKnownUsersResponse{}
if err := rsAPI.QueryKnownUsers(ctx, stateReq, stateRes); err != nil && err != sql.ErrNoRows {
errRes := util.ErrorResponse(fmt.Errorf("rsAPI.QueryKnownUsers: %w", err))
return &errRes
}
for _, user := range stateRes.Users {
if len(results) == limit {
response.Limited = true
break
// TODO: We should probably cache/store this
fedProfile, fedErr := federation.LookupProfile(ctx, serverName, userID, "")
if fedErr != nil {
if x, ok := fedErr.(gomatrix.HTTPError); ok {
if x.Code == http.StatusNotFound {
continue
}
}
}
if _, ok := results[user.UserID]; !ok {
results[user.UserID] = user
if strings.Contains(fedProfile.DisplayName, searchString) {
profile.DisplayName = fedProfile.DisplayName
profile.AvatarURL = fedProfile.AvatarURL
results[userID] = profile
if len(results) == limit {
response.Limited = true
break knownUsersLoop
}
}
}
}
@ -114,7 +130,7 @@ func SearchUserDirectory(
response.Results = append(response.Results, result)
}
return &util.JSONResponse{
return util.JSONResponse{
Code: 200,
JSON: response,
}

10
clientapi/routing/voip.go
View file

@ -22,15 +22,17 @@ import (
"net/http"
"time"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/util"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/util"
)
// RequestTurnServer implements:
// GET /voip/turnServer
//
// GET /voip/turnServer
func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.ClientAPI) util.JSONResponse {
turnConfig := cfg.TURN
@ -52,6 +54,7 @@ func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.Client
if turnConfig.SharedSecret != "" {
expiry := time.Now().Add(duration).Unix()
resp.Username = fmt.Sprintf("%d:%s", expiry, device.UserID)
mac := hmac.New(sha1.New, []byte(turnConfig.SharedSecret))
_, err := mac.Write([]byte(resp.Username))
@ -60,7 +63,6 @@ func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.Client
return jsonerror.InternalServerError()
}
resp.Username = fmt.Sprintf("%d:%s", expiry, device.UserID)
resp.Password = base64.StdEncoding.EncodeToString(mac.Sum(nil))
} else if turnConfig.Username != "" && turnConfig.Password != "" {
resp.Username = turnConfig.Username

14
clientapi/threepid/invites.go
View file

@ -86,7 +86,7 @@ var (
func CheckAndProcessInvite(
ctx context.Context,
device *userapi.Device, body *MembershipRequest, cfg *config.ClientAPI,
rsAPI api.RoomserverInternalAPI, db userapi.UserProfileAPI,
rsAPI api.ClientRoomserverAPI, db userapi.ClientUserAPI,
roomID string,
evTime time.Time,
) (inviteStoredOnIDServer bool, err error) {
@ -136,7 +136,7 @@ func CheckAndProcessInvite(
// Returns an error if a check or a request failed.
func queryIDServer(
ctx context.Context,
db userapi.UserProfileAPI, cfg *config.ClientAPI, device *userapi.Device,
userAPI userapi.ClientUserAPI, cfg *config.ClientAPI, device *userapi.Device,
body *MembershipRequest, roomID string,
) (lookupRes *idServerLookupResponse, storeInviteRes *idServerStoreInviteResponse, err error) {
if err = isTrusted(body.IDServer, cfg); err != nil {
@ -152,7 +152,7 @@ func queryIDServer(
if lookupRes.MXID == "" {
// No Matrix ID matches with the given 3PID, ask the server to store the
// invite and return a token
storeInviteRes, err = queryIDServerStoreInvite(ctx, db, cfg, device, body, roomID)
storeInviteRes, err = queryIDServerStoreInvite(ctx, userAPI, cfg, device, body, roomID)
return
}
@ -163,7 +163,7 @@ func queryIDServer(
if lookupRes.NotBefore > now || now > lookupRes.NotAfter {
// If the current timestamp isn't in the time frame in which the association
// is known to be valid, re-run the query
return queryIDServer(ctx, db, cfg, device, body, roomID)
return queryIDServer(ctx, userAPI, cfg, device, body, roomID)
}
// Check the request signatures and send an error if one isn't valid
@ -205,7 +205,7 @@ func queryIDServerLookup(ctx context.Context, body *MembershipRequest) (*idServe
// Returns an error if the request failed to send or if the response couldn't be parsed.
func queryIDServerStoreInvite(
ctx context.Context,
db userapi.UserProfileAPI, cfg *config.ClientAPI, device *userapi.Device,
userAPI userapi.ClientUserAPI, cfg *config.ClientAPI, device *userapi.Device,
body *MembershipRequest, roomID string,
) (*idServerStoreInviteResponse, error) {
// Retrieve the sender's profile to get their display name
@ -217,7 +217,7 @@ func queryIDServerStoreInvite(
var profile *authtypes.Profile
if serverName == cfg.Matrix.ServerName {
res := &userapi.QueryProfileResponse{}
err = db.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: device.UserID}, res)
err = userAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: device.UserID}, res)
if err != nil {
return nil, err
}
@ -337,7 +337,7 @@ func emit3PIDInviteEvent(
ctx context.Context,
body *MembershipRequest, res *idServerStoreInviteResponse,
device *userapi.Device, roomID string, cfg *config.ClientAPI,
rsAPI api.RoomserverInternalAPI,
rsAPI api.ClientRoomserverAPI,
evTime time.Time,
) error {
builder := &gomatrixserverlib.EventBuilder{

167
cmd/create-account/main.go
View file

@ -15,20 +15,26 @@
package main
import (
"context"
"bytes"
"crypto/hmac"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"flag"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"regexp"
"strings"
"time"
"github.com/tidwall/gjson"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/sirupsen/logrus"
"golang.org/x/term"
"github.com/matrix-org/dendrite/setup"
)
const usage = `Usage: %s
@ -46,8 +52,6 @@ Example:
# read password from stdin
%s --config dendrite.yaml -username alice -passwordstdin < my.pass
cat my.pass | %s --config dendrite.yaml -username alice -passwordstdin
# reset password for a user, can be used with a combination above to read the password
%s --config dendrite.yaml -reset-password -username alice -password foobarbaz
Arguments:
@ -58,29 +62,39 @@ var (
password = flag.String("password", "", "The password to associate with the account")
pwdFile = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)")
pwdStdin = flag.Bool("passwordstdin", false, "Reads the password from stdin")
pwdLess = flag.Bool("passwordless", false, "Create a passwordless account, e.g. if only an accesstoken is required")
isAdmin = flag.Bool("admin", false, "Create an admin account")
resetPassword = flag.Bool("reset-password", false, "Resets the password for the given username")
resetPassword = flag.Bool("reset-password", false, "Deprecated")
serverURL = flag.String("url", "https://localhost:8448", "The URL to connect to.")
validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
timeout = flag.Duration("timeout", time.Second*30, "Timeout for the http client when connecting to the server")
)
var cl = http.Client{
Timeout: time.Second * 30,
Transport: http.DefaultTransport,
}
func main() {
name := os.Args[0]
flag.Usage = func() {
_, _ = fmt.Fprintf(os.Stderr, usage, name, name, name, name, name, name, name)
_, _ = fmt.Fprintf(os.Stderr, usage, name, name, name, name, name, name)
flag.PrintDefaults()
}
cfg := setup.ParseFlags(true)
if *resetPassword {
logrus.Fatalf("The reset-password flag has been replaced by the POST /_dendrite/admin/resetPassword/{localpart} admin API.")
}
if cfg.ClientAPI.RegistrationSharedSecret == "" {
logrus.Fatalln("Shared secret registration is not enabled, enable it by setting a shared secret in the config: 'client_api.registration_shared_secret'")
}
if *username == "" {
flag.Usage()
os.Exit(1)
}
if *pwdLess && *resetPassword {
logrus.Fatalf("Can not reset to an empty password, unable to login afterwards.")
}
if !validUsernameRegex.MatchString(*username) {
logrus.Warn("Username can only contain characters a-z, 0-9, or '_-./='")
os.Exit(1)
@ -90,57 +104,102 @@ func main() {
logrus.Fatalf("Username can not be longer than 255 characters: %s", fmt.Sprintf("@%s:%s", *username, cfg.Global.ServerName))
}
var pass string
var err error
if !*pwdLess {
pass, err = getPassword(*password, *pwdFile, *pwdStdin, os.Stdin)
if err != nil {
logrus.Fatalln(err)
}
}
b := base.NewBaseDendrite(cfg, "Monolith")
accountDB := b.CreateAccountsDB()
accType := api.AccountTypeUser
if *isAdmin {
accType = api.AccountTypeAdmin
}
available, err := accountDB.CheckAccountAvailability(context.Background(), *username)
pass, err := getPassword(*password, *pwdFile, *pwdStdin, os.Stdin)
if err != nil {
logrus.Fatalln("Unable check username existence.")
}
if *resetPassword {
if available {
logrus.Fatalln("Username could not be found.")
}
err = accountDB.SetPassword(context.Background(), *username, pass)
if err != nil {
logrus.Fatalf("Failed to update password for user %s: %s", *username, err.Error())
}
if _, err = accountDB.RemoveAllDevices(context.Background(), *username, ""); err != nil {
logrus.Fatalf("Failed to remove all devices: %s", err.Error())
}
logrus.Infof("Updated password for user %s and invalidated all logins\n", *username)
return
}
if !available {
logrus.Fatalln("Username is already in use.")
logrus.Fatalln(err)
}
_, err = accountDB.CreateAccount(context.Background(), *username, pass, "", accType)
cl.Timeout = *timeout
accessToken, err := sharedSecretRegister(cfg.ClientAPI.RegistrationSharedSecret, *serverURL, *username, pass, *isAdmin)
if err != nil {
logrus.Fatalln("Failed to create the account:", err.Error())
}
logrus.Infoln("Created account", *username)
logrus.Infof("Created account: %s (AccessToken: %s)", *username, accessToken)
}
type sharedSecretRegistrationRequest struct {
User string `json:"username"`
Password string `json:"password"`
Nonce string `json:"nonce"`
MacStr string `json:"mac"`
Admin bool `json:"admin"`
}
func sharedSecretRegister(sharedSecret, serverURL, localpart, password string, admin bool) (accessToken string, err error) {
registerURL := fmt.Sprintf("%s/_synapse/admin/v1/register", strings.Trim(serverURL, "/"))
nonceReq, err := http.NewRequest(http.MethodGet, registerURL, nil)
if err != nil {
return "", fmt.Errorf("unable to create http request: %w", err)
}
nonceResp, err := cl.Do(nonceReq)
if err != nil {
return "", fmt.Errorf("unable to get nonce: %w", err)
}
body, err := io.ReadAll(nonceResp.Body)
if err != nil {
return "", fmt.Errorf("failed to read response body: %w", err)
}
defer nonceResp.Body.Close() // nolint: errcheck
nonce := gjson.GetBytes(body, "nonce").Str
adminStr := "notadmin"
if admin {
adminStr = "admin"
}
reg := sharedSecretRegistrationRequest{
User: localpart,
Password: password,
Nonce: nonce,
Admin: admin,
}
macStr, err := getRegisterMac(sharedSecret, nonce, localpart, password, adminStr)
if err != nil {
return "", err
}
reg.MacStr = macStr
js, err := json.Marshal(reg)
if err != nil {
return "", fmt.Errorf("unable to marshal json: %w", err)
}
registerReq, err := http.NewRequest(http.MethodPost, registerURL, bytes.NewBuffer(js))
if err != nil {
return "", fmt.Errorf("unable to create http request: %w", err)
}
regResp, err := cl.Do(registerReq)
if err != nil {
return "", fmt.Errorf("unable to create account: %w", err)
}
defer regResp.Body.Close() // nolint: errcheck
if regResp.StatusCode < 200 || regResp.StatusCode >= 300 {
body, _ = io.ReadAll(regResp.Body)
return "", fmt.Errorf(gjson.GetBytes(body, "error").Str)
}
r, _ := io.ReadAll(regResp.Body)
return gjson.GetBytes(r, "access_token").Str, nil
}
func getRegisterMac(sharedSecret, nonce, localpart, password, adminStr string) (string, error) {
joined := strings.Join([]string{nonce, localpart, password, adminStr}, "\x00")
mac := hmac.New(sha1.New, []byte(sharedSecret))
_, err := mac.Write([]byte(joined))
if err != nil {
return "", fmt.Errorf("unable to construct mac: %w", err)
}
regMac := mac.Sum(nil)
return hex.EncodeToString(regMac), nil
}
func getPassword(password, pwdFile string, pwdStdin bool, r io.Reader) (string, error) {
// read password from file
if pwdFile != "" {
pw, err := ioutil.ReadFile(pwdFile)
pw, err := os.ReadFile(pwdFile)
if err != nil {
return "", fmt.Errorf("Unable to read password from file: %v", err)
}
@ -149,7 +208,7 @@ func getPassword(password, pwdFile string, pwdStdin bool, r io.Reader) (string,
// read password from stdin
if pwdStdin {
data, err := ioutil.ReadAll(r)
data, err := io.ReadAll(r)
if err != nil {
return "", fmt.Errorf("Unable to read password from stdin: %v", err)
}

230
cmd/dendrite-demo-libp2p/main.go
View file

@ -1,230 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"crypto/ed25519"
"flag"
"fmt"
"io/ioutil"
"net/http"
"os"
"time"
"github.com/gorilla/mux"
gostream "github.com/libp2p/go-libp2p-gostream"
p2phttp "github.com/libp2p/go-libp2p-http"
p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery"
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed"
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/mscs"
"github.com/matrix-org/dendrite/userapi"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
_ "github.com/mattn/go-sqlite3"
)
func createKeyDB(
base *P2PDendrite,
db *gomatrixserverlib.KeyRing,
) {
mdns := mDNSListener{
host: base.LibP2P,
keydb: db,
}
serv, err := p2pdisc.NewMdnsService(
base.LibP2PContext,
base.LibP2P,
time.Second*10,
"_matrix-dendrite-p2p._tcp",
)
if err != nil {
panic(err)
}
serv.RegisterNotifee(&mdns)
}
func createFederationClient(
base *P2PDendrite,
) *gomatrixserverlib.FederationClient {
fmt.Println("Running in libp2p federation mode")
fmt.Println("Warning: Federation with non-libp2p homeservers will not work in this mode yet!")
tr := &http.Transport{}
tr.RegisterProtocol(
"matrix",
p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")),
)
return gomatrixserverlib.NewFederationClient(
base.Base.Cfg.Global.ServerName, base.Base.Cfg.Global.KeyID,
base.Base.Cfg.Global.PrivateKey,
gomatrixserverlib.WithTransport(tr),
)
}
func createClient(
base *P2PDendrite,
) *gomatrixserverlib.Client {
tr := &http.Transport{}
tr.RegisterProtocol(
"matrix",
p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")),
)
return gomatrixserverlib.NewClient(
gomatrixserverlib.WithTransport(tr),
)
}
func main() {
instanceName := flag.String("name", "dendrite-p2p", "the name of this P2P demo instance")
instancePort := flag.Int("port", 8080, "the port that the client API will listen on")
flag.Parse()
filename := fmt.Sprintf("%s-private.key", *instanceName)
_, err := os.Stat(filename)
var privKey ed25519.PrivateKey
if os.IsNotExist(err) {
_, privKey, _ = ed25519.GenerateKey(nil)
if err = ioutil.WriteFile(filename, privKey, 0600); err != nil {
fmt.Printf("Couldn't write private key to file '%s': %s\n", filename, err)
}
} else {
privKey, err = ioutil.ReadFile(filename)
if err != nil {
fmt.Printf("Couldn't read private key from file '%s': %s\n", filename, err)
_, privKey, _ = ed25519.GenerateKey(nil)
}
}
cfg := config.Dendrite{}
cfg.Defaults(true)
cfg.Global.ServerName = "p2p"
cfg.Global.PrivateKey = privKey
cfg.Global.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName))
cfg.FederationAPI.FederationMaxRetries = 6
cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName))
cfg.MSCs.MSCs = []string{"msc2836"}
cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
if err = cfg.Derive(); err != nil {
panic(err)
}
base := NewP2PDendrite(&cfg, "Monolith")
defer base.Base.Close() // nolint: errcheck
accountDB := base.Base.CreateAccountsDB()
federation := createFederationClient(base)
keyAPI := keyserver.NewInternalAPI(&base.Base, &base.Base.Cfg.KeyServer, federation)
rsAPI := roomserver.NewInternalAPI(
&base.Base,
)
userAPI := userapi.NewInternalAPI(&base.Base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.Base.PushGatewayHTTPClient())
keyAPI.SetUserAPI(userAPI)
asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI)
rsAPI.SetAppserviceAPI(asAPI)
fsAPI := federationapi.NewInternalAPI(
&base.Base, federation, rsAPI, base.Base.Caches, nil, true,
)
keyRing := fsAPI.KeyRing()
rsAPI.SetFederationAPI(fsAPI, keyRing)
provider := newPublicRoomsProvider(base.LibP2PPubsub, rsAPI)
err = provider.Start()
if err != nil {
panic("failed to create new public rooms provider: " + err.Error())
}
createKeyDB(
base, keyRing,
)
monolith := setup.Monolith{
Config: base.Base.Cfg,
AccountDB: accountDB,
Client: createClient(base),
FedClient: federation,
KeyRing: keyRing,
AppserviceAPI: asAPI,
FederationAPI: fsAPI,
RoomserverAPI: rsAPI,
UserAPI: userAPI,
KeyAPI: keyAPI,
ExtPublicRoomsProvider: provider,
}
monolith.AddAllPublicRoutes(
base.Base.ProcessContext,
base.Base.PublicClientAPIMux,
base.Base.PublicFederationAPIMux,
base.Base.PublicKeyAPIMux,
base.Base.PublicWellKnownAPIMux,
base.Base.PublicMediaAPIMux,
base.Base.SynapseAdminMux,
)
if err := mscs.Enable(&base.Base, &monolith); err != nil {
logrus.WithError(err).Fatalf("Failed to enable MSCs")
}
httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.Base.InternalAPIMux)
httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.Base.PublicClientAPIMux)
httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Base.PublicMediaAPIMux)
embed.Embed(httpRouter, *instancePort, "Yggdrasil Demo")
libp2pRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
libp2pRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.Base.PublicFederationAPIMux)
libp2pRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(base.Base.PublicKeyAPIMux)
libp2pRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.Base.PublicMediaAPIMux)
// Expose the matrix APIs directly rather than putting them under a /api path.
go func() {
httpBindAddr := fmt.Sprintf(":%d", *instancePort)
logrus.Info("Listening on ", httpBindAddr)
logrus.Fatal(http.ListenAndServe(httpBindAddr, httpRouter))
}()
// Expose the matrix APIs also via libp2p
if base.LibP2P != nil {
go func() {
logrus.Info("Listening on libp2p host ID ", base.LibP2P.ID())
listener, err := gostream.Listen(base.LibP2P, "/matrix")
if err != nil {
panic(err)
}
defer func() {
logrus.Fatal(listener.Close())
}()
logrus.Fatal(http.Serve(listener, libp2pRouter))
}()
}
// We want to block forever to let the HTTP and HTTPS handler serve the APIs
base.Base.WaitForShutdown()
}

62
cmd/dendrite-demo-libp2p/mdnslistener.go
View file

@ -1,62 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"fmt"
"math"
"github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/matrix-org/gomatrixserverlib"
)
type mDNSListener struct {
keydb *gomatrixserverlib.KeyRing
host host.Host
}
func (n *mDNSListener) HandlePeerFound(p peer.AddrInfo) {
if err := n.host.Connect(context.Background(), p); err != nil {
fmt.Println("Error adding peer", p.ID.String(), "via mDNS:", err)
}
if pubkey, err := p.ID.ExtractPublicKey(); err == nil {
raw, _ := pubkey.Raw()
if err := n.keydb.KeyDatabase.StoreKeys(
context.Background(),
map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{
{
ServerName: gomatrixserverlib.ServerName(p.ID.String()),
KeyID: "ed25519:p2pdemo",
}: {
VerifyKey: gomatrixserverlib.VerifyKey{
Key: gomatrixserverlib.Base64Bytes(raw),
},
ValidUntilTS: math.MaxUint64 >> 1,
ExpiredTS: gomatrixserverlib.PublicKeyNotExpired,
},
},
); err != nil {
fmt.Println("Failed to store keys:", err)
}
}
fmt.Println("Discovered", len(n.host.Peerstore().Peers())-1, "other libp2p peer(s):")
for _, peer := range n.host.Peerstore().Peers() {
if peer != n.host.ID() {
fmt.Println("-", peer)
}
}
}

126
cmd/dendrite-demo-libp2p/p2pdendrite.go
View file

@ -1,126 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"fmt"
"errors"
pstore "github.com/libp2p/go-libp2p-core/peerstore"
record "github.com/libp2p/go-libp2p-record"
"github.com/libp2p/go-libp2p"
circuit "github.com/libp2p/go-libp2p-circuit"
crypto "github.com/libp2p/go-libp2p-core/crypto"
routing "github.com/libp2p/go-libp2p-core/routing"
host "github.com/libp2p/go-libp2p-core/host"
dht "github.com/libp2p/go-libp2p-kad-dht"
pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
)
// P2PDendrite is a Peer-to-Peer variant of BaseDendrite.
type P2PDendrite struct {
Base base.BaseDendrite
// Store our libp2p object so that we can make outgoing connections from it
// later
LibP2P host.Host
LibP2PContext context.Context
LibP2PCancel context.CancelFunc
LibP2PDHT *dht.IpfsDHT
LibP2PPubsub *pubsub.PubSub
}
// NewP2PDendrite creates a new instance to be used by a component.
// The componentName is used for logging purposes, and should be a friendly name
// of the component running, e.g. SyncAPI.
func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite {
baseDendrite := base.NewBaseDendrite(cfg, componentName)
ctx, cancel := context.WithCancel(context.Background())
privKey, err := crypto.UnmarshalEd25519PrivateKey(cfg.Global.PrivateKey[:])
if err != nil {
panic(err)
}
//defaultIP6ListenAddr, _ := multiaddr.NewMultiaddr("/ip6/::/tcp/0")
var libp2pdht *dht.IpfsDHT
libp2p, err := libp2p.New(ctx,
libp2p.Identity(privKey),
libp2p.DefaultListenAddrs,
//libp2p.ListenAddrs(defaultIP6ListenAddr),
libp2p.DefaultTransports,
libp2p.Routing(func(h host.Host) (r routing.PeerRouting, err error) {
libp2pdht, err = dht.New(ctx, h)
if err != nil {
return nil, err
}
libp2pdht.Validator = libP2PValidator{}
r = libp2pdht
return
}),
libp2p.EnableAutoRelay(),
libp2p.EnableRelay(circuit.OptHop),
)
if err != nil {
panic(err)
}
libp2ppubsub, err := pubsub.NewFloodSub(context.Background(), libp2p, []pubsub.Option{
pubsub.WithMessageSigning(true),
}...)
if err != nil {
panic(err)
}
fmt.Println("Our public key:", privKey.GetPublic())
fmt.Println("Our node ID:", libp2p.ID())
fmt.Println("Our addresses:", libp2p.Addrs())
cfg.Global.ServerName = gomatrixserverlib.ServerName(libp2p.ID().String())
return &P2PDendrite{
Base: *baseDendrite,
LibP2P: libp2p,
LibP2PContext: ctx,
LibP2PCancel: cancel,
LibP2PDHT: libp2pdht,
LibP2PPubsub: libp2ppubsub,
}
}
type libP2PValidator struct {
KeyBook pstore.KeyBook
}
func (v libP2PValidator) Validate(key string, value []byte) error {
ns, _, err := record.SplitKey(key)
if err != nil || ns != "matrix" {
return errors.New("not Matrix path")
}
return nil
}
func (v libP2PValidator) Select(k string, vals [][]byte) (int, error) {
return 0, nil
}

153
cmd/dendrite-demo-libp2p/publicrooms.go
View file

@ -1,153 +0,0 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"encoding/json"
"fmt"
"sync"
"sync/atomic"
"time"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
const MaintenanceInterval = time.Second * 10
type discoveredRoom struct {
time time.Time
room gomatrixserverlib.PublicRoom
}
type publicRoomsProvider struct {
pubsub *pubsub.PubSub
topic *pubsub.Topic
subscription *pubsub.Subscription
foundRooms map[string]discoveredRoom // additional rooms we have learned about from the DHT
foundRoomsMutex sync.RWMutex // protects foundRooms
maintenanceTimer *time.Timer //
roomsAdvertised atomic.Value // stores int
rsAPI roomserverAPI.RoomserverInternalAPI
}
func newPublicRoomsProvider(ps *pubsub.PubSub, rsAPI roomserverAPI.RoomserverInternalAPI) *publicRoomsProvider {
return &publicRoomsProvider{
foundRooms: make(map[string]discoveredRoom),
pubsub: ps,
rsAPI: rsAPI,
}
}
func (p *publicRoomsProvider) Start() error {
if topic, err := p.pubsub.Join("/matrix/publicRooms"); err != nil {
return err
} else if sub, err := topic.Subscribe(); err == nil {
p.topic = topic
p.subscription = sub
go p.MaintenanceTimer()
go p.FindRooms()
p.roomsAdvertised.Store(0)
} else {
return err
}
return nil
}
func (p *publicRoomsProvider) MaintenanceTimer() {
if p.maintenanceTimer != nil && !p.maintenanceTimer.Stop() {
<-p.maintenanceTimer.C
}
p.Interval()
}
func (p *publicRoomsProvider) Interval() {
p.foundRoomsMutex.Lock()
for k, v := range p.foundRooms {
if time.Since(v.time) > time.Minute {
delete(p.foundRooms, k)
}
}
p.foundRoomsMutex.Unlock()
if err := p.AdvertiseRooms(); err != nil {
fmt.Println("Failed to advertise room in DHT:", err)
}
p.foundRoomsMutex.RLock()
defer p.foundRoomsMutex.RUnlock()
fmt.Println("Found", len(p.foundRooms), "room(s), advertised", p.roomsAdvertised.Load(), "room(s)")
p.maintenanceTimer = time.AfterFunc(MaintenanceInterval, p.Interval)
}
func (p *publicRoomsProvider) AdvertiseRooms() error {
ctx := context.Background()
var queryRes roomserverAPI.QueryPublishedRoomsResponse
// Query published rooms on our server. This will not invoke clientapi.ExtraPublicRoomsProvider
err := p.rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed")
return err
}
ourRooms, err := roomserverAPI.PopulatePublicRooms(ctx, queryRes.RoomIDs, p.rsAPI)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("PopulatePublicRooms failed")
return err
}
advertised := 0
for _, room := range ourRooms {
if j, err := json.Marshal(room); err == nil {
if err := p.topic.Publish(context.TODO(), j); err != nil {
fmt.Println("Failed to publish public room:", err)
} else {
advertised++
}
}
}
p.roomsAdvertised.Store(advertised)
return nil
}
func (p *publicRoomsProvider) FindRooms() {
for {
msg, err := p.subscription.Next(context.Background())
if err != nil {
continue
}
received := discoveredRoom{
time: time.Now(),
}
if err := json.Unmarshal(msg.Data, &received.room); err != nil {
fmt.Println("Unmarshal error:", err)
continue
}
fmt.Printf("received %+v \n", received)
p.foundRoomsMutex.Lock()
p.foundRooms[received.room.RoomID] = received
p.foundRoomsMutex.Unlock()
}
}
func (p *publicRoomsProvider) Rooms() (rooms []gomatrixserverlib.PublicRoom) {
p.foundRoomsMutex.RLock()
defer p.foundRoomsMutex.RUnlock()
for _, dr := range p.foundRooms {
rooms = append(rooms, dr.room)
}
return
}

139
cmd/dendrite-demo-pinecone/main.go
View file

@ -21,10 +21,11 @@ import (
"encoding/hex"
"flag"
"fmt"
"io/ioutil"
"net"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/gorilla/mux"
@ -43,6 +44,7 @@ import (
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/userapi"
"github.com/matrix-org/gomatrixserverlib"
@ -61,6 +63,7 @@ var (
instancePort = flag.Int("port", 8008, "the port that the client API will listen on")
instancePeer = flag.String("peer", "", "the static Pinecone peers to connect to, comma separated-list")
instanceListen = flag.String("listen", ":0", "the port Pinecone peers can connect to")
instanceDir = flag.String("dir", ".", "the directory to store the databases in (if --config not specified)")
)
// nolint:gocyclo
@ -71,31 +74,94 @@ func main() {
var pk ed25519.PublicKey
var sk ed25519.PrivateKey
keyfile := *instanceName + ".key"
if _, err := os.Stat(keyfile); os.IsNotExist(err) {
if pk, sk, err = ed25519.GenerateKey(nil); err != nil {
panic(err)
// iterate through the cli args and check if the config flag was set
configFlagSet := false
for _, arg := range os.Args {
if arg == "--config" || arg == "-config" {
configFlagSet = true
break
}
if err = ioutil.WriteFile(keyfile, sk, 0644); err != nil {
panic(err)
}
} else if err == nil {
if sk, err = ioutil.ReadFile(keyfile); err != nil {
panic(err)
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
pk = sk.Public().(ed25519.PublicKey)
}
pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk, false)
cfg := &config.Dendrite{}
// use custom config if config flag is set
if configFlagSet {
cfg = setup.ParseFlags(true)
sk = cfg.Global.PrivateKey
} else {
keyfile := filepath.Join(*instanceDir, *instanceName) + ".pem"
if _, err := os.Stat(keyfile); os.IsNotExist(err) {
oldkeyfile := *instanceName + ".key"
if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
if err = test.NewMatrixKey(keyfile); err != nil {
panic("failed to generate a new PEM key: " + err.Error())
}
if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
panic("failed to load PEM key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
} else {
if sk, err = os.ReadFile(oldkeyfile); err != nil {
panic("failed to read the old private key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
if err := test.SaveMatrixKey(keyfile, sk); err != nil {
panic("failed to convert the private key to PEM format: " + err.Error())
}
}
} else {
var err error
if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
panic("failed to load PEM key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
}
pk = sk.Public().(ed25519.PublicKey)
cfg.Defaults(config.DefaultOpts{
Generate: true,
Monolithic: true,
})
cfg.Global.PrivateKey = sk
cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", filepath.Join(*instanceDir, *instanceName)))
cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(*instanceDir, *instanceName)))
cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(*instanceDir, *instanceName)))
cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(*instanceDir, *instanceName)))
cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(*instanceDir, *instanceName)))
cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(*instanceDir, *instanceName)))
cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", filepath.Join(*instanceDir, *instanceName)))
cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName)))
cfg.ClientAPI.RegistrationDisabled = false
cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
if err := cfg.Derive(); err != nil {
panic(err)
}
}
cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
base := base.NewBaseDendrite(cfg, "Monolith")
defer base.Close() // nolint: errcheck
pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk)
pQUIC := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"})
pMulticast := pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), pRouter)
pManager := pineconeConnections.NewConnectionManager(pRouter)
pManager := pineconeConnections.NewConnectionManager(pRouter, nil)
pMulticast.Start()
if instancePeer != nil && *instancePeer != "" {
pManager.AddPeer(*instancePeer)
for _, peer := range strings.Split(*instancePeer, ",") {
pManager.AddPeer(strings.Trim(peer, " \t\r\n"))
}
}
go func() {
@ -126,28 +192,6 @@ func main() {
}
}()
cfg := &config.Dendrite{}
cfg.Defaults(true)
cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
cfg.Global.PrivateKey = sk
cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
if err := cfg.Derive(); err != nil {
panic(err)
}
base := base.NewBaseDendrite(cfg, "Monolith")
defer base.Close() // nolint: errcheck
accountDB := base.CreateAccountsDB()
federation := conn.CreateFederationClient(base, pQUIC)
serverKeyAPI := &signing.YggdrasilKeys{}
@ -160,7 +204,7 @@ func main() {
)
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
keyAPI.SetUserAPI(userAPI)
asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
@ -172,7 +216,6 @@ func main() {
monolith := setup.Monolith{
Config: base.Cfg,
AccountDB: accountDB,
Client: conn.CreateClient(base, pQUIC),
FedClient: federation,
KeyRing: keyRing,
@ -185,15 +228,7 @@ func main() {
ExtPublicRoomsProvider: roomProvider,
ExtUserDirectoryProvider: userProvider,
}
monolith.AddAllPublicRoutes(
base.ProcessContext,
base.PublicClientAPIMux,
base.PublicFederationAPIMux,
base.PublicKeyAPIMux,
base.PublicWellKnownAPIMux,
base.PublicMediaAPIMux,
base.SynapseAdminMux,
)
monolith.AddAllPublicRoutes(base)
wsUpgrader := websocket.Upgrader{
CheckOrigin: func(_ *http.Request) bool {

4
cmd/dendrite-demo-pinecone/users/users.go
View file

@ -37,7 +37,7 @@ import (
type PineconeUserProvider struct {
r *pineconeRouter.Router
s *pineconeSessions.Sessions
userAPI userapi.UserProfileAPI
userAPI userapi.QuerySearchProfilesAPI
fedClient *gomatrixserverlib.FederationClient
}
@ -46,7 +46,7 @@ const PublicURL = "/_matrix/p2p/profiles"
func NewPineconeUserProvider(
r *pineconeRouter.Router,
s *pineconeSessions.Sessions,
userAPI userapi.UserProfileAPI,
userAPI userapi.QuerySearchProfilesAPI,
fedClient *gomatrixserverlib.FederationClient,
) *PineconeUserProvider {
p := &PineconeUserProvider{

6
cmd/dendrite-demo-yggdrasil/README.md
View file

@ -1,6 +1,6 @@
# Yggdrasil Demo
This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.16 or later.
This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.18 or later.
To run the homeserver, start at the root of the Dendrite repository and run:
@ -13,10 +13,10 @@ The following command line arguments are accepted:
* `-peer tcp://a.b.c.d:e` to specify a static Yggdrasil peer to connect to - you will need to supply this if you do not have another Yggdrasil node on your network
* `-port 12345` to specify a port to listen on for client connections
If you need to find an internet peer, take a look at [this list](https://publicpeers.neilalexander.dev/).
If you need to find an internet peer, take a look at [this list](https://publicpeers.neilalexander.dev/).
Then point your favourite Matrix client to the homeserver URL`http://localhost:8008` (or whichever `-port` you specified), create an account and log in.
If your peering connection is operational then you should see a `Connected TCP:` line in the log output. If not then try a different peer.
Once logged in, you should be able to open the room directory or join a room by its ID.
Once logged in, you should be able to open the room directory or join a room by its ID.

108
cmd/dendrite-demo-yggdrasil/main.go
View file

@ -16,12 +16,15 @@ package main
import (
"context"
"crypto/ed25519"
"crypto/tls"
"encoding/hex"
"flag"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"time"
"github.com/matrix-org/gomatrixserverlib"
@ -42,6 +45,7 @@ import (
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/mscs"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/userapi"
"github.com/sirupsen/logrus"
@ -49,19 +53,19 @@ import (
)
var (
instanceName = flag.String("name", "dendrite-p2p-ygg", "the name of this P2P demo instance")
instancePort = flag.Int("port", 8008, "the port that the client API will listen on")
instancePeer = flag.String("peer", "", "an internet Yggdrasil peer to connect to")
instanceName = flag.String("name", "dendrite-p2p-ygg", "the name of this P2P demo instance")
instancePort = flag.Int("port", 8008, "the port that the client API will listen on")
instancePeer = flag.String("peer", "", "the static Yggdrasil peers to connect to, comma separated-list")
instanceListen = flag.String("listen", "tcp://:0", "the port Yggdrasil peers can connect to")
instanceDir = flag.String("dir", ".", "the directory to store the databases in (if --config not specified)")
)
func main() {
flag.Parse()
internal.SetupPprof()
ygg, err := yggconn.Setup(*instanceName, ".", *instancePeer)
if err != nil {
panic(err)
}
var pk ed25519.PublicKey
var sk ed25519.PrivateKey
// iterate through the cli args and check if the config flag was set
configFlagSet := false
@ -74,35 +78,78 @@ func main() {
cfg := &config.Dendrite{}
keyfile := filepath.Join(*instanceDir, *instanceName) + ".pem"
if _, err := os.Stat(keyfile); os.IsNotExist(err) {
oldkeyfile := *instanceName + ".key"
if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
if err = test.NewMatrixKey(keyfile); err != nil {
panic("failed to generate a new PEM key: " + err.Error())
}
if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
panic("failed to load PEM key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
} else {
if sk, err = os.ReadFile(oldkeyfile); err != nil {
panic("failed to read the old private key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
if err := test.SaveMatrixKey(keyfile, sk); err != nil {
panic("failed to convert the private key to PEM format: " + err.Error())
}
}
} else {
var err error
if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
panic("failed to load PEM key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
}
pk = sk.Public().(ed25519.PublicKey)
// use custom config if config flag is set
if configFlagSet {
cfg = setup.ParseFlags(true)
} else {
cfg.Defaults(true)
cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
cfg.MSCs.MSCs = []string{"msc2836"}
cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
if err = cfg.Derive(); err != nil {
cfg.Defaults(config.DefaultOpts{
Generate: true,
Monolithic: true,
})
cfg.Global.PrivateKey = sk
cfg.Global.JetStream.StoragePath = config.Path(filepath.Join(*instanceDir, *instanceName))
cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(*instanceDir, *instanceName)))
cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(*instanceDir, *instanceName)))
cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(*instanceDir, *instanceName)))
cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(*instanceDir, *instanceName)))
cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(*instanceDir, *instanceName)))
cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", filepath.Join(*instanceDir, *instanceName)))
cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName)))
cfg.ClientAPI.RegistrationDisabled = false
cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
if err := cfg.Derive(); err != nil {
panic(err)
}
}
// always override ServerName, PrivateKey and KeyID
cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
cfg.Global.PrivateKey = ygg.PrivateKey()
cfg.Global.KeyID = signing.KeyID
cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
base := base.NewBaseDendrite(cfg, "Monolith")
defer base.Close() // nolint: errcheck
accountDB := base.CreateAccountsDB()
ygg, err := yggconn.Setup(sk, *instanceName, ".", *instancePeer, *instanceListen)
if err != nil {
panic(err)
}
federation := ygg.CreateFederationClient(base)
serverKeyAPI := &signing.YggdrasilKeys{}
@ -115,7 +162,7 @@ func main() {
)
rsAPI := rsComponent
userAPI := userapi.NewInternalAPI(base, accountDB, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
keyAPI.SetUserAPI(userAPI)
asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
@ -128,7 +175,6 @@ func main() {
monolith := setup.Monolith{
Config: base.Cfg,
AccountDB: accountDB,
Client: ygg.CreateClient(base),
FedClient: federation,
KeyRing: keyRing,
@ -142,15 +188,7 @@ func main() {
ygg, fsAPI, federation,
),
}
monolith.AddAllPublicRoutes(
base.ProcessContext,
base.PublicClientAPIMux,
base.PublicFederationAPIMux,
base.PublicKeyAPIMux,
base.PublicWellKnownAPIMux,
base.PublicMediaAPIMux,
base.SynapseAdminMux,
)
monolith.AddAllPublicRoutes(base)
if err := mscs.Enable(base, &monolith); err != nil {
logrus.WithError(err).Fatalf("Failed to enable MSCs")
}

Some files were not shown because too many files have changed in this diff Show more