From 5324826bfa5a567e1705bf2065fd48f8dcf6696d Mon Sep 17 00:00:00 2001 From: Dan Peleg Date: Tue, 31 Oct 2023 00:44:08 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=80=20Merge=200.13.4-upstream?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 3 +- .golangci.yml | 2 +- CHANGES.md | 64 ++++ appservice/appservice_test.go | 3 +- appservice/query/query.go | 8 +- are-we-synapse-yet.list | 10 +- build/docker/Dockerfile.demo-pinecone | 2 +- build/docker/Dockerfile.demo-yggdrasil | 2 +- build/scripts/complement.sh | 2 +- clientapi/admin_test.go | 18 +- clientapi/clientapi_test.go | 27 +- clientapi/routing/capabilities.go | 5 +- clientapi/routing/createroom.go | 2 +- clientapi/routing/device.go | 10 +- clientapi/routing/joinroom_test.go | 2 +- clientapi/routing/key_crosssigning.go | 2 +- clientapi/routing/login_test.go | 1 + clientapi/routing/register.go | 6 +- clientapi/routing/register_test.go | 3 + clientapi/routing/room_hierarchy.go | 180 ++++++++++ clientapi/routing/routing.go | 70 ++-- clientapi/routing/sendevent_test.go | 275 ++++++++++++++ clientapi/routing/state_test.go | 253 +++++++++++++ cmd/dendrite-demo-yggdrasil/README.md | 2 +- cmd/dendrite-upgrade-tests/main.go | 23 +- cmd/generate-config/main.go | 2 +- dendrite-sample.yaml | 5 +- docs/FAQ.md | 10 +- docs/Gemfile.lock | 2 +- docs/administration/4_adminapi.md | 2 +- docs/administration/5_optimisation.md | 2 +- docs/development/CONTRIBUTING.md | 2 +- docs/installation/1_planning.md | 2 +- federationapi/internal/api.go | 5 +- federationapi/internal/perform_test.go | 27 +- federationapi/queue/queue_test.go | 2 +- federationapi/routing/invite.go | 78 ++++ federationapi/routing/profile.go | 8 + federationapi/routing/query.go | 64 ++++ federationapi/routing/routing.go | 39 ++ go.mod | 36 +- go.sum | 68 ++-- helm/dendrite/Chart.yaml | 4 +- helm/dendrite/README.md | 15 +- helm/dendrite/templates/deployment.yaml | 2 +- helm/dendrite/templates/ingress.yaml | 4 +- helm/dendrite/templates/pvc.yaml | 27 +- helm/dendrite/values.yaml | 38 +- internal/pushrules/action_test.go | 2 - internal/pushrules/default_override.go | 11 +- internal/pushrules/default_pushrules_test.go | 6 +- internal/pushrules/evaluate_test.go | 85 +++-- internal/pushrules/util.go | 5 +- internal/pushrules/util_test.go | 5 +- internal/pushrules/validate.go | 2 +- internal/sqlutil/migrate.go | 8 +- internal/validate.go | 2 +- internal/version.go | 32 +- mediaapi/routing/download.go | 12 + roomserver/internal/query/query_test.go | 1 + roomserver/roomserver_test.go | 30 +- roomserver/version/version.go | 6 - setup/config/config_clientapi.go | 20 +- setup/config/config_global.go | 4 + setup/config/config_mscs.go | 1 - setup/jetstream/streams.go | 1 + setup/mscs/msc2836/msc2836_test.go | 5 +- syncapi/internal/keychange_test.go | 28 +- syncapi/routing/search_test.go | 2 +- .../postgres/current_room_state_table.go | 2 +- syncapi/storage/postgres/invites_table.go | 2 +- syncapi/storage/postgres/memberships_table.go | 2 +- .../sqlite3/current_room_state_table.go | 2 +- syncapi/storage/sqlite3/invites_table.go | 2 +- syncapi/storage/sqlite3/memberships_table.go | 2 +- syncapi/storage/storage_test.go | 38 +- syncapi/storage/tables/topology_test.go | 50 +-- syncapi/syncapi_test.go | 167 ++++++++- syncapi/synctypes/clientevent_test.go | 334 +++++++++--------- syncapi/types/types_test.go | 29 +- userapi/consumers/roomserver_test.go | 91 ++++- userapi/internal/device_list_update.go | 45 ++- userapi/internal/device_list_update_test.go | 46 +++ userapi/internal/user_api.go | 13 +- userapi/storage/storage_test.go | 51 +++ 85 files changed, 2056 insertions(+), 507 deletions(-) create mode 100644 clientapi/routing/room_hierarchy.go create mode 100644 clientapi/routing/sendevent_test.go create mode 100644 clientapi/routing/state_test.go diff --git a/.dockerignore b/.dockerignore index 76547e9ee..80ccf937c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,2 @@ bin -*.wasm -.git \ No newline at end of file +*.wasm \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index bb8d38a8b..5bee0a885 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -180,7 +180,6 @@ linters-settings: linters: enable: - errcheck - - goconst - gocyclo - goimports # Does everything gofmt does - gosimple @@ -211,6 +210,7 @@ linters: - stylecheck - typecheck # Should turn back on soon - unconvert # Should turn back on soon + - goconst # Slightly annoying, as it reports "issues" in SQL statements disable-all: false presets: fast: false diff --git a/CHANGES.md b/CHANGES.md index c99ed2255..57e3a3d4f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,69 @@ # Changelog +## Dendrite 0.13.4 (2023-10-25) + +Upgrading to this version is **highly** recommended, as it fixes a long-standing bug in the state resolution +algorithm. + +### Fixes: + +- The "device list updater" now de-duplicates the servers to fetch devices from on startup. (This also + avoids spamming the logs when shutting down.) +- A bug in the state resolution algorithm has been fixed. This bug could result in users "being reset" + out of rooms and other missing state events due to calculating the wrong state. +- A bug when setting notifications from Element Android has been fixed by implementing MSC3987 + +### Features + +- Updated dependencies + - Internal NATS Server has been updated from v2.9.19 to v2.9.23 + +## Dendrite 0.13.3 (2023-09-28) + +### Fixes: + +- The `user_id` query parameter when authenticating is now used correctly (contributed by [tulir](https://github.com/tulir)) +- Invitations are now correctly pushed to devices +- A bug which could result in the corruption of `m.direct` account data has been fixed + +### Features + +- [Sliding Sync proxy](https://github.com/matrix-org/sliding-sync) can be configured in the `/.well-known/matrix/client` response +- Room version 11 is now supported +- Clients can request the `federation` `event_format` when creating filters +- Many under the hood improvements for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md) + +### Other + +- Dendrite now requires Go 1.20 if building from source + +## Dendrite 0.13.2 (2023-08-23) + +### Fixes: + +- Migrations in SQLite are now prepared on the correct context (transaction or database) +- The `InputRoomEvent` stream now has a maximum age of 24h, which should help with slow start up times of NATS JetStream (contributed by [neilalexander](https://github.com/neilalexander)) +- Event size checks are more in line with Synapse +- Requests to `/messages` have been optimized, possibly reducing database round trips +- Re-add the revision of Dendrite when building from source (Note: This only works if git is installed) +- Getting local members to notify has been optimized, which should significantly reduce memory allocation and cache usage +- When getting queried about user profiles, we now return HTTP404 if the user/profiles does not exist +- Background federated joins should now be fixed and not timeout after a short time +- Database connections are now correctly re-used +- Restored the old behavior of the `/purgeRoom` admin endpoint (does not evacuate the room before purging) +- Don't expose information about the system when trying to download files that don't exist + +### Features + +- Further improvements and fixes for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md) + - Lookup correct prev events in the sync API + - Populate `prev_sender` correctly in the sync API + - Event federation should work better +- Added new `dendrite_up` Prometheus metric, containing the version of Dendrite +- Space summaries ([MSC2946](https://github.com/matrix-org/matrix-spec-proposals/pull/2946)) have been moved from MSC to being natively supported +- For easier issue investigation, logs for application services now contain the application service ID (contributed by [maxberger](https://github.com/maxberger)) +- The default room version to use when creating rooms can now be configured using `room_server.default_room_version` + ## Dendrite 0.13.1 (2023-07-06) This releases fixes a long-standing "off-by-one" error which could result in state resets. Upgrading to this version is **highly** recommended. diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go index 878ca5666..ddc24477b 100644 --- a/appservice/appservice_test.go +++ b/appservice/appservice_test.go @@ -134,7 +134,6 @@ func TestAppserviceInternalAPI(t *testing.T) { } as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation) cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as} - t.Cleanup(func() { ctx.ShutdownDendrite() ctx.WaitForShutdown() @@ -144,6 +143,7 @@ func TestAppserviceInternalAPI(t *testing.T) { natsInstance := jetstream.NATSInstance{} cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) @@ -238,6 +238,7 @@ func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) { natsInstance := jetstream.NATSInstance{} cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) diff --git a/appservice/query/query.go b/appservice/query/query.go index ca8d7b3a3..5c736f379 100644 --- a/appservice/query/query.go +++ b/appservice/query/query.go @@ -217,7 +217,7 @@ func (a *AppServiceQueryAPI) Locations( } if err := requestDo[[]api.ASLocationResponse](as.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil { - log.WithError(err).Error("unable to get 'locations' from application service") + log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'locations' from application service") continue } @@ -252,7 +252,7 @@ func (a *AppServiceQueryAPI) User( } if err := requestDo[[]api.ASUserResponse](as.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil { - log.WithError(err).Error("unable to get 'user' from application service") + log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'user' from application service") continue } @@ -290,7 +290,7 @@ func (a *AppServiceQueryAPI) Protocols( for _, as := range a.Cfg.Derived.ApplicationServices { var proto api.ASProtocolResponse if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+req.Protocol, &proto); err != nil { - log.WithError(err).Error("unable to get 'protocol' from application service") + log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service") continue } @@ -320,7 +320,7 @@ func (a *AppServiceQueryAPI) Protocols( for _, p := range as.Protocols { var proto api.ASProtocolResponse if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+p, &proto); err != nil { - log.WithError(err).Error("unable to get 'protocol' from application service") + log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service") continue } existing, ok := response[p] diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index 585374738..80c0dbff6 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -944,4 +944,12 @@ rmv remote user can join room with version 10 rmv User can invite remote user to room with version 10 rmv Remote user can backfill in a room with version 10 rmv Can reject invites over federation for rooms with version 10 -rmv Can receive redactions from regular users over federation in room version 10 \ No newline at end of file +rmv Can receive redactions from regular users over federation in room version 10 +rmv User can create and send/receive messages in a room with version 11 +rmv local user can join room with version 11 +rmv User can invite local user to room with version 11 +rmv remote user can join room with version 11 +rmv User can invite remote user to room with version 11 +rmv Remote user can backfill in a room with version 11 +rmv Can reject invites over federation for rooms with version 11 +rmv Can receive redactions from regular users over federation in room version 11 \ No newline at end of file diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone index 90f515167..ab50cf318 100644 --- a/build/docker/Dockerfile.demo-pinecone +++ b/build/docker/Dockerfile.demo-pinecone @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.19-alpine AS base +FROM docker.io/golang:1.21-alpine AS base # # Needs to be separate from the main Dockerfile for OpenShift, diff --git a/build/docker/Dockerfile.demo-yggdrasil b/build/docker/Dockerfile.demo-yggdrasil index efae5496c..b9e387666 100644 --- a/build/docker/Dockerfile.demo-yggdrasil +++ b/build/docker/Dockerfile.demo-yggdrasil @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.19-alpine AS base +FROM docker.io/golang:1.21-alpine AS base # # Needs to be separate from the main Dockerfile for OpenShift, diff --git a/build/scripts/complement.sh b/build/scripts/complement.sh index 29feff304..8608d8fa5 100755 --- a/build/scripts/complement.sh +++ b/build/scripts/complement.sh @@ -15,5 +15,5 @@ tar -xzf master.tar.gz # Run the tests! cd complement-master -COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests +COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests ./tests/csapi diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index 9d2acd68e..66667b03c 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -44,6 +44,7 @@ func TestAdminCreateToken(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ @@ -194,6 +195,7 @@ func TestAdminListRegistrationTokens(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ @@ -311,6 +313,7 @@ func TestAdminGetRegistrationToken(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ @@ -411,6 +414,7 @@ func TestAdminDeleteRegistrationToken(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ @@ -504,6 +508,7 @@ func TestAdminUpdateRegistrationToken(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ @@ -686,6 +691,7 @@ func TestAdminResetPassword(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) // Needed for changing the password/login userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. @@ -780,13 +786,14 @@ func TestPurgeRoom(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // this starts the JetStream consumers - syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) + // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { t.Fatalf("failed to send events: %v", err) @@ -851,12 +858,13 @@ func TestAdminEvacuateRoom(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // this starts the JetStream consumers fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { t.Fatalf("failed to send events: %v", err) @@ -951,12 +959,13 @@ func TestAdminEvacuateUser(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // this starts the JetStream consumers fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { t.Fatalf("failed to send events: %v", err) @@ -1045,6 +1054,7 @@ func TestAdminMarkAsStale(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go index 2bd7eb791..dfc7ebb2b 100644 --- a/clientapi/clientapi_test.go +++ b/clientapi/clientapi_test.go @@ -117,6 +117,7 @@ func TestGetPutDevices(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. @@ -165,6 +166,7 @@ func TestDeleteDevice(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. @@ -268,6 +270,7 @@ func TestDeleteDevices(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. @@ -897,13 +900,17 @@ func TestCapabilities(t *testing.T) { } } + var tempRoomServerCfg config.RoomServer + tempRoomServerCfg.Defaults(config.DefaultOpts{}) + defaultRoomVersion := tempRoomServerCfg.DefaultRoomVersion + expectedMap := map[string]interface{}{ "capabilities": map[string]interface{}{ "m.change_password": map[string]bool{ "enabled": true, }, "m.room_versions": map[string]interface{}{ - "default": version.DefaultRoomVersion(), + "default": defaultRoomVersion, "available": versionsMap, }, }, @@ -924,6 +931,7 @@ func TestCapabilities(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -970,6 +978,7 @@ func TestTurnserver(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) //rsAPI.SetUserAPI(userAPI) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. @@ -1066,11 +1075,12 @@ func TestTurnserver(t *testing.T) { // routers := httputil.NewRouters() // cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) -// // Needed to create accounts -// rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) -// userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) -// // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. -// AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + // Needed to create accounts + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) // // Create the users in the userapi and login // accessTokens := map[*test.User]userDevice{ @@ -1243,6 +1253,7 @@ func TestPushRules(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. @@ -1385,7 +1396,7 @@ func TestPushRules(t *testing.T) { validateFunc: func(t *testing.T, respBody *bytes.Buffer) { actions := gjson.GetBytes(respBody.Bytes(), "actions").Array() // only a basic check - assert.Equal(t, 1, len(actions)) + assert.Equal(t, 0, len(actions)) }, }, { @@ -1629,6 +1640,7 @@ func TestKeys(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. @@ -2108,6 +2120,7 @@ func TestKeyBackup(t *testing.T) { routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. diff --git a/clientapi/routing/capabilities.go b/clientapi/routing/capabilities.go index fa50fa1aa..38e5bd467 100644 --- a/clientapi/routing/capabilities.go +++ b/clientapi/routing/capabilities.go @@ -17,6 +17,7 @@ package routing import ( "net/http" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -24,7 +25,7 @@ import ( // GetCapabilities returns information about the server's supported feature set // and other relevant capabilities to an authenticated user. -func GetCapabilities() util.JSONResponse { +func GetCapabilities(rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { versionsMap := map[gomatrixserverlib.RoomVersion]string{} for v, desc := range version.SupportedRoomVersions() { if desc.Stable() { @@ -40,7 +41,7 @@ func GetCapabilities() util.JSONResponse { "enabled": true, }, "m.room_versions": map[string]interface{}{ - "default": version.DefaultRoomVersion(), + "default": rsAPI.DefaultRoomVersion(), "available": versionsMap, }, }, diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 320f236cb..47e3ba1c3 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -171,7 +171,7 @@ func createRoom( // Clobber keys: creator, room_version - roomVersion := roomserverVersion.DefaultRoomVersion() + roomVersion := rsAPI.DefaultRoomVersion() if createRequest.RoomVersion != "" { candidateVersion := gomatrixserverlib.RoomVersion(createRequest.RoomVersion) _, roomVersionError := roomserverVersion.SupportedRoomVersion(candidateVersion) diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go index 6c59475e3..ea24454a3 100644 --- a/clientapi/routing/device.go +++ b/clientapi/routing/device.go @@ -161,12 +161,6 @@ func UpdateDeviceByID( JSON: spec.Forbidden("device not owned by current user"), } } - if performRes.Forbidden { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: spec.Forbidden("device not owned by current user"), - } - } return util.JSONResponse{ Code: http.StatusOK, @@ -266,13 +260,13 @@ func DeleteDevices( req *http.Request, userAPI api.ClientUserAPI, device *api.Device, ) util.JSONResponse { ctx := req.Context() - payload := devicesDeleteJSON{} + payload := devicesDeleteJSON{} if resErr := httputil.UnmarshalJSONRequest(req, &payload); resErr != nil { return *resErr } - defer req.Body.Close() // nolint: errcheck + defer req.Body.Close() // nolint:errcheck var res api.PerformDeviceDeletionResponse if err := userAPI.PerformDeviceDeletion(ctx, &api.PerformDeviceDeletionRequest{ diff --git a/clientapi/routing/joinroom_test.go b/clientapi/routing/joinroom_test.go index 0ddff8a95..933ea8d3a 100644 --- a/clientapi/routing/joinroom_test.go +++ b/clientapi/routing/joinroom_test.go @@ -35,9 +35,9 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) natsInstance := jetstream.NATSInstance{} rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) - rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc // Create the users in the userapi for _, u := range []*test.User{alice, bob, charlie} { diff --git a/clientapi/routing/key_crosssigning.go b/clientapi/routing/key_crosssigning.go index 200db1a41..fd00c17c8 100644 --- a/clientapi/routing/key_crosssigning.go +++ b/clientapi/routing/key_crosssigning.go @@ -35,7 +35,7 @@ type crossSigningRequest struct { } func UploadCrossSigningDeviceKeys( - req *http.Request, + req *http.Request, userInteractiveAuth *auth.UserInteractive, keyserverAPI api.ClientKeyAPI, device *api.Device, accountAPI api.ClientUserAPI, cfg *config.ClientAPI, ) util.JSONResponse { diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go index bff676826..252017db2 100644 --- a/clientapi/routing/login_test.go +++ b/clientapi/routing/login_test.go @@ -47,6 +47,7 @@ func TestLogin(t *testing.T) { routers := httputil.NewRouters() caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) // Needed for /login userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 2921c9cb6..0ae502e68 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -238,7 +238,7 @@ type authDict struct { ThreePidCreds threepid.Credentials `json:"threepid_creds"` } -// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#user-interactive-authentication-api +// https://spec.matrix.org/v1.7/client-server-api/#user-interactive-authentication-api type userInteractiveResponse struct { Flows []authtypes.Flow `json:"flows"` Completed []authtypes.LoginType `json:"completed"` @@ -258,7 +258,7 @@ func newUserInteractiveResponse( } } -// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register +// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register type registerResponse struct { UserID string `json:"user_id"` AccessToken string `json:"access_token,omitempty"` @@ -464,7 +464,7 @@ func validateApplicationService( } // Register processes a /register request. -// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register +// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register func Register( req *http.Request, userAPI userapi.ClientUserAPI, diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index 5b7855ad6..c9a35d738 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -415,6 +415,7 @@ func Test_register(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) for _, tc := range testCases { @@ -594,6 +595,7 @@ func TestRegisterUserWithDisplayName(t *testing.T) { natsInstance := jetstream.NATSInstance{} cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) deviceName, deviceID := "deviceName", "deviceID" expectedDisplayName := "DisplayName" @@ -635,6 +637,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) expectedDisplayName := "rabbit" diff --git a/clientapi/routing/room_hierarchy.go b/clientapi/routing/room_hierarchy.go new file mode 100644 index 000000000..2884d2c32 --- /dev/null +++ b/clientapi/routing/room_hierarchy.go @@ -0,0 +1,180 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + "strconv" + "sync" + + "github.com/google/uuid" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib/fclient" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" + log "github.com/sirupsen/logrus" +) + +// For storing pagination information for room hierarchies +type RoomHierarchyPaginationCache struct { + cache map[string]roomserverAPI.RoomHierarchyWalker + mu sync.Mutex +} + +// Create a new, empty, pagination cache. +func NewRoomHierarchyPaginationCache() RoomHierarchyPaginationCache { + return RoomHierarchyPaginationCache{ + cache: map[string]roomserverAPI.RoomHierarchyWalker{}, + } +} + +// Get a cached page, or nil if there is no associated page in the cache. +func (c *RoomHierarchyPaginationCache) Get(token string) *roomserverAPI.RoomHierarchyWalker { + c.mu.Lock() + defer c.mu.Unlock() + line, ok := c.cache[token] + if ok { + return &line + } else { + return nil + } +} + +// Add a cache line to the pagination cache. +func (c *RoomHierarchyPaginationCache) AddLine(line roomserverAPI.RoomHierarchyWalker) string { + c.mu.Lock() + defer c.mu.Unlock() + token := uuid.NewString() + c.cache[token] = line + return token +} + +// Query the hierarchy of a room/space +// +// Implements /_matrix/client/v1/rooms/{roomID}/hierarchy +func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr string, rsAPI roomserverAPI.ClientRoomserverAPI, paginationCache *RoomHierarchyPaginationCache) util.JSONResponse { + parsedRoomID, err := spec.NewRoomID(roomIDStr) + if err != nil { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.InvalidParam("room is unknown/forbidden"), + } + } + roomID := *parsedRoomID + + suggestedOnly := false // Defaults to false (spec-defined) + switch req.URL.Query().Get("suggested_only") { + case "true": + suggestedOnly = true + case "false": + case "": // Empty string is returned when query param is not set + default: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("query parameter 'suggested_only', if set, must be 'true' or 'false'"), + } + } + + limit := 1000 // Default to 1000 + limitStr := req.URL.Query().Get("limit") + if limitStr != "" { + var maybeLimit int + maybeLimit, err = strconv.Atoi(limitStr) + if err != nil || maybeLimit < 0 { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("query parameter 'limit', if set, must be a positive integer"), + } + } + limit = maybeLimit + if limit > 1000 { + limit = 1000 // Maximum limit of 1000 + } + } + + maxDepth := -1 // '-1' representing no maximum depth + maxDepthStr := req.URL.Query().Get("max_depth") + if maxDepthStr != "" { + var maybeMaxDepth int + maybeMaxDepth, err = strconv.Atoi(maxDepthStr) + if err != nil || maybeMaxDepth < 0 { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("query parameter 'max_depth', if set, must be a positive integer"), + } + } + maxDepth = maybeMaxDepth + } + + from := req.URL.Query().Get("from") + + var walker roomserverAPI.RoomHierarchyWalker + if from == "" { // No pagination token provided, so start new hierarchy walker + walker = roomserverAPI.NewRoomHierarchyWalker(types.NewDeviceNotServerName(*device), roomID, suggestedOnly, maxDepth) + } else { // Attempt to resume cached walker + cachedWalker := paginationCache.Get(from) + + if cachedWalker == nil || cachedWalker.SuggestedOnly != suggestedOnly || cachedWalker.MaxDepth != maxDepth { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("pagination not found for provided token ('from') with given 'max_depth', 'suggested_only' and room ID"), + } + } + + walker = *cachedWalker + } + + discoveredRooms, nextWalker, err := rsAPI.QueryNextRoomHierarchyPage(req.Context(), walker, limit) + + if err != nil { + switch err.(type) { + case roomserverAPI.ErrRoomUnknownOrNotAllowed: + util.GetLogger(req.Context()).WithError(err).Debugln("room unknown/forbidden when handling CS room hierarchy request") + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("room is unknown/forbidden"), + } + default: + log.WithError(err).Errorf("failed to fetch next page of room hierarchy (CS API)") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + } + + nextBatch := "" + // nextWalker will be nil if there's no more rooms left to walk + if nextWalker != nil { + nextBatch = paginationCache.AddLine(*nextWalker) + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: RoomHierarchyClientResponse{ + Rooms: discoveredRooms, + NextBatch: nextBatch, + }, + } + +} + +// Success response for /_matrix/client/v1/rooms/{roomID}/hierarchy +type RoomHierarchyClientResponse struct { + Rooms []fclient.RoomHierarchyRoom `json:"rooms"` + NextBatch string `json:"next_batch,omitempty"` +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 5af421fd7..2581ccd33 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -45,6 +45,19 @@ import ( "github.com/matrix-org/dendrite/setup/jetstream" ) +type WellKnownClientHomeserver struct { + BaseUrl string `json:"base_url"` +} + +type WellKnownSlidingSyncProxy struct { + Url string `json:"url"` +} + +type WellKnownClientResponse struct { + Homeserver WellKnownClientHomeserver `json:"m.homeserver"` + SlidingSyncProxy *WellKnownSlidingSyncProxy `json:"org.matrix.msc3575.proxy,omitempty"` +} + // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. // @@ -98,20 +111,22 @@ func Setup( if cfg.Matrix.WellKnownClientName != "" { logrus.Infof("Setting m.homeserver base_url as %s at /.well-known/matrix/client", cfg.Matrix.WellKnownClientName) + if cfg.Matrix.WellKnownSlidingSyncProxy != "" { + logrus.Infof("Setting org.matrix.msc3575.proxy url as %s at /.well-known/matrix/client", cfg.Matrix.WellKnownSlidingSyncProxy) + } wkMux.Handle("/client", httputil.MakeExternalAPI("wellknown", func(r *http.Request) util.JSONResponse { + response := WellKnownClientResponse{ + Homeserver: WellKnownClientHomeserver{cfg.Matrix.WellKnownClientName}, + } + if cfg.Matrix.WellKnownSlidingSyncProxy != "" { + response.SlidingSyncProxy = &WellKnownSlidingSyncProxy{ + Url: cfg.Matrix.WellKnownSlidingSyncProxy, + } + } + return util.JSONResponse{ Code: http.StatusOK, - JSON: struct { - HomeserverName struct { - BaseUrl string `json:"base_url"` - } `json:"m.homeserver"` - }{ - HomeserverName: struct { - BaseUrl string `json:"base_url"` - }{ - BaseUrl: cfg.Matrix.WellKnownClientName, - }, - }, + JSON: response, } })).Methods(http.MethodGet, http.MethodOptions) } @@ -290,6 +305,8 @@ func Setup( // Note that 'apiversion' is chosen because it must not collide with a variable used in any of the routing! v3mux := publicAPIMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter() + v1mux := publicAPIMux.PathPrefix("/v1/").Subrouter() + unstableMux := publicAPIMux.PathPrefix("/unstable").Subrouter() v3mux.Handle("/createRoom", @@ -507,6 +524,26 @@ func Setup( }, httputil.WithAllowGuests()), ).Methods(http.MethodPut, http.MethodOptions) + // Defined outside of handler to persist between calls + // TODO: clear based on some criteria + roomHierarchyPaginationCache := NewRoomHierarchyPaginationCache() + v1mux.Handle("/rooms/{roomID}/hierarchy", + httputil.MakeAuthAPI("spaces", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return QueryRoomHierarchy(req, device, vars["roomID"], rsAPI, &roomHierarchyPaginationCache) + }, httputil.WithAllowGuests()), + ).Methods(http.MethodGet, http.MethodOptions) + + v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { + if r := rateLimits.Limit(req, nil); r != nil { + return *r + } + return Register(req, userAPI, cfg) + })).Methods(http.MethodPost, http.MethodOptions) + v3mux.Handle("/multiroom/{dataType}", httputil.MakeAuthAPI("send_multiroom", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -518,13 +555,6 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) - v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { - if r := rateLimits.Limit(req, nil); r != nil { - return *r - } - return Register(req, userAPI, cfg) - })).Methods(http.MethodPost, http.MethodOptions) - v3mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse { if r := rateLimits.Limit(req, nil); r != nil { return *r @@ -1254,7 +1284,7 @@ func Setup( if r := rateLimits.Limit(req, device); r != nil { return *r } - return GetCapabilities() + return GetCapabilities(rsAPI) }, httputil.WithAllowGuests()), ).Methods(http.MethodGet, http.MethodOptions) @@ -1431,7 +1461,7 @@ func Setup( // Cross-signing device keys postDeviceSigningKeys := httputil.MakeAuthAPI("post_device_signing_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return UploadCrossSigningDeviceKeys(req, userAPI, device, userAPI, cfg) + return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, userAPI, device, userAPI, cfg) }) postDeviceSigningSignatures := httputil.MakeAuthAPI("post_device_signing_signatures", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { diff --git a/clientapi/routing/sendevent_test.go b/clientapi/routing/sendevent_test.go new file mode 100644 index 000000000..9cdd75358 --- /dev/null +++ b/clientapi/routing/sendevent_test.go @@ -0,0 +1,275 @@ +package routing + +import ( + "context" + "crypto/ed25519" + "fmt" + "io" + "net/http" + "strings" + "testing" + + rsapi "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/config" + uapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" + "github.com/matrix-org/gomatrixserverlib/spec" + "gotest.tools/v3/assert" +) + +// Mock roomserver API for testing +// +// Currently pretty specialised for the pseudo ID test, so will need +// editing if future (other) sendevent tests are using this. +type sendEventTestRoomserverAPI struct { + rsapi.ClientRoomserverAPI + t *testing.T + roomIDStr string + roomVersion gomatrixserverlib.RoomVersion + roomState []*types.HeaderedEvent + + // userID -> room key + senderMapping map[string]ed25519.PrivateKey + + savedInputRoomEvents []rsapi.InputRoomEvent +} + +func (s *sendEventTestRoomserverAPI) QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) { + if roomID == s.roomIDStr { + return s.roomVersion, nil + } else { + s.t.Logf("room version queried for %s", roomID) + return "", fmt.Errorf("unknown room") + } +} + +func (s *sendEventTestRoomserverAPI) QueryCurrentState(ctx context.Context, req *rsapi.QueryCurrentStateRequest, res *rsapi.QueryCurrentStateResponse) error { + res.StateEvents = map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{} + for _, stateKeyTuple := range req.StateTuples { + for _, stateEv := range s.roomState { + if stateEv.Type() == stateKeyTuple.EventType && stateEv.StateKey() != nil && *stateEv.StateKey() == stateKeyTuple.StateKey { + res.StateEvents[stateKeyTuple] = stateEv + } + } + } + return nil +} + +func (s *sendEventTestRoomserverAPI) QueryLatestEventsAndState(ctx context.Context, req *rsapi.QueryLatestEventsAndStateRequest, res *rsapi.QueryLatestEventsAndStateResponse) error { + if req.RoomID == s.roomIDStr { + res.RoomExists = true + res.RoomVersion = s.roomVersion + + res.StateEvents = make([]*types.HeaderedEvent, len(s.roomState)) + copy(res.StateEvents, s.roomState) + + res.LatestEvents = []string{} + res.Depth = 1 + return nil + } else { + s.t.Logf("room event/state queried for %s", req.RoomID) + return fmt.Errorf("unknown room") + } + +} + +func (s *sendEventTestRoomserverAPI) QuerySenderIDForUser( + ctx context.Context, + roomID spec.RoomID, + userID spec.UserID, +) (*spec.SenderID, error) { + if roomID.String() == s.roomIDStr { + if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs { + roomKey, ok := s.senderMapping[userID.String()] + if ok { + sender := spec.SenderIDFromPseudoIDKey(roomKey) + return &sender, nil + } else { + return nil, nil + } + } else { + senderID := spec.SenderIDFromUserID(userID) + return &senderID, nil + } + } + + return nil, fmt.Errorf("room not found") +} + +func (s *sendEventTestRoomserverAPI) QueryUserIDForSender( + ctx context.Context, + roomID spec.RoomID, + senderID spec.SenderID, +) (*spec.UserID, error) { + if roomID.String() == s.roomIDStr { + if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs { + for uID, roomKey := range s.senderMapping { + if string(spec.SenderIDFromPseudoIDKey(roomKey)) == string(senderID) { + parsedUserID, err := spec.NewUserID(uID, true) + if err != nil { + s.t.Fatalf("Mock QueryUserIDForSender failed: %s", err) + } + return parsedUserID, nil + } + } + } else { + userID := senderID.ToUserID() + if userID == nil { + return nil, fmt.Errorf("bad sender ID") + } + return userID, nil + } + } + + return nil, fmt.Errorf("room not found") +} + +func (s *sendEventTestRoomserverAPI) SigningIdentityFor(ctx context.Context, roomID spec.RoomID, sender spec.UserID) (fclient.SigningIdentity, error) { + if s.roomIDStr == roomID.String() { + if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs { + roomKey, ok := s.senderMapping[sender.String()] + if !ok { + s.t.Logf("SigningIdentityFor used with unknown user ID: %v", sender.String()) + return fclient.SigningIdentity{}, fmt.Errorf("could not get signing identity for %v", sender.String()) + } + return fclient.SigningIdentity{PrivateKey: roomKey}, nil + } else { + return fclient.SigningIdentity{PrivateKey: ed25519.NewKeyFromSeed(make([]byte, 32))}, nil + } + } + + return fclient.SigningIdentity{}, fmt.Errorf("room not found") +} + +func (s *sendEventTestRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) { + s.savedInputRoomEvents = req.InputRoomEvents +} + +// Test that user ID state keys are translated correctly +func Test_SendEvent_PseudoIDStateKeys(t *testing.T) { + nonpseudoIDRoomVersion := gomatrixserverlib.RoomVersionV10 + pseudoIDRoomVersion := gomatrixserverlib.RoomVersionPseudoIDs + + senderKeySeed := make([]byte, 32) + senderUserID := "@testuser:domain" + senderPrivKey := ed25519.NewKeyFromSeed(senderKeySeed) + senderPseudoID := string(spec.SenderIDFromPseudoIDKey(senderPrivKey)) + + eventType := "com.example.test" + roomIDStr := "!id:domain" + + device := &uapi.Device{ + UserID: senderUserID, + } + + t.Run("user ID state key are not translated to room key in non-pseudo ID room", func(t *testing.T) { + eventsJSON := []string{ + fmt.Sprintf(`{"type":"m.room.create","state_key":"","room_id":"%v","sender":"%v","content":{"creator":"%v","room_version":"%v"}}`, roomIDStr, senderUserID, senderUserID, nonpseudoIDRoomVersion), + fmt.Sprintf(`{"type":"m.room.member","state_key":"%v","room_id":"%v","sender":"%v","content":{"membership":"join"}}`, senderUserID, roomIDStr, senderUserID), + } + + roomState, err := createEvents(eventsJSON, nonpseudoIDRoomVersion) + if err != nil { + t.Fatalf("failed to prepare state events: %s", err.Error()) + } + + rsAPI := &sendEventTestRoomserverAPI{ + t: t, + roomIDStr: roomIDStr, + roomVersion: nonpseudoIDRoomVersion, + roomState: roomState, + } + + req, err := http.NewRequest("POST", "https://domain", io.NopCloser(strings.NewReader("{}"))) + if err != nil { + t.Fatalf("failed to make new request: %s", err.Error()) + } + + cfg := &config.ClientAPI{} + + resp := SendEvent(req, device, roomIDStr, eventType, nil, &senderUserID, cfg, rsAPI, nil) + + if resp.Code != http.StatusOK { + t.Fatalf("non-200 HTTP code returned: %v\nfull response: %v", resp.Code, resp) + } + + assert.Equal(t, len(rsAPI.savedInputRoomEvents), 1) + + ev := rsAPI.savedInputRoomEvents[0] + stateKey := ev.Event.StateKey() + if stateKey == nil { + t.Fatalf("submitted InputRoomEvent has nil state key, when it should be %v", senderUserID) + } + if *stateKey != senderUserID { + t.Fatalf("expected submitted InputRoomEvent to have user ID state key\nfound: %v\nexpected: %v", *stateKey, senderUserID) + } + }) + + t.Run("user ID state key are translated to room key in pseudo ID room", func(t *testing.T) { + eventsJSON := []string{ + fmt.Sprintf(`{"type":"m.room.create","state_key":"","room_id":"%v","sender":"%v","content":{"creator":"%v","room_version":"%v"}}`, roomIDStr, senderPseudoID, senderPseudoID, pseudoIDRoomVersion), + fmt.Sprintf(`{"type":"m.room.member","state_key":"%v","room_id":"%v","sender":"%v","content":{"membership":"join"}}`, senderPseudoID, roomIDStr, senderPseudoID), + } + + roomState, err := createEvents(eventsJSON, pseudoIDRoomVersion) + if err != nil { + t.Fatalf("failed to prepare state events: %s", err.Error()) + } + + rsAPI := &sendEventTestRoomserverAPI{ + t: t, + roomIDStr: roomIDStr, + roomVersion: pseudoIDRoomVersion, + senderMapping: map[string]ed25519.PrivateKey{ + senderUserID: senderPrivKey, + }, + roomState: roomState, + } + + req, err := http.NewRequest("POST", "https://domain", io.NopCloser(strings.NewReader("{}"))) + if err != nil { + t.Fatalf("failed to make new request: %s", err.Error()) + } + + cfg := &config.ClientAPI{} + + resp := SendEvent(req, device, roomIDStr, eventType, nil, &senderUserID, cfg, rsAPI, nil) + + if resp.Code != http.StatusOK { + t.Fatalf("non-200 HTTP code returned: %v\nfull response: %v", resp.Code, resp) + } + + assert.Equal(t, len(rsAPI.savedInputRoomEvents), 1) + + ev := rsAPI.savedInputRoomEvents[0] + stateKey := ev.Event.StateKey() + if stateKey == nil { + t.Fatalf("submitted InputRoomEvent has nil state key, when it should be %v", senderPseudoID) + } + if *stateKey != senderPseudoID { + t.Fatalf("expected submitted InputRoomEvent to have pseudo ID state key\nfound: %v\nexpected: %v", *stateKey, senderPseudoID) + } + }) +} + +func createEvents(eventsJSON []string, roomVer gomatrixserverlib.RoomVersion) ([]*types.HeaderedEvent, error) { + events := make([]*types.HeaderedEvent, len(eventsJSON)) + + roomVerImpl, err := gomatrixserverlib.GetRoomVersion(roomVer) + if err != nil { + return nil, fmt.Errorf("no roomver impl: %s", err.Error()) + } + + for i, eventJSON := range eventsJSON { + pdu, evErr := roomVerImpl.NewEventFromTrustedJSON([]byte(eventJSON), false) + if evErr != nil { + return nil, fmt.Errorf("failed to make event: %s", err.Error()) + } + ev := types.HeaderedEvent{PDU: pdu} + events[i] = &ev + } + + return events, nil +} diff --git a/clientapi/routing/state_test.go b/clientapi/routing/state_test.go new file mode 100644 index 000000000..93b043723 --- /dev/null +++ b/clientapi/routing/state_test.go @@ -0,0 +1,253 @@ +package routing + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "testing" + + rsapi "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/config" + uapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" + "gotest.tools/v3/assert" +) + +var () + +type stateTestRoomserverAPI struct { + rsapi.RoomserverInternalAPI + t *testing.T + roomState map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent + roomIDStr string + roomVersion gomatrixserverlib.RoomVersion + userIDStr string + // userID -> senderID + senderMapping map[string]string +} + +func (s stateTestRoomserverAPI) QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) { + if roomID == s.roomIDStr { + return s.roomVersion, nil + } else { + s.t.Logf("room version queried for %s", roomID) + return "", fmt.Errorf("unknown room") + } +} + +func (s stateTestRoomserverAPI) QueryLatestEventsAndState( + ctx context.Context, + req *rsapi.QueryLatestEventsAndStateRequest, + res *rsapi.QueryLatestEventsAndStateResponse, +) error { + res.RoomExists = req.RoomID == s.roomIDStr + if !res.RoomExists { + return nil + } + + res.StateEvents = []*types.HeaderedEvent{} + for _, stateKeyTuple := range req.StateToFetch { + val, ok := s.roomState[stateKeyTuple] + if ok && val != nil { + res.StateEvents = append(res.StateEvents, val) + } + } + + return nil +} + +func (s stateTestRoomserverAPI) QueryMembershipForUser( + ctx context.Context, + req *rsapi.QueryMembershipForUserRequest, + res *rsapi.QueryMembershipForUserResponse, +) error { + if req.UserID.String() == s.userIDStr { + res.HasBeenInRoom = true + res.IsInRoom = true + res.RoomExists = true + res.Membership = spec.Join + } + + return nil +} + +func (s stateTestRoomserverAPI) QuerySenderIDForUser( + ctx context.Context, + roomID spec.RoomID, + userID spec.UserID, +) (*spec.SenderID, error) { + sID, ok := s.senderMapping[userID.String()] + if ok { + sender := spec.SenderID(sID) + return &sender, nil + } else { + return nil, nil + } +} + +func (s stateTestRoomserverAPI) QueryUserIDForSender( + ctx context.Context, + roomID spec.RoomID, + senderID spec.SenderID, +) (*spec.UserID, error) { + for uID, sID := range s.senderMapping { + if sID == string(senderID) { + parsedUserID, err := spec.NewUserID(uID, true) + if err != nil { + s.t.Fatalf("Mock QueryUserIDForSender failed: %s", err) + } + return parsedUserID, nil + } + } + return nil, nil +} + +func (s stateTestRoomserverAPI) QueryStateAfterEvents( + ctx context.Context, + req *rsapi.QueryStateAfterEventsRequest, + res *rsapi.QueryStateAfterEventsResponse, +) error { + return nil +} + +func Test_OnIncomingStateTypeRequest(t *testing.T) { + var tempRoomServerCfg config.RoomServer + tempRoomServerCfg.Defaults(config.DefaultOpts{}) + defaultRoomVersion := tempRoomServerCfg.DefaultRoomVersion + pseudoIDRoomVersion := gomatrixserverlib.RoomVersionPseudoIDs + nonPseudoIDRoomVersion := gomatrixserverlib.RoomVersionV10 + + userIDStr := "@testuser:domain" + eventType := "com.example.test" + stateKey := "testStateKey" + roomIDStr := "!id:domain" + + device := &uapi.Device{ + UserID: userIDStr, + } + + t.Run("request simple state key", func(t *testing.T) { + ctx := context.Background() + + rsAPI := stateTestRoomserverAPI{ + roomVersion: defaultRoomVersion, + roomIDStr: roomIDStr, + roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{ + { + EventType: eventType, + StateKey: stateKey, + }: mustCreateStatePDU(t, defaultRoomVersion, roomIDStr, eventType, stateKey, map[string]interface{}{ + "foo": "bar", + }), + }, + userIDStr: userIDStr, + } + + jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateKey, false) + + assert.DeepEqual(t, jsonResp, util.JSONResponse{ + Code: http.StatusOK, + JSON: spec.RawJSON(`{"foo":"bar"}`), + }) + }) + + t.Run("user ID key translated to room key in pseudo ID rooms", func(t *testing.T) { + ctx := context.Background() + + stateSenderUserID := "@sender:domain" + stateSenderRoomKey := "testsenderkey" + + rsAPI := stateTestRoomserverAPI{ + roomVersion: pseudoIDRoomVersion, + roomIDStr: roomIDStr, + roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{ + { + EventType: eventType, + StateKey: stateSenderRoomKey, + }: mustCreateStatePDU(t, pseudoIDRoomVersion, roomIDStr, eventType, stateSenderRoomKey, map[string]interface{}{ + "foo": "bar", + }), + { + EventType: eventType, + StateKey: stateSenderUserID, + }: mustCreateStatePDU(t, pseudoIDRoomVersion, roomIDStr, eventType, stateSenderUserID, map[string]interface{}{ + "not": "thisone", + }), + }, + userIDStr: userIDStr, + senderMapping: map[string]string{ + stateSenderUserID: stateSenderRoomKey, + }, + } + + jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateSenderUserID, false) + + assert.DeepEqual(t, jsonResp, util.JSONResponse{ + Code: http.StatusOK, + JSON: spec.RawJSON(`{"foo":"bar"}`), + }) + }) + + t.Run("user ID key not translated to room key in non-pseudo ID rooms", func(t *testing.T) { + ctx := context.Background() + + stateSenderUserID := "@sender:domain" + stateSenderRoomKey := "testsenderkey" + + rsAPI := stateTestRoomserverAPI{ + roomVersion: nonPseudoIDRoomVersion, + roomIDStr: roomIDStr, + roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{ + { + EventType: eventType, + StateKey: stateSenderRoomKey, + }: mustCreateStatePDU(t, nonPseudoIDRoomVersion, roomIDStr, eventType, stateSenderRoomKey, map[string]interface{}{ + "not": "thisone", + }), + { + EventType: eventType, + StateKey: stateSenderUserID, + }: mustCreateStatePDU(t, nonPseudoIDRoomVersion, roomIDStr, eventType, stateSenderUserID, map[string]interface{}{ + "foo": "bar", + }), + }, + userIDStr: userIDStr, + senderMapping: map[string]string{ + stateSenderUserID: stateSenderUserID, + }, + } + + jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateSenderUserID, false) + + assert.DeepEqual(t, jsonResp, util.JSONResponse{ + Code: http.StatusOK, + JSON: spec.RawJSON(`{"foo":"bar"}`), + }) + }) +} + +func mustCreateStatePDU(t *testing.T, roomVer gomatrixserverlib.RoomVersion, roomID string, stateType string, stateKey string, stateContent map[string]interface{}) *types.HeaderedEvent { + t.Helper() + roomVerImpl := gomatrixserverlib.MustGetRoomVersion(roomVer) + + evBytes, err := json.Marshal(map[string]interface{}{ + "room_id": roomID, + "type": stateType, + "state_key": stateKey, + "content": stateContent, + }) + if err != nil { + t.Fatalf("failed to create event: %v", err) + } + + ev, err := roomVerImpl.NewEventFromTrustedJSON(evBytes, false) + if err != nil { + t.Fatalf("failed to create event: %v", err) + } + + return &types.HeaderedEvent{PDU: ev} +} diff --git a/cmd/dendrite-demo-yggdrasil/README.md b/cmd/dendrite-demo-yggdrasil/README.md index 14fc3a2db..23304c214 100644 --- a/cmd/dendrite-demo-yggdrasil/README.md +++ b/cmd/dendrite-demo-yggdrasil/README.md @@ -1,6 +1,6 @@ # Yggdrasil Demo -This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.18 or later. +This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.20 or later. To run the homeserver, start at the root of the Dendrite repository and run: diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go index dcc45bdcc..b78c5f605 100644 --- a/cmd/dendrite-upgrade-tests/main.go +++ b/cmd/dendrite-upgrade-tests/main.go @@ -7,7 +7,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" "net/http" "os" @@ -55,7 +54,7 @@ var latest, _ = semver.NewVersion("v6.6.6") // Dummy version, used as "HEAD" // due to the error: // When using COPY with more than one source file, the destination must be a directory and end with a / // We need to run a postgres anyway, so use the dockerfile associated with Complement instead. -const DockerfilePostgreSQL = `FROM golang:1.18-buster as build +const DockerfilePostgreSQL = `FROM golang:1.20-bookworm as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build ARG BINARY @@ -74,16 +73,16 @@ RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password RUN sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml # No password when connecting over localhost -RUN sed -i "s%127.0.0.1/32 md5%127.0.0.1/32 trust%g" /etc/postgresql/11/main/pg_hba.conf +RUN sed -i "s%127.0.0.1/32 scram-sha-256%127.0.0.1/32 trust%g" /etc/postgresql/15/main/pg_hba.conf # Bump up max conns for moar concurrency -RUN sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/11/main/postgresql.conf +RUN sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/15/main/postgresql.conf RUN sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml # This entry script starts postgres, waits for it to be up then starts dendrite RUN echo '\ #!/bin/bash -eu \n\ pg_lsclusters \n\ -pg_ctlcluster 11 main start \n\ +pg_ctlcluster 15 main start \n\ \n\ until pg_isready \n\ do \n\ @@ -101,7 +100,7 @@ ENV BINARY=dendrite EXPOSE 8008 8448 CMD /build/run_dendrite.sh` -const DockerfileSQLite = `FROM golang:1.18-buster as build +const DockerfileSQLite = `FROM golang:1.20-bookworm as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build ARG BINARY @@ -119,7 +118,7 @@ RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key # Make sure the SQLite databases are in a persistent location, we're already mapping # the postgresql folder so let's just use that for simplicity -RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postgresql\/11\/main\/%g" dendrite.yaml +RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postgresql\/15\/main\/%g" dendrite.yaml # This entry script starts postgres, waits for it to be up then starts dendrite RUN echo '\ @@ -402,7 +401,7 @@ func runImage(dockerClient *client.Client, volumeName string, branchNameToImageI { Type: mount.TypeVolume, Source: volumeName, - Target: "/var/lib/postgresql/11/main", + Target: "/var/lib/postgresql/15/main", }, }, }, nil, nil, "dendrite_upgrade_test_"+branchName) @@ -515,7 +514,7 @@ func testCreateAccount(dockerClient *client.Client, version *semver.Version, con } defer response.Close() - data, err := ioutil.ReadAll(response.Reader) + data, err := io.ReadAll(response.Reader) if err != nil { return err } @@ -557,8 +556,8 @@ func cleanup(dockerClient *client.Client) { }) for _, c := range containers { log.Printf("Removing container: %v %v\n", c.ID, c.Names) - s := time.Second - _ = dockerClient.ContainerStop(context.Background(), c.ID, &s) + timeout := 1 + _ = dockerClient.ContainerStop(context.Background(), c.ID, container.StopOptions{Timeout: &timeout}) _ = dockerClient.ContainerRemove(context.Background(), c.ID, types.ContainerRemoveOptions{ Force: true, }) @@ -592,7 +591,7 @@ func main() { branchToImageID := buildDendriteImages(httpClient, dockerClient, *flagTempDir, *flagBuildConcurrency, versions) // make a shared postgres volume - volume, err := dockerClient.VolumeCreate(context.Background(), volume.VolumeCreateBody{ + volume, err := dockerClient.VolumeCreate(context.Background(), volume.CreateOptions{ Name: "dendrite_upgrade_test", Labels: map[string]string{ dendriteUpgradeTestLabel: "yes", diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index b7ac5751e..508e4be8e 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -73,7 +73,7 @@ func main() { // don't hit matrix.org when running tests!!! cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{} cfg.MediaAPI.BasePath = config.Path(filepath.Join(*dirPath, "media")) - cfg.MSCs.MSCs = []string{"msc2836", "msc2946", "msc2444", "msc2753"} + cfg.MSCs.MSCs = []string{"msc2836", "msc2444", "msc2753"} cfg.Logging[0].Level = "trace" cfg.Logging[0].Type = "std" cfg.UserAPI.BCryptCost = bcrypt.MinCost diff --git a/dendrite-sample.yaml b/dendrite-sample.yaml index 96143d85f..7affc2599 100644 --- a/dendrite-sample.yaml +++ b/dendrite-sample.yaml @@ -72,6 +72,10 @@ global: # The base URL to delegate client-server communications to e.g. https://localhost well_known_client_name: "" + # The server name to delegate sliding sync communications to, with optional port. + # Requires `well_known_client_name` to also be configured. + well_known_sliding_sync_proxy: "" + # Lists of domains that the server will trust as identity servers to verify third # party identifiers such as phone numbers and email addresses. trusted_third_party_id_servers: @@ -276,7 +280,6 @@ media_api: mscs: mscs: # - msc2836 # (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) - # - msc2946 # (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946) # Configuration for the Sync API. sync_api: diff --git a/docs/FAQ.md b/docs/FAQ.md index 757bf9625..570ba677e 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -64,16 +64,18 @@ Use [dendrite.matrix.org](https://dendrite.matrix.org) which we officially suppo ## Does Dendrite support Space Summaries? -Yes, [Space Summaries](https://github.com/matrix-org/matrix-spec-proposals/pull/2946) were merged into the Matrix Spec as of 2022-01-17 however, they are still treated as an MSC (Matrix Specification Change) in Dendrite. In order to enable Space Summaries in Dendrite, you must add the MSC to the MSC configuration section in the configuration YAML. If the MSC is not enabled, a user will typically see a perpetual loading icon on the summary page. See below for a demonstration of how to add to the Dendrite configuration: +Yes + +## Does Dendrite support Threads? + +Yes, to enable them [msc2836](https://github.com/matrix-org/matrix-spec-proposals/pull/2836) would need to be added to mscs configuration in order to support Threading. Other MSCs are not currently supported. ``` mscs: mscs: - - msc2946 + - msc2836 ``` -Similarly, [msc2836](https://github.com/matrix-org/matrix-spec-proposals/pull/2836) would need to be added to mscs configuration in order to support Threading. Other MSCs are not currently supported. - Please note that MSCs should be considered experimental and can result in significant usability issues when enabled. If you'd like more details on how MSCs are ratified or the current status of MSCs, please see the [Matrix specification documentation](https://spec.matrix.org/proposals/) on the subject. ## Does Dendrite support push notifications? diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index 0901965c1..195f60c6f 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -14,7 +14,7 @@ GEM execjs coffee-script-source (1.11.1) colorator (1.1.0) - commonmarker (0.23.9) + commonmarker (0.23.10) concurrent-ruby (1.2.0) dnsruby (1.61.9) simpleidn (~> 0.1) diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md index 6f6458997..40d02622b 100644 --- a/docs/administration/4_adminapi.md +++ b/docs/administration/4_adminapi.md @@ -75,7 +75,7 @@ This endpoint instructs Dendrite to immediately query `/devices/{userID}` on a f ## POST `/_dendrite/admin/purgeRoom/{roomID}` -This endpoint instructs Dendrite to remove the given room from its database. Before doing so, it will evacuate all local users from the room. It does **NOT** remove media files. Depending on the size of the room, this may take a while. Will return an empty JSON once other components were instructed to delete the room. +This endpoint instructs Dendrite to remove the given room from its database. It does **NOT** remove media files. Depending on the size of the room, this may take a while. Will return an empty JSON once other components were instructed to delete the room. ## POST `/_synapse/admin/v1/send_server_notice` diff --git a/docs/administration/5_optimisation.md b/docs/administration/5_optimisation.md index b327171eb..57b7924d3 100644 --- a/docs/administration/5_optimisation.md +++ b/docs/administration/5_optimisation.md @@ -95,7 +95,7 @@ Consider enabling the DNS cache by modifying the `global` section of your config ## Time synchronisation Matrix relies heavily on TLS which requires the system time to be correct. If the clock -drifts then you may find that federation no works reliably (or at all) and clients may +drifts then you may find that federation will not work reliably (or at all) and clients may struggle to connect to your Dendrite server. Ensure that the time is synchronised on your system by enabling NTP sync. diff --git a/docs/development/CONTRIBUTING.md b/docs/development/CONTRIBUTING.md index 71e7516a2..caab1e749 100644 --- a/docs/development/CONTRIBUTING.md +++ b/docs/development/CONTRIBUTING.md @@ -109,7 +109,7 @@ To configure the connection to a remote Postgres, you can use the following envi ```bash POSTGRES_USER=postgres -POSTGERS_PASSWORD=yourPostgresPassword +POSTGRES_PASSWORD=yourPostgresPassword POSTGRES_HOST=localhost POSTGRES_DB=postgres # the superuser database to use ``` diff --git a/docs/installation/1_planning.md b/docs/installation/1_planning.md index 354003aef..37ca5702a 100644 --- a/docs/installation/1_planning.md +++ b/docs/installation/1_planning.md @@ -59,7 +59,7 @@ In order to install Dendrite, you will need to satisfy the following dependencie ### Go -At this time, Dendrite supports being built with Go 1.18 or later. We do not support building +At this time, Dendrite supports being built with Go 1.20 or later. We do not support building Dendrite with older versions of Go than this. If you are installing Go using a package manager, you should check (by running `go version`) that you are using a suitable version before you start. diff --git a/federationapi/internal/api.go b/federationapi/internal/api.go index aa501f63c..3e6f39566 100644 --- a/federationapi/internal/api.go +++ b/federationapi/internal/api.go @@ -54,11 +54,14 @@ func NewFederationInternalAPI( KeyDatabase: serverKeyDB, } + pubKey := cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey) addDirectFetcher := func() { keyRing.KeyFetchers = append( keyRing.KeyFetchers, &gomatrixserverlib.DirectKeyFetcher{ - Client: federation, + Client: federation, + IsLocalServerName: cfg.Matrix.IsLocalServerName, + LocalPublicKey: []byte(pubKey), }, ) } diff --git a/federationapi/internal/perform_test.go b/federationapi/internal/perform_test.go index 2f61235ae..656755f96 100644 --- a/federationapi/internal/perform_test.go +++ b/federationapi/internal/perform_test.go @@ -16,6 +16,7 @@ package internal import ( "context" + "crypto/ed25519" "testing" "github.com/matrix-org/dendrite/federationapi/api" @@ -53,10 +54,14 @@ func TestPerformWakeupServers(t *testing.T) { assert.NoError(t, err) assert.True(t, offline) + _, key, err := ed25519.GenerateKey(nil) + assert.NoError(t, err) cfg := config.FederationAPI{ Matrix: &config.Global{ SigningIdentity: fclient.SigningIdentity{ ServerName: "relay", + KeyID: "ed25519:1", + PrivateKey: key, }, }, } @@ -95,10 +100,14 @@ func TestQueryRelayServers(t *testing.T) { err := testDB.P2PAddRelayServersForServer(context.Background(), server, relayServers) assert.NoError(t, err) + _, key, err := ed25519.GenerateKey(nil) + assert.NoError(t, err) cfg := config.FederationAPI{ Matrix: &config.Global{ SigningIdentity: fclient.SigningIdentity{ ServerName: "relay", + KeyID: "ed25519:1", + PrivateKey: key, }, }, } @@ -132,10 +141,14 @@ func TestRemoveRelayServers(t *testing.T) { err := testDB.P2PAddRelayServersForServer(context.Background(), server, relayServers) assert.NoError(t, err) + _, key, err := ed25519.GenerateKey(nil) + assert.NoError(t, err) cfg := config.FederationAPI{ Matrix: &config.Global{ SigningIdentity: fclient.SigningIdentity{ ServerName: "relay", + KeyID: "ed25519:1", + PrivateKey: key, }, }, } @@ -168,10 +181,14 @@ func TestRemoveRelayServers(t *testing.T) { func TestPerformDirectoryLookup(t *testing.T) { testDB := test.NewInMemoryFederationDatabase() + _, key, err := ed25519.GenerateKey(nil) + assert.NoError(t, err) cfg := config.FederationAPI{ Matrix: &config.Global{ SigningIdentity: fclient.SigningIdentity{ ServerName: "relay", + KeyID: "ed25519:1", + PrivateKey: key, }, }, } @@ -192,7 +209,7 @@ func TestPerformDirectoryLookup(t *testing.T) { ServerName: "server", } res := api.PerformDirectoryLookupResponse{} - err := fedAPI.PerformDirectoryLookup(context.Background(), &req, &res) + err = fedAPI.PerformDirectoryLookup(context.Background(), &req, &res) assert.NoError(t, err) } @@ -203,10 +220,14 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) { testDB.SetServerAssumedOffline(context.Background(), server) testDB.P2PAddRelayServersForServer(context.Background(), server, []spec.ServerName{"relay"}) + _, key, err := ed25519.GenerateKey(nil) + assert.NoError(t, err) cfg := config.FederationAPI{ Matrix: &config.Global{ SigningIdentity: fclient.SigningIdentity{ - ServerName: server, + ServerName: "relay", + KeyID: "ed25519:1", + PrivateKey: key, }, }, } @@ -227,6 +248,6 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) { ServerName: server, } res := api.PerformDirectoryLookupResponse{} - err := fedAPI.PerformDirectoryLookup(context.Background(), &req, &res) + err = fedAPI.PerformDirectoryLookup(context.Background(), &req, &res) assert.Error(t, err) } diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go index cc38e136f..e75615e05 100644 --- a/federationapi/queue/queue_test.go +++ b/federationapi/queue/queue_test.go @@ -104,7 +104,7 @@ func (f *stubFederationClient) P2PSendTransactionToRelay(ctx context.Context, u func mustCreatePDU(t *testing.T) *types.HeaderedEvent { t.Helper() - content := `{"type":"m.room.message"}` + content := `{"type":"m.room.message", "room_id":"!room:a"}` ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionV10).NewEventFromTrustedJSON([]byte(content), false) if err != nil { t.Fatalf("failed to create event: %v", err) diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index e45209a2f..76fadaa93 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -16,6 +16,7 @@ package routing import ( "context" + "crypto/ed25519" "encoding/json" "fmt" "net/http" @@ -29,6 +30,73 @@ import ( "github.com/matrix-org/util" ) +// InviteV3 implements /_matrix/federation/v2/invite/{roomID}/{userID} +func InviteV3( + httpReq *http.Request, + request *fclient.FederationRequest, + roomID spec.RoomID, + invitedUser spec.UserID, + cfg *config.FederationAPI, + rsAPI api.FederationRoomserverAPI, + keys gomatrixserverlib.JSONVerifier, +) util.JSONResponse { + inviteReq := fclient.InviteV3Request{} + err := json.Unmarshal(request.Content(), &inviteReq) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.BadJSON(err.Error()), + } + } + if !cfg.Matrix.IsLocalServerName(invitedUser.Domain()) { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("The invited user domain does not belong to this server"), + } + } + + input := gomatrixserverlib.HandleInviteV3Input{ + HandleInviteInput: gomatrixserverlib.HandleInviteInput{ + RoomVersion: inviteReq.RoomVersion(), + RoomID: roomID, + InvitedUser: invitedUser, + KeyID: cfg.Matrix.KeyID, + PrivateKey: cfg.Matrix.PrivateKey, + Verifier: keys, + RoomQuerier: rsAPI, + MembershipQuerier: &api.MembershipQuerier{Roomserver: rsAPI}, + StateQuerier: rsAPI.StateQuerier(), + InviteEvent: nil, + StrippedState: inviteReq.InviteRoomState(), + UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(httpReq.Context(), roomID, senderID) + }, + }, + InviteProtoEvent: inviteReq.Event(), + GetOrCreateSenderID: func(ctx context.Context, userID spec.UserID, roomID spec.RoomID, roomVersion string) (spec.SenderID, ed25519.PrivateKey, error) { + // assign a roomNID, otherwise we can't create a private key for the user + _, nidErr := rsAPI.AssignRoomNID(ctx, roomID, gomatrixserverlib.RoomVersion(roomVersion)) + if nidErr != nil { + return "", nil, nidErr + } + key, keyErr := rsAPI.GetOrCreateUserRoomPrivateKey(ctx, userID, roomID) + if keyErr != nil { + return "", nil, keyErr + } + + return spec.SenderIDFromPseudoIDKey(key), key, nil + }, + } + event, jsonErr := handleInviteV3(httpReq.Context(), input, rsAPI) + if jsonErr != nil { + return *jsonErr + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: fclient.RespInviteV2{Event: event.JSON()}, + } +} + // InviteV2 implements /_matrix/federation/v2/invite/{roomID}/{eventID} func InviteV2( httpReq *http.Request, @@ -204,6 +272,15 @@ func InviteV1( func handleInvite(ctx context.Context, input gomatrixserverlib.HandleInviteInput, rsAPI api.FederationRoomserverAPI) (gomatrixserverlib.PDU, *util.JSONResponse) { inviteEvent, err := gomatrixserverlib.HandleInvite(ctx, input) + return handleInviteResult(ctx, inviteEvent, err, rsAPI) +} + +func handleInviteV3(ctx context.Context, input gomatrixserverlib.HandleInviteV3Input, rsAPI api.FederationRoomserverAPI) (gomatrixserverlib.PDU, *util.JSONResponse) { + inviteEvent, err := gomatrixserverlib.HandleInviteV3(ctx, input) + return handleInviteResult(ctx, inviteEvent, err, rsAPI) +} + +func handleInviteResult(ctx context.Context, inviteEvent gomatrixserverlib.PDU, err error, rsAPI api.FederationRoomserverAPI) (gomatrixserverlib.PDU, *util.JSONResponse) { switch e := err.(type) { case nil: case spec.InternalServerError: @@ -245,4 +322,5 @@ func handleInvite(ctx context.Context, input gomatrixserverlib.HandleInviteInput } } return inviteEvent, nil + } diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go index e6a488ba3..e8c7ff793 100644 --- a/federationapi/routing/profile.go +++ b/federationapi/routing/profile.go @@ -15,9 +15,11 @@ package routing import ( + "errors" "fmt" "net/http" + appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -52,6 +54,12 @@ func GetProfile( profile, err := userAPI.QueryProfile(httpReq.Context(), userID) if err != nil { + if errors.Is(err, appserviceAPI.ErrProfileNotExists) { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.NotFound("The user does not exist or does not have a profile."), + } + } util.GetLogger(httpReq.Context()).WithError(err).Error("userAPI.QueryProfile failed") return util.JSONResponse{ Code: http.StatusInternalServerError, diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 2e845f32c..327ba9b08 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -20,12 +20,14 @@ import ( federationAPI "github.com/matrix-org/dendrite/federationapi/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" + log "github.com/sirupsen/logrus" ) // RoomAliasToID converts the queried alias into a room ID and returns it @@ -116,3 +118,65 @@ func RoomAliasToID( JSON: resp, } } + +// Query the immediate children of a room/space +// +// Implements /_matrix/federation/v1/hierarchy/{roomID} +func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationRequest, roomIDStr string, rsAPI roomserverAPI.FederationRoomserverAPI) util.JSONResponse { + parsedRoomID, err := spec.NewRoomID(roomIDStr) + if err != nil { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.InvalidParam("room is unknown/forbidden"), + } + } + roomID := *parsedRoomID + + suggestedOnly := false // Defaults to false (spec-defined) + switch httpReq.URL.Query().Get("suggested_only") { + case "true": + suggestedOnly = true + case "false": + case "": // Empty string is returned when query param is not set + default: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("query parameter 'suggested_only', if set, must be 'true' or 'false'"), + } + } + + walker := roomserverAPI.NewRoomHierarchyWalker(types.NewServerNameNotDevice(request.Origin()), roomID, suggestedOnly, 1) + discoveredRooms, _, err := rsAPI.QueryNextRoomHierarchyPage(httpReq.Context(), walker, -1) + + if err != nil { + switch err.(type) { + case roomserverAPI.ErrRoomUnknownOrNotAllowed: + util.GetLogger(httpReq.Context()).WithError(err).Debugln("room unknown/forbidden when handling SS room hierarchy request") + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.NotFound("room is unknown/forbidden"), + } + default: + log.WithError(err).Errorf("failed to fetch next page of room hierarchy (SS API)") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + } + + if len(discoveredRooms) == 0 { + util.GetLogger(httpReq.Context()).Debugln("no rooms found when handling SS room hierarchy request") + return util.JSONResponse{ + Code: 404, + JSON: spec.NotFound("room is unknown/forbidden"), + } + } + return util.JSONResponse{ + Code: 200, + JSON: fclient.RoomHierarchyResponse{ + Room: discoveredRooms[0], + Children: discoveredRooms[1:], + }, + } +} diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 8865022ff..dc7a363e7 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -78,6 +78,7 @@ func Setup( v2keysmux := keyMux.PathPrefix("/v2").Subrouter() v1fedmux := fedMux.PathPrefix("/v1").Subrouter() v2fedmux := fedMux.PathPrefix("/v2").Subrouter() + v3fedmux := fedMux.PathPrefix("/v3").Subrouter() wakeup := &FederationWakeups{ FsAPI: fsAPI, @@ -191,6 +192,37 @@ func Setup( }, )).Methods(http.MethodPut, http.MethodOptions) + v3fedmux.Handle("/invite/{roomID}/{userID}", MakeFedAPI( + "federation_invite", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, + func(httpReq *http.Request, request *fclient.FederationRequest, vars map[string]string) util.JSONResponse { + if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("Forbidden by server ACLs"), + } + } + + userID, err := spec.NewUserID(vars["userID"], true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("Invalid UserID"), + } + } + roomID, err := spec.NewRoomID(vars["roomID"]) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("Invalid RoomID"), + } + } + return InviteV3( + httpReq, request, *roomID, *userID, + cfg, rsAPI, keys, + ) + }, + )).Methods(http.MethodPut, http.MethodOptions) + v1fedmux.Handle("/3pid/onbind", httputil.MakeExternalAPI("3pid_onbind", func(req *http.Request) util.JSONResponse { return CreateInvitesFrom3PIDInvites(req, rsAPI, cfg, federation, userAPI) @@ -564,6 +596,13 @@ func Setup( return GetOpenIDUserInfo(req, userAPI) }), ).Methods(http.MethodGet) + + v1fedmux.Handle("/hierarchy/{roomID}", MakeFedAPI( + "federation_room_hierarchy", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, + func(httpReq *http.Request, request *fclient.FederationRequest, vars map[string]string) util.JSONResponse { + return QueryRoomHierarchy(httpReq, request, vars["roomID"], rsAPI) + }, + )).Methods(http.MethodGet) } func ErrorIfLocalServerNotInRoom( diff --git a/go.mod b/go.mod index f3cf2c584..5781b0068 100644 --- a/go.mod +++ b/go.mod @@ -4,16 +4,19 @@ require ( github.com/Arceliar/ironwood v0.0.0-20221025225125-45b4281814c2 github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 github.com/DATA-DOG/go-sqlmock v1.5.0 + github.com/MFAshby/stdemuxerhook v1.0.0 github.com/Masterminds/semver/v3 v3.1.1 github.com/blevesearch/bleve/v2 v2.3.8 github.com/codeclysm/extract v2.2.0+incompatible github.com/dgraph-io/ristretto v0.1.1 - github.com/docker/docker v20.10.24+incompatible + github.com/docker/docker v24.0.5+incompatible github.com/docker/go-connections v0.4.0 - github.com/getsentry/sentry-go v0.22.0 + github.com/getsentry/sentry-go v0.14.0 + github.com/go-ldap/ldap/v3 v3.4.6 + github.com/golang-jwt/jwt/v4 v4.5.0 github.com/gologme/log v1.3.0 github.com/google/go-cmp v0.5.9 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.3.1 github.com/gorilla/mux v1.8.0 github.com/kardianos/minwinsvc v1.0.2 github.com/lib/pq v1.10.9 @@ -21,6 +24,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 + github.com/matryer/is v1.4.1 github.com/mattn/go-sqlite3 v1.14.17 github.com/nats-io/nats-server/v2 v2.9.23 github.com/nats-io/nats.go v1.28.0 @@ -32,17 +36,17 @@ require ( github.com/prometheus/client_golang v1.16.0 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.2 - github.com/tidwall/gjson v1.14.4 + github.com/tidwall/gjson v1.17.0 github.com/tidwall/sjson v1.2.5 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible github.com/yggdrasil-network/yggdrasil-go v0.4.6 go.uber.org/atomic v1.10.0 golang.org/x/crypto v0.14.0 - golang.org/x/exp v0.0.0-20221205204356-47842c84f3db + golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 golang.org/x/image v0.5.0 golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e - golang.org/x/sync v0.2.0 + golang.org/x/sync v0.3.0 golang.org/x/term v0.13.0 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 @@ -51,13 +55,6 @@ require ( modernc.org/sqlite v1.23.1 ) -require ( - github.com/MFAshby/stdemuxerhook v1.0.0 - github.com/go-ldap/ldap/v3 v3.4.5 - github.com/golang-jwt/jwt/v4 v4.5.0 - github.com/matryer/is v1.4.0 -) - require ( github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect @@ -85,12 +82,13 @@ require ( github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect + github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/pprof v0.0.0-20230808223545-4887780b67fb // indirect github.com/h2non/filetype v1.1.3 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/juju/errors v1.0.0 // indirect @@ -115,17 +113,17 @@ require ( github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/rs/zerolog v1.29.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect - go.etcd.io/bbolt v1.3.5 // indirect - golang.org/x/mod v0.9.0 // indirect + go.etcd.io/bbolt v1.3.6 // indirect + golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.6.0 // indirect + golang.org/x/tools v0.12.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/macaroon.v2 v2.1.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -141,4 +139,4 @@ require ( modernc.org/token v1.0.1 // indirect ) -go 1.18 +go 1.20 diff --git a/go.sum b/go.sum index 73a377767..415ee8604 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.24+incompatible h1:Ugvxm7a8+Gz6vqQYQQ2W7GYq5EUPaAiuPgIfVyI3dYE= -github.com/docker/docker v20.10.24+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY= +github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -107,16 +107,16 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/getsentry/sentry-go v0.22.0 h1:XNX9zKbv7baSEI65l+H1GEJgSeIC1c7EN5kluWaP6dM= -github.com/getsentry/sentry-go v0.22.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/getsentry/sentry-go v0.14.0 h1:rlOBkuFZRKKdUnKO+0U3JclRDQKlRu5vVQtkWSQvC70= +github.com/getsentry/sentry-go v0.14.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2HtRckJk9XVxJ9I= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= -github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A= -github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA= +github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8= -github.com/go-ldap/ldap/v3 v3.4.5/go.mod h1:bMGIq3AGbytbaMwf8wdv5Phdxz0FWHTIYMSzyrYgnQs= +github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A= +github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -147,9 +147,10 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/pprof v0.0.0-20230808223545-4887780b67fb h1:oqpb3Cwpc7EOml5PVGMYbSGmwNui2R7i8IW83gs4W0c= +github.com/google/pprof v0.0.0-20230808223545-4887780b67fb/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= @@ -188,8 +189,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca h1:JC github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca/go.mod h1:M8m7seOroO5ePlgxA7AFZymnG90Cnh94rYQyngSrZkk= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66/go.mod h1:iBI1foelCqA09JJgPV0FYz4qA5dUXYOxMi57FxKBdd4= -github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= -github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= +github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ= +github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -257,8 +258,8 @@ github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPH github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= @@ -283,8 +284,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= +github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -303,8 +304,8 @@ github.com/yggdrasil-network/yggdrasil-go v0.4.6/go.mod h1:PBMoAOvQjA9geNEeGyMXA github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -314,7 +315,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -322,8 +323,8 @@ golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20221205204356-47842c84f3db h1:D/cFflL63o2KSLJIwjlcIt8PR064j/xsmdEJL/YvY/o= -golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 h1:EDuYyU/MkFXllv9QF9819VlI9a4tzGuCbhG0ExK9o1U= +golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -337,8 +338,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -347,7 +348,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -356,14 +357,14 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -378,20 +379,22 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= @@ -406,8 +409,9 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml index 668fd84ec..32f479960 100644 --- a/helm/dendrite/Chart.yaml +++ b/helm/dendrite/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: dendrite -version: "0.13.0" -appVersion: "0.13.0" +version: "0.13.5" +appVersion: "0.13.4" description: Dendrite Matrix Homeserver type: application keywords: diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md index 562d1e235..22daa1813 100644 --- a/helm/dendrite/README.md +++ b/helm/dendrite/README.md @@ -1,7 +1,7 @@ # dendrite -![Version: 0.13.0](https://img.shields.io/badge/Version-0.13.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.0](https://img.shields.io/badge/AppVersion-0.13.0-informational?style=flat-square) +![Version: 0.13.5](https://img.shields.io/badge/Version-0.13.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.4](https://img.shields.io/badge/AppVersion-0.13.4-informational?style=flat-square) Dendrite Matrix Homeserver Status: **NOT PRODUCTION READY** @@ -48,7 +48,7 @@ Create a folder `appservices` and place your configurations in there. The confi | signing_key.create | bool | `true` | Create a new signing key, if not exists | | signing_key.existingSecret | string | `""` | Use an existing secret | | resources | object | sets some sane default values | Default resource requests/limits. | -| persistence.storageClass | string | `""` | The storage class to use for volume claims. Defaults to the cluster default storage class. | +| persistence.jetstream | object | `{"capacity":"1Gi","existingClaim":""}` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. # If defined, storageClassName: # If set to "-", storageClassName: "", which disables dynamic provisioning # If undefined (the default) or set to null, no storageClassName spec is # set, choosing the default provisioner. (gp2 on AWS, standard on # GKE, AWS & OpenStack) # storageClass: "" | | persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream | | persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume | | persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files | @@ -97,7 +97,7 @@ Create a folder `appservices` and place your configurations in there. The confi | dendrite_config.global.dns_cache.cache_lifetime | string | `"10m"` | Duration for how long DNS cache items should be considered valid ([see time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more) | | dendrite_config.global.profiling.enabled | bool | `false` | Enable pprof. You will need to manually create a port forwarding to the deployment to access PPROF, as it will only listen on localhost and the defined port. e.g. `kubectl port-forward deployments/dendrite 65432:65432` | | dendrite_config.global.profiling.port | int | `65432` | pprof port, if enabled | -| dendrite_config.mscs | object | `{"mscs":["msc2946"]}` | Configuration for experimental MSC's. (Valid values are: msc2836 and msc2946) | +| dendrite_config.mscs | object | `{"mscs":[]}` | Configuration for experimental MSC's. (Valid values are: msc2836) | | dendrite_config.app_service_api.disable_tls_validation | bool | `false` | Disable the validation of TLS certificates of appservices. This is not recommended in production since it may allow appservice traffic to be sent to an insecure endpoint. | | dendrite_config.app_service_api.config_files | list | `[]` | Appservice config files to load on startup. (**NOTE**: This is overriden by Helm, if a folder `./appservices/` exists) | | dendrite_config.client_api.registration_disabled | bool | `true` | Prevents new users from being able to register on this homeserver, except when using the registration shared secret below. | @@ -144,12 +144,11 @@ Create a folder `appservices` and place your configurations in there. The confi | postgresql.auth.password | string | `"changeme"` | | | postgresql.auth.database | string | `"dendrite"` | | | postgresql.persistence.enabled | bool | `false` | | -| ingress.enabled | bool | `false` | Create an ingress for a monolith deployment | -| ingress.hosts | list | `[]` | | -| ingress.className | string | `""` | | -| ingress.hostName | string | `""` | | +| ingress.enabled | bool | `false` | Create an ingress for the deployment | +| ingress.className | string | `""` | The ingressClass to use. Will be converted to annotation if not yet supported. | | ingress.annotations | object | `{}` | Extra, custom annotations | -| ingress.tls | list | `[]` | | +| ingress.hostName | string | `""` | The ingress hostname for your matrix server. Should align with the server_name and well_known_* hosts. If not set, generated from the dendrite_config values. | +| ingress.tls | list | `[]` | TLS configuration. Should contain information for the server_name and well-known hosts. Alternatively, set tls.generate=true to generate defaults based on the dendrite_config. | | service.type | string | `"ClusterIP"` | | | service.port | int | `8008` | | | prometheus.servicemonitor.enabled | bool | `false` | Enable ServiceMonitor for Prometheus-Operator for scrape metric-endpoint | diff --git a/helm/dendrite/templates/deployment.yaml b/helm/dendrite/templates/deployment.yaml index df7dbbdc3..e3f84cdae 100644 --- a/helm/dendrite/templates/deployment.yaml +++ b/helm/dendrite/templates/deployment.yaml @@ -109,4 +109,4 @@ spec: failureThreshold: 10 httpGet: path: /_dendrite/monitor/up - port: http \ No newline at end of file + port: http diff --git a/helm/dendrite/templates/ingress.yaml b/helm/dendrite/templates/ingress.yaml index 9ef413dc9..4bcaee12d 100644 --- a/helm/dendrite/templates/ingress.yaml +++ b/helm/dendrite/templates/ingress.yaml @@ -1,8 +1,8 @@ {{- if .Values.ingress.enabled -}} {{- $fullName := include "dendrite.fullname" . -}} {{- $serverNameHost := .Values.dendrite_config.global.server_name -}} -{{- $wellKnownServerHost := default $serverNameHost (regexFind "^[^:]+" .Values.dendrite_config.global.well_known_server_name) -}} -{{- $wellKnownClientHost := default $serverNameHost (regexFind "^[^:]+" .Values.dendrite_config.global.well_known_client_name) -}} +{{- $wellKnownServerHost := default $serverNameHost (regexFind "^(\\[.+\\])?[^:]*" .Values.dendrite_config.global.well_known_server_name) -}} +{{- $wellKnownClientHost := default $serverNameHost (regexFind "//(\\[.+\\])?[^:/]*" .Values.dendrite_config.global.well_known_client_name | trimAll "/") -}} {{- $allHosts := list $serverNameHost $wellKnownServerHost $wellKnownClientHost | uniq -}} {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1 diff --git a/helm/dendrite/templates/pvc.yaml b/helm/dendrite/templates/pvc.yaml index 88eff3bed..70b1ce563 100644 --- a/helm/dendrite/templates/pvc.yaml +++ b/helm/dendrite/templates/pvc.yaml @@ -12,7 +12,14 @@ spec: resources: requests: storage: {{ .Values.persistence.media.capacity }} - storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.media.storageClass }} + {{ $storageClass := .Values.persistence.media.storageClass | default .Values.persistence.storageClass }} + {{- if $storageClass }} + {{- if (eq "-" $storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ $storageClass }}" + {{- end }} + {{- end }} {{ end }} {{ if not .Values.persistence.jetstream.existingClaim }} --- @@ -28,7 +35,14 @@ spec: resources: requests: storage: {{ .Values.persistence.jetstream.capacity }} - storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.jetstream.storageClass }} + {{ $storageClass := .Values.persistence.jetstream.storageClass | default .Values.persistence.storageClass }} + {{- if $storageClass }} + {{- if (eq "-" $storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ $storageClass }}" + {{- end }} + {{- end }} {{ end }} {{ if not .Values.persistence.search.existingClaim }} --- @@ -44,5 +58,12 @@ spec: resources: requests: storage: {{ .Values.persistence.search.capacity }} - storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.search.storageClass }} + {{ $storageClass := .Values.persistence.search.storageClass | default .Values.persistence.storageClass }} + {{- if $storageClass }} + {{- if (eq "-" $storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ $storageClass }}" + {{- end }} + {{- end }} {{ end }} diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml index 2b009c7d6..afce1d930 100644 --- a/helm/dendrite/values.yaml +++ b/helm/dendrite/values.yaml @@ -26,7 +26,13 @@ persistence: # -- The storage class to use for volume claims. # Used unless specified at the specific component. # Defaults to the cluster default storage class. - storageClass: "" + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "" jetstream: # -- Use an existing volume claim for jetstream existingClaim: "" @@ -34,7 +40,13 @@ persistence: capacity: "1Gi" # -- The storage class to use for volume claims. # Defaults to persistence.storageClass - storageClass: "" + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "" media: # -- Use an existing volume claim for media files existingClaim: "" @@ -42,7 +54,13 @@ persistence: capacity: "1Gi" # -- The storage class to use for volume claims. # Defaults to persistence.storageClass - storageClass: "" + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "" search: # -- Use an existing volume claim for the fulltext search index existingClaim: "" @@ -50,7 +68,13 @@ persistence: capacity: "1Gi" # -- The storage class to use for volume claims. # Defaults to persistence.storageClass - storageClass: "" + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "" # -- Add additional volumes to the Dendrite Pod extraVolumes: [] @@ -211,14 +235,12 @@ dendrite_config: # -- pprof port, if enabled port: 65432 - # -- Configuration for experimental MSC's. (Valid values are: msc2836 and msc2946) + # -- Configuration for experimental MSC's. (Valid values are: msc2836) mscs: - mscs: - - msc2946 + mscs: [] # A list of enabled MSC's # Currently valid values are: # - msc2836 (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) - # - msc2946 (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946) app_service_api: # -- Disable the validation of TLS certificates of appservices. This is diff --git a/internal/pushrules/action_test.go b/internal/pushrules/action_test.go index 72db9c998..d1eb16ef1 100644 --- a/internal/pushrules/action_test.go +++ b/internal/pushrules/action_test.go @@ -13,8 +13,6 @@ func TestActionJSON(t *testing.T) { Want Action }{ {Action{Kind: NotifyAction}}, - {Action{Kind: DontNotifyAction}}, - {Action{Kind: CoalesceAction}}, {Action{Kind: SetTweakAction}}, {Action{Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"}}, diff --git a/internal/pushrules/default_override.go b/internal/pushrules/default_override.go index f97427b71..cb825af5f 100644 --- a/internal/pushrules/default_override.go +++ b/internal/pushrules/default_override.go @@ -10,6 +10,7 @@ func defaultOverrideRules(userID string) []*Rule { &mRuleRoomNotifDefinition, &mRuleTombstoneDefinition, &mRuleReactionDefinition, + &mRuleACLsDefinition, } } @@ -30,7 +31,7 @@ var ( RuleID: MRuleMaster, Default: true, Enabled: false, - Actions: []*Action{{Kind: DontNotifyAction}}, + Actions: []*Action{}, } mRuleSuppressNoticesDefinition = Rule{ RuleID: MRuleSuppressNotices, @@ -43,7 +44,7 @@ var ( Pattern: pointer("m.notice"), }, }, - Actions: []*Action{{Kind: DontNotifyAction}}, + Actions: []*Action{}, } mRuleMemberEventDefinition = Rule{ RuleID: MRuleMemberEvent, @@ -56,7 +57,7 @@ var ( Pattern: pointer("m.room.member"), }, }, - Actions: []*Action{{Kind: DontNotifyAction}}, + Actions: []*Action{}, } mRuleContainsDisplayNameDefinition = Rule{ RuleID: MRuleContainsDisplayName, @@ -152,9 +153,7 @@ var ( Pattern: pointer("m.reaction"), }, }, - Actions: []*Action{ - {Kind: DontNotifyAction}, - }, + Actions: []*Action{}, } ) diff --git a/internal/pushrules/default_pushrules_test.go b/internal/pushrules/default_pushrules_test.go index dea829842..506fe3cc8 100644 --- a/internal/pushrules/default_pushrules_test.go +++ b/internal/pushrules/default_pushrules_test.go @@ -21,12 +21,12 @@ func TestDefaultRules(t *testing.T) { // Default override rules { name: ".m.rule.master", - inputBytes: []byte(`{"rule_id":".m.rule.master","default":true,"enabled":false,"actions":["dont_notify"]}`), + inputBytes: []byte(`{"rule_id":".m.rule.master","default":true,"enabled":false,"actions":[]}`), want: mRuleMasterDefinition, }, { name: ".m.rule.suppress_notices", - inputBytes: []byte(`{"rule_id":".m.rule.suppress_notices","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"content.msgtype","pattern":"m.notice"}],"actions":["dont_notify"]}`), + inputBytes: []byte(`{"rule_id":".m.rule.suppress_notices","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"content.msgtype","pattern":"m.notice"}],"actions":[]}`), want: mRuleSuppressNoticesDefinition, }, { @@ -36,7 +36,7 @@ func TestDefaultRules(t *testing.T) { }, { name: ".m.rule.member_event", - inputBytes: []byte(`{"rule_id":".m.rule.member_event","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"type","pattern":"m.room.member"}],"actions":["dont_notify"]}`), + inputBytes: []byte(`{"rule_id":".m.rule.member_event","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"type","pattern":"m.room.member"}],"actions":[]}`), want: mRuleMemberEventDefinition, }, { diff --git a/internal/pushrules/evaluate_test.go b/internal/pushrules/evaluate_test.go index a4ccc3d0f..fbc88b2e7 100644 --- a/internal/pushrules/evaluate_test.go +++ b/internal/pushrules/evaluate_test.go @@ -13,7 +13,7 @@ func UserIDForSender(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, } func TestRuleSetEvaluatorMatchEvent(t *testing.T) { - ev := mustEventFromJSON(t, `{}`) + ev := mustEventFromJSON(t, `{"room_id":"!room:a"}`) defaultEnabled := &Rule{ RuleID: ".default.enabled", Default: true, @@ -44,8 +44,8 @@ func TestRuleSetEvaluatorMatchEvent(t *testing.T) { {"overrideRoom", RuleSet{Override: []*Rule{userEnabled}, Room: []*Rule{userEnabled2}}, userEnabled, ev}, {"overrideSender", RuleSet{Override: []*Rule{userEnabled}, Sender: []*Rule{userEnabled2}}, userEnabled, ev}, {"overrideUnderride", RuleSet{Override: []*Rule{userEnabled}, Underride: []*Rule{userEnabled2}}, userEnabled, ev}, - {"reactions don't notify", *defaultRuleset, &mRuleReactionDefinition, mustEventFromJSON(t, `{"type":"m.reaction"}`)}, - {"receipts don't notify", *defaultRuleset, nil, mustEventFromJSON(t, `{"type":"m.receipt"}`)}, + {"reactions don't notify", *defaultRuleset, &mRuleReactionDefinition, mustEventFromJSON(t, `{"room_id":"!room:a","type":"m.reaction"}`)}, + {"receipts don't notify", *defaultRuleset, nil, mustEventFromJSON(t, `{"room_id":"!room:a","type":"m.receipt"}`)}, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { @@ -70,28 +70,27 @@ func TestRuleMatches(t *testing.T) { EventJSON string Want bool }{ - {"emptyOverride", OverrideKind, emptyRule, `{}`, true}, - {"emptyContent", ContentKind, emptyRule, `{}`, false}, - {"emptyRoom", RoomKind, emptyRule, `{}`, true}, + {"emptyOverride", OverrideKind, emptyRule, `{"room_id":"!room:example.com"}`, true}, + {"emptyContent", ContentKind, emptyRule, `{"room_id":"!room:example.com"}`, false}, {"emptySender", SenderKind, emptyRule, `{"room_id":"!room:example.com"}`, true}, - {"emptyUnderride", UnderrideKind, emptyRule, `{}`, true}, + {"emptyUnderride", UnderrideKind, emptyRule, `{"room_id":"!room:example.com"}`, true}, - {"disabled", OverrideKind, Rule{}, `{}`, false}, + {"disabled", OverrideKind, Rule{}, `{"room_id":"!room:example.com"}`, false}, - {"overrideConditionMatch", OverrideKind, Rule{Enabled: true}, `{}`, true}, - {"overrideConditionNoMatch", OverrideKind, Rule{Enabled: true, Conditions: []*Condition{{}}}, `{}`, false}, + {"overrideConditionMatch", OverrideKind, Rule{Enabled: true}, `{"room_id":"!room:example.com"}`, true}, + {"overrideConditionNoMatch", OverrideKind, Rule{Enabled: true, Conditions: []*Condition{{}}}, `{"room_id":"!room:example.com"}`, false}, - {"underrideConditionMatch", UnderrideKind, Rule{Enabled: true}, `{}`, true}, - {"underrideConditionNoMatch", UnderrideKind, Rule{Enabled: true, Conditions: []*Condition{{}}}, `{}`, false}, + {"underrideConditionMatch", UnderrideKind, Rule{Enabled: true}, `{"room_id":"!room:example.com"}`, true}, + {"underrideConditionNoMatch", UnderrideKind, Rule{Enabled: true, Conditions: []*Condition{{}}}, `{"room_id":"!room:example.com"}`, false}, - {"contentMatch", ContentKind, Rule{Enabled: true, Pattern: pointer("b")}, `{"content":{"body":"abc"}}`, true}, - {"contentNoMatch", ContentKind, Rule{Enabled: true, Pattern: pointer("d")}, `{"content":{"body":"abc"}}`, false}, + {"contentMatch", ContentKind, Rule{Enabled: true, Pattern: pointer("b")}, `{"room_id":"!room:example.com","content":{"body":"abc"}}`, true}, + {"contentNoMatch", ContentKind, Rule{Enabled: true, Pattern: pointer("d")}, `{"room_id":"!room:example.com","content":{"body":"abc"}}`, false}, {"roomMatch", RoomKind, Rule{Enabled: true, RuleID: "!room:example.com"}, `{"room_id":"!room:example.com"}`, true}, {"roomNoMatch", RoomKind, Rule{Enabled: true, RuleID: "!room:example.com"}, `{"room_id":"!otherroom:example.com"}`, false}, - {"senderMatch", SenderKind, Rule{Enabled: true, RuleID: "@user:example.com"}, `{"sender":"@user:example.com","room_id":"!room:example.com"}`, true}, - {"senderNoMatch", SenderKind, Rule{Enabled: true, RuleID: "@user:example.com"}, `{"sender":"@otheruser:example.com","room_id":"!room:example.com"}`, false}, + {"senderMatch", SenderKind, Rule{Enabled: true, RuleID: "@user:example.com"}, `{"room_id":"!room:example.com","sender":"@user:example.com","room_id":"!room:example.com"}`, true}, + {"senderNoMatch", SenderKind, Rule{Enabled: true, RuleID: "@user:example.com"}, `{"room_id":"!room:example.com","sender":"@otheruser:example.com","room_id":"!room:example.com"}`, false}, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { @@ -114,32 +113,32 @@ func TestConditionMatches(t *testing.T) { WantMatch bool WantErr bool }{ - {Name: "empty", Cond: Condition{}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "empty", Cond: Condition{Kind: "unknownstring"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, + {Name: "empty", Cond: Condition{}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "empty", Cond: Condition{Kind: "unknownstring"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, // Neither of these should match because `content` is not a full string match, // and `content.body` is not a string value. - {Name: "eventMatch", Cond: Condition{Kind: EventMatchCondition, Key: "content", Pattern: pointer("")}, EventJSON: `{"content":{}}`, WantMatch: false, WantErr: false}, - {Name: "eventBodyMatch", Cond: Condition{Kind: EventMatchCondition, Key: "content.body", Is: "3", Pattern: pointer("")}, EventJSON: `{"content":{"body": "3"}}`, WantMatch: false, WantErr: false}, - {Name: "eventBodyMatch matches", Cond: Condition{Kind: EventMatchCondition, Key: "content.body", Pattern: pointer("world")}, EventJSON: `{"content":{"body": "hello world!"}}`, WantMatch: true, WantErr: false}, - {Name: "EventMatch missing pattern", Cond: Condition{Kind: EventMatchCondition, Key: "content.body"}, EventJSON: `{"content":{"body": "hello world!"}}`, WantMatch: false, WantErr: true}, + {Name: "eventMatch", Cond: Condition{Kind: EventMatchCondition, Key: "content", Pattern: pointer("")}, EventJSON: `{"room_id":"!room:example.com","content":{}}`, WantMatch: false, WantErr: false}, + {Name: "eventBodyMatch", Cond: Condition{Kind: EventMatchCondition, Key: "content.body", Is: "3", Pattern: pointer("")}, EventJSON: `{"room_id":"!room:example.com","content":{"body": "3"}}`, WantMatch: false, WantErr: false}, + {Name: "eventBodyMatch matches", Cond: Condition{Kind: EventMatchCondition, Key: "content.body", Pattern: pointer("world")}, EventJSON: `{"room_id":"!room:example.com","content":{"body": "hello world!"}}`, WantMatch: true, WantErr: false}, + {Name: "EventMatch missing pattern", Cond: Condition{Kind: EventMatchCondition, Key: "content.body"}, EventJSON: `{"room_id":"!room:example.com","content":{"body": "hello world!"}}`, WantMatch: false, WantErr: true}, - {Name: "displayNameNoMatch", Cond: Condition{Kind: ContainsDisplayNameCondition}, EventJSON: `{"content":{"body":"something without displayname"}}`, WantMatch: false, WantErr: false}, - {Name: "displayNameMatch", Cond: Condition{Kind: ContainsDisplayNameCondition}, EventJSON: `{"content":{"body":"hello Dear User, how are you?"}}`, WantMatch: true, WantErr: false}, + {Name: "displayNameNoMatch", Cond: Condition{Kind: ContainsDisplayNameCondition}, EventJSON: `{"room_id":"!room:example.com","content":{"body":"something without displayname"}}`, WantMatch: false, WantErr: false}, + {Name: "displayNameMatch", Cond: Condition{Kind: ContainsDisplayNameCondition}, EventJSON: `{"room_id":"!room:example.com","content":{"body":"hello Dear User, how are you?"}}`, WantMatch: true, WantErr: false}, - {Name: "roomMemberCountLessNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<2"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "roomMemberCountLessMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<3"}, EventJSON: `{}`, WantMatch: true, WantErr: false}, - {Name: "roomMemberCountLessEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<=1"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "roomMemberCountLessEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<=2"}, EventJSON: `{}`, WantMatch: true, WantErr: false}, - {Name: "roomMemberCountEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "==1"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "roomMemberCountEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "==2"}, EventJSON: `{}`, WantMatch: true, WantErr: false}, - {Name: "roomMemberCountGreaterEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">=3"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "roomMemberCountGreaterEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">=2"}, EventJSON: `{}`, WantMatch: true, WantErr: false}, - {Name: "roomMemberCountGreaterNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">2"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "roomMemberCountGreaterMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">1"}, EventJSON: `{}`, WantMatch: true, WantErr: false}, + {Name: "roomMemberCountLessNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "roomMemberCountLessMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<3"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false}, + {Name: "roomMemberCountLessEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<=1"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "roomMemberCountLessEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<=2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false}, + {Name: "roomMemberCountEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "==1"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "roomMemberCountEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "==2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false}, + {Name: "roomMemberCountGreaterEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">=3"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "roomMemberCountGreaterEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">=2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false}, + {Name: "roomMemberCountGreaterNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "roomMemberCountGreaterMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">1"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false}, - {Name: "senderNotificationPermissionMatch", Cond: Condition{Kind: SenderNotificationPermissionCondition, Key: "powerlevel"}, EventJSON: `{"sender":"@poweruser:example.com"}`, WantMatch: true, WantErr: false}, - {Name: "senderNotificationPermissionNoMatch", Cond: Condition{Kind: SenderNotificationPermissionCondition, Key: "powerlevel"}, EventJSON: `{"sender":"@nobody:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "senderNotificationPermissionMatch", Cond: Condition{Kind: SenderNotificationPermissionCondition, Key: "powerlevel"}, EventJSON: `{"room_id":"!room:example.com","sender":"@poweruser:example.com"}`, WantMatch: true, WantErr: false}, + {Name: "senderNotificationPermissionNoMatch", Cond: Condition{Kind: SenderNotificationPermissionCondition, Key: "powerlevel"}, EventJSON: `{"room_id":"!room:example.com","sender":"@nobody:example.com"}`, WantMatch: false, WantErr: false}, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { @@ -170,15 +169,15 @@ func TestPatternMatches(t *testing.T) { EventJSON string Want bool }{ - {"empty", "", "", `{}`, false}, + {"empty", "", "", `{"room_id":"!room:a"}`, false}, - {"patternEmpty", "content", "", `{"content":{}}`, false}, + {"patternEmpty", "content", "", `{"room_id":"!room:a","content":{}}`, false}, - {"literal", "content.creator", "acreator", `{"content":{"creator":"acreator"}}`, true}, - {"substring", "content.creator", "reat", `{"content":{"creator":"acreator"}}`, true}, - {"singlePattern", "content.creator", "acr?ator", `{"content":{"creator":"acreator"}}`, true}, - {"multiPattern", "content.creator", "a*ea*r", `{"content":{"creator":"acreator"}}`, true}, - {"patternNoSubstring", "content.creator", "r*t", `{"content":{"creator":"acreator"}}`, false}, + {"literal", "content.creator", "acreator", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, true}, + {"substring", "content.creator", "reat", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, true}, + {"singlePattern", "content.creator", "acr?ator", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, true}, + {"multiPattern", "content.creator", "a*ea*r", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, true}, + {"patternNoSubstring", "content.creator", "r*t", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, false}, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { diff --git a/internal/pushrules/util.go b/internal/pushrules/util.go index de8fe5cd0..e2821b57a 100644 --- a/internal/pushrules/util.go +++ b/internal/pushrules/util.go @@ -16,10 +16,7 @@ func ActionsToTweaks(as []*Action) (ActionKind, map[string]interface{}, error) { for _, a := range as { switch a.Kind { - case DontNotifyAction: - // Don't bother processing any further - return DontNotifyAction, nil, nil - + case DontNotifyAction: // Ignored case SetTweakAction: if tweaks == nil { tweaks = map[string]interface{}{} diff --git a/internal/pushrules/util_test.go b/internal/pushrules/util_test.go index 89f8243d9..83eee7ede 100644 --- a/internal/pushrules/util_test.go +++ b/internal/pushrules/util_test.go @@ -17,17 +17,16 @@ func TestActionsToTweaks(t *testing.T) { {"empty", nil, UnknownAction, nil}, {"zero", []*Action{{}}, UnknownAction, nil}, {"onlyPrimary", []*Action{{Kind: NotifyAction}}, NotifyAction, nil}, - {"onlyPrimaryDontNotify", []*Action{{Kind: DontNotifyAction}}, DontNotifyAction, nil}, + {"onlyPrimaryDontNotify", []*Action{}, UnknownAction, nil}, {"onlyTweak", []*Action{{Kind: SetTweakAction, Tweak: HighlightTweak}}, UnknownAction, map[string]interface{}{"highlight": nil}}, {"onlyTweakWithValue", []*Action{{Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"}}, UnknownAction, map[string]interface{}{"sound": "default"}}, { "all", []*Action{ - {Kind: CoalesceAction}, {Kind: SetTweakAction, Tweak: HighlightTweak}, {Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"}, }, - CoalesceAction, + UnknownAction, map[string]interface{}{"highlight": nil, "sound": "default"}, }, } diff --git a/internal/pushrules/validate.go b/internal/pushrules/validate.go index b54ec3fb0..4cc479345 100644 --- a/internal/pushrules/validate.go +++ b/internal/pushrules/validate.go @@ -18,7 +18,7 @@ func ValidateRule(kind Kind, rule *Rule) []error { errs = append(errs, fmt.Errorf("invalid rule ID: %s", rule.RuleID)) } - if len(rule.Actions) == 0 { + if rule.Actions == nil { errs = append(errs, fmt.Errorf("missing actions")) } for _, action := range rule.Actions { diff --git a/internal/sqlutil/migrate.go b/internal/sqlutil/migrate.go index a66a75826..735fb4927 100644 --- a/internal/sqlutil/migrate.go +++ b/internal/sqlutil/migrate.go @@ -112,7 +112,13 @@ func (m *Migrator) Up(ctx context.Context) error { func (m *Migrator) insertMigration(ctx context.Context, txn *sql.Tx, migrationName string) error { if m.insertStmt == nil { - stmt, err := m.db.Prepare(insertVersionSQL) + var stmt *sql.Stmt + var err error + if txn == nil { + stmt, err = m.db.PrepareContext(ctx, insertVersionSQL) + } else { + stmt, err = txn.PrepareContext(ctx, insertVersionSQL) + } if err != nil { return fmt.Errorf("unable to prepare insert statement: %w", err) } diff --git a/internal/validate.go b/internal/validate.go index 99088f240..7f0d8b9e6 100644 --- a/internal/validate.go +++ b/internal/validate.go @@ -25,7 +25,7 @@ import ( ) const ( - maxUsernameLength = 254 // http://matrix.org/speculator/spec/HEAD/intro.html#user-identifiers TODO account for domain + maxUsernameLength = 254 // https://spec.matrix.org/v1.7/appendices/#user-identifiers TODO account for domain minPasswordLength = 8 // http://matrix.org/docs/spec/client_server/r0.2.0.html#password-based maxPasswordLength = 512 // https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161 diff --git a/internal/version.go b/internal/version.go index c42b20390..55726ddcb 100644 --- a/internal/version.go +++ b/internal/version.go @@ -2,6 +2,7 @@ package internal import ( "fmt" + "runtime/debug" "strings" ) @@ -17,8 +18,10 @@ var build string const ( VersionMajor = 0 VersionMinor = 13 - VersionPatch = 1 + VersionPatch = 4 VersionTag = "" // example: "rc1" + + gitRevLen = 7 // 7 matches the displayed characters on github.com ) func VersionString() string { @@ -37,7 +40,30 @@ func init() { if branch != "" { parts = append(parts, branch) } - if len(parts) > 0 { - version += "+" + strings.Join(parts, ".") + + defer func() { + if len(parts) > 0 { + version += "+" + strings.Join(parts, ".") + } + }() + + // Try to get the revision Dendrite was build from. + // If we can't, e.g. Dendrite wasn't built (go run) or no VCS version is present, + // we just use the provided version above. + info, ok := debug.ReadBuildInfo() + if !ok { + return + } + + for _, setting := range info.Settings { + if setting.Key == "vcs.revision" { + revLen := len(setting.Value) + if revLen >= gitRevLen { + parts = append(parts, setting.Value[:gitRevLen]) + } else { + parts = append(parts, setting.Value[:revLen]) + } + break + } } } diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 8fb1b6534..51afa1f9f 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" "io" + "io/fs" "mime" "net/http" "net/url" @@ -126,6 +127,17 @@ func Download( activeRemoteRequests, activeThumbnailGeneration, ) if err != nil { + // If we bubbled up a os.PathError, e.g. no such file or directory, don't send + // it to the client, be more generic. + var perr *fs.PathError + if errors.As(err, &perr) { + dReq.Logger.WithError(err).Error("failed to open file") + dReq.jsonErrorResponse(w, util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.NotFound("File not found"), + }) + return + } // TODO: Handle the fact we might have started writing the response dReq.jsonErrorResponse(w, util.JSONResponse{ Code: http.StatusNotFound, diff --git a/roomserver/internal/query/query_test.go b/roomserver/internal/query/query_test.go index 619d93030..296960b2f 100644 --- a/roomserver/internal/query/query_test.go +++ b/roomserver/internal/query/query_test.go @@ -49,6 +49,7 @@ func (db *getEventDB) addFakeEvent(eventID string, authIDs []string) error { } builder := map[string]interface{}{ "event_id": eventID, + "room_id": "!room:a", "auth_events": authEvents, } diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index cae375653..90e67b699 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -11,7 +11,6 @@ import ( "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/stretchr/testify/assert" "github.com/tidwall/gjson" @@ -227,6 +226,11 @@ func TestPurgeRoom(t *testing.T) { bob := test.NewUser(t) room := test.NewRoom(t, alice, test.RoomPreset(test.PresetTrustedPrivateChat)) + roomID, err := spec.NewRoomID(room.ID) + if err != nil { + t.Fatal(err) + } + // Invite Bob inviteEvent := room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{ "membership": "invite", @@ -249,13 +253,14 @@ func TestPurgeRoom(t *testing.T) { defer jetstream.DeleteAllStreams(jsCtx, &cfg.Global.JetStream) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // this starts the JetStream consumers - syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) + // Create the room if err = api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { t.Fatalf("failed to send events: %v", err) @@ -273,9 +278,7 @@ func TestPurgeRoom(t *testing.T) { if !isPublished { t.Fatalf("room should be published before purging") } - - aliasResp := &api.SetRoomAliasResponse{} - if err = rsAPI.SetRoomAlias(ctx, &api.SetRoomAliasRequest{RoomID: room.ID, Alias: "myalias", UserID: alice.ID}, aliasResp); err != nil { + if _, err = rsAPI.SetRoomAlias(ctx, spec.SenderID(alice.ID), *roomID, "myalias"); err != nil { t.Fatal(err) } // check the alias is actually there @@ -937,14 +940,17 @@ func TestUpgrade(t *testing.T) { upgradeUser: alice.ID, roomFunc: func(rsAPI api.RoomserverInternalAPI) string { r := test.NewRoom(t, alice) + roomID, err := spec.NewRoomID(r.ID) + if err != nil { + t.Fatal(err) + } if err := api.SendEvents(ctx, rsAPI, api.KindNew, r.Events(), "test", "test", "test", nil, false); err != nil { t.Errorf("failed to send events: %v", err) } - if err := rsAPI.SetRoomAlias(ctx, &api.SetRoomAliasRequest{ - RoomID: r.ID, - Alias: "#myroomalias:test", - }, &api.SetRoomAliasResponse{}); err != nil { + if _, err := rsAPI.SetRoomAlias(ctx, spec.SenderID(alice.ID), + *roomID, + "#myroomalias:test"); err != nil { t.Fatal(err) } @@ -1043,8 +1049,8 @@ func TestUpgrade(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) rsAPI.SetFederationAPI(nil, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) rsAPI.SetUserAPI(userAPI) for _, tc := range testCases { @@ -1061,7 +1067,7 @@ func TestUpgrade(t *testing.T) { if err != nil { t.Fatalf("upgrade userID is invalid") } - newRoomID, err := rsAPI.PerformRoomUpgrade(processCtx.Context(), roomID, *userID, version.DefaultRoomVersion()) + newRoomID, err := rsAPI.PerformRoomUpgrade(processCtx.Context(), roomID, *userID, rsAPI.DefaultRoomVersion()) if err != nil && tc.wantNewRoom { t.Fatal(err) } diff --git a/roomserver/version/version.go b/roomserver/version/version.go index 270d42897..e8bd890a0 100644 --- a/roomserver/version/version.go +++ b/roomserver/version/version.go @@ -20,12 +20,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// DefaultRoomVersion contains the room version that will, by -// default, be used to create new rooms on this server. -func DefaultRoomVersion() gomatrixserverlib.RoomVersion { - return gomatrixserverlib.RoomVersionV10 -} - // RoomVersions returns a map of all known room versions to this // server. func RoomVersions() map[gomatrixserverlib.RoomVersion]gomatrixserverlib.IRoomVersion { diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go index 0a602cea6..ab4f5a64d 100644 --- a/setup/config/config_clientapi.go +++ b/setup/config/config_clientapi.go @@ -127,17 +127,15 @@ func (c *ClientAPI) Verify(configErrs *ConfigErrors) { checkNotEmpty(configErrs, "client_api.recaptcha_sitekey_class", c.RecaptchaSitekeyClass) } // Ensure there is any spam counter measure when enabling registration - if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled { - if !c.RecaptchaEnabled { - configErrs.Add( - "You have tried to enable open registration without any secondary verification methods " + - "(such as reCAPTCHA). By enabling open registration, you are SIGNIFICANTLY " + - "increasing the risk that your server will be used to send spam or abuse, and may result in " + - "your server being banned from some rooms. If you are ABSOLUTELY CERTAIN you want to do this, " + - "start Dendrite with the -really-enable-open-registration command line flag. Otherwise, you " + - "should set the registration_disabled option in your Dendrite config.", - ) - } + if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled && !c.RecaptchaEnabled { + configErrs.Add( + "You have tried to enable open registration without any secondary verification methods " + + "(such as reCAPTCHA). By enabling open registration, you are SIGNIFICANTLY " + + "increasing the risk that your server will be used to send spam or abuse, and may result in " + + "your server being banned from some rooms. If you are ABSOLUTELY CERTAIN you want to do this, " + + "start Dendrite with the -really-enable-open-registration command line flag. Otherwise, you " + + "should set the registration_disabled option in your Dendrite config.", + ) } } diff --git a/setup/config/config_global.go b/setup/config/config_global.go index 1622bf357..5b4ccf400 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -48,6 +48,10 @@ type Global struct { // The server name to delegate client-server communications to, with optional port WellKnownClientName string `yaml:"well_known_client_name"` + // The server name to delegate sliding sync communications to, with optional port. + // Requires `well_known_client_name` to also be configured. + WellKnownSlidingSyncProxy string `yaml:"well_known_sliding_sync_proxy"` + // Disables federation. Dendrite will not be able to make any outbound HTTP requests // to other servers and the federation API will not be exposed. DisableFederation bool `yaml:"disable_federation"` diff --git a/setup/config/config_mscs.go b/setup/config/config_mscs.go index 21d4b4da0..ce491cd72 100644 --- a/setup/config/config_mscs.go +++ b/setup/config/config_mscs.go @@ -7,7 +7,6 @@ type MSCs struct { // 'msc2444': Peeking over federation - https://github.com/matrix-org/matrix-doc/pull/2444 // 'msc2753': Peeking via /sync - https://github.com/matrix-org/matrix-doc/pull/2753 // 'msc2836': Threading - https://github.com/matrix-org/matrix-doc/pull/2836 - // 'msc2946': Spaces Summary - https://github.com/matrix-org/matrix-doc/pull/2946 MSCs []string `yaml:"mscs"` Database DatabaseOptions `yaml:"database,omitempty"` diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go index b68ca498d..2a8aaf314 100644 --- a/setup/jetstream/streams.go +++ b/setup/jetstream/streams.go @@ -49,6 +49,7 @@ var streams = []*nats.StreamConfig{ Name: InputRoomEvent, Retention: nats.InterestPolicy, Storage: nats.FileStorage, + MaxAge: time.Hour * 24, }, { Name: InputDeviceListUpdate, diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go index 16fb3efe1..ecbab706f 100644 --- a/setup/mscs/msc2836/msc2836_test.go +++ b/setup/mscs/msc2836/msc2836_test.go @@ -529,8 +529,9 @@ func (r *testRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spe return spec.NewUserID(string(senderID), true) } -func (r *testRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (spec.SenderID, error) { - return spec.SenderID(userID.String()), nil +func (r *testRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + senderID := spec.SenderID(userID.String()) + return &senderID, nil } func (r *testRoomserverAPI) QueryEventsByID(ctx context.Context, req *roomserver.QueryEventsByIDRequest, res *roomserver.QueryEventsByIDResponse) error { diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go index 3f5e990c4..56954cfa0 100644 --- a/syncapi/internal/keychange_test.go +++ b/syncapi/internal/keychange_test.go @@ -59,22 +59,22 @@ func (k *mockKeyAPI) QueryDeviceMessages(ctx context.Context, req *userapi.Query func (k *mockKeyAPI) QuerySignatures(ctx context.Context, req *userapi.QuerySignaturesRequest, res *userapi.QuerySignaturesResponse) { } -type mockRoomserverAPI struct { +type keyChangeMockRoomserverAPI struct { api.RoomserverInternalAPI roomIDToJoinedMembers map[string][]string } -func (s *mockRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { +func (s *keyChangeMockRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return spec.NewUserID(string(senderID), true) } // QueryRoomsForUser retrieves a list of room IDs matching the given query. -func (s *mockRoomserverAPI) QueryRoomsForUser(ctx context.Context, req *api.QueryRoomsForUserRequest, res *api.QueryRoomsForUserResponse) error { - return nil +func (s *keyChangeMockRoomserverAPI) QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) { + return nil, nil } // QueryBulkStateContent does a bulk query for state event content in the given rooms. -func (s *mockRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *api.QueryBulkStateContentRequest, res *api.QueryBulkStateContentResponse) error { +func (s *keyChangeMockRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *api.QueryBulkStateContentRequest, res *api.QueryBulkStateContentResponse) error { res.Rooms = make(map[string]map[gomatrixserverlib.StateKeyTuple]string) if req.AllowWildcards && len(req.StateTuples) == 1 && req.StateTuples[0].EventType == spec.MRoomMember && req.StateTuples[0].StateKey == "*" { for _, roomID := range req.RoomIDs { @@ -91,7 +91,7 @@ func (s *mockRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *api. } // QuerySharedUsers returns a list of users who share at least 1 room in common with the given user. -func (s *mockRoomserverAPI) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUsersRequest, res *api.QuerySharedUsersResponse) error { +func (s *keyChangeMockRoomserverAPI) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUsersRequest, res *api.QuerySharedUsersResponse) error { roomsToQuery := req.IncludeRoomIDs for roomID, members := range s.roomIDToJoinedMembers { exclude := false @@ -123,7 +123,7 @@ func (s *mockRoomserverAPI) QuerySharedUsers(ctx context.Context, req *api.Query // This is actually a database function, but seeing as we track the state inside the // *mockRoomserverAPI, we'll just comply with the interface here instead. -func (s *mockRoomserverAPI) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) { +func (s *keyChangeMockRoomserverAPI) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) { commonUsers := []string{} for _, members := range s.roomIDToJoinedMembers { for _, member := range members { @@ -211,7 +211,7 @@ func TestKeyChangeCatchupOnJoinShareNewUser(t *testing.T) { syncResponse := types.NewResponse() syncResponse = joinResponseWithRooms(syncResponse, syncingUser, []string{newlyJoinedRoom}) - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ newlyJoinedRoom: {syncingUser, newShareUser}, "!another:room": {syncingUser}, @@ -234,7 +234,7 @@ func TestKeyChangeCatchupOnLeaveShareLeftUser(t *testing.T) { syncResponse := types.NewResponse() syncResponse = leaveResponseWithRooms(syncResponse, syncingUser, []string{newlyLeftRoom}) - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ newlyLeftRoom: {removeUser}, "!another:room": {syncingUser}, @@ -257,7 +257,7 @@ func TestKeyChangeCatchupOnJoinShareNoNewUsers(t *testing.T) { syncResponse := types.NewResponse() syncResponse = joinResponseWithRooms(syncResponse, syncingUser, []string{newlyJoinedRoom}) - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ newlyJoinedRoom: {syncingUser, existingUser}, "!another:room": {syncingUser, existingUser}, @@ -279,7 +279,7 @@ func TestKeyChangeCatchupOnLeaveShareNoUsers(t *testing.T) { syncResponse := types.NewResponse() syncResponse = leaveResponseWithRooms(syncResponse, syncingUser, []string{newlyLeftRoom}) - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ newlyLeftRoom: {existingUser}, "!another:room": {syncingUser, existingUser}, @@ -343,7 +343,7 @@ func TestKeyChangeCatchupNoNewJoinsButMessages(t *testing.T) { jr.Timeline = &types.Timeline{Events: roomTimelineEvents} syncResponse.Rooms.Join[roomID] = jr - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ roomID: {syncingUser, existingUser}, }, @@ -369,7 +369,7 @@ func TestKeyChangeCatchupChangeAndLeft(t *testing.T) { syncResponse = joinResponseWithRooms(syncResponse, syncingUser, []string{newlyJoinedRoom}) syncResponse = leaveResponseWithRooms(syncResponse, syncingUser, []string{newlyLeftRoom}) - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ newlyJoinedRoom: {syncingUser, newShareUser, newShareUser2}, newlyLeftRoom: {newlyLeftUser, newlyLeftUser2}, @@ -459,7 +459,7 @@ func TestKeyChangeCatchupChangeAndLeftSameRoom(t *testing.T) { lr.Timeline = &types.Timeline{Events: roomEvents} syncResponse.Rooms.Leave[roomID] = lr - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ roomID: {newShareUser, newShareUser2}, "!another:room": {syncingUser}, diff --git a/syncapi/routing/search_test.go b/syncapi/routing/search_test.go index ab47da129..d9eff9de3 100644 --- a/syncapi/routing/search_test.go +++ b/syncapi/routing/search_test.go @@ -238,7 +238,7 @@ func TestSearch(t *testing.T) { } elements = append(elements, fulltext.IndexElement{ EventID: x.EventID(), - RoomID: x.RoomID(), + RoomID: x.RoomID().String(), Content: string(x.Content()), ContentType: x.Type(), StreamPosition: int64(sp), diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 112fa9d4a..b0148bef5 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -340,7 +340,7 @@ func (s *currentRoomStateStatements) UpsertRoomState( stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt) _, err = stmt.ExecContext( ctx, - event.RoomID(), + event.RoomID().String(), event.EventID(), event.Type(), event.UserID.String(), diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go index 7b8d2d733..1f46cd09d 100644 --- a/syncapi/storage/postgres/invites_table.go +++ b/syncapi/storage/postgres/invites_table.go @@ -99,7 +99,7 @@ func (s *inviteEventsStatements) InsertInviteEvent( err = sqlutil.TxStmt(txn, s.insertInviteEventStmt).QueryRowContext( ctx, - inviteEvent.RoomID(), + inviteEvent.RoomID().String(), inviteEvent.EventID(), inviteEvent.UserID.String(), headeredJSON, diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go index 9c51493df..4fe4260da 100644 --- a/syncapi/storage/postgres/memberships_table.go +++ b/syncapi/storage/postgres/memberships_table.go @@ -108,7 +108,7 @@ func (s *membershipsStatements) UpsertMembership( } _, err = sqlutil.TxStmt(txn, s.upsertMembershipStmt).ExecContext( ctx, - event.RoomID(), + event.RoomID().String(), event.StateKeyResolved, membership, event.EventID(), diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 3bd19b367..78b2e397c 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -339,7 +339,7 @@ func (s *currentRoomStateStatements) UpsertRoomState( stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt) _, err = stmt.ExecContext( ctx, - event.RoomID(), + event.RoomID().String(), event.EventID(), event.Type(), event.UserID.String(), diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index 7e0d895f1..ebb469d24 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -106,7 +106,7 @@ func (s *inviteEventsStatements) InsertInviteEvent( _, err = stmt.ExecContext( ctx, streamPos, - inviteEvent.RoomID(), + inviteEvent.RoomID().String(), inviteEvent.EventID(), inviteEvent.UserID.String(), headeredJSON, diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go index f93393d60..a1b16306c 100644 --- a/syncapi/storage/sqlite3/memberships_table.go +++ b/syncapi/storage/sqlite3/memberships_table.go @@ -111,7 +111,7 @@ func (s *membershipsStatements) UpsertMembership( } _, err = sqlutil.TxStmt(txn, s.upsertMembershipStmt).ExecContext( ctx, - event.RoomID(), + event.RoomID().String(), event.StateKeyResolved, membership, event.EventID(), diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 992466450..62e925d20 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -213,12 +213,48 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) { // backpaginate 5 messages starting at the latest position. filter := &synctypes.RoomEventFilter{Limit: 5} - paginatedEvents, err := snapshot.GetEventsInTopologicalRange(ctx, &from, &to, r.ID, filter, true) + paginatedEvents, start, end, err := snapshot.GetEventsInTopologicalRange(ctx, &from, &to, r.ID, filter, true) if err != nil { t.Fatalf("GetEventsInTopologicalRange returned an error: %s", err) } gots := snapshot.StreamEventsToEvents(context.Background(), nil, paginatedEvents, nil) test.AssertEventsEqual(t, gots, test.Reversed(events[len(events)-5:])) + assert.Equal(t, types.TopologyToken{Depth: 15, PDUPosition: 15}, start) + assert.Equal(t, types.TopologyToken{Depth: 11, PDUPosition: 11}, end) + }) + }) +} + +// The purpose of this test is to ensure that backfilling returns no start/end if a given filter removes +// all events. +func TestGetEventsInRangeWithTopologyTokenNoEventsForFilter(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := MustCreateDatabase(t, dbType) + defer close() + alice := test.NewUser(t) + r := test.NewRoom(t, alice) + for i := 0; i < 10; i++ { + r.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": fmt.Sprintf("hi %d", i)}) + } + events := r.Events() + _ = MustWriteEvents(t, db, events) + + WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) { + from := types.TopologyToken{Depth: math.MaxInt64, PDUPosition: math.MaxInt64} + t.Logf("max topo pos = %+v", from) + // head towards the beginning of time + to := types.TopologyToken{} + + // backpaginate 20 messages starting at the latest position. + notTypes := []string{spec.MRoomRedaction} + senders := []string{alice.ID} + filter := &synctypes.RoomEventFilter{Limit: 20, NotTypes: ¬Types, Senders: &senders} + paginatedEvents, start, end, err := snapshot.GetEventsInTopologicalRange(ctx, &from, &to, r.ID, filter, true) + assert.NoError(t, err) + assert.Equal(t, 0, len(paginatedEvents)) + // Even if we didn't get anything back due to the filter, we should still have start/end + assert.Equal(t, types.TopologyToken{Depth: 15, PDUPosition: 15}, start) + assert.Equal(t, types.TopologyToken{Depth: 1, PDUPosition: 1}, end) }) }) } diff --git a/syncapi/storage/tables/topology_test.go b/syncapi/storage/tables/topology_test.go index f4f75bdf3..7691cc5f8 100644 --- a/syncapi/storage/tables/topology_test.go +++ b/syncapi/storage/tables/topology_test.go @@ -13,6 +13,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/test" + "github.com/stretchr/testify/assert" ) func newTopologyTable(t *testing.T, dbType test.DBType) (tables.Topology, *sql.DB, func()) { @@ -60,28 +61,37 @@ func TestTopologyTable(t *testing.T) { highestPos = topoPos + 1 } // check ordering works without limit - eventIDs, err := tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 100, true) - if err != nil { - return fmt.Errorf("failed to SelectEventIDsInRange: %s", err) - } + eventIDs, start, end, err := tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 100, true) + assert.NoError(t, err, "failed to SelectEventIDsInRange") test.AssertEventIDsEqual(t, eventIDs, events[:]) - eventIDs, err = tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 100, false) - if err != nil { - return fmt.Errorf("failed to SelectEventIDsInRange: %s", err) - } - test.AssertEventIDsEqual(t, eventIDs, test.Reversed(events[:])) - // check ordering works with limit - eventIDs, err = tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 3, true) - if err != nil { - return fmt.Errorf("failed to SelectEventIDsInRange: %s", err) - } - test.AssertEventIDsEqual(t, eventIDs, events[:3]) - eventIDs, err = tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 3, false) - if err != nil { - return fmt.Errorf("failed to SelectEventIDsInRange: %s", err) - } - test.AssertEventIDsEqual(t, eventIDs, test.Reversed(events[len(events)-3:])) + assert.Equal(t, types.TopologyToken{Depth: 1, PDUPosition: 0}, start) + assert.Equal(t, types.TopologyToken{Depth: 5, PDUPosition: 4}, end) + eventIDs, start, end, err = tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 100, false) + assert.NoError(t, err, "failed to SelectEventIDsInRange") + test.AssertEventIDsEqual(t, eventIDs, test.Reversed(events[:])) + assert.Equal(t, types.TopologyToken{Depth: 5, PDUPosition: 4}, start) + assert.Equal(t, types.TopologyToken{Depth: 1, PDUPosition: 0}, end) + + // check ordering works with limit + eventIDs, start, end, err = tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 3, true) + assert.NoError(t, err, "failed to SelectEventIDsInRange") + test.AssertEventIDsEqual(t, eventIDs, events[:3]) + assert.Equal(t, types.TopologyToken{Depth: 1, PDUPosition: 0}, start) + assert.Equal(t, types.TopologyToken{Depth: 3, PDUPosition: 2}, end) + + eventIDs, start, end, err = tab.SelectEventIDsInRange(ctx, txn, room.ID, 0, highestPos, highestPos, 3, false) + assert.NoError(t, err, "failed to SelectEventIDsInRange") + test.AssertEventIDsEqual(t, eventIDs, test.Reversed(events[len(events)-3:])) + assert.Equal(t, types.TopologyToken{Depth: 5, PDUPosition: 4}, start) + assert.Equal(t, types.TopologyToken{Depth: 3, PDUPosition: 2}, end) + + // Check that we return no values for invalid rooms + eventIDs, start, end, err = tab.SelectEventIDsInRange(ctx, txn, "!doesnotexist:localhost", 0, highestPos, highestPos, 10, false) + assert.NoError(t, err, "failed to SelectEventIDsInRange") + assert.Equal(t, 0, len(eventIDs)) + assert.Equal(t, types.TopologyToken{}, start) + assert.Equal(t, types.TopologyToken{}, end) return nil }) if err != nil { diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go index 1080cdbb8..5a447ad36 100644 --- a/syncapi/syncapi_test.go +++ b/syncapi/syncapi_test.go @@ -44,6 +44,11 @@ func (s *syncRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spe return spec.NewUserID(string(senderID), true) } +func (s *syncRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + senderID := spec.SenderID(userID.String()) + return &senderID, nil +} + func (s *syncRoomserverAPI) QueryLatestEventsAndState(ctx context.Context, req *rsapi.QueryLatestEventsAndStateRequest, res *rsapi.QueryLatestEventsAndStateResponse) error { var room *test.Room for _, r := range s.rooms { @@ -74,8 +79,13 @@ func (s *syncRoomserverAPI) QueryMembershipForUser(ctx context.Context, req *rsa return nil } -func (s *syncRoomserverAPI) QueryMembershipAtEvent(ctx context.Context, req *rsapi.QueryMembershipAtEventRequest, res *rsapi.QueryMembershipAtEventResponse) error { - return nil +func (s *syncRoomserverAPI) QueryMembershipAtEvent( + ctx context.Context, + roomID spec.RoomID, + eventIDs []string, + senderID spec.SenderID, +) (map[string]*rstypes.HeaderedEvent, error) { + return map[string]*rstypes.HeaderedEvent{}, nil } type syncUserAPI struct { @@ -199,6 +209,156 @@ func testSyncAccessTokens(t *testing.T, dbType test.DBType) { } } +func TestSyncAPIEventFormatPowerLevels(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + testSyncEventFormatPowerLevels(t, dbType) + }) +} + +func testSyncEventFormatPowerLevels(t *testing.T, dbType test.DBType) { + user := test.NewUser(t) + setRoomVersion := func(t *testing.T, r *test.Room) { r.Version = gomatrixserverlib.RoomVersionPseudoIDs } + room := test.NewRoom(t, user, setRoomVersion) + alice := userapi.Device{ + ID: "ALICEID", + UserID: user.ID, + AccessToken: "ALICE_BEARER_TOKEN", + DisplayName: "Alice", + AccountType: userapi.AccountTypeUser, + } + + room.CreateAndInsert(t, user, spec.MRoomPowerLevels, gomatrixserverlib.PowerLevelContent{ + Users: map[string]int64{ + user.ID: 100, + }, + }, test.WithStateKey("")) + + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + natsInstance := jetstream.NATSInstance{} + defer close() + + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) + msgs := toNATSMsgs(t, cfg, room.Events()...) + AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, caches, caching.DisableMetrics) + testrig.MustPublishMsgs(t, jsctx, msgs...) + + testCases := []struct { + name string + wantCode int + wantJoinedRooms []string + eventFormat synctypes.ClientEventFormat + }{ + { + name: "Client format", + wantCode: 200, + wantJoinedRooms: []string{room.ID}, + eventFormat: synctypes.FormatSync, + }, + { + name: "Federation format", + wantCode: 200, + wantJoinedRooms: []string{room.ID}, + eventFormat: synctypes.FormatSyncFederation, + }, + } + + syncUntil(t, routers, alice.AccessToken, false, func(syncBody string) bool { + // wait for the last sent eventID to come down sync + path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(event_id=="%s")`, room.ID, room.Events()[len(room.Events())-1].EventID()) + return gjson.Get(syncBody, path).Exists() + }) + + for _, tc := range testCases { + format := "" + if tc.eventFormat == synctypes.FormatSyncFederation { + format = "federation" + } + + w := httptest.NewRecorder() + routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + "access_token": alice.AccessToken, + "timeout": "0", + "filter": fmt.Sprintf(`{"event_format":"%s"}`, format), + }))) + if w.Code != tc.wantCode { + t.Fatalf("%s: got HTTP %d want %d", tc.name, w.Code, tc.wantCode) + } + if tc.wantJoinedRooms != nil { + var res types.Response + if err := json.NewDecoder(w.Body).Decode(&res); err != nil { + t.Fatalf("%s: failed to decode response body: %s", tc.name, err) + } + if len(res.Rooms.Join) != len(tc.wantJoinedRooms) { + t.Errorf("%s: got %v joined rooms, want %v.\nResponse: %+v", tc.name, len(res.Rooms.Join), len(tc.wantJoinedRooms), res) + } + t.Logf("res: %+v", res.Rooms.Join[room.ID]) + + gotEventIDs := make([]string, len(res.Rooms.Join[room.ID].Timeline.Events)) + for i, ev := range res.Rooms.Join[room.ID].Timeline.Events { + gotEventIDs[i] = ev.EventID + } + test.AssertEventIDsEqual(t, gotEventIDs, room.Events()) + + event := room.CreateAndInsert(t, user, spec.MRoomPowerLevels, gomatrixserverlib.PowerLevelContent{ + Users: map[string]int64{ + user.ID: 100, + "@otheruser:localhost": 50, + }, + }, test.WithStateKey("")) + + msgs := toNATSMsgs(t, cfg, event) + testrig.MustPublishMsgs(t, jsctx, msgs...) + + syncUntil(t, routers, alice.AccessToken, false, func(syncBody string) bool { + // wait for the last sent eventID to come down sync + path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(event_id=="%s")`, room.ID, room.Events()[len(room.Events())-1].EventID()) + return gjson.Get(syncBody, path).Exists() + }) + + since := res.NextBatch.String() + w := httptest.NewRecorder() + routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + "access_token": alice.AccessToken, + "timeout": "0", + "filter": fmt.Sprintf(`{"event_format":"%s"}`, format), + "since": since, + }))) + if w.Code != 200 { + t.Errorf("since=%s got HTTP %d want 200", since, w.Code) + } + + res = *types.NewResponse() + if err := json.NewDecoder(w.Body).Decode(&res); err != nil { + t.Errorf("failed to decode response body: %s", err) + } + if len(res.Rooms.Join) != 1 { + t.Fatalf("since=%s got %d joined rooms, want 1", since, len(res.Rooms.Join)) + } + gotEventIDs = make([]string, len(res.Rooms.Join[room.ID].Timeline.Events)) + for j, ev := range res.Rooms.Join[room.ID].Timeline.Events { + gotEventIDs[j] = ev.EventID + if ev.Type == spec.MRoomPowerLevels { + content := gomatrixserverlib.PowerLevelContent{} + err := json.Unmarshal(ev.Content, &content) + if err != nil { + t.Errorf("failed to unmarshal power level content: %s", err) + } + otherUserLevel := content.UserLevel("@otheruser:localhost") + if otherUserLevel != 50 { + t.Errorf("Expected user PL of %d but got %d", 50, otherUserLevel) + } + } + } + events := []*rstypes.HeaderedEvent{room.Events()[len(room.Events())-1]} + test.AssertEventIDsEqual(t, gotEventIDs, events) + } + } +} + // Tests what happens when we create a room and then /sync before all events from /createRoom have // been sent to the syncapi func TestSyncAPICreateRoomSyncEarly(t *testing.T) { @@ -433,6 +593,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) { } cfg, processCtx, close := testrig.CreateConfig(t, dbType) + cfg.ClientAPI.RateLimiting = config.RateLimiting{Enabled: false} routers := httputil.NewRouters() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) @@ -1240,7 +1401,7 @@ func toNATSMsgs(t *testing.T, cfg *config.Dendrite, input ...*rstypes.HeaderedEv if ev.StateKey() != nil { addsStateIDs = append(addsStateIDs, ev.EventID()) } - result[i] = testrig.NewOutputEventMsg(t, cfg, ev.RoomID(), api.OutputEvent{ + result[i] = testrig.NewOutputEventMsg(t, cfg, ev.RoomID().String(), api.OutputEvent{ Type: rsapi.OutputTypeNewRoomEvent, NewRoomEvent: &rsapi.OutputNewRoomEvent{ Event: ev, diff --git a/syncapi/synctypes/clientevent_test.go b/syncapi/synctypes/clientevent_test.go index 559517117..662f9ea43 100644 --- a/syncapi/synctypes/clientevent_test.go +++ b/syncapi/synctypes/clientevent_test.go @@ -91,21 +91,21 @@ func verifyEventFields(t *testing.T, got EventFieldsToVerify, want EventFieldsTo func TestToClientEvent(t *testing.T) { // nolint: gocyclo ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionV1).NewEventFromTrustedJSON([]byte(`{ - "type": "m.room.name", - "state_key": "", - "event_id": "$test:localhost", - "room_id": "!test:localhost", - "sender": "@test:localhost", - "content": { - "name": "Hello World" - }, - "origin_server_ts": 123456, - "unsigned": { - "prev_content": { - "name": "Goodbye World" - } - } - }`), false) + "type": "m.room.name", + "state_key": "", + "event_id": "$test:localhost", + "room_id": "!test:localhost", + "sender": "@test:localhost", + "content": { + "name": "Hello World" + }, + "origin_server_ts": 123456, + "unsigned": { + "prev_content": { + "name": "Goodbye World" + } + } + }`), false) if err != nil { t.Fatalf("failed to create Event: %s", err) } @@ -156,21 +156,21 @@ func TestToClientEvent(t *testing.T) { // nolint: gocyclo func TestToClientFormatSync(t *testing.T) { ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionV1).NewEventFromTrustedJSON([]byte(`{ - "type": "m.room.name", - "state_key": "", - "event_id": "$test:localhost", - "room_id": "!test:localhost", - "sender": "@test:localhost", - "content": { - "name": "Hello World" - }, - "origin_server_ts": 123456, - "unsigned": { - "prev_content": { - "name": "Goodbye World" - } - } - }`), false) + "type": "m.room.name", + "state_key": "", + "event_id": "$test:localhost", + "room_id": "!test:localhost", + "sender": "@test:localhost", + "content": { + "name": "Hello World" + }, + "origin_server_ts": 123456, + "unsigned": { + "prev_content": { + "name": "Goodbye World" + } + } + }`), false) if err != nil { t.Fatalf("failed to create Event: %s", err) } @@ -187,30 +187,30 @@ func TestToClientFormatSync(t *testing.T) { func TestToClientEventFormatSyncFederation(t *testing.T) { // nolint: gocyclo ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionV10).NewEventFromTrustedJSON([]byte(`{ - "type": "m.room.name", - "state_key": "", - "event_id": "$test:localhost", - "room_id": "!test:localhost", - "sender": "@test:localhost", - "content": { - "name": "Hello World" - }, - "origin_server_ts": 123456, - "unsigned": { - "prev_content": { - "name": "Goodbye World" - } - }, - "depth": 8, - "prev_events": [ - "$f597Tp0Mm1PPxEgiprzJc2cZAjVhxCxACOGuwJb33Oo" - ], - "auth_events": [ - "$Bj0ZGgX6VTqAQdqKH4ZG3l6rlbxY3rZlC5D3MeuK1OQ", - "$QsMs6A1PUVUhgSvmHBfpqEYJPgv4DXt96r8P2AK7iXQ", - "$tBteKtlnFiwlmPJsv0wkKTMEuUVWpQH89H7Xskxve1Q" - ] - }`), false) + "type": "m.room.name", + "state_key": "", + "event_id": "$test:localhost", + "room_id": "!test:localhost", + "sender": "@test:localhost", + "content": { + "name": "Hello World" + }, + "origin_server_ts": 123456, + "unsigned": { + "prev_content": { + "name": "Goodbye World" + } + }, + "depth": 8, + "prev_events": [ + "$f597Tp0Mm1PPxEgiprzJc2cZAjVhxCxACOGuwJb33Oo" + ], + "auth_events": [ + "$Bj0ZGgX6VTqAQdqKH4ZG3l6rlbxY3rZlC5D3MeuK1OQ", + "$QsMs6A1PUVUhgSvmHBfpqEYJPgv4DXt96r8P2AK7iXQ", + "$tBteKtlnFiwlmPJsv0wkKTMEuUVWpQH89H7Xskxve1Q" + ] + }`), false) if err != nil { t.Fatalf("failed to create Event: %s", err) } @@ -262,59 +262,59 @@ func userIDForSender(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, func TestToClientEventsFormatSyncFederation(t *testing.T) { // nolint: gocyclo ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ - "type": "m.room.name", - "state_key": "testSenderID", - "event_id": "$test:localhost", - "room_id": "!test:localhost", - "sender": "testSenderID", - "content": { - "name": "Hello World" - }, - "origin_server_ts": 123456, - "unsigned": { - "prev_content": { - "name": "Goodbye World" - } - }, - "depth": 8, - "prev_events": [ - "$f597Tp0Mm1PPxEgiprzJc2cZAjVhxCxACOGuwJb33Oo" - ], - "auth_events": [ - "$Bj0ZGgX6VTqAQdqKH4ZG3l6rlbxY3rZlC5D3MeuK1OQ", - "$QsMs6A1PUVUhgSvmHBfpqEYJPgv4DXt96r8P2AK7iXQ", - "$tBteKtlnFiwlmPJsv0wkKTMEuUVWpQH89H7Xskxve1Q" - ] - }`), false) + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World" + }, + "origin_server_ts": 123456, + "unsigned": { + "prev_content": { + "name": "Goodbye World" + } + }, + "depth": 8, + "prev_events": [ + "$f597Tp0Mm1PPxEgiprzJc2cZAjVhxCxACOGuwJb33Oo" + ], + "auth_events": [ + "$Bj0ZGgX6VTqAQdqKH4ZG3l6rlbxY3rZlC5D3MeuK1OQ", + "$QsMs6A1PUVUhgSvmHBfpqEYJPgv4DXt96r8P2AK7iXQ", + "$tBteKtlnFiwlmPJsv0wkKTMEuUVWpQH89H7Xskxve1Q" + ] + }`), false) if err != nil { t.Fatalf("failed to create Event: %s", err) } ev2, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ - "type": "m.room.name", - "state_key": "testSenderID", - "event_id": "$test2:localhost", - "room_id": "!test:localhost", - "sender": "testSenderID", - "content": { - "name": "Hello World 2" - }, - "origin_server_ts": 1234567, - "unsigned": { - "prev_content": { - "name": "Goodbye World 2" - }, - "prev_sender": "testSenderID" - }, - "depth": 9, - "prev_events": [ - "$f597Tp0Mm1PPxEgiprzJc2cZAjVhxCxACOGuwJb33Oo" - ], - "auth_events": [ - "$Bj0ZGgX6VTqAQdqKH4ZG3l6rlbxY3rZlC5D3MeuK1OQ", - "$QsMs6A1PUVUhgSvmHBfpqEYJPgv4DXt96r8P2AK7iXQ", - "$tBteKtlnFiwlmPJsv0wkKTMEuUVWpQH89H7Xskxve1Q" - ] - }`), false) + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test2:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World 2" + }, + "origin_server_ts": 1234567, + "unsigned": { + "prev_content": { + "name": "Goodbye World 2" + }, + "prev_sender": "testSenderID" + }, + "depth": 9, + "prev_events": [ + "$f597Tp0Mm1PPxEgiprzJc2cZAjVhxCxACOGuwJb33Oo" + ], + "auth_events": [ + "$Bj0ZGgX6VTqAQdqKH4ZG3l6rlbxY3rZlC5D3MeuK1OQ", + "$QsMs6A1PUVUhgSvmHBfpqEYJPgv4DXt96r8P2AK7iXQ", + "$tBteKtlnFiwlmPJsv0wkKTMEuUVWpQH89H7Xskxve1Q" + ] + }`), false) if err != nil { t.Fatalf("failed to create Event: %s", err) } @@ -378,42 +378,42 @@ func TestToClientEventsFormatSyncFederation(t *testing.T) { // nolint: gocyclo func TestToClientEventsFormatSync(t *testing.T) { // nolint: gocyclo ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ - "type": "m.room.name", - "state_key": "testSenderID", - "event_id": "$test:localhost", - "room_id": "!test:localhost", - "sender": "testSenderID", - "content": { - "name": "Hello World" - }, - "origin_server_ts": 123456, - "unsigned": { - "prev_content": { - "name": "Goodbye World" - } - } - }`), false) + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World" + }, + "origin_server_ts": 123456, + "unsigned": { + "prev_content": { + "name": "Goodbye World" + } + } + }`), false) if err != nil { t.Fatalf("failed to create Event: %s", err) } ev2, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ - "type": "m.room.name", - "state_key": "testSenderID", - "event_id": "$test2:localhost", - "room_id": "!test:localhost", - "sender": "testSenderID", - "content": { - "name": "Hello World 2" - }, - "origin_server_ts": 1234567, - "unsigned": { - "prev_content": { - "name": "Goodbye World 2" - }, - "prev_sender": "testSenderID" - }, - "depth": 9 - }`), false) + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test2:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World 2" + }, + "origin_server_ts": 1234567, + "unsigned": { + "prev_content": { + "name": "Goodbye World 2" + }, + "prev_sender": "testSenderID" + }, + "depth": 9 + }`), false) if err != nil { t.Fatalf("failed to create Event: %s", err) } @@ -470,42 +470,42 @@ func TestToClientEventsFormatSync(t *testing.T) { // nolint: gocyclo func TestToClientEventsFormatSyncUnknownPrevSender(t *testing.T) { // nolint: gocyclo ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ - "type": "m.room.name", - "state_key": "testSenderID", - "event_id": "$test:localhost", - "room_id": "!test:localhost", - "sender": "testSenderID", - "content": { - "name": "Hello World" - }, - "origin_server_ts": 123456, - "unsigned": { - "prev_content": { - "name": "Goodbye World" - } - } - }`), false) + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World" + }, + "origin_server_ts": 123456, + "unsigned": { + "prev_content": { + "name": "Goodbye World" + } + } + }`), false) if err != nil { t.Fatalf("failed to create Event: %s", err) } ev2, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ - "type": "m.room.name", - "state_key": "testSenderID", - "event_id": "$test2:localhost", - "room_id": "!test:localhost", - "sender": "testSenderID", - "content": { - "name": "Hello World 2" - }, - "origin_server_ts": 1234567, - "unsigned": { - "prev_content": { - "name": "Goodbye World 2" - }, - "prev_sender": "unknownSenderID" - }, - "depth": 9 - }`), false) + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test2:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World 2" + }, + "origin_server_ts": 1234567, + "unsigned": { + "prev_content": { + "name": "Goodbye World 2" + }, + "prev_sender": "unknownSenderID" + }, + "depth": 9 + }`), false) if err != nil { t.Fatalf("failed to create Event: %s", err) } diff --git a/syncapi/types/types_test.go b/syncapi/types/types_test.go index 45fc44a47..aa22c86b0 100644 --- a/syncapi/types/types_test.go +++ b/syncapi/types/types_test.go @@ -1,6 +1,7 @@ package types import ( + "context" "encoding/json" "reflect" "testing" @@ -11,8 +12,19 @@ import ( "github.com/matrix-org/gomatrixserverlib/spec" ) -func UserIDForSender(roomID string, senderID string) (*spec.UserID, error) { - return spec.NewUserID(senderID, true) +type FakeRoomserverAPI struct{} + +func (f *FakeRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + if senderID == "" { + return nil, nil + } + + return spec.NewUserID(string(senderID), true) +} + +func (f *FakeRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + sender := spec.SenderID(userID.String()) + return &sender, nil } func TestSyncTokens(t *testing.T) { @@ -61,25 +73,18 @@ func TestNewInviteResponse(t *testing.T) { t.Fatal(err) } - sender, err := spec.NewUserID("@neilalexander:matrix.org", true) + rsAPI := FakeRoomserverAPI{} + res, err := NewInviteResponse(context.Background(), &rsAPI, &types.HeaderedEvent{PDU: ev}, synctypes.FormatSync) if err != nil { t.Fatal(err) } - skUserID, err := spec.NewUserID("@neilalexander:dendrite.neilalexander.dev", true) - if err != nil { - t.Fatal(err) - } - skString := skUserID.String() - sk := &skString - - res := NewInviteResponse(&types.HeaderedEvent{PDU: ev}, *sender, sk) j, err := json.Marshal(res) if err != nil { t.Fatal(err) } if string(j) != expected { - t.Fatalf("Invite response didn't contain correct info") + t.Fatalf("Invite response didn't contain correct info, \nexpected: %s \ngot: %s", expected, string(j)) } } diff --git a/userapi/consumers/roomserver_test.go b/userapi/consumers/roomserver_test.go index 4dc81e74a..7b7c08618 100644 --- a/userapi/consumers/roomserver_test.go +++ b/userapi/consumers/roomserver_test.go @@ -2,16 +2,22 @@ package consumers import ( "context" + "crypto/ed25519" "reflect" "sync" "testing" "time" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/stretchr/testify/assert" + "golang.org/x/crypto/bcrypt" "github.com/matrix-org/dendrite/internal/pushrules" rsapi "github.com/matrix-org/dendrite/roomserver/api" @@ -75,12 +81,8 @@ func Test_evaluatePushRules(t *testing.T) { { name: "m.reaction doesn't notify", eventContent: `{"type":"m.reaction","room_id":"!room:example.com"}`, - wantAction: pushrules.DontNotifyAction, - wantActions: []*pushrules.Action{ - { - Kind: pushrules.DontNotifyAction, - }, - }, + wantAction: pushrules.UnknownAction, + wantActions: []*pushrules.Action{}, }, { name: "m.room.message notifies", @@ -130,7 +132,7 @@ func Test_evaluatePushRules(t *testing.T) { t.Fatalf("expected action to be '%s', got '%s'", tc.wantAction, gotAction) } // this is taken from `notifyLocal` - if tc.wantNotify && gotAction != pushrules.NotifyAction && gotAction != pushrules.CoalesceAction { + if tc.wantNotify && gotAction != pushrules.NotifyAction { t.Fatalf("expected to notify but didn't") } }) @@ -139,6 +141,42 @@ func Test_evaluatePushRules(t *testing.T) { }) } +func TestLocalRoomMembers(t *testing.T) { + alice := test.NewUser(t) + _, sk, err := ed25519.GenerateKey(nil) + assert.NoError(t, err) + bob := test.NewUser(t, test.WithSigningServer("notlocalhost", "ed25519:abc", sk)) + charlie := test.NewUser(t, test.WithSigningServer("notlocalhost", "ed25519:abc", sk)) + + room := test.NewRoom(t, alice) + room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]string{"membership": spec.Join}, test.WithStateKey(bob.ID)) + room.CreateAndInsert(t, charlie, spec.MRoomMember, map[string]string{"membership": spec.Join}, test.WithStateKey(charlie.ID)) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + defer close() + + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + natsInstance := &jetstream.NATSInstance{} + caches := caching.NewRistrettoCache(8*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + db, err := storage.NewUserDatabase(processCtx.Context(), cm, &cfg.UserAPI.AccountDatabase, cfg.Global.ServerName, bcrypt.MinCost, 1000, 1000, "") + assert.NoError(t, err) + + err = rsapi.SendEvents(processCtx.Context(), rsAPI, rsapi.KindNew, room.Events(), "", "test", "test", nil, false) + assert.NoError(t, err) + + consumer := OutputRoomEventConsumer{db: db, rsAPI: rsAPI, serverName: "test", cfg: &cfg.UserAPI} + members, count, err := consumer.localRoomMembers(processCtx.Context(), room.ID) + assert.NoError(t, err) + assert.Equal(t, 3, count) + expectedLocalMember := &localMembership{UserID: alice.ID, Localpart: alice.Localpart, Domain: "test", MemberContent: gomatrixserverlib.MemberContent{Membership: spec.Join}} + assert.Equal(t, expectedLocalMember, members[0]) + }) + +} + func TestMessageStats(t *testing.T) { type args struct { eventType string @@ -257,3 +295,42 @@ func TestMessageStats(t *testing.T) { } }) } + +func BenchmarkLocalRoomMembers(b *testing.B) { + t := &testing.T{} + + cfg, processCtx, close := testrig.CreateConfig(t, test.DBTypePostgres) + defer close() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + natsInstance := &jetstream.NATSInstance{} + caches := caching.NewRistrettoCache(8*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + db, err := storage.NewUserDatabase(processCtx.Context(), cm, &cfg.UserAPI.AccountDatabase, cfg.Global.ServerName, bcrypt.MinCost, 1000, 1000, "") + assert.NoError(b, err) + + consumer := OutputRoomEventConsumer{db: db, rsAPI: rsAPI, serverName: "test", cfg: &cfg.UserAPI} + _, sk, err := ed25519.GenerateKey(nil) + assert.NoError(b, err) + + alice := test.NewUser(t) + room := test.NewRoom(t, alice) + + for i := 0; i < 100; i++ { + user := test.NewUser(t, test.WithSigningServer("notlocalhost", "ed25519:abc", sk)) + room.CreateAndInsert(t, user, spec.MRoomMember, map[string]string{"membership": spec.Join}, test.WithStateKey(user.ID)) + } + + err = rsapi.SendEvents(processCtx.Context(), rsAPI, rsapi.KindNew, room.Events(), "", "test", "test", nil, false) + assert.NoError(b, err) + + expectedLocalMember := &localMembership{UserID: alice.ID, Localpart: alice.Localpart, Domain: "test", MemberContent: gomatrixserverlib.MemberContent{Membership: spec.Join}} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + members, count, err := consumer.localRoomMembers(processCtx.Context(), room.ID) + assert.NoError(b, err) + assert.Equal(b, 101, count) + assert.Equal(b, expectedLocalMember, members[0]) + } +} diff --git a/userapi/internal/device_list_update.go b/userapi/internal/device_list_update.go index 3fccf56bb..2f33589fe 100644 --- a/userapi/internal/device_list_update.go +++ b/userapi/internal/device_list_update.go @@ -180,11 +180,13 @@ func (u *DeviceListUpdater) Start() error { if err != nil { return err } + + newStaleLists := dedupeStaleLists(staleLists) offset, step := time.Second*10, time.Second - if max := len(staleLists); max > 120 { + if max := len(newStaleLists); max > 120 { step = (time.Second * 120) / time.Duration(max) } - for _, userID := range staleLists { + for _, userID := range newStaleLists { userID := userID // otherwise we are only sending the last entry time.AfterFunc(offset, func() { u.notifyWorkers(userID) @@ -416,6 +418,12 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName) { func (u *DeviceListUpdater) processServer(serverName spec.ServerName) (time.Duration, bool) { ctx := u.process.Context() + // If the process.Context is canceled, there is no need to go further. + // This avoids spamming the logs when shutting down + if errors.Is(ctx.Err(), context.Canceled) { + return defaultWaitTime, false + } + logger := util.GetLogger(ctx).WithField("server_name", serverName) deviceListUpdateCount.WithLabelValues(string(serverName)).Inc() @@ -428,13 +436,6 @@ func (u *DeviceListUpdater) processServer(serverName spec.ServerName) (time.Dura return waitTime, true } - defer func() { - for _, userID := range userIDs { - // always clear the channel to unblock Update calls regardless of success/failure - u.clearChannel(userID) - } - }() - for _, userID := range userIDs { userWait, err := u.processServerUser(ctx, serverName, userID) if err != nil { @@ -461,6 +462,11 @@ func (u *DeviceListUpdater) processServer(serverName spec.ServerName) (time.Dura func (u *DeviceListUpdater) processServerUser(ctx context.Context, serverName spec.ServerName, userID string) (time.Duration, error) { ctx, cancel := context.WithTimeout(ctx, requestTimeout) defer cancel() + + // If we are processing more than one user per server, this unblocks further calls to Update + // immediately instead of just after **all** users have been processed. + defer u.clearChannel(userID) + logger := util.GetLogger(ctx).WithFields(logrus.Fields{ "server_name": serverName, "user_id": userID, @@ -579,3 +585,24 @@ func (u *DeviceListUpdater) updateDeviceList(res *fclient.RespUserDevices) error } return nil } + +// dedupeStaleLists de-duplicates the stateList entries using the domain. +// This is used on startup, processServer is getting all users anyway, so +// there is no need to send every user to the workers. +func dedupeStaleLists(staleLists []string) []string { + seenDomains := make(map[spec.ServerName]struct{}) + newStaleLists := make([]string, 0, len(staleLists)) + for _, userID := range staleLists { + _, domain, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + // non-fatal and should not block starting up + continue + } + if _, ok := seenDomains[domain]; ok { + continue + } + newStaleLists = append(newStaleLists, userID) + seenDomains[domain] = struct{}{} + } + return newStaleLists +} diff --git a/userapi/internal/device_list_update_test.go b/userapi/internal/device_list_update_test.go index 10b9c6521..38fd8b583 100644 --- a/userapi/internal/device_list_update_test.go +++ b/userapi/internal/device_list_update_test.go @@ -428,3 +428,49 @@ func TestDeviceListUpdater_CleanUp(t *testing.T) { } }) } + +func Test_dedupeStateList(t *testing.T) { + alice := "@alice:localhost" + bob := "@bob:localhost" + charlie := "@charlie:notlocalhost" + invalidUserID := "iaminvalid:localhost" + + tests := []struct { + name string + staleLists []string + want []string + }{ + { + name: "empty stateLists", + staleLists: []string{}, + want: []string{}, + }, + { + name: "single entry", + staleLists: []string{alice}, + want: []string{alice}, + }, + { + name: "multiple entries without dupe servers", + staleLists: []string{alice, charlie}, + want: []string{alice, charlie}, + }, + { + name: "multiple entries with dupe servers", + staleLists: []string{alice, bob, charlie}, + want: []string{alice, charlie}, + }, + { + name: "list with invalid userID", + staleLists: []string{alice, bob, invalidUserID}, + want: []string{alice}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := dedupeStaleLists(tt.staleLists); !reflect.DeepEqual(got, tt.want) { + t.Errorf("dedupeStaleLists() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/userapi/internal/user_api.go b/userapi/internal/user_api.go index 367d1ccc9..3f552df12 100644 --- a/userapi/internal/user_api.go +++ b/userapi/internal/user_api.go @@ -568,12 +568,15 @@ func (a *UserInternalAPI) QueryAccountData(ctx context.Context, req *api.QueryAc func (a *UserInternalAPI) QueryAccessToken(ctx context.Context, req *api.QueryAccessTokenRequest, res *api.QueryAccessTokenResponse) error { if req.AppServiceUserID != "" { appServiceDevice, err := a.queryAppServiceToken(ctx, req.AccessToken, req.AppServiceUserID) - if err != nil { - res.Err = err.Error() - } - res.Device = appServiceDevice + if err != nil || appServiceDevice != nil { + if err != nil { + res.Err = err.Error() + } + res.Device = appServiceDevice - return nil + return nil + } + // If the provided token wasn't an as_token (both err and appServiceDevice are nil), continue with normal auth. } device, err := a.DB.GetDeviceByAccessToken(ctx, req.AccessToken) if err != nil { diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index 87913e0f8..db1b944ac 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -1,6 +1,7 @@ package storage_test import ( + "bytes" "context" "encoding/json" "fmt" @@ -763,3 +764,53 @@ func TestDeviceKeysStreamIDGeneration(t *testing.T) { } }) } + +func TestOneTimeKeys(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, clean := mustCreateKeyDatabase(t, dbType) + defer clean() + userID := "@alice:localhost" + deviceID := "alice_device" + otk := api.OneTimeKeys{ + UserID: userID, + DeviceID: deviceID, + KeyJSON: map[string]json.RawMessage{"curve25519:KEY1": []byte(`{"key":"v1"}`)}, + } + + // Add a one time key to the DB + _, err := db.StoreOneTimeKeys(ctx, otk) + MustNotError(t, err) + + // Check the count of one time keys is correct + count, err := db.OneTimeKeysCount(ctx, userID, deviceID) + MustNotError(t, err) + if count.KeyCount["curve25519"] != 1 { + t.Fatalf("Expected 1 key, got %d", count.KeyCount["curve25519"]) + } + + // Check the actual key contents are correct + keysJSON, err := db.ExistingOneTimeKeys(ctx, userID, deviceID, []string{"curve25519:KEY1"}) + MustNotError(t, err) + keyJSON, err := keysJSON["curve25519:KEY1"].MarshalJSON() + MustNotError(t, err) + if !bytes.Equal(keyJSON, []byte(`{"key":"v1"}`)) { + t.Fatalf("Existing keys do not match expected. Got %v", keysJSON["curve25519:KEY1"]) + } + + // Claim a one time key from the database. This should remove it from the database. + claimedKeys, err := db.ClaimKeys(ctx, map[string]map[string]string{userID: {deviceID: "curve25519"}}) + MustNotError(t, err) + + // Check the claimed key contents are correct + if !reflect.DeepEqual(claimedKeys[0], otk) { + t.Fatalf("Expected to claim stored key %v. Got %v", otk, claimedKeys[0]) + } + + // Check the count of one time keys is now zero + count, err = db.OneTimeKeysCount(ctx, userID, deviceID) + MustNotError(t, err) + if count.KeyCount["curve25519"] != 0 { + t.Fatalf("Expected 0 keys, got %d", count.KeyCount["curve25519"]) + } + }) +}