diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md index a3b8c0754..e18968baf 100644 --- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -62,6 +62,6 @@ If you can identify any relevant log snippets from server logs, please include those (please be careful to remove any personal or private data). Please surround them with ``` (three backticks, on a line on their own), so that they are formatted legibly. -Alternatively, please send logs to @kegan:matrix.org or @neilalexander:matrix.org +Alternatively, please send logs to @kegan:matrix.org, @s7evink:matrix.org or @devonh:one.ems.host with a link to the respective Github issue, thanks! --> diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 85fd355f2..ac40f06b0 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -123,7 +123,7 @@ jobs: with: # Optional: pass GITHUB_TOKEN to avoid rate limiting. token: ${{ secrets.GITHUB_TOKEN }} - - run: go test -json -v ./... 2>&1 | gotestfmt + - run: go test -json -v ./... 2>&1 | gotestfmt -hide all env: POSTGRES_HOST: localhost POSTGRES_USER: postgres @@ -255,7 +255,7 @@ jobs: key: ${{ runner.os }}-go-stable-test-race-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go-stable-test-race- - - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt + - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt -hide all env: POSTGRES_HOST: localhost POSTGRES_USER: postgres @@ -280,6 +280,8 @@ jobs: with: go-version: "stable" cache: true + - name: Docker version + run: docker version - name: Build upgrade-tests run: go build ./cmd/dendrite-upgrade-tests - name: Test upgrade (PostgreSQL) @@ -300,6 +302,8 @@ jobs: with: go-version: "stable" cache: true + - name: Docker version + run: docker version - name: Build upgrade-tests run: go build ./cmd/dendrite-upgrade-tests - name: Test upgrade (PostgreSQL) @@ -436,7 +440,7 @@ jobs: # Run Complement - run: | set -o pipefail && - go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt + go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all shell: bash name: Run Complement Tests env: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8448d8e23..8d3a8d674 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -32,10 +32,6 @@ jobs: if: github.event_name == 'release' # Only for GitHub releases run: | echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -60,7 +56,6 @@ jobs: cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -75,7 +70,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -109,10 +103,6 @@ jobs: if: github.event_name == 'release' # Only for GitHub releases run: | echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -137,7 +127,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} file: ./build/docker/Dockerfile.demo-pinecone platforms: ${{ env.PLATFORMS }} push: true @@ -153,7 +142,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} file: ./build/docker/Dockerfile.demo-pinecone platforms: ${{ env.PLATFORMS }} push: true @@ -176,10 +164,6 @@ jobs: if: github.event_name == 'release' # Only for GitHub releases run: | echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -204,7 +188,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} file: ./build/docker/Dockerfile.demo-yggdrasil platforms: ${{ env.PLATFORMS }} push: true @@ -220,7 +203,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} file: ./build/docker/Dockerfile.demo-yggdrasil platforms: ${{ env.PLATFORMS }} push: true diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index bf62a1c19..9a5eb2b62 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -32,7 +32,7 @@ jobs: version: v3.10.0 - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.4.1 + uses: helm/chart-releaser-action@ed43eb303604cbc0eeec8390544f7748dc6c790d # specific commit, since `mark_as_latest` is not yet in a release env: CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" with: diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index e76cc82f3..509861860 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -128,7 +128,7 @@ jobs: # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64 run: | sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev - go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest + go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest - name: Run actions/checkout@v3 for dendrite uses: actions/checkout@v3 with: diff --git a/.golangci.yml b/.golangci.yml index bb8d38a8b..5bee0a885 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -180,7 +180,6 @@ linters-settings: linters: enable: - errcheck - - goconst - gocyclo - goimports # Does everything gofmt does - gosimple @@ -211,6 +210,7 @@ linters: - stylecheck - typecheck # Should turn back on soon - unconvert # Should turn back on soon + - goconst # Slightly annoying, as it reports "issues" in SQL statements disable-all: false presets: fast: false diff --git a/CHANGES.md b/CHANGES.md index c99ed2255..57e3a3d4f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,69 @@ # Changelog +## Dendrite 0.13.4 (2023-10-25) + +Upgrading to this version is **highly** recommended, as it fixes a long-standing bug in the state resolution +algorithm. + +### Fixes: + +- The "device list updater" now de-duplicates the servers to fetch devices from on startup. (This also + avoids spamming the logs when shutting down.) +- A bug in the state resolution algorithm has been fixed. This bug could result in users "being reset" + out of rooms and other missing state events due to calculating the wrong state. +- A bug when setting notifications from Element Android has been fixed by implementing MSC3987 + +### Features + +- Updated dependencies + - Internal NATS Server has been updated from v2.9.19 to v2.9.23 + +## Dendrite 0.13.3 (2023-09-28) + +### Fixes: + +- The `user_id` query parameter when authenticating is now used correctly (contributed by [tulir](https://github.com/tulir)) +- Invitations are now correctly pushed to devices +- A bug which could result in the corruption of `m.direct` account data has been fixed + +### Features + +- [Sliding Sync proxy](https://github.com/matrix-org/sliding-sync) can be configured in the `/.well-known/matrix/client` response +- Room version 11 is now supported +- Clients can request the `federation` `event_format` when creating filters +- Many under the hood improvements for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md) + +### Other + +- Dendrite now requires Go 1.20 if building from source + +## Dendrite 0.13.2 (2023-08-23) + +### Fixes: + +- Migrations in SQLite are now prepared on the correct context (transaction or database) +- The `InputRoomEvent` stream now has a maximum age of 24h, which should help with slow start up times of NATS JetStream (contributed by [neilalexander](https://github.com/neilalexander)) +- Event size checks are more in line with Synapse +- Requests to `/messages` have been optimized, possibly reducing database round trips +- Re-add the revision of Dendrite when building from source (Note: This only works if git is installed) +- Getting local members to notify has been optimized, which should significantly reduce memory allocation and cache usage +- When getting queried about user profiles, we now return HTTP404 if the user/profiles does not exist +- Background federated joins should now be fixed and not timeout after a short time +- Database connections are now correctly re-used +- Restored the old behavior of the `/purgeRoom` admin endpoint (does not evacuate the room before purging) +- Don't expose information about the system when trying to download files that don't exist + +### Features + +- Further improvements and fixes for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md) + - Lookup correct prev events in the sync API + - Populate `prev_sender` correctly in the sync API + - Event federation should work better +- Added new `dendrite_up` Prometheus metric, containing the version of Dendrite +- Space summaries ([MSC2946](https://github.com/matrix-org/matrix-spec-proposals/pull/2946)) have been moved from MSC to being natively supported +- For easier issue investigation, logs for application services now contain the application service ID (contributed by [maxberger](https://github.com/maxberger)) +- The default room version to use when creating rooms can now be configured using `room_server.default_room_version` + ## Dendrite 0.13.1 (2023-07-06) This releases fixes a long-standing "off-by-one" error which could result in state resets. Upgrading to this version is **highly** recommended. diff --git a/Dockerfile b/Dockerfile index 4ee20933a..8c8f1588f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # # base installs required dependencies and runs go mod download to cache dependencies # -FROM --platform=${BUILDPLATFORM} docker.io/golang:1.20-alpine AS base +FROM --platform=${BUILDPLATFORM} docker.io/golang:1.21-alpine AS base RUN apk --update --no-cache add bash build-base curl git # @@ -13,7 +13,6 @@ FROM --platform=${BUILDPLATFORM} base AS build WORKDIR /src ARG TARGETOS ARG TARGETARCH -ARG FLAGS RUN --mount=target=. \ --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ @@ -21,7 +20,7 @@ RUN --mount=target=. \ GOARCH="$TARGETARCH" \ GOOS="linux" \ CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0") \ - go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/... + go build -v -trimpath -o /out/ ./cmd/... # diff --git a/README.md b/README.md index 34604eff9..bde19b07e 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j See the [Planning your Installation](https://matrix-org.github.io/dendrite/installation/planning) page for more information on requirements. -To build Dendrite, you will need Go 1.18 or later. +To build Dendrite, you will need Go 1.20 or later. For a usable federating Dendrite deployment, you will also need: diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go index ddc24477b..eca63371d 100644 --- a/appservice/appservice_test.go +++ b/appservice/appservice_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/stretchr/testify/assert" "github.com/matrix-org/dendrite/appservice" @@ -32,6 +33,10 @@ import ( "github.com/matrix-org/dendrite/test/testrig" ) +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + func TestAppserviceInternalAPI(t *testing.T) { // Set expected results @@ -144,7 +149,7 @@ func TestAppserviceInternalAPI(t *testing.T) { cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) runCases(t, asAPI) @@ -239,7 +244,7 @@ func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) { cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) t.Run("UserIDExists", func(t *testing.T) { @@ -378,7 +383,7 @@ func TestRoomserverConsumerOneInvite(t *testing.T) { // Create required internal APIs rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // start the consumer appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 1877de37a..e8b9211c4 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -128,7 +128,7 @@ func (s *OutputRoomEventConsumer) onMessage( if len(output.NewRoomEvent.AddsStateEventIDs) > 0 { newEventID := output.NewRoomEvent.Event.EventID() eventsReq := &api.QueryEventsByIDRequest{ - RoomID: output.NewRoomEvent.Event.RoomID(), + RoomID: output.NewRoomEvent.Event.RoomID().String(), EventIDs: make([]string, 0, len(output.NewRoomEvent.AddsStateEventIDs)), } eventsRes := &api.QueryEventsByIDResponse{} @@ -236,11 +236,7 @@ func (s *appserviceState) backoffAndPause(err error) error { // TODO: This should be cached, see https://github.com/matrix-org/dendrite/issues/1682 func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event *types.HeaderedEvent, appservice *config.ApplicationService) bool { user := "" - validRoomID, err := spec.NewRoomID(event.RoomID()) - if err != nil { - return false - } - userID, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, event.SenderID()) + userID, err := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID()) if err == nil { user = userID.String() } @@ -250,7 +246,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont return false case appservice.IsInterestedInUserID(user): return true - case appservice.IsInterestedInRoomID(event.RoomID()): + case appservice.IsInterestedInRoomID(event.RoomID().String()): return true } @@ -261,7 +257,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont } // Check all known room aliases of the room the event came from - queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()} + queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID().String()} var queryRes api.GetAliasesForRoomIDResponse if err := s.rsAPI.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil { for _, alias := range queryRes.Aliases { @@ -272,7 +268,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont } else { log.WithFields(log.Fields{ "appservice": appservice.ID, - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), }).WithError(err).Errorf("Unable to get aliases for room") } @@ -288,7 +284,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e // until we have a lighter way of checking the state before the event that // doesn't involve state res, then this is probably OK. membershipReq := &api.QueryMembershipsForRoomRequest{ - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), JoinedOnly: true, } membershipRes := &api.QueryMembershipsForRoomResponse{} @@ -317,7 +313,7 @@ func (s *OutputRoomEventConsumer) appserviceJoinedAtEvent(ctx context.Context, e } else { log.WithFields(log.Fields{ "appservice": appservice.ID, - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), }).WithError(err).Errorf("Unable to get membership for room") } return false diff --git a/appservice/query/query.go b/appservice/query/query.go index ca8d7b3a3..5c736f379 100644 --- a/appservice/query/query.go +++ b/appservice/query/query.go @@ -217,7 +217,7 @@ func (a *AppServiceQueryAPI) Locations( } if err := requestDo[[]api.ASLocationResponse](as.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil { - log.WithError(err).Error("unable to get 'locations' from application service") + log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'locations' from application service") continue } @@ -252,7 +252,7 @@ func (a *AppServiceQueryAPI) User( } if err := requestDo[[]api.ASUserResponse](as.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil { - log.WithError(err).Error("unable to get 'user' from application service") + log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'user' from application service") continue } @@ -290,7 +290,7 @@ func (a *AppServiceQueryAPI) Protocols( for _, as := range a.Cfg.Derived.ApplicationServices { var proto api.ASProtocolResponse if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+req.Protocol, &proto); err != nil { - log.WithError(err).Error("unable to get 'protocol' from application service") + log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service") continue } @@ -320,7 +320,7 @@ func (a *AppServiceQueryAPI) Protocols( for _, p := range as.Protocols { var proto api.ASProtocolResponse if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+p, &proto); err != nil { - log.WithError(err).Error("unable to get 'protocol' from application service") + log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service") continue } existing, ok := response[p] diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index 585374738..80c0dbff6 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -944,4 +944,12 @@ rmv remote user can join room with version 10 rmv User can invite remote user to room with version 10 rmv Remote user can backfill in a room with version 10 rmv Can reject invites over federation for rooms with version 10 -rmv Can receive redactions from regular users over federation in room version 10 \ No newline at end of file +rmv Can receive redactions from regular users over federation in room version 10 +rmv User can create and send/receive messages in a room with version 11 +rmv local user can join room with version 11 +rmv User can invite local user to room with version 11 +rmv remote user can join room with version 11 +rmv User can invite remote user to room with version 11 +rmv Remote user can backfill in a room with version 11 +rmv Can reject invites over federation for rooms with version 11 +rmv Can receive redactions from regular users over federation in room version 11 \ No newline at end of file diff --git a/build/dendritejs-pinecone/main.go b/build/dendritejs-pinecone/main.go index 61baed902..6acc93c7b 100644 --- a/build/dendritejs-pinecone/main.go +++ b/build/dendritejs-pinecone/main.go @@ -38,6 +38,7 @@ import ( "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/userapi" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/gomatrixserverlib" @@ -190,13 +191,13 @@ func startup() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) + fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fedSenderAPI.IsBlacklistedOrBackingOff) asQuery := appservice.NewInternalAPI( processCtx, cfg, &natsInstance, userAPI, rsAPI, ) rsAPI.SetAppserviceAPI(asQuery) - fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true) rsAPI.SetFederationAPI(fedSenderAPI, keyRing) monolith := setup.Monolith{ diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone index 90f515167..ab50cf318 100644 --- a/build/docker/Dockerfile.demo-pinecone +++ b/build/docker/Dockerfile.demo-pinecone @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.19-alpine AS base +FROM docker.io/golang:1.21-alpine AS base # # Needs to be separate from the main Dockerfile for OpenShift, diff --git a/build/docker/Dockerfile.demo-yggdrasil b/build/docker/Dockerfile.demo-yggdrasil index efae5496c..b9e387666 100644 --- a/build/docker/Dockerfile.demo-yggdrasil +++ b/build/docker/Dockerfile.demo-yggdrasil @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.19-alpine AS base +FROM docker.io/golang:1.21-alpine AS base # # Needs to be separate from the main Dockerfile for OpenShift, diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 720ce37eb..2b227d373 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -216,7 +216,7 @@ func (m *DendriteMonolith) Start() { processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true, ) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) diff --git a/build/scripts/complement.sh b/build/scripts/complement.sh index 29feff304..8608d8fa5 100755 --- a/build/scripts/complement.sh +++ b/build/scripts/complement.sh @@ -15,5 +15,5 @@ tar -xzf master.tar.gz # Run the tests! cd complement-master -COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests +COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v -count=1 ./tests ./tests/csapi diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index 66667b03c..f0e5f004d 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -45,7 +45,7 @@ func TestAdminCreateToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -196,7 +196,7 @@ func TestAdminListRegistrationTokens(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -314,7 +314,7 @@ func TestAdminGetRegistrationToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -415,7 +415,7 @@ func TestAdminDeleteRegistrationToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -509,7 +509,7 @@ func TestAdminUpdateRegistrationToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -693,7 +693,7 @@ func TestAdminResetPassword(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed for changing the password/login - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -791,7 +791,7 @@ func TestPurgeRoom(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) // Create the room @@ -863,7 +863,7 @@ func TestAdminEvacuateRoom(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { @@ -964,7 +964,7 @@ func TestAdminEvacuateUser(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { @@ -1055,7 +1055,7 @@ func TestAdminMarkAsStale(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go index ae14d271d..2ff4b6503 100644 --- a/clientapi/clientapi_test.go +++ b/clientapi/clientapi_test.go @@ -17,6 +17,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/routing" "github.com/matrix-org/dendrite/clientapi/threepid" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/pushrules" @@ -49,6 +50,10 @@ type userDevice struct { password string } +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + func TestGetPutDevices(t *testing.T) { alice := test.NewUser(t) bob := test.NewUser(t) @@ -121,7 +126,7 @@ func TestGetPutDevices(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -170,7 +175,7 @@ func TestDeleteDevice(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -275,7 +280,7 @@ func TestDeleteDevices(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -442,7 +447,7 @@ func TestSetDisplayname(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI) AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -554,7 +559,7 @@ func TestSetAvatarURL(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI) AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -632,7 +637,7 @@ func TestTyping(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed to create accounts - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -716,7 +721,7 @@ func TestMembership(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed to create accounts - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) rsAPI.SetUserAPI(userAPI) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -923,13 +928,17 @@ func TestCapabilities(t *testing.T) { } } + var tempRoomServerCfg config.RoomServer + tempRoomServerCfg.Defaults(config.DefaultOpts{}) + defaultRoomVersion := tempRoomServerCfg.DefaultRoomVersion + expectedMap := map[string]interface{}{ "capabilities": map[string]interface{}{ "m.change_password": map[string]bool{ "enabled": true, }, "m.room_versions": map[string]interface{}{ - "default": version.DefaultRoomVersion(), + "default": defaultRoomVersion, "available": versionsMap, }, }, @@ -951,7 +960,7 @@ func TestCapabilities(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -998,7 +1007,7 @@ func TestTurnserver(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) //rsAPI.SetUserAPI(userAPI) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1096,7 +1105,7 @@ func Test3PID(t *testing.T) { // Needed to create accounts rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1272,7 +1281,7 @@ func TestPushRules(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1414,7 +1423,7 @@ func TestPushRules(t *testing.T) { validateFunc: func(t *testing.T, respBody *bytes.Buffer) { actions := gjson.GetBytes(respBody.Bytes(), "actions").Array() // only a basic check - assert.Equal(t, 1, len(actions)) + assert.Equal(t, 0, len(actions)) }, }, { @@ -1659,7 +1668,7 @@ func TestKeys(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -2121,7 +2130,7 @@ func TestKeyBackup(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) diff --git a/clientapi/routing/capabilities.go b/clientapi/routing/capabilities.go index fa50fa1aa..38e5bd467 100644 --- a/clientapi/routing/capabilities.go +++ b/clientapi/routing/capabilities.go @@ -17,6 +17,7 @@ package routing import ( "net/http" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -24,7 +25,7 @@ import ( // GetCapabilities returns information about the server's supported feature set // and other relevant capabilities to an authenticated user. -func GetCapabilities() util.JSONResponse { +func GetCapabilities(rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { versionsMap := map[gomatrixserverlib.RoomVersion]string{} for v, desc := range version.SupportedRoomVersions() { if desc.Stable() { @@ -40,7 +41,7 @@ func GetCapabilities() util.JSONResponse { "enabled": true, }, "m.room_versions": map[string]interface{}{ - "default": version.DefaultRoomVersion(), + "default": rsAPI.DefaultRoomVersion(), "available": versionsMap, }, }, diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 320f236cb..47e3ba1c3 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -171,7 +171,7 @@ func createRoom( // Clobber keys: creator, room_version - roomVersion := roomserverVersion.DefaultRoomVersion() + roomVersion := rsAPI.DefaultRoomVersion() if createRequest.RoomVersion != "" { candidateVersion := gomatrixserverlib.RoomVersion(createRequest.RoomVersion) _, roomVersionError := roomserverVersion.SupportedRoomVersion(candidateVersion) diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index d9129d1bd..907727662 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -181,13 +181,39 @@ func SetLocalAlias( return *resErr } - queryReq := roomserverAPI.SetRoomAliasRequest{ - UserID: device.UserID, - RoomID: r.RoomID, - Alias: alias, + roomID, err := spec.NewRoomID(r.RoomID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("invalid room ID"), + } } - var queryRes roomserverAPI.SetRoomAliasResponse - if err := rsAPI.SetRoomAlias(req.Context(), &queryReq, &queryRes); err != nil { + + userID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + + senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *roomID, *userID) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("QuerySenderIDForUser failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } else if senderID == nil { + util.GetLogger(req.Context()).WithField("roomID", *roomID).WithField("userID", *userID).Error("Sender ID not found") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + + aliasAlreadyExists, err := rsAPI.SetRoomAlias(req.Context(), *senderID, *roomID, alias) + if err != nil { util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed") return util.JSONResponse{ Code: http.StatusInternalServerError, @@ -195,7 +221,7 @@ func SetLocalAlias( } } - if queryRes.AliasExists { + if aliasAlreadyExists { return util.JSONResponse{ Code: http.StatusConflict, JSON: spec.Unknown("The alias " + alias + " already exists."), @@ -240,6 +266,31 @@ func RemoveLocalAlias( JSON: spec.NotFound("The alias does not exist."), } } + + // This seems like the kind of auth check that should be done in the roomserver, but + // if this check fails (user is not in the room), then there will be no SenderID for the user + // for pseudo-ID rooms - it will just return "". However, we can't use lack of a sender ID + // as meaning they are not in the room, since lacking a sender ID could be caused by other bugs. + // TODO: maybe have QuerySenderIDForUser return richer errors? + var queryResp roomserverAPI.QueryMembershipForUserResponse + err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{ + RoomID: validRoomID.String(), + UserID: *userID, + }, &queryResp) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("roomserverAPI.QueryMembershipForUser failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + if !queryResp.IsInRoom { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("You do not have permission to remove this alias."), + } + } + deviceSenderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *userID) if err != nil { return util.JSONResponse{ @@ -247,28 +298,31 @@ func RemoveLocalAlias( JSON: spec.NotFound("The alias does not exist."), } } - - queryReq := roomserverAPI.RemoveRoomAliasRequest{ - Alias: alias, - SenderID: deviceSenderID, - } - var queryRes roomserverAPI.RemoveRoomAliasResponse - if err := rsAPI.RemoveRoomAlias(req.Context(), &queryReq, &queryRes); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.RemoveRoomAlias failed") + // TODO: how to handle this case? missing user/room keys seem to be a whole new class of errors + if deviceSenderID == nil { return util.JSONResponse{ Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, + JSON: spec.Unknown("internal server error"), } } - if !queryRes.Found { + aliasFound, aliasRemoved, err := rsAPI.RemoveRoomAlias(req.Context(), *deviceSenderID, alias) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.RemoveRoomAlias failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + + if !aliasFound { return util.JSONResponse{ Code: http.StatusNotFound, JSON: spec.NotFound("The alias does not exist."), } } - if !queryRes.Removed { + if !aliasRemoved { return util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Forbidden("You do not have permission to remove this alias."), @@ -337,7 +391,7 @@ func SetVisibility( } } senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID) - if err != nil { + if err != nil || senderID == nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.Unknown("failed to find senderID for this user"), @@ -368,7 +422,7 @@ func SetVisibility( // NOTSPEC: Check if the user's power is greater than power required to change m.room.canonical_alias event power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].PDU) - if power.UserLevel(senderID) < power.EventLevel(spec.MRoomCanonicalAlias, true) { + if power.UserLevel(*senderID) < power.EventLevel(spec.MRoomCanonicalAlias, true) { return util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Forbidden("userID doesn't have power level to change visibility"), diff --git a/clientapi/routing/joined_rooms.go b/clientapi/routing/joined_rooms.go index f664183f8..3fe0d8b4d 100644 --- a/clientapi/routing/joined_rooms.go +++ b/clientapi/routing/joined_rooms.go @@ -33,23 +33,36 @@ func GetJoinedRooms( device *userapi.Device, rsAPI api.ClientRoomserverAPI, ) util.JSONResponse { - var res api.QueryRoomsForUserResponse - err := rsAPI.QueryRoomsForUser(req.Context(), &api.QueryRoomsForUserRequest{ - UserID: device.UserID, - WantMembership: "join", - }, &res) + deviceUserID, err := spec.NewUserID(device.UserID, true) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("Invalid device user ID") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + + rooms, err := rsAPI.QueryRoomsForUser(req.Context(), *deviceUserID, "join") if err != nil { util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed") return util.JSONResponse{ Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, + JSON: spec.Unknown("internal server error"), } } - if res.RoomIDs == nil { - res.RoomIDs = []string{} + + var roomIDStrs []string + if rooms == nil { + roomIDStrs = []string{} + } else { + roomIDStrs = make([]string, len(rooms)) + for i, roomID := range rooms { + roomIDStrs[i] = roomID.String() + } } + return util.JSONResponse{ Code: http.StatusOK, - JSON: getJoinedRoomsResponse{res.RoomIDs}, + JSON: getJoinedRoomsResponse{roomIDStrs}, } } diff --git a/clientapi/routing/joinroom_test.go b/clientapi/routing/joinroom_test.go index 933ea8d3a..bd854efa8 100644 --- a/clientapi/routing/joinroom_test.go +++ b/clientapi/routing/joinroom_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/jetstream" @@ -21,6 +22,10 @@ import ( uapi "github.com/matrix-org/dendrite/userapi/api" ) +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + func TestJoinRoomByIDOrAlias(t *testing.T) { alice := test.NewUser(t) bob := test.NewUser(t) @@ -36,7 +41,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { natsInstance := jetstream.NATSInstance{} rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) // Create the users in the userapi diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go index 252017db2..1c628b196 100644 --- a/clientapi/routing/login_test.go +++ b/clientapi/routing/login_test.go @@ -49,7 +49,7 @@ func TestLogin(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed for /login - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. Setup(routers, cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, caching.DisableMetrics) diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 9668421c9..06683c47d 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -71,7 +71,7 @@ func SendBan( } } senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID) - if err != nil { + if err != nil || senderID == nil { return util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Forbidden("You don't have permission to ban this user, unknown senderID"), @@ -87,7 +87,7 @@ func SendBan( if errRes != nil { return *errRes } - allowedToBan := pl.UserLevel(senderID) >= pl.Ban + allowedToBan := pl.UserLevel(*senderID) >= pl.Ban if !allowedToBan { return util.JSONResponse{ Code: http.StatusForbidden, @@ -169,7 +169,7 @@ func SendKick( } } senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *deviceUserID) - if err != nil { + if err != nil || senderID == nil { return util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Forbidden("You don't have permission to kick this user, unknown senderID"), @@ -193,7 +193,7 @@ func SendKick( if errRes != nil { return *errRes } - allowedToKick := pl.UserLevel(senderID) >= pl.Kick || bodyUserID.String() == deviceUserID.String() + allowedToKick := pl.UserLevel(*senderID) >= pl.Kick || bodyUserID.String() == deviceUserID.String() if !allowedToKick { return util.JSONResponse{ Code: http.StatusForbidden, @@ -477,6 +477,8 @@ func buildMembershipEvent( senderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *userID) if err != nil { return nil, err + } else if senderID == nil { + return nil, fmt.Errorf("no sender ID for %s in %s", *userID, *validRoomID) } targetID, err := spec.NewUserID(targetUserID, true) @@ -486,6 +488,8 @@ func buildMembershipEvent( targetSenderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *targetID) if err != nil { return nil, err + } else if targetSenderID == nil { + return nil, fmt.Errorf("no sender ID for %s in %s", *targetID, *validRoomID) } identity, err := rsAPI.SigningIdentityFor(ctx, *validRoomID, *userID) @@ -493,8 +497,8 @@ func buildMembershipEvent( return nil, err } - return buildMembershipEventDirect(ctx, targetSenderID, reason, profile.DisplayName, profile.AvatarURL, - senderID, device.UserDomain(), membership, roomID, isDirect, identity.KeyID, identity.PrivateKey, evTime, rsAPI) + return buildMembershipEventDirect(ctx, *targetSenderID, reason, profile.DisplayName, profile.AvatarURL, + *senderID, device.UserDomain(), membership, roomID, isDirect, identity.KeyID, identity.PrivateKey, evTime, rsAPI) } // loadProfile lookups the profile of a given user from the database and returns diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 35da15e0e..9959144c8 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -16,6 +16,7 @@ package routing import ( "context" + "fmt" "net/http" "time" @@ -250,11 +251,15 @@ func updateProfile( profile *authtypes.Profile, userID string, evTime time.Time, ) (util.JSONResponse, error) { - var res api.QueryRoomsForUserResponse - err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{ - UserID: device.UserID, - WantMembership: "join", - }, &res) + deviceUserID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + }, err + } + + rooms, err := rsAPI.QueryRoomsForUser(ctx, *deviceUserID, "join") if err != nil { util.GetLogger(ctx).WithError(err).Error("QueryRoomsForUser failed") return util.JSONResponse{ @@ -263,6 +268,11 @@ func updateProfile( }, err } + roomIDStrs := make([]string, len(rooms)) + for i, room := range rooms { + roomIDStrs[i] = room.String() + } + _, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.SplitID failed") @@ -273,7 +283,7 @@ func updateProfile( } events, err := buildMembershipEvents( - ctx, res.RoomIDs, *profile, userID, evTime, rsAPI, + ctx, roomIDStrs, *profile, userID, evTime, rsAPI, ) switch e := err.(type) { case nil: @@ -290,7 +300,7 @@ func updateProfile( }, e } - if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, true); err != nil { + if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, false); err != nil { util.GetLogger(ctx).WithError(err).Error("SendEvents failed") return util.JSONResponse{ Code: http.StatusInternalServerError, @@ -362,8 +372,10 @@ func buildMembershipEvents( senderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *fullUserID) if err != nil { return nil, err + } else if senderID == nil { + return nil, fmt.Errorf("sender ID not found for %s in %s", *fullUserID, *validRoomID) } - senderIDString := string(senderID) + senderIDString := string(*senderID) proto := gomatrixserverlib.ProtoEvent{ SenderID: senderIDString, RoomID: roomID, diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go index 1b9a5a818..f331a73c7 100644 --- a/clientapi/routing/redaction.go +++ b/clientapi/routing/redaction.go @@ -34,7 +34,8 @@ import ( ) type redactionContent struct { - Reason string `json:"reason"` + Reason string `json:"reason"` + Redacts string `json:"redacts"` } type redactionResponse struct { @@ -74,6 +75,16 @@ func SendRedaction( return *resErr } + // if user is member of room, and sender ID is nil, then this user doesn't have a pseudo ID for some reason, + // which is unexpected. + if senderID == nil { + util.GetLogger(req.Context()).WithField("userID", *deviceUserID).WithField("roomID", roomID).Error("missing sender ID for user, despite having membership") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + if txnID != nil { // Try to fetch response from transactionsCache if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID, req.URL); ok { @@ -88,7 +99,7 @@ func SendRedaction( JSON: spec.NotFound("unknown event ID"), // TODO: is it ok to leak existence? } } - if ev.RoomID() != roomID { + if ev.RoomID().String() != roomID { return util.JSONResponse{ Code: 400, JSON: spec.NotFound("cannot redact event in another room"), @@ -98,7 +109,7 @@ func SendRedaction( // "Users may redact their own events, and any user with a power level greater than or equal // to the redact power level of the room may redact events there" // https://matrix.org/docs/spec/client_server/r0.6.1#put-matrix-client-r0-rooms-roomid-redact-eventid-txnid - allowedToRedact := ev.SenderID() == senderID + allowedToRedact := ev.SenderID() == *senderID if !allowedToRedact { plEvent := roomserverAPI.GetStateEvent(req.Context(), rsAPI, roomID, gomatrixserverlib.StateKeyTuple{ EventType: spec.MRoomPowerLevels, @@ -119,7 +130,7 @@ func SendRedaction( ), } } - allowedToRedact = pl.UserLevel(senderID) >= pl.Redact + allowedToRedact = pl.UserLevel(*senderID) >= pl.Redact } if !allowedToRedact { return util.JSONResponse{ @@ -136,11 +147,16 @@ func SendRedaction( // create the new event and set all the fields we can proto := gomatrixserverlib.ProtoEvent{ - SenderID: string(senderID), + SenderID: string(*senderID), RoomID: roomID, Type: spec.MRoomRedaction, Redacts: eventID, } + + // Room version 11 expects the "redacts" field on the + // content field, so add it here as well + r.Redacts = eventID + err = proto.SetContent(r) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("proto.SetContent failed") diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 565c41533..090a2fc20 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -236,7 +236,7 @@ type authDict struct { // TODO: Lots of custom keys depending on the type } -// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#user-interactive-authentication-api +// https://spec.matrix.org/v1.7/client-server-api/#user-interactive-authentication-api type userInteractiveResponse struct { Flows []authtypes.Flow `json:"flows"` Completed []authtypes.LoginType `json:"completed"` @@ -256,7 +256,7 @@ func newUserInteractiveResponse( } } -// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register +// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register type registerResponse struct { UserID string `json:"user_id"` AccessToken string `json:"access_token,omitempty"` @@ -462,7 +462,7 @@ func validateApplicationService( } // Register processes a /register request. -// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register +// https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3register func Register( req *http.Request, userAPI userapi.ClientUserAPI, diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index 0a1986cf7..98455f80a 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -416,7 +416,7 @@ func Test_register(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -596,7 +596,7 @@ func TestRegisterUserWithDisplayName(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) deviceName, deviceID := "deviceName", "deviceID" expectedDisplayName := "DisplayName" response := completeRegistration( @@ -637,7 +637,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) expectedDisplayName := "rabbit" jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 8cd207b7a..d4aa1d08d 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -44,6 +44,19 @@ import ( "github.com/matrix-org/dendrite/setup/jetstream" ) +type WellKnownClientHomeserver struct { + BaseUrl string `json:"base_url"` +} + +type WellKnownSlidingSyncProxy struct { + Url string `json:"url"` +} + +type WellKnownClientResponse struct { + Homeserver WellKnownClientHomeserver `json:"m.homeserver"` + SlidingSyncProxy *WellKnownSlidingSyncProxy `json:"org.matrix.msc3575.proxy,omitempty"` +} + // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. // @@ -96,20 +109,22 @@ func Setup( if cfg.Matrix.WellKnownClientName != "" { logrus.Infof("Setting m.homeserver base_url as %s at /.well-known/matrix/client", cfg.Matrix.WellKnownClientName) + if cfg.Matrix.WellKnownSlidingSyncProxy != "" { + logrus.Infof("Setting org.matrix.msc3575.proxy url as %s at /.well-known/matrix/client", cfg.Matrix.WellKnownSlidingSyncProxy) + } wkMux.Handle("/client", httputil.MakeExternalAPI("wellknown", func(r *http.Request) util.JSONResponse { + response := WellKnownClientResponse{ + Homeserver: WellKnownClientHomeserver{cfg.Matrix.WellKnownClientName}, + } + if cfg.Matrix.WellKnownSlidingSyncProxy != "" { + response.SlidingSyncProxy = &WellKnownSlidingSyncProxy{ + Url: cfg.Matrix.WellKnownSlidingSyncProxy, + } + } + return util.JSONResponse{ Code: http.StatusOK, - JSON: struct { - HomeserverName struct { - BaseUrl string `json:"base_url"` - } `json:"m.homeserver"` - }{ - HomeserverName: struct { - BaseUrl string `json:"base_url"` - }{ - BaseUrl: cfg.Matrix.WellKnownClientName, - }, - }, + JSON: response, } })).Methods(http.MethodGet, http.MethodOptions) } @@ -1256,7 +1271,7 @@ func Setup( if r := rateLimits.Limit(req, device); r != nil { return *r } - return GetCapabilities() + return GetCapabilities(rsAPI) }, httputil.WithAllowGuests()), ).Methods(http.MethodGet, http.MethodOptions) diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 41a3793ae..69131966b 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -29,6 +29,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/synctypes" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" @@ -92,6 +93,30 @@ func SendEvent( } } + // Translate user ID state keys to room keys in pseudo ID rooms + if roomVersion == gomatrixserverlib.RoomVersionPseudoIDs && stateKey != nil { + parsedRoomID, innerErr := spec.NewRoomID(roomID) + if innerErr != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("invalid room ID"), + } + } + + newStateKey, innerErr := synctypes.FromClientStateKey(*parsedRoomID, *stateKey, func(roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + return rsAPI.QuerySenderIDForUser(req.Context(), roomID, userID) + }) + if innerErr != nil { + // TODO: work out better logic for failure cases (e.g. sender ID not found) + util.GetLogger(req.Context()).WithError(innerErr).Error("synctypes.FromClientStateKey failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + stateKey = newStateKey + } + // create a mutex for the specific user in the specific room // this avoids a situation where events that are received in quick succession are sent to the roomserver in a jumbled order userID := device.UserID @@ -238,7 +263,11 @@ func SendEvent( } func updatePowerLevels(req *http.Request, r map[string]interface{}, roomID string, rsAPI api.ClientRoomserverAPI) error { - userMap := r["users"].(map[string]interface{}) + users, ok := r["users"] + if !ok { + return nil + } + userMap := users.(map[string]interface{}) validRoomID, err := spec.NewRoomID(roomID) if err != nil { return err @@ -251,8 +280,11 @@ func updatePowerLevels(req *http.Request, r map[string]interface{}, roomID strin senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *uID) if err != nil { return err + } else if senderID == nil { + util.GetLogger(req.Context()).Warnf("sender ID not found for %s in %s", uID, *validRoomID) + continue } - userMap[string(senderID)] = level + userMap[string(*senderID)] = level delete(userMap, user) } r["users"] = userMap @@ -316,14 +348,21 @@ func generateSendEvent( senderID, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *fullUserID) if err != nil { return nil, &util.JSONResponse{ - Code: http.StatusNotFound, - JSON: spec.NotFound("Unable to find senderID for user"), + Code: http.StatusInternalServerError, + JSON: spec.NotFound("internal server error"), + } + } else if senderID == nil { + // TODO: is it always the case that lack of a sender ID means they're not joined? + // And should this logic be deferred to the roomserver somehow? + return nil, &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("not joined to room"), } } // create the new event and set all the fields we can proto := gomatrixserverlib.ProtoEvent{ - SenderID: string(senderID), + SenderID: string(*senderID), RoomID: roomID, Type: eventType, StateKey: stateKey, @@ -403,7 +442,7 @@ func generateSendEvent( JSON: spec.BadJSON("Cannot unmarshal the event content."), } } - if content["replacement_room"] == e.RoomID() { + if content["replacement_room"] == e.RoomID().String() { return nil, &util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.InvalidParam("Cannot send tombstone event that points to the same room."), diff --git a/clientapi/routing/sendevent_test.go b/clientapi/routing/sendevent_test.go new file mode 100644 index 000000000..9cdd75358 --- /dev/null +++ b/clientapi/routing/sendevent_test.go @@ -0,0 +1,275 @@ +package routing + +import ( + "context" + "crypto/ed25519" + "fmt" + "io" + "net/http" + "strings" + "testing" + + rsapi "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/config" + uapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/fclient" + "github.com/matrix-org/gomatrixserverlib/spec" + "gotest.tools/v3/assert" +) + +// Mock roomserver API for testing +// +// Currently pretty specialised for the pseudo ID test, so will need +// editing if future (other) sendevent tests are using this. +type sendEventTestRoomserverAPI struct { + rsapi.ClientRoomserverAPI + t *testing.T + roomIDStr string + roomVersion gomatrixserverlib.RoomVersion + roomState []*types.HeaderedEvent + + // userID -> room key + senderMapping map[string]ed25519.PrivateKey + + savedInputRoomEvents []rsapi.InputRoomEvent +} + +func (s *sendEventTestRoomserverAPI) QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) { + if roomID == s.roomIDStr { + return s.roomVersion, nil + } else { + s.t.Logf("room version queried for %s", roomID) + return "", fmt.Errorf("unknown room") + } +} + +func (s *sendEventTestRoomserverAPI) QueryCurrentState(ctx context.Context, req *rsapi.QueryCurrentStateRequest, res *rsapi.QueryCurrentStateResponse) error { + res.StateEvents = map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{} + for _, stateKeyTuple := range req.StateTuples { + for _, stateEv := range s.roomState { + if stateEv.Type() == stateKeyTuple.EventType && stateEv.StateKey() != nil && *stateEv.StateKey() == stateKeyTuple.StateKey { + res.StateEvents[stateKeyTuple] = stateEv + } + } + } + return nil +} + +func (s *sendEventTestRoomserverAPI) QueryLatestEventsAndState(ctx context.Context, req *rsapi.QueryLatestEventsAndStateRequest, res *rsapi.QueryLatestEventsAndStateResponse) error { + if req.RoomID == s.roomIDStr { + res.RoomExists = true + res.RoomVersion = s.roomVersion + + res.StateEvents = make([]*types.HeaderedEvent, len(s.roomState)) + copy(res.StateEvents, s.roomState) + + res.LatestEvents = []string{} + res.Depth = 1 + return nil + } else { + s.t.Logf("room event/state queried for %s", req.RoomID) + return fmt.Errorf("unknown room") + } + +} + +func (s *sendEventTestRoomserverAPI) QuerySenderIDForUser( + ctx context.Context, + roomID spec.RoomID, + userID spec.UserID, +) (*spec.SenderID, error) { + if roomID.String() == s.roomIDStr { + if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs { + roomKey, ok := s.senderMapping[userID.String()] + if ok { + sender := spec.SenderIDFromPseudoIDKey(roomKey) + return &sender, nil + } else { + return nil, nil + } + } else { + senderID := spec.SenderIDFromUserID(userID) + return &senderID, nil + } + } + + return nil, fmt.Errorf("room not found") +} + +func (s *sendEventTestRoomserverAPI) QueryUserIDForSender( + ctx context.Context, + roomID spec.RoomID, + senderID spec.SenderID, +) (*spec.UserID, error) { + if roomID.String() == s.roomIDStr { + if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs { + for uID, roomKey := range s.senderMapping { + if string(spec.SenderIDFromPseudoIDKey(roomKey)) == string(senderID) { + parsedUserID, err := spec.NewUserID(uID, true) + if err != nil { + s.t.Fatalf("Mock QueryUserIDForSender failed: %s", err) + } + return parsedUserID, nil + } + } + } else { + userID := senderID.ToUserID() + if userID == nil { + return nil, fmt.Errorf("bad sender ID") + } + return userID, nil + } + } + + return nil, fmt.Errorf("room not found") +} + +func (s *sendEventTestRoomserverAPI) SigningIdentityFor(ctx context.Context, roomID spec.RoomID, sender spec.UserID) (fclient.SigningIdentity, error) { + if s.roomIDStr == roomID.String() { + if s.roomVersion == gomatrixserverlib.RoomVersionPseudoIDs { + roomKey, ok := s.senderMapping[sender.String()] + if !ok { + s.t.Logf("SigningIdentityFor used with unknown user ID: %v", sender.String()) + return fclient.SigningIdentity{}, fmt.Errorf("could not get signing identity for %v", sender.String()) + } + return fclient.SigningIdentity{PrivateKey: roomKey}, nil + } else { + return fclient.SigningIdentity{PrivateKey: ed25519.NewKeyFromSeed(make([]byte, 32))}, nil + } + } + + return fclient.SigningIdentity{}, fmt.Errorf("room not found") +} + +func (s *sendEventTestRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) { + s.savedInputRoomEvents = req.InputRoomEvents +} + +// Test that user ID state keys are translated correctly +func Test_SendEvent_PseudoIDStateKeys(t *testing.T) { + nonpseudoIDRoomVersion := gomatrixserverlib.RoomVersionV10 + pseudoIDRoomVersion := gomatrixserverlib.RoomVersionPseudoIDs + + senderKeySeed := make([]byte, 32) + senderUserID := "@testuser:domain" + senderPrivKey := ed25519.NewKeyFromSeed(senderKeySeed) + senderPseudoID := string(spec.SenderIDFromPseudoIDKey(senderPrivKey)) + + eventType := "com.example.test" + roomIDStr := "!id:domain" + + device := &uapi.Device{ + UserID: senderUserID, + } + + t.Run("user ID state key are not translated to room key in non-pseudo ID room", func(t *testing.T) { + eventsJSON := []string{ + fmt.Sprintf(`{"type":"m.room.create","state_key":"","room_id":"%v","sender":"%v","content":{"creator":"%v","room_version":"%v"}}`, roomIDStr, senderUserID, senderUserID, nonpseudoIDRoomVersion), + fmt.Sprintf(`{"type":"m.room.member","state_key":"%v","room_id":"%v","sender":"%v","content":{"membership":"join"}}`, senderUserID, roomIDStr, senderUserID), + } + + roomState, err := createEvents(eventsJSON, nonpseudoIDRoomVersion) + if err != nil { + t.Fatalf("failed to prepare state events: %s", err.Error()) + } + + rsAPI := &sendEventTestRoomserverAPI{ + t: t, + roomIDStr: roomIDStr, + roomVersion: nonpseudoIDRoomVersion, + roomState: roomState, + } + + req, err := http.NewRequest("POST", "https://domain", io.NopCloser(strings.NewReader("{}"))) + if err != nil { + t.Fatalf("failed to make new request: %s", err.Error()) + } + + cfg := &config.ClientAPI{} + + resp := SendEvent(req, device, roomIDStr, eventType, nil, &senderUserID, cfg, rsAPI, nil) + + if resp.Code != http.StatusOK { + t.Fatalf("non-200 HTTP code returned: %v\nfull response: %v", resp.Code, resp) + } + + assert.Equal(t, len(rsAPI.savedInputRoomEvents), 1) + + ev := rsAPI.savedInputRoomEvents[0] + stateKey := ev.Event.StateKey() + if stateKey == nil { + t.Fatalf("submitted InputRoomEvent has nil state key, when it should be %v", senderUserID) + } + if *stateKey != senderUserID { + t.Fatalf("expected submitted InputRoomEvent to have user ID state key\nfound: %v\nexpected: %v", *stateKey, senderUserID) + } + }) + + t.Run("user ID state key are translated to room key in pseudo ID room", func(t *testing.T) { + eventsJSON := []string{ + fmt.Sprintf(`{"type":"m.room.create","state_key":"","room_id":"%v","sender":"%v","content":{"creator":"%v","room_version":"%v"}}`, roomIDStr, senderPseudoID, senderPseudoID, pseudoIDRoomVersion), + fmt.Sprintf(`{"type":"m.room.member","state_key":"%v","room_id":"%v","sender":"%v","content":{"membership":"join"}}`, senderPseudoID, roomIDStr, senderPseudoID), + } + + roomState, err := createEvents(eventsJSON, pseudoIDRoomVersion) + if err != nil { + t.Fatalf("failed to prepare state events: %s", err.Error()) + } + + rsAPI := &sendEventTestRoomserverAPI{ + t: t, + roomIDStr: roomIDStr, + roomVersion: pseudoIDRoomVersion, + senderMapping: map[string]ed25519.PrivateKey{ + senderUserID: senderPrivKey, + }, + roomState: roomState, + } + + req, err := http.NewRequest("POST", "https://domain", io.NopCloser(strings.NewReader("{}"))) + if err != nil { + t.Fatalf("failed to make new request: %s", err.Error()) + } + + cfg := &config.ClientAPI{} + + resp := SendEvent(req, device, roomIDStr, eventType, nil, &senderUserID, cfg, rsAPI, nil) + + if resp.Code != http.StatusOK { + t.Fatalf("non-200 HTTP code returned: %v\nfull response: %v", resp.Code, resp) + } + + assert.Equal(t, len(rsAPI.savedInputRoomEvents), 1) + + ev := rsAPI.savedInputRoomEvents[0] + stateKey := ev.Event.StateKey() + if stateKey == nil { + t.Fatalf("submitted InputRoomEvent has nil state key, when it should be %v", senderPseudoID) + } + if *stateKey != senderPseudoID { + t.Fatalf("expected submitted InputRoomEvent to have pseudo ID state key\nfound: %v\nexpected: %v", *stateKey, senderPseudoID) + } + }) +} + +func createEvents(eventsJSON []string, roomVer gomatrixserverlib.RoomVersion) ([]*types.HeaderedEvent, error) { + events := make([]*types.HeaderedEvent, len(eventsJSON)) + + roomVerImpl, err := gomatrixserverlib.GetRoomVersion(roomVer) + if err != nil { + return nil, fmt.Errorf("no roomver impl: %s", err.Error()) + } + + for i, eventJSON := range eventsJSON { + pdu, evErr := roomVerImpl.NewEventFromTrustedJSON([]byte(eventJSON), false) + if evErr != nil { + return nil, fmt.Errorf("failed to make event: %s", err.Error()) + } + ev := types.HeaderedEvent{PDU: pdu} + events[i] = &ev + } + + return events, nil +} diff --git a/clientapi/routing/server_notices.go b/clientapi/routing/server_notices.go index 66258a68a..5deb559df 100644 --- a/clientapi/routing/server_notices.go +++ b/clientapi/routing/server_notices.go @@ -28,7 +28,6 @@ import ( "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/dendrite/roomserver/version" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/httputil" @@ -95,34 +94,42 @@ func SendServerNotice( } } + userID, err := spec.NewUserID(r.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("invalid user ID"), + } + } + // get rooms for specified user - allUserRooms := []string{} - userRooms := api.QueryRoomsForUserResponse{} + allUserRooms := []spec.RoomID{} // Get rooms the user is either joined, invited or has left. for _, membership := range []string{"join", "invite", "leave"} { - if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{ - UserID: r.UserID, - WantMembership: membership, - }, &userRooms); err != nil { + userRooms, queryErr := rsAPI.QueryRoomsForUser(ctx, *userID, membership) + if queryErr != nil { return util.ErrorResponse(err) } - allUserRooms = append(allUserRooms, userRooms.RoomIDs...) + allUserRooms = append(allUserRooms, userRooms...) } // get rooms of the sender - senderUserID := fmt.Sprintf("@%s:%s", cfgNotices.LocalPart, cfgClient.Matrix.ServerName) - senderRooms := api.QueryRoomsForUserResponse{} - if err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{ - UserID: senderUserID, - WantMembership: "join", - }, &senderRooms); err != nil { + senderUserID, err := spec.NewUserID(fmt.Sprintf("@%s:%s", cfgNotices.LocalPart, cfgClient.Matrix.ServerName), true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + senderRooms, err := rsAPI.QueryRoomsForUser(ctx, *senderUserID, "join") + if err != nil { return util.ErrorResponse(err) } // check if we have rooms in common - commonRooms := []string{} + commonRooms := []spec.RoomID{} for _, userRoomID := range allUserRooms { - for _, senderRoomID := range senderRooms.RoomIDs { + for _, senderRoomID := range senderRooms { if userRoomID == senderRoomID { commonRooms = append(commonRooms, senderRoomID) } @@ -135,12 +142,12 @@ func SendServerNotice( var ( roomID string - roomVersion = version.DefaultRoomVersion() + roomVersion = rsAPI.DefaultRoomVersion() ) // create a new room for the user if len(commonRooms) == 0 { - powerLevelContent := eventutil.InitialPowerLevelsContent(senderUserID) + powerLevelContent := eventutil.InitialPowerLevelsContent(senderUserID.String()) powerLevelContent.Users[r.UserID] = -10 // taken from Synapse pl, err := json.Marshal(powerLevelContent) if err != nil { @@ -196,7 +203,7 @@ func SendServerNotice( } } - roomID = commonRooms[0] + roomID = commonRooms[0].String() membershipRes := api.QueryMembershipForUserResponse{} err = rsAPI.QueryMembershipForUser(ctx, &api.QueryMembershipForUserRequest{UserID: *deviceUserID, RoomID: roomID}, &membershipRes) if err != nil { diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go index f53cb3013..18f9a0e9c 100644 --- a/clientapi/routing/state.go +++ b/clientapi/routing/state.go @@ -172,28 +172,16 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a } } for _, ev := range stateAfterRes.StateEvents { - sender := spec.UserID{} - evRoomID, err := spec.NewRoomID(ev.RoomID()) + clientEvent, err := synctypes.ToClientEvent(ev, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) if err != nil { - util.GetLogger(ctx).WithError(err).Error("Event roomID is invalid") + util.GetLogger(ctx).WithError(err).Error("Failed converting to ClientEvent") continue } - userID, err := rsAPI.QueryUserIDForSender(ctx, *evRoomID, ev.SenderID()) - if err == nil && userID != nil { - sender = *userID - } - - sk := ev.StateKey() - if sk != nil && *sk != "" { - skUserID, err := rsAPI.QueryUserIDForSender(ctx, *evRoomID, spec.SenderID(*ev.StateKey())) - if err == nil && skUserID != nil { - skString := skUserID.String() - sk = &skString - } - } stateEvents = append( stateEvents, - synctypes.ToClientEvent(ev, synctypes.FormatAll, sender, sk), + *clientEvent, ) } } @@ -217,6 +205,37 @@ func OnIncomingStateTypeRequest( var worldReadable bool var wantLatestState bool + roomVer, err := rsAPI.QueryRoomVersionForRoom(ctx, roomID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)), + } + } + + // Translate user ID state keys to room keys in pseudo ID rooms + if roomVer == gomatrixserverlib.RoomVersionPseudoIDs { + parsedRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.InvalidParam("invalid room ID"), + } + } + newStateKey, err := synctypes.FromClientStateKey(*parsedRoomID, stateKey, func(roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + return rsAPI.QuerySenderIDForUser(ctx, roomID, userID) + }) + if err != nil { + // TODO: work out better logic for failure cases (e.g. sender ID not found) + util.GetLogger(ctx).WithError(err).Error("synctypes.FromClientStateKey failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + stateKey = *newStateKey + } + // Always fetch visibility so that we can work out whether to show // the latest events or the last event from when the user was joined. // Then include the requested event type and state key, assuming it diff --git a/clientapi/routing/state_test.go b/clientapi/routing/state_test.go new file mode 100644 index 000000000..93b043723 --- /dev/null +++ b/clientapi/routing/state_test.go @@ -0,0 +1,253 @@ +package routing + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "testing" + + rsapi "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/setup/config" + uapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" + "gotest.tools/v3/assert" +) + +var () + +type stateTestRoomserverAPI struct { + rsapi.RoomserverInternalAPI + t *testing.T + roomState map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent + roomIDStr string + roomVersion gomatrixserverlib.RoomVersion + userIDStr string + // userID -> senderID + senderMapping map[string]string +} + +func (s stateTestRoomserverAPI) QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) { + if roomID == s.roomIDStr { + return s.roomVersion, nil + } else { + s.t.Logf("room version queried for %s", roomID) + return "", fmt.Errorf("unknown room") + } +} + +func (s stateTestRoomserverAPI) QueryLatestEventsAndState( + ctx context.Context, + req *rsapi.QueryLatestEventsAndStateRequest, + res *rsapi.QueryLatestEventsAndStateResponse, +) error { + res.RoomExists = req.RoomID == s.roomIDStr + if !res.RoomExists { + return nil + } + + res.StateEvents = []*types.HeaderedEvent{} + for _, stateKeyTuple := range req.StateToFetch { + val, ok := s.roomState[stateKeyTuple] + if ok && val != nil { + res.StateEvents = append(res.StateEvents, val) + } + } + + return nil +} + +func (s stateTestRoomserverAPI) QueryMembershipForUser( + ctx context.Context, + req *rsapi.QueryMembershipForUserRequest, + res *rsapi.QueryMembershipForUserResponse, +) error { + if req.UserID.String() == s.userIDStr { + res.HasBeenInRoom = true + res.IsInRoom = true + res.RoomExists = true + res.Membership = spec.Join + } + + return nil +} + +func (s stateTestRoomserverAPI) QuerySenderIDForUser( + ctx context.Context, + roomID spec.RoomID, + userID spec.UserID, +) (*spec.SenderID, error) { + sID, ok := s.senderMapping[userID.String()] + if ok { + sender := spec.SenderID(sID) + return &sender, nil + } else { + return nil, nil + } +} + +func (s stateTestRoomserverAPI) QueryUserIDForSender( + ctx context.Context, + roomID spec.RoomID, + senderID spec.SenderID, +) (*spec.UserID, error) { + for uID, sID := range s.senderMapping { + if sID == string(senderID) { + parsedUserID, err := spec.NewUserID(uID, true) + if err != nil { + s.t.Fatalf("Mock QueryUserIDForSender failed: %s", err) + } + return parsedUserID, nil + } + } + return nil, nil +} + +func (s stateTestRoomserverAPI) QueryStateAfterEvents( + ctx context.Context, + req *rsapi.QueryStateAfterEventsRequest, + res *rsapi.QueryStateAfterEventsResponse, +) error { + return nil +} + +func Test_OnIncomingStateTypeRequest(t *testing.T) { + var tempRoomServerCfg config.RoomServer + tempRoomServerCfg.Defaults(config.DefaultOpts{}) + defaultRoomVersion := tempRoomServerCfg.DefaultRoomVersion + pseudoIDRoomVersion := gomatrixserverlib.RoomVersionPseudoIDs + nonPseudoIDRoomVersion := gomatrixserverlib.RoomVersionV10 + + userIDStr := "@testuser:domain" + eventType := "com.example.test" + stateKey := "testStateKey" + roomIDStr := "!id:domain" + + device := &uapi.Device{ + UserID: userIDStr, + } + + t.Run("request simple state key", func(t *testing.T) { + ctx := context.Background() + + rsAPI := stateTestRoomserverAPI{ + roomVersion: defaultRoomVersion, + roomIDStr: roomIDStr, + roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{ + { + EventType: eventType, + StateKey: stateKey, + }: mustCreateStatePDU(t, defaultRoomVersion, roomIDStr, eventType, stateKey, map[string]interface{}{ + "foo": "bar", + }), + }, + userIDStr: userIDStr, + } + + jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateKey, false) + + assert.DeepEqual(t, jsonResp, util.JSONResponse{ + Code: http.StatusOK, + JSON: spec.RawJSON(`{"foo":"bar"}`), + }) + }) + + t.Run("user ID key translated to room key in pseudo ID rooms", func(t *testing.T) { + ctx := context.Background() + + stateSenderUserID := "@sender:domain" + stateSenderRoomKey := "testsenderkey" + + rsAPI := stateTestRoomserverAPI{ + roomVersion: pseudoIDRoomVersion, + roomIDStr: roomIDStr, + roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{ + { + EventType: eventType, + StateKey: stateSenderRoomKey, + }: mustCreateStatePDU(t, pseudoIDRoomVersion, roomIDStr, eventType, stateSenderRoomKey, map[string]interface{}{ + "foo": "bar", + }), + { + EventType: eventType, + StateKey: stateSenderUserID, + }: mustCreateStatePDU(t, pseudoIDRoomVersion, roomIDStr, eventType, stateSenderUserID, map[string]interface{}{ + "not": "thisone", + }), + }, + userIDStr: userIDStr, + senderMapping: map[string]string{ + stateSenderUserID: stateSenderRoomKey, + }, + } + + jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateSenderUserID, false) + + assert.DeepEqual(t, jsonResp, util.JSONResponse{ + Code: http.StatusOK, + JSON: spec.RawJSON(`{"foo":"bar"}`), + }) + }) + + t.Run("user ID key not translated to room key in non-pseudo ID rooms", func(t *testing.T) { + ctx := context.Background() + + stateSenderUserID := "@sender:domain" + stateSenderRoomKey := "testsenderkey" + + rsAPI := stateTestRoomserverAPI{ + roomVersion: nonPseudoIDRoomVersion, + roomIDStr: roomIDStr, + roomState: map[gomatrixserverlib.StateKeyTuple]*types.HeaderedEvent{ + { + EventType: eventType, + StateKey: stateSenderRoomKey, + }: mustCreateStatePDU(t, nonPseudoIDRoomVersion, roomIDStr, eventType, stateSenderRoomKey, map[string]interface{}{ + "not": "thisone", + }), + { + EventType: eventType, + StateKey: stateSenderUserID, + }: mustCreateStatePDU(t, nonPseudoIDRoomVersion, roomIDStr, eventType, stateSenderUserID, map[string]interface{}{ + "foo": "bar", + }), + }, + userIDStr: userIDStr, + senderMapping: map[string]string{ + stateSenderUserID: stateSenderUserID, + }, + } + + jsonResp := OnIncomingStateTypeRequest(ctx, device, rsAPI, roomIDStr, eventType, stateSenderUserID, false) + + assert.DeepEqual(t, jsonResp, util.JSONResponse{ + Code: http.StatusOK, + JSON: spec.RawJSON(`{"foo":"bar"}`), + }) + }) +} + +func mustCreateStatePDU(t *testing.T, roomVer gomatrixserverlib.RoomVersion, roomID string, stateType string, stateKey string, stateContent map[string]interface{}) *types.HeaderedEvent { + t.Helper() + roomVerImpl := gomatrixserverlib.MustGetRoomVersion(roomVer) + + evBytes, err := json.Marshal(map[string]interface{}{ + "room_id": roomID, + "type": stateType, + "state_key": stateKey, + "content": stateContent, + }) + if err != nil { + t.Fatalf("failed to create event: %v", err) + } + + ev, err := roomVerImpl.NewEventFromTrustedJSON(evBytes, false) + if err != nil { + t.Fatalf("failed to create event: %v", err) + } + + return &types.HeaderedEvent{PDU: ev} +} diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index d15cc6d46..365e9f869 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -366,9 +366,11 @@ func emit3PIDInviteEvent( sender, err := rsAPI.QuerySenderIDForUser(ctx, *validRoomID, *userID) if err != nil { return err + } else if sender == nil { + return fmt.Errorf("sender ID not found for %s in %s", *userID, *validRoomID) } proto := &gomatrixserverlib.ProtoEvent{ - SenderID: string(sender), + SenderID: string(*sender), RoomID: roomID, Type: "m.room.third_party_invite", StateKey: &res.Token, diff --git a/cmd/dendrite-demo-pinecone/monolith/monolith.go b/cmd/dendrite-demo-pinecone/monolith/monolith.go index abeea19d4..d9f44b5cc 100644 --- a/cmd/dendrite-demo-pinecone/monolith/monolith.go +++ b/cmd/dendrite-demo-pinecone/monolith/monolith.go @@ -145,7 +145,7 @@ func (p *P2PMonolith) SetupDendrite( ) rsAPI.SetFederationAPI(fsAPI, keyRing) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, enableMetrics, fsAPI.IsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) @@ -221,8 +221,8 @@ func (p *P2PMonolith) closeAllResources() { p.httpServerMu.Lock() if p.httpServer != nil { _ = p.httpServer.Shutdown(context.Background()) - p.httpServerMu.Unlock() } + p.httpServerMu.Unlock() select { case p.stopHandlingEvents <- true: diff --git a/cmd/dendrite-demo-yggdrasil/README.md b/cmd/dendrite-demo-yggdrasil/README.md index 14fc3a2db..23304c214 100644 --- a/cmd/dendrite-demo-yggdrasil/README.md +++ b/cmd/dendrite-demo-yggdrasil/README.md @@ -1,6 +1,6 @@ # Yggdrasil Demo -This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.18 or later. +This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.20 or later. To run the homeserver, start at the root of the Dendrite repository and run: diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 3ec550113..1e5186348 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -213,14 +213,15 @@ func main() { natsInstance := jetstream.NATSInstance{} rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) - - asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) - rsAPI.SetAppserviceAPI(asAPI) fsAPI := federationapi.NewInternalAPI( processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true, ) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff) + + asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) + rsAPI.SetAppserviceAPI(asAPI) + rsAPI.SetFederationAPI(fsAPI, keyRing) monolith := setup.Monolith{ diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go index dcc45bdcc..b78c5f605 100644 --- a/cmd/dendrite-upgrade-tests/main.go +++ b/cmd/dendrite-upgrade-tests/main.go @@ -7,7 +7,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" "net/http" "os" @@ -55,7 +54,7 @@ var latest, _ = semver.NewVersion("v6.6.6") // Dummy version, used as "HEAD" // due to the error: // When using COPY with more than one source file, the destination must be a directory and end with a / // We need to run a postgres anyway, so use the dockerfile associated with Complement instead. -const DockerfilePostgreSQL = `FROM golang:1.18-buster as build +const DockerfilePostgreSQL = `FROM golang:1.20-bookworm as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build ARG BINARY @@ -74,16 +73,16 @@ RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key # Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password RUN sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml # No password when connecting over localhost -RUN sed -i "s%127.0.0.1/32 md5%127.0.0.1/32 trust%g" /etc/postgresql/11/main/pg_hba.conf +RUN sed -i "s%127.0.0.1/32 scram-sha-256%127.0.0.1/32 trust%g" /etc/postgresql/15/main/pg_hba.conf # Bump up max conns for moar concurrency -RUN sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/11/main/postgresql.conf +RUN sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/15/main/postgresql.conf RUN sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml # This entry script starts postgres, waits for it to be up then starts dendrite RUN echo '\ #!/bin/bash -eu \n\ pg_lsclusters \n\ -pg_ctlcluster 11 main start \n\ +pg_ctlcluster 15 main start \n\ \n\ until pg_isready \n\ do \n\ @@ -101,7 +100,7 @@ ENV BINARY=dendrite EXPOSE 8008 8448 CMD /build/run_dendrite.sh` -const DockerfileSQLite = `FROM golang:1.18-buster as build +const DockerfileSQLite = `FROM golang:1.20-bookworm as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build ARG BINARY @@ -119,7 +118,7 @@ RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key # Make sure the SQLite databases are in a persistent location, we're already mapping # the postgresql folder so let's just use that for simplicity -RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postgresql\/11\/main\/%g" dendrite.yaml +RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postgresql\/15\/main\/%g" dendrite.yaml # This entry script starts postgres, waits for it to be up then starts dendrite RUN echo '\ @@ -402,7 +401,7 @@ func runImage(dockerClient *client.Client, volumeName string, branchNameToImageI { Type: mount.TypeVolume, Source: volumeName, - Target: "/var/lib/postgresql/11/main", + Target: "/var/lib/postgresql/15/main", }, }, }, nil, nil, "dendrite_upgrade_test_"+branchName) @@ -515,7 +514,7 @@ func testCreateAccount(dockerClient *client.Client, version *semver.Version, con } defer response.Close() - data, err := ioutil.ReadAll(response.Reader) + data, err := io.ReadAll(response.Reader) if err != nil { return err } @@ -557,8 +556,8 @@ func cleanup(dockerClient *client.Client) { }) for _, c := range containers { log.Printf("Removing container: %v %v\n", c.ID, c.Names) - s := time.Second - _ = dockerClient.ContainerStop(context.Background(), c.ID, &s) + timeout := 1 + _ = dockerClient.ContainerStop(context.Background(), c.ID, container.StopOptions{Timeout: &timeout}) _ = dockerClient.ContainerRemove(context.Background(), c.ID, types.ContainerRemoveOptions{ Force: true, }) @@ -592,7 +591,7 @@ func main() { branchToImageID := buildDendriteImages(httpClient, dockerClient, *flagTempDir, *flagBuildConcurrency, versions) // make a shared postgres volume - volume, err := dockerClient.VolumeCreate(context.Background(), volume.VolumeCreateBody{ + volume, err := dockerClient.VolumeCreate(context.Background(), volume.CreateOptions{ Name: "dendrite_upgrade_test", Labels: map[string]string{ dendriteUpgradeTestLabel: "yes", diff --git a/cmd/dendrite/main.go b/cmd/dendrite/main.go index f3140b4e2..5234b7504 100644 --- a/cmd/dendrite/main.go +++ b/cmd/dendrite/main.go @@ -162,7 +162,7 @@ func main() { // dependency. Other components also need updating after their dependencies are up. rsAPI.SetFederationAPI(fsAPI, keyRing) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go index 3ffcac9e6..d6db72436 100644 --- a/cmd/resolve-state/main.go +++ b/cmd/resolve-state/main.go @@ -11,13 +11,11 @@ import ( "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" @@ -35,6 +33,19 @@ var roomVersion = flag.String("roomversion", "5", "the room version to parse eve var filterType = flag.String("filtertype", "", "the event types to filter on") var difference = flag.Bool("difference", false, "whether to calculate the difference between snapshots") +// dummyQuerier implements QuerySenderIDAPI. Does **NOT** do any "magic" for pseudoID rooms +// to avoid having to "start" a full roomserver API. +type dummyQuerier struct{} + +func (d dummyQuerier) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + s := spec.SenderIDFromUserID(userID) + return &s, nil +} + +func (d dummyQuerier) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return senderID.ToUserID(), nil +} + // nolint:gocyclo func main() { ctx := context.Background() @@ -56,27 +67,32 @@ func main() { } } - fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs") - processCtx := process.NewProcessContext() cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + + dbOpts := cfg.RoomServer.Database + if dbOpts.ConnectionString == "" { + dbOpts = cfg.Global.DatabaseOptions + } + + fmt.Println("Opening database") roomserverDB, err := storage.Open( - processCtx.Context(), cm, &cfg.RoomServer.Database, - caching.NewRistrettoCache(128*1024*1024, time.Hour, true), + processCtx.Context(), cm, &dbOpts, + caching.NewRistrettoCache(8*1024*1024, time.Minute*5, caching.DisableMetrics), ) if err != nil { panic(err) } - natsInstance := &jetstream.NATSInstance{} - rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, - natsInstance, caching.NewRistrettoCache(128*1024*1024, time.Hour, true), false) + rsAPI := dummyQuerier{} roomInfo := &types.RoomInfo{ RoomVersion: gomatrixserverlib.RoomVersion(*roomVersion), } stateres := state.NewStateResolution(roomserverDB, roomInfo, rsAPI) + fmt.Println("Fetching", len(snapshotNIDs), "snapshot NIDs") + if *difference { if len(snapshotNIDs) != 2 { panic("need exactly two state snapshot NIDs to calculate difference") @@ -186,12 +202,25 @@ func main() { authEvents[i] = authEventEntries[i].PDU } + // Get the roomNID + roomInfo, err = roomserverDB.RoomInfo(ctx, authEvents[0].RoomID().String()) + if err != nil { + panic(err) + } + fmt.Println("Resolving state") var resolved Events resolved, err = gomatrixserverlib.ResolveConflicts( gomatrixserverlib.RoomVersion(*roomVersion), events, authEvents, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) }, + func(eventID string) bool { + isRejected, rejectedErr := roomserverDB.IsEventRejected(ctx, roomInfo.RoomNID, eventID) + if rejectedErr != nil { + return true + } + return isRejected + }, ) if err != nil { panic(err) diff --git a/dendrite-sample.yaml b/dendrite-sample.yaml index 8abc23011..e143a7398 100644 --- a/dendrite-sample.yaml +++ b/dendrite-sample.yaml @@ -72,6 +72,10 @@ global: # The base URL to delegate client-server communications to e.g. https://localhost well_known_client_name: "" + # The server name to delegate sliding sync communications to, with optional port. + # Requires `well_known_client_name` to also be configured. + well_known_sliding_sync_proxy: "" + # Lists of domains that the server will trust as identity servers to verify third # party identifiers such as phone numbers and email addresses. trusted_third_party_id_servers: @@ -321,6 +325,10 @@ user_api: auto_join_rooms: # - "#main:matrix.org" + # The number of workers to start for the DeviceListUpdater. Defaults to 8. + # This only needs updating if the "InputDeviceListUpdate" stream keeps growing indefinitely. + # worker_count: 8 + # Configuration for Opentracing. # See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on # how this works and how to set it up. diff --git a/docs/FAQ.md b/docs/FAQ.md index 570ba677e..82b1581ea 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -24,7 +24,7 @@ No, although a good portion of the Matrix specification has been implemented. Mo Dendrite development is currently supported by a small team of developers and due to those limited resources, the majority of the effort is focused on getting Dendrite to be specification complete. If there are major features you're requesting (e.g. new administration endpoints), we'd like to strongly encourage you to join the community in supporting -the development efforts through [contributing](../development/contributing). +the development efforts through [contributing](./development/CONTRIBUTING.md). ## Is there a migration path from Synapse to Dendrite? @@ -105,7 +105,7 @@ This can be done by performing a room upgrade. Use the command `/upgraderoom 0.1) diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md index 6f6458997..40d02622b 100644 --- a/docs/administration/4_adminapi.md +++ b/docs/administration/4_adminapi.md @@ -75,7 +75,7 @@ This endpoint instructs Dendrite to immediately query `/devices/{userID}` on a f ## POST `/_dendrite/admin/purgeRoom/{roomID}` -This endpoint instructs Dendrite to remove the given room from its database. Before doing so, it will evacuate all local users from the room. It does **NOT** remove media files. Depending on the size of the room, this may take a while. Will return an empty JSON once other components were instructed to delete the room. +This endpoint instructs Dendrite to remove the given room from its database. It does **NOT** remove media files. Depending on the size of the room, this may take a while. Will return an empty JSON once other components were instructed to delete the room. ## POST `/_synapse/admin/v1/send_server_notice` diff --git a/docs/administration/5_optimisation.md b/docs/administration/5_optimisation.md index b327171eb..57b7924d3 100644 --- a/docs/administration/5_optimisation.md +++ b/docs/administration/5_optimisation.md @@ -95,7 +95,7 @@ Consider enabling the DNS cache by modifying the `global` section of your config ## Time synchronisation Matrix relies heavily on TLS which requires the system time to be correct. If the clock -drifts then you may find that federation no works reliably (or at all) and clients may +drifts then you may find that federation will not work reliably (or at all) and clients may struggle to connect to your Dendrite server. Ensure that the time is synchronised on your system by enabling NTP sync. diff --git a/docs/development/CONTRIBUTING.md b/docs/development/CONTRIBUTING.md index 71e7516a2..caab1e749 100644 --- a/docs/development/CONTRIBUTING.md +++ b/docs/development/CONTRIBUTING.md @@ -109,7 +109,7 @@ To configure the connection to a remote Postgres, you can use the following envi ```bash POSTGRES_USER=postgres -POSTGERS_PASSWORD=yourPostgresPassword +POSTGRES_PASSWORD=yourPostgresPassword POSTGRES_HOST=localhost POSTGRES_DB=postgres # the superuser database to use ``` diff --git a/docs/installation/1_planning.md b/docs/installation/1_planning.md index 354003aef..37ca5702a 100644 --- a/docs/installation/1_planning.md +++ b/docs/installation/1_planning.md @@ -59,7 +59,7 @@ In order to install Dendrite, you will need to satisfy the following dependencie ### Go -At this time, Dendrite supports being built with Go 1.18 or later. We do not support building +At this time, Dendrite supports being built with Go 1.20 or later. We do not support building Dendrite with older versions of Go than this. If you are installing Go using a package manager, you should check (by running `go version`) that you are using a suitable version before you start. diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go index 3fdc835bb..6210bddb6 100644 --- a/federationapi/consumers/keychange.go +++ b/federationapi/consumers/keychange.go @@ -117,19 +117,27 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { return true } - var queryRes roomserverAPI.QueryRoomsForUserResponse - err = t.rsAPI.QueryRoomsForUser(t.ctx, &roomserverAPI.QueryRoomsForUserRequest{ - UserID: m.UserID, - WantMembership: "join", - }, &queryRes) + userID, err := spec.NewUserID(m.UserID, true) + if err != nil { + sentry.CaptureException(err) + logger.WithError(err).Error("invalid user ID") + return true + } + + roomIDs, err := t.rsAPI.QueryRoomsForUser(t.ctx, *userID, "join") if err != nil { sentry.CaptureException(err) logger.WithError(err).Error("failed to calculate joined rooms for user") return true } + roomIDStrs := make([]string, len(roomIDs)) + for i, room := range roomIDs { + roomIDStrs[i] = room.String() + } + // send this key change to all servers who share rooms with this user. - destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true) + destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, roomIDStrs, true, true) if err != nil { sentry.CaptureException(err) logger.WithError(err).Error("failed to calculate joined hosts for rooms user is in") @@ -179,18 +187,27 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool { } logger := logrus.WithField("user_id", output.UserID) - var queryRes roomserverAPI.QueryRoomsForUserResponse - err = t.rsAPI.QueryRoomsForUser(t.ctx, &roomserverAPI.QueryRoomsForUserRequest{ - UserID: output.UserID, - WantMembership: "join", - }, &queryRes) + outputUserID, err := spec.NewUserID(output.UserID, true) + if err != nil { + sentry.CaptureException(err) + logrus.WithError(err).Errorf("invalid user ID") + return true + } + + rooms, err := t.rsAPI.QueryRoomsForUser(t.ctx, *outputUserID, "join") if err != nil { sentry.CaptureException(err) logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined rooms for user") return true } + + roomIDStrs := make([]string, len(rooms)) + for i, room := range rooms { + roomIDStrs[i] = room.String() + } + // send this key change to all servers who share rooms with this user. - destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true) + destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, roomIDStrs, true, true) if err != nil { sentry.CaptureException(err) logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined hosts for rooms user is in") diff --git a/federationapi/consumers/presence.go b/federationapi/consumers/presence.go index e751b65d4..dd100bc08 100644 --- a/federationapi/consumers/presence.go +++ b/federationapi/consumers/presence.go @@ -29,6 +29,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" "github.com/nats-io/nats.go" log "github.com/sirupsen/logrus" ) @@ -94,16 +95,23 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg return true } - var queryRes roomserverAPI.QueryRoomsForUserResponse - err = t.rsAPI.QueryRoomsForUser(t.ctx, &roomserverAPI.QueryRoomsForUserRequest{ - UserID: userID, - WantMembership: "join", - }, &queryRes) + parsedUserID, err := spec.NewUserID(userID, true) + if err != nil { + util.GetLogger(ctx).WithError(err).WithField("user_id", userID).Error("invalid user ID") + return true + } + + roomIDs, err := t.rsAPI.QueryRoomsForUser(t.ctx, *parsedUserID, "join") if err != nil { log.WithError(err).Error("failed to calculate joined rooms for user") return true } + roomIDStrs := make([]string, len(roomIDs)) + for i, roomID := range roomIDs { + roomIDStrs[i] = roomID.String() + } + presence := msg.Header.Get("presence") ts, err := strconv.Atoi(msg.Header.Get("last_active_ts")) @@ -112,7 +120,7 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg } // send this presence to all servers who share rooms with this user. - joined, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true) + joined, err := t.db.GetJoinedHostsForRooms(t.ctx, roomIDStrs, true, true) if err != nil { log.WithError(err).Error("failed to get joined hosts") return true diff --git a/federationapi/consumers/roomserver.go b/federationapi/consumers/roomserver.go index 6c0580322..f1dcb1175 100644 --- a/federationapi/consumers/roomserver.go +++ b/federationapi/consumers/roomserver.go @@ -176,7 +176,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew // Finally, work out if there are any more events missing. if len(missingEventIDs) > 0 { eventsReq := &api.QueryEventsByIDRequest{ - RoomID: ore.Event.RoomID(), + RoomID: ore.Event.RoomID().String(), EventIDs: missingEventIDs, } eventsRes := &api.QueryEventsByIDResponse{} @@ -205,7 +205,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew // talking to the roomserver oldJoinedHosts, err := s.db.UpdateRoom( s.ctx, - ore.Event.RoomID(), + ore.Event.RoomID().String(), addsJoinedHosts, ore.RemovesStateEventIDs, rewritesState, // if we're re-writing state, nuke all joined hosts before adding @@ -218,7 +218,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew if s.cfg.Matrix.Presence.EnableOutbound && len(addsJoinedHosts) > 0 && ore.Event.Type() == spec.MRoomMember && ore.Event.StateKey() != nil { membership, _ := ore.Event.Membership() if membership == spec.Join { - s.sendPresence(ore.Event.RoomID(), addsJoinedHosts) + s.sendPresence(ore.Event.RoomID().String(), addsJoinedHosts) } } @@ -376,7 +376,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent( } // handle peeking hosts - inboundPeeks, err := s.db.GetInboundPeeks(s.ctx, ore.Event.PDU.RoomID()) + inboundPeeks, err := s.db.GetInboundPeeks(s.ctx, ore.Event.PDU.RoomID().String()) if err != nil { return nil, err } @@ -409,12 +409,8 @@ func JoinedHostsFromEvents(ctx context.Context, evs []gomatrixserverlib.PDU, rsA if membership != spec.Join { continue } - validRoomID, err := spec.NewRoomID(ev.RoomID()) - if err != nil { - return nil, err - } var domain spec.ServerName - userID, err := rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*ev.StateKey())) + userID, err := rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), spec.SenderID(*ev.StateKey())) if err != nil { if errors.As(err, new(base64.CorruptInputError)) { // Fallback to using the "old" way of getting the user domain, avoids @@ -510,7 +506,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // At this point the missing events are neither the event itself nor are // they present in our local database. Our only option is to fetch them // from the roomserver using the query API. - eventReq := api.QueryEventsByIDRequest{EventIDs: missing, RoomID: event.RoomID()} + eventReq := api.QueryEventsByIDRequest{EventIDs: missing, RoomID: event.RoomID().String()} var eventResp api.QueryEventsByIDResponse if err := s.rsAPI.QueryEventsByID(s.ctx, &eventReq, &eventResp); err != nil { return nil, err diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index e148199fb..e2524f66a 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -24,7 +24,6 @@ import ( "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/sirupsen/logrus" - "github.com/matrix-org/dendrite/federationapi/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/consumers" "github.com/matrix-org/dendrite/federationapi/internal" @@ -102,7 +101,7 @@ func NewInternalAPI( caches *caching.Caches, keyRing *gomatrixserverlib.KeyRing, resetBlacklist bool, -) api.FederationInternalAPI { +) *internal.FederationInternalAPI { cfg := &dendriteCfg.FederationAPI federationDB, err := storage.NewDatabase(processContext.Context(), cm, &cfg.Database, caches, dendriteCfg.Global.IsLocalServerName) diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index 5d167c0ee..1ea8c40ea 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -33,15 +33,16 @@ import ( type fedRoomserverAPI struct { rsapi.FederationRoomserverAPI inputRoomEvents func(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) - queryRoomsForUser func(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error + queryRoomsForUser func(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) } func (f *fedRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return spec.NewUserID(string(senderID), true) } -func (f *fedRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (spec.SenderID, error) { - return spec.SenderID(userID.String()), nil +func (f *fedRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + senderID := spec.SenderID(userID.String()) + return &senderID, nil } // PerformJoin will call this function @@ -53,11 +54,11 @@ func (f *fedRoomserverAPI) InputRoomEvents(ctx context.Context, req *rsapi.Input } // keychange consumer calls this -func (f *fedRoomserverAPI) QueryRoomsForUser(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error { +func (f *fedRoomserverAPI) QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) { if f.queryRoomsForUser == nil { - return nil + return nil, nil } - return f.queryRoomsForUser(ctx, req, res) + return f.queryRoomsForUser(ctx, userID, desiredMembership) } // TODO: This struct isn't generic, only works for TestFederationAPIJoinThenKeyUpdate @@ -145,7 +146,7 @@ func (f *fedClient) SendJoin(ctx context.Context, origin, s spec.ServerName, eve f.fedClientMutex.Lock() defer f.fedClientMutex.Unlock() for _, r := range f.allowJoins { - if r.ID == event.RoomID() { + if r.ID == event.RoomID().String() { r.InsertEvent(f.t, &types.HeaderedEvent{PDU: event}) f.t.Logf("Join event: %v", event.EventID()) res.StateEvents = types.NewEventJSONsFromHeaderedEvents(r.CurrentState()) @@ -198,18 +199,22 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) { fmt.Printf("creator: %v joining user: %v\n", creator.ID, joiningUser.ID) room := test.NewRoom(t, creator) + roomID, err := spec.NewRoomID(room.ID) + if err != nil { + t.Fatalf("Invalid room ID: %q", roomID) + } + rsapi := &fedRoomserverAPI{ inputRoomEvents: func(ctx context.Context, req *rsapi.InputRoomEventsRequest, res *rsapi.InputRoomEventsResponse) { if req.Asynchronous { t.Errorf("InputRoomEvents from PerformJoin MUST be synchronous") } }, - queryRoomsForUser: func(ctx context.Context, req *rsapi.QueryRoomsForUserRequest, res *rsapi.QueryRoomsForUserResponse) error { - if req.UserID == joiningUser.ID && req.WantMembership == "join" { - res.RoomIDs = []string{room.ID} - return nil + queryRoomsForUser: func(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) { + if userID.String() == joiningUser.ID && desiredMembership == "join" { + return []spec.RoomID{*roomID}, nil } - return fmt.Errorf("unexpected queryRoomsForUser: %+v", *req) + return nil, fmt.Errorf("unexpected queryRoomsForUser: %v, %v", userID, desiredMembership) }, } fc := &fedClient{ diff --git a/federationapi/internal/api.go b/federationapi/internal/api.go index 3e6f39566..67388a102 100644 --- a/federationapi/internal/api.go +++ b/federationapi/internal/api.go @@ -112,7 +112,7 @@ func NewFederationInternalAPI( } } -func (a *FederationInternalAPI) isBlacklistedOrBackingOff(s spec.ServerName) (*statistics.ServerStatistics, error) { +func (a *FederationInternalAPI) IsBlacklistedOrBackingOff(s spec.ServerName) (*statistics.ServerStatistics, error) { stats := a.statistics.ForServer(s) if stats.Blacklisted() { return stats, &api.FederationClientError{ @@ -151,7 +151,7 @@ func failBlacklistableError(err error, stats *statistics.ServerStatistics) (unti func (a *FederationInternalAPI) doRequestIfNotBackingOffOrBlacklisted( s spec.ServerName, request func() (interface{}, error), ) (interface{}, error) { - stats, err := a.isBlacklistedOrBackingOff(s) + stats, err := a.IsBlacklistedOrBackingOff(s) if err != nil { return nil, err } diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index ff00305bf..0200cf69b 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -481,8 +481,10 @@ func (r *FederationInternalAPI) PerformLeave( senderID, err := r.rsAPI.QuerySenderIDForUser(ctx, *roomID, *userID) if err != nil { return err + } else if senderID == nil { + return fmt.Errorf("sender ID not found for %s in %s", *userID, *roomID) } - senderIDString := string(senderID) + senderIDString := string(*senderID) respMakeLeave.LeaveEvent.Type = spec.MRoomMember respMakeLeave.LeaveEvent.SenderID = senderIDString respMakeLeave.LeaveEvent.StateKey = &senderIDString @@ -546,11 +548,7 @@ func (r *FederationInternalAPI) SendInvite( event gomatrixserverlib.PDU, strippedState []gomatrixserverlib.InviteStrippedState, ) (gomatrixserverlib.PDU, error) { - validRoomID, err := spec.NewRoomID(event.RoomID()) - if err != nil { - return nil, err - } - inviter, err := r.rsAPI.QueryUserIDForSender(ctx, *validRoomID, event.SenderID()) + inviter, err := r.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID()) if err != nil { return nil, err } @@ -573,7 +571,7 @@ func (r *FederationInternalAPI) SendInvite( logrus.WithFields(logrus.Fields{ "event_id": event.EventID(), "user_id": *event.StateKey(), - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), "room_version": event.Version(), "destination": destination, }).Info("Sending invite") diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 26305ed7a..24b3efd2d 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -218,7 +218,7 @@ func (oqs *OutgoingQueues) SendEvent( if api.IsServerBannedFromRoom( oqs.process.Context(), oqs.rsAPI, - ev.RoomID(), + ev.RoomID().String(), destination, ) { delete(destmap, destination) diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go index cc38e136f..e75615e05 100644 --- a/federationapi/queue/queue_test.go +++ b/federationapi/queue/queue_test.go @@ -104,7 +104,7 @@ func (f *stubFederationClient) P2PSendTransactionToRelay(ctx context.Context, u func mustCreatePDU(t *testing.T) *types.HeaderedEvent { t.Helper() - content := `{"type":"m.room.message"}` + content := `{"type":"m.room.message", "room_id":"!room:a"}` ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionV10).NewEventFromTrustedJSON([]byte(content), false) if err != nil { t.Fatalf("failed to create event: %v", err) diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 552c4eac2..75a007265 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -109,7 +109,7 @@ func Backfill( var ev *types.HeaderedEvent for _, ev = range res.Events { - if ev.RoomID() == roomID { + if ev.RoomID().String() == roomID { evs = append(evs, ev.PDU) } } diff --git a/federationapi/routing/eventauth.go b/federationapi/routing/eventauth.go index c26aa2f15..2be3ecdb1 100644 --- a/federationapi/routing/eventauth.go +++ b/federationapi/routing/eventauth.go @@ -42,10 +42,10 @@ func GetEventAuth( return *resErr } - if event.RoomID() != roomID { + if event.RoomID().String() != roomID { return util.JSONResponse{Code: http.StatusNotFound, JSON: spec.NotFound("event does not belong to this room")} } - resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID()) + resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID().String()) if resErr != nil { return *resErr } diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go index d3f0e81c3..f4659f528 100644 --- a/federationapi/routing/events.go +++ b/federationapi/routing/events.go @@ -42,7 +42,7 @@ func GetEvent( return *err } - err = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID()) + err = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID().String()) if err != nil { return *err } diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index a090dbc8d..ce7ad30ff 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -99,7 +99,7 @@ func MakeJoin( Roomserver: rsAPI, } - senderID, err := rsAPI.QuerySenderIDForUser(httpReq.Context(), roomID, userID) + senderIDPtr, err := rsAPI.QuerySenderIDForUser(httpReq.Context(), roomID, userID) if err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QuerySenderIDForUser failed") return util.JSONResponse{ @@ -108,8 +108,11 @@ func MakeJoin( } } - if senderID == "" { + var senderID spec.SenderID + if senderIDPtr == nil { senderID = spec.SenderID(userID.String()) + } else { + senderID = *senderIDPtr } input := gomatrixserverlib.HandleMakeJoinInput{ diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 5c8dd00f3..7c86ba69b 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -94,11 +94,17 @@ func MakeLeave( Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}, } + } else if senderID == nil { + util.GetLogger(httpReq.Context()).WithField("roomID", roomID).WithField("userID", userID).Error("rsAPI.QuerySenderIDForUser returned nil sender ID") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } } input := gomatrixserverlib.HandleMakeLeaveInput{ UserID: userID, - SenderID: senderID, + SenderID: *senderID, RoomID: roomID, RoomVersion: roomVersion, RequestOrigin: request.Origin(), @@ -205,7 +211,7 @@ func SendLeave( } // Check that the room ID is correct. - if event.RoomID() != roomID { + if event.RoomID().String() != roomID { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.BadJSON("The room ID in the request path must match the room ID in the leave event JSON"), @@ -236,14 +242,7 @@ func SendLeave( // Check that the sender belongs to the server that is sending us // the request. By this point we've already asserted that the sender // and the state key are equal so we don't need to check both. - validRoomID, err := spec.NewRoomID(event.RoomID()) - if err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.BadJSON("Room ID is invalid."), - } - } - sender, err := rsAPI.QueryUserIDForSender(httpReq.Context(), *validRoomID, event.SenderID()) + sender, err := rsAPI.QueryUserIDForSender(httpReq.Context(), event.RoomID(), event.SenderID()) if err != nil { return util.JSONResponse{ Code: http.StatusForbidden, diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go index f57d30204..b1cefe7b4 100644 --- a/federationapi/routing/missingevents.go +++ b/federationapi/routing/missingevents.go @@ -87,7 +87,7 @@ func filterEvents( ) []*types.HeaderedEvent { ref := events[:0] for _, ev := range events { - if ev.RoomID() == roomID { + if ev.RoomID().String() == roomID { ref = append(ref, ev) } } diff --git a/federationapi/routing/profile_test.go b/federationapi/routing/profile_test.go index a31b206c1..ba13e07fc 100644 --- a/federationapi/routing/profile_test.go +++ b/federationapi/routing/profile_test.go @@ -26,7 +26,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" fedAPI "github.com/matrix-org/dendrite/federationapi" - fedInternal "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" @@ -67,11 +66,8 @@ func TestHandleQueryProfile(t *testing.T) { keyRing := serverKeyAPI.KeyRing() fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true) userapi := fakeUserAPI{} - r, ok := fedapi.(*fedInternal.FederationInternalAPI) - if !ok { - panic("This is a programming error.") - } - routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics) + + routing.Setup(routers, cfg, nil, fedapi, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics) handler := fedMux.Get(routing.QueryProfileRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/federationapi/routing/query_test.go b/federationapi/routing/query_test.go index bb14ab031..fd0894d15 100644 --- a/federationapi/routing/query_test.go +++ b/federationapi/routing/query_test.go @@ -25,7 +25,6 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" fedAPI "github.com/matrix-org/dendrite/federationapi" - fedInternal "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" @@ -65,11 +64,8 @@ func TestHandleQueryDirectory(t *testing.T) { keyRing := serverKeyAPI.KeyRing() fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true) userapi := fakeUserAPI{} - r, ok := fedapi.(*fedInternal.FederationInternalAPI) - if !ok { - panic("This is a programming error.") - } - routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics) + + routing.Setup(routers, cfg, nil, fedapi, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics) handler := fedMux.Get(routing.QueryDirectoryRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index f629479da..ff4f7bd06 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -23,7 +23,6 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" fedAPI "github.com/matrix-org/dendrite/federationapi" - fedInternal "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" @@ -62,11 +61,8 @@ func TestHandleSend(t *testing.T) { fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, nil, nil, nil, true) serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - r, ok := fedapi.(*fedInternal.FederationInternalAPI) - if !ok { - panic("This is a programming error.") - } - routing.Setup(routers, cfg, nil, r, keyRing, nil, nil, &cfg.MSCs, nil, caching.DisableMetrics) + + routing.Setup(routers, cfg, nil, fedapi, keyRing, nil, nil, &cfg.MSCs, nil, caching.DisableMetrics) handler := fedMux.Get(routing.SendRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 11ad1ebfc..d10910573 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -113,10 +113,10 @@ func getState( return nil, nil, resErr } - if event.RoomID() != roomID { + if event.RoomID().String() != roomID { return nil, nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: spec.NotFound("event does not belong to this room")} } - resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID()) + resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID, event.RoomID().String()) if resErr != nil { return nil, nil, resErr } diff --git a/federationapi/storage/postgres/notary_server_keys_metadata_table.go b/federationapi/storage/postgres/notary_server_keys_metadata_table.go index 7a1ec4122..47aa82b48 100644 --- a/federationapi/storage/postgres/notary_server_keys_metadata_table.go +++ b/federationapi/storage/postgres/notary_server_keys_metadata_table.go @@ -151,7 +151,7 @@ func (s *notaryServerKeysMetadataStatements) SelectKeys(ctx context.Context, txn } results = append(results, sk) } - return results, nil + return results, rows.Err() } func (s *notaryServerKeysMetadataStatements) DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error { diff --git a/federationapi/storage/postgres/queue_json_table.go b/federationapi/storage/postgres/queue_json_table.go index 563738dd5..f92e33d5e 100644 --- a/federationapi/storage/postgres/queue_json_table.go +++ b/federationapi/storage/postgres/queue_json_table.go @@ -109,5 +109,5 @@ func (s *queueJSONStatements) SelectQueueJSON( } blobs[nid] = blob } - return blobs, err + return blobs, rows.Err() } diff --git a/federationapi/storage/postgres/relay_servers_table.go b/federationapi/storage/postgres/relay_servers_table.go index 9e1bc5d40..1a47816e2 100644 --- a/federationapi/storage/postgres/relay_servers_table.go +++ b/federationapi/storage/postgres/relay_servers_table.go @@ -110,7 +110,7 @@ func (s *relayServersStatements) SelectRelayServers( } result = append(result, spec.ServerName(relayServer)) } - return result, nil + return result, rows.Err() } func (s *relayServersStatements) DeleteRelayServers( diff --git a/federationapi/storage/sqlite3/joined_hosts_table.go b/federationapi/storage/sqlite3/joined_hosts_table.go index 2412cacdb..418194312 100644 --- a/federationapi/storage/sqlite3/joined_hosts_table.go +++ b/federationapi/storage/sqlite3/joined_hosts_table.go @@ -216,5 +216,5 @@ func joinedHostsFromStmt( }) } - return result, nil + return result, rows.Err() } diff --git a/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go b/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go index 2fd9ef211..d9b98fc4f 100644 --- a/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go +++ b/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go @@ -154,7 +154,7 @@ func (s *notaryServerKeysMetadataStatements) SelectKeys(ctx context.Context, txn } results = append(results, sk) } - return results, nil + return results, rows.Err() } func (s *notaryServerKeysMetadataStatements) DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error { diff --git a/federationapi/storage/sqlite3/queue_json_table.go b/federationapi/storage/sqlite3/queue_json_table.go index 0e2806d56..33ae06131 100644 --- a/federationapi/storage/sqlite3/queue_json_table.go +++ b/federationapi/storage/sqlite3/queue_json_table.go @@ -135,5 +135,5 @@ func (s *queueJSONStatements) SelectQueueJSON( } blobs[nid] = blob } - return blobs, err + return blobs, rows.Err() } diff --git a/federationapi/storage/sqlite3/relay_servers_table.go b/federationapi/storage/sqlite3/relay_servers_table.go index 36cabeb4d..232db32af 100644 --- a/federationapi/storage/sqlite3/relay_servers_table.go +++ b/federationapi/storage/sqlite3/relay_servers_table.go @@ -109,7 +109,7 @@ func (s *relayServersStatements) SelectRelayServers( } result = append(result, spec.ServerName(relayServer)) } - return result, nil + return result, rows.Err() } func (s *relayServersStatements) DeleteRelayServers( diff --git a/go.mod b/go.mod index 0e77f903e..8dad4ebf0 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/blevesearch/bleve/v2 v2.3.8 github.com/codeclysm/extract v2.2.0+incompatible github.com/dgraph-io/ristretto v0.1.1 - github.com/docker/docker v20.10.24+incompatible + github.com/docker/docker v24.0.7+incompatible github.com/docker/go-connections v0.4.0 github.com/getsentry/sentry-go v0.14.0 github.com/gologme/log v1.3.0 @@ -22,12 +22,12 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20230721154317-b5b0448aa378 - github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a + github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca + github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 github.com/mattn/go-sqlite3 v1.14.17 - github.com/nats-io/nats-server/v2 v2.9.19 - github.com/nats-io/nats.go v1.27.0 + github.com/nats-io/nats-server/v2 v2.9.23 + github.com/nats-io/nats.go v1.28.0 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/opentracing/opentracing-go v1.2.0 @@ -36,18 +36,18 @@ require ( github.com/prometheus/client_golang v1.16.0 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.2 - github.com/tidwall/gjson v1.14.4 + github.com/tidwall/gjson v1.17.0 github.com/tidwall/sjson v1.2.5 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible github.com/yggdrasil-network/yggdrasil-go v0.4.6 go.uber.org/atomic v1.10.0 - golang.org/x/crypto v0.11.0 - golang.org/x/exp v0.0.0-20221205204356-47842c84f3db - golang.org/x/image v0.5.0 + golang.org/x/crypto v0.14.0 + golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 + golang.org/x/image v0.10.0 golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e - golang.org/x/sync v0.2.0 - golang.org/x/term v0.10.0 + golang.org/x/sync v0.3.0 + golang.org/x/term v0.13.0 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 gotest.tools/v3 v3.4.0 @@ -82,19 +82,19 @@ require ( github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect + github.com/google/pprof v0.0.0-20230808223545-4887780b67fb // indirect github.com/h2non/filetype v1.1.3 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/juju/errors v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/compress v1.16.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect @@ -104,33 +104,30 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mschoch/smat v0.2.0 // indirect - github.com/nats-io/jwt/v2 v2.4.1 // indirect - github.com/nats-io/nkeys v0.4.4 // indirect + github.com/nats-io/jwt/v2 v2.5.0 // indirect + github.com/nats-io/nkeys v0.4.6 // indirect github.com/nats-io/nuid v1.0.1 // indirect - github.com/onsi/ginkgo/v2 v2.3.0 // indirect - github.com/onsi/gomega v1.22.1 // indirect + github.com/onsi/ginkgo/v2 v2.11.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect - github.com/quic-go/qtls-go1-18 v0.2.0 // indirect - github.com/quic-go/qtls-go1-19 v0.2.0 // indirect - github.com/quic-go/qtls-go1-20 v0.1.0 // indirect - github.com/quic-go/quic-go v0.32.0 // indirect + github.com/quic-go/qtls-go1-20 v0.3.2 // indirect + github.com/quic-go/quic-go v0.37.4 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/rs/zerolog v1.29.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect go.etcd.io/bbolt v1.3.6 // indirect - golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.6.0 // indirect + golang.org/x/tools v0.12.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/macaroon.v2 v2.1.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -146,4 +143,4 @@ require ( modernc.org/token v1.0.1 // indirect ) -go 1.18 +go 1.20 diff --git a/go.sum b/go.sum index 28c60df43..b173f6b85 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.24+incompatible h1:Ugvxm7a8+Gz6vqQYQQ2W7GYq5EUPaAiuPgIfVyI3dYE= -github.com/docker/docker v20.10.24+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= +github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -113,6 +113,7 @@ github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= @@ -120,8 +121,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= @@ -160,8 +161,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= -github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/pprof v0.0.0-20230808223545-4887780b67fb h1:oqpb3Cwpc7EOml5PVGMYbSGmwNui2R7i8IW83gs4W0c= +github.com/google/pprof v0.0.0-20230808223545-4887780b67fb/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -189,8 +190,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -207,10 +208,10 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230721154317-b5b0448aa378 h1:a6sfiJiNZWVbPRHvEB/YlpqSg+Dh7El+824mzccSk68= -github.com/matrix-org/gomatrixserverlib v0.0.0-20230721154317-b5b0448aa378/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU= -github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a h1:awrPDf9LEFySxTLKYBMCiObelNx/cBuv/wzllvCCH3A= -github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a/go.mod h1:HchJX9oKMXaT2xYFs0Ha/6Zs06mxLU8k6F1ODnrGkeQ= +github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca h1:JCP72vU4Vcmur2071RwYVOSoekR+ZjbC03wZD5lAAK0= +github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca/go.mod h1:M8m7seOroO5ePlgxA7AFZymnG90Cnh94rYQyngSrZkk= +github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4= +github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66/go.mod h1:iBI1foelCqA09JJgPV0FYz4qA5dUXYOxMi57FxKBdd4= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= @@ -241,14 +242,14 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= -github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4= -github.com/nats-io/jwt/v2 v2.4.1/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= -github.com/nats-io/nats-server/v2 v2.9.19 h1:OF9jSKZGo425C/FcVVIvNgpd36CUe7aVTTXEZRJk6kA= -github.com/nats-io/nats-server/v2 v2.9.19/go.mod h1:aTb/xtLCGKhfTFLxP591CMWfkdgBmcUUSkiSOe5A3gw= -github.com/nats-io/nats.go v1.27.0 h1:3o9fsPhmoKm+yK7rekH2GtWoE+D9jFbw8N3/ayI1C00= -github.com/nats-io/nats.go v1.27.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= -github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= -github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= +github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak= +github.com/nats-io/jwt/v2 v2.5.0/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= +github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU= +github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0= +github.com/nats-io/nats.go v1.28.0 h1:Th4G6zdsz2d0OqXdfzKLClo6bOfoI/b1kInhRtFIy5c= +github.com/nats-io/nats.go v1.28.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= +github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY= +github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ= @@ -256,10 +257,9 @@ github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHG github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/ginkgo/v2 v2.3.0 h1:kUMoxMoQG3ogk/QWyKh3zibV7BKZ+xBpWil1cTylVqc= -github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= -github.com/onsi/gomega v1.22.1 h1:pY8O4lBfsHKZHM/6nrxkhVPUznOlIu3quZcKP/M20KI= -github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= @@ -284,14 +284,10 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/quic-go/qtls-go1-18 v0.2.0 h1:5ViXqBZ90wpUcZS0ge79rf029yx0dYB0McyPJwqqj7U= -github.com/quic-go/qtls-go1-18 v0.2.0/go.mod h1:moGulGHK7o6O8lSPSZNoOwcLvJKJ85vVNc7oJFD65bc= -github.com/quic-go/qtls-go1-19 v0.2.0 h1:Cvn2WdhyViFUHoOqK52i51k4nDX8EwIh5VJiVM4nttk= -github.com/quic-go/qtls-go1-19 v0.2.0/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= -github.com/quic-go/qtls-go1-20 v0.1.0 h1:d1PK3ErFy9t7zxKsG3NXBJXZjp/kMLoIb3y/kV54oAI= -github.com/quic-go/qtls-go1-20 v0.1.0/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= -github.com/quic-go/quic-go v0.32.0 h1:lY02md31s1JgPiiyfqJijpu/UX/Iun304FI3yUqX7tA= -github.com/quic-go/quic-go v0.32.0/go.mod h1:/fCsKANhQIeD5l76c2JFU+07gVE3KaA0FP+0zMWwfwo= +github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI= +github.com/quic-go/qtls-go1-20 v0.3.2/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= +github.com/quic-go/quic-go v0.37.4 h1:ke8B73yMCWGq9MfrCCAw0Uzdm7GaViC3i39dsIdDlH4= +github.com/quic-go/quic-go v0.37.4/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= @@ -315,15 +311,15 @@ github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= +github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -358,20 +354,20 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20221205204356-47842c84f3db h1:D/cFflL63o2KSLJIwjlcIt8PR064j/xsmdEJL/YvY/o= -golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 h1:EDuYyU/MkFXllv9QF9819VlI9a4tzGuCbhG0ExK9o1U= +golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= -golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M= +golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e h1:zSgtO19fpg781xknwqiQPmOHaASr6E7ZVlTseLd9Fx4= golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e/go.mod h1:aAjjkJNdrh3PMckS4B10TGS2nag27cbKR1y2BpUxsiY= @@ -380,8 +376,9 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -390,16 +387,18 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -422,19 +421,22 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -449,8 +451,9 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml index 8fa06dd97..32f479960 100644 --- a/helm/dendrite/Chart.yaml +++ b/helm/dendrite/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: dendrite -version: "0.13.1" -appVersion: "0.13.1" +version: "0.13.5" +appVersion: "0.13.4" description: Dendrite Matrix Homeserver type: application keywords: diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md index 7eabe88e6..22daa1813 100644 --- a/helm/dendrite/README.md +++ b/helm/dendrite/README.md @@ -1,7 +1,7 @@ # dendrite -![Version: 0.13.1](https://img.shields.io/badge/Version-0.13.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.1](https://img.shields.io/badge/AppVersion-0.13.1-informational?style=flat-square) +![Version: 0.13.5](https://img.shields.io/badge/Version-0.13.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.4](https://img.shields.io/badge/AppVersion-0.13.4-informational?style=flat-square) Dendrite Matrix Homeserver Status: **NOT PRODUCTION READY** @@ -48,16 +48,13 @@ Create a folder `appservices` and place your configurations in there. The confi | signing_key.create | bool | `true` | Create a new signing key, if not exists | | signing_key.existingSecret | string | `""` | Use an existing secret | | resources | object | sets some sane default values | Default resource requests/limits. | -| persistence.storageClass | string | `""` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. | +| persistence.jetstream | object | `{"capacity":"1Gi","existingClaim":""}` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. # If defined, storageClassName: # If set to "-", storageClassName: "", which disables dynamic provisioning # If undefined (the default) or set to null, no storageClassName spec is # set, choosing the default provisioner. (gp2 on AWS, standard on # GKE, AWS & OpenStack) # storageClass: "" | | persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream | | persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume | -| persistence.jetstream.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass | | persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files | | persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume | -| persistence.media.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass | | persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index | | persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume | -| persistence.search.storageClass | string | `""` | The storage class to use for volume claims. Defaults to persistence.storageClass | | extraVolumes | list | `[]` | Add additional volumes to the Dendrite Pod | | extraVolumeMounts | list | `[]` | Configure additional mount points volumes in the Dendrite Pod | | strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate | @@ -189,5 +186,3 @@ grafana: ``` PS: The label `release=kube-prometheus-stack` is setup with the helmchart of the Prometheus Operator. For Grafana Dashboards it may be necessary to enable scanning in the correct namespaces (or ALL), enabled by `sidecar.dashboards.searchNamespace` in [Helmchart of grafana](https://artifacthub.io/packages/helm/grafana/grafana) (which is part of PrometheusOperator, so `grafana.sidecar.dashboards.searchNamespace`) ----------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0) \ No newline at end of file diff --git a/helm/dendrite/grafana_dashboards/dendrite-rev2.json b/helm/dendrite/grafana_dashboards/dendrite-rev2.json index 817f950b3..420d8bf1b 100644 --- a/helm/dendrite/grafana_dashboards/dendrite-rev2.json +++ b/helm/dendrite/grafana_dashboards/dendrite-rev2.json @@ -119,7 +119,7 @@ "refId": "A" } ], - "title": "Registerd Users", + "title": "Registered Users", "type": "stat" }, { diff --git a/helm/dendrite/templates/deployment.yaml b/helm/dendrite/templates/deployment.yaml index df7dbbdc3..e3f84cdae 100644 --- a/helm/dendrite/templates/deployment.yaml +++ b/helm/dendrite/templates/deployment.yaml @@ -109,4 +109,4 @@ spec: failureThreshold: 10 httpGet: path: /_dendrite/monitor/up - port: http \ No newline at end of file + port: http diff --git a/helm/dendrite/templates/ingress.yaml b/helm/dendrite/templates/ingress.yaml index 9ef413dc9..4bcaee12d 100644 --- a/helm/dendrite/templates/ingress.yaml +++ b/helm/dendrite/templates/ingress.yaml @@ -1,8 +1,8 @@ {{- if .Values.ingress.enabled -}} {{- $fullName := include "dendrite.fullname" . -}} {{- $serverNameHost := .Values.dendrite_config.global.server_name -}} -{{- $wellKnownServerHost := default $serverNameHost (regexFind "^[^:]+" .Values.dendrite_config.global.well_known_server_name) -}} -{{- $wellKnownClientHost := default $serverNameHost (regexFind "^[^:]+" .Values.dendrite_config.global.well_known_client_name) -}} +{{- $wellKnownServerHost := default $serverNameHost (regexFind "^(\\[.+\\])?[^:]*" .Values.dendrite_config.global.well_known_server_name) -}} +{{- $wellKnownClientHost := default $serverNameHost (regexFind "//(\\[.+\\])?[^:/]*" .Values.dendrite_config.global.well_known_client_name | trimAll "/") -}} {{- $allHosts := list $serverNameHost $wellKnownServerHost $wellKnownClientHost | uniq -}} {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1 diff --git a/helm/dendrite/templates/pvc.yaml b/helm/dendrite/templates/pvc.yaml index 88eff3bed..70b1ce563 100644 --- a/helm/dendrite/templates/pvc.yaml +++ b/helm/dendrite/templates/pvc.yaml @@ -12,7 +12,14 @@ spec: resources: requests: storage: {{ .Values.persistence.media.capacity }} - storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.media.storageClass }} + {{ $storageClass := .Values.persistence.media.storageClass | default .Values.persistence.storageClass }} + {{- if $storageClass }} + {{- if (eq "-" $storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ $storageClass }}" + {{- end }} + {{- end }} {{ end }} {{ if not .Values.persistence.jetstream.existingClaim }} --- @@ -28,7 +35,14 @@ spec: resources: requests: storage: {{ .Values.persistence.jetstream.capacity }} - storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.jetstream.storageClass }} + {{ $storageClass := .Values.persistence.jetstream.storageClass | default .Values.persistence.storageClass }} + {{- if $storageClass }} + {{- if (eq "-" $storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ $storageClass }}" + {{- end }} + {{- end }} {{ end }} {{ if not .Values.persistence.search.existingClaim }} --- @@ -44,5 +58,12 @@ spec: resources: requests: storage: {{ .Values.persistence.search.capacity }} - storageClassName: {{ default .Values.persistence.storageClass .Values.persistence.search.storageClass }} + {{ $storageClass := .Values.persistence.search.storageClass | default .Values.persistence.storageClass }} + {{- if $storageClass }} + {{- if (eq "-" $storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ $storageClass }}" + {{- end }} + {{- end }} {{ end }} diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml index 8a72f6693..afce1d930 100644 --- a/helm/dendrite/values.yaml +++ b/helm/dendrite/values.yaml @@ -26,7 +26,13 @@ persistence: # -- The storage class to use for volume claims. # Used unless specified at the specific component. # Defaults to the cluster default storage class. - storageClass: "" + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "" jetstream: # -- Use an existing volume claim for jetstream existingClaim: "" @@ -34,7 +40,13 @@ persistence: capacity: "1Gi" # -- The storage class to use for volume claims. # Defaults to persistence.storageClass - storageClass: "" + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "" media: # -- Use an existing volume claim for media files existingClaim: "" @@ -42,7 +54,13 @@ persistence: capacity: "1Gi" # -- The storage class to use for volume claims. # Defaults to persistence.storageClass - storageClass: "" + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "" search: # -- Use an existing volume claim for the fulltext search index existingClaim: "" @@ -50,7 +68,13 @@ persistence: capacity: "1Gi" # -- The storage class to use for volume claims. # Defaults to persistence.storageClass - storageClass: "" + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "" # -- Add additional volumes to the Dendrite Pod extraVolumes: [] diff --git a/internal/eventutil/events.go b/internal/eventutil/events.go index 56ee576a0..40d62fd68 100644 --- a/internal/eventutil/events.go +++ b/internal/eventutil/events.go @@ -176,15 +176,13 @@ func RedactEvent(ctx context.Context, redactionEvent, redactedEvent gomatrixserv return fmt.Errorf("RedactEvent: redactionEvent isn't a redaction event, is '%s'", redactionEvent.Type()) } redactedEvent.Redact() - validRoomID, err := spec.NewRoomID(redactionEvent.RoomID()) + clientEvent, err := synctypes.ToClientEvent(redactionEvent, synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return querier.QueryUserIDForSender(ctx, roomID, senderID) + }) if err != nil { return err } - senderID, err := querier.QueryUserIDForSender(ctx, *validRoomID, redactionEvent.SenderID()) - if err != nil { - return err - } - redactedBecause := synctypes.ToClientEvent(redactionEvent, synctypes.FormatSync, *senderID, redactionEvent.StateKey()) + redactedBecause := clientEvent if err := redactedEvent.SetUnsignedField("redacted_because", redactedBecause); err != nil { return err } diff --git a/internal/pushrules/action_test.go b/internal/pushrules/action_test.go index 72db9c998..d1eb16ef1 100644 --- a/internal/pushrules/action_test.go +++ b/internal/pushrules/action_test.go @@ -13,8 +13,6 @@ func TestActionJSON(t *testing.T) { Want Action }{ {Action{Kind: NotifyAction}}, - {Action{Kind: DontNotifyAction}}, - {Action{Kind: CoalesceAction}}, {Action{Kind: SetTweakAction}}, {Action{Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"}}, diff --git a/internal/pushrules/default_override.go b/internal/pushrules/default_override.go index f97427b71..cb825af5f 100644 --- a/internal/pushrules/default_override.go +++ b/internal/pushrules/default_override.go @@ -10,6 +10,7 @@ func defaultOverrideRules(userID string) []*Rule { &mRuleRoomNotifDefinition, &mRuleTombstoneDefinition, &mRuleReactionDefinition, + &mRuleACLsDefinition, } } @@ -30,7 +31,7 @@ var ( RuleID: MRuleMaster, Default: true, Enabled: false, - Actions: []*Action{{Kind: DontNotifyAction}}, + Actions: []*Action{}, } mRuleSuppressNoticesDefinition = Rule{ RuleID: MRuleSuppressNotices, @@ -43,7 +44,7 @@ var ( Pattern: pointer("m.notice"), }, }, - Actions: []*Action{{Kind: DontNotifyAction}}, + Actions: []*Action{}, } mRuleMemberEventDefinition = Rule{ RuleID: MRuleMemberEvent, @@ -56,7 +57,7 @@ var ( Pattern: pointer("m.room.member"), }, }, - Actions: []*Action{{Kind: DontNotifyAction}}, + Actions: []*Action{}, } mRuleContainsDisplayNameDefinition = Rule{ RuleID: MRuleContainsDisplayName, @@ -152,9 +153,7 @@ var ( Pattern: pointer("m.reaction"), }, }, - Actions: []*Action{ - {Kind: DontNotifyAction}, - }, + Actions: []*Action{}, } ) diff --git a/internal/pushrules/default_pushrules_test.go b/internal/pushrules/default_pushrules_test.go index dea829842..506fe3cc8 100644 --- a/internal/pushrules/default_pushrules_test.go +++ b/internal/pushrules/default_pushrules_test.go @@ -21,12 +21,12 @@ func TestDefaultRules(t *testing.T) { // Default override rules { name: ".m.rule.master", - inputBytes: []byte(`{"rule_id":".m.rule.master","default":true,"enabled":false,"actions":["dont_notify"]}`), + inputBytes: []byte(`{"rule_id":".m.rule.master","default":true,"enabled":false,"actions":[]}`), want: mRuleMasterDefinition, }, { name: ".m.rule.suppress_notices", - inputBytes: []byte(`{"rule_id":".m.rule.suppress_notices","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"content.msgtype","pattern":"m.notice"}],"actions":["dont_notify"]}`), + inputBytes: []byte(`{"rule_id":".m.rule.suppress_notices","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"content.msgtype","pattern":"m.notice"}],"actions":[]}`), want: mRuleSuppressNoticesDefinition, }, { @@ -36,7 +36,7 @@ func TestDefaultRules(t *testing.T) { }, { name: ".m.rule.member_event", - inputBytes: []byte(`{"rule_id":".m.rule.member_event","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"type","pattern":"m.room.member"}],"actions":["dont_notify"]}`), + inputBytes: []byte(`{"rule_id":".m.rule.member_event","default":true,"enabled":true,"conditions":[{"kind":"event_match","key":"type","pattern":"m.room.member"}],"actions":[]}`), want: mRuleMemberEventDefinition, }, { diff --git a/internal/pushrules/evaluate.go b/internal/pushrules/evaluate.go index 28dea97c4..6baef4347 100644 --- a/internal/pushrules/evaluate.go +++ b/internal/pushrules/evaluate.go @@ -111,15 +111,11 @@ func ruleMatches(rule *Rule, kind Kind, event gomatrixserverlib.PDU, ec Evaluati return patternMatches("content.body", *rule.Pattern, event) case RoomKind: - return rule.RuleID == event.RoomID(), nil + return rule.RuleID == event.RoomID().String(), nil case SenderKind: userID := "" - validRoomID, err := spec.NewRoomID(event.RoomID()) - if err != nil { - return false, err - } - sender, err := userIDForSender(*validRoomID, event.SenderID()) + sender, err := userIDForSender(event.RoomID(), event.SenderID()) if err == nil { userID = sender.String() } diff --git a/internal/pushrules/evaluate_test.go b/internal/pushrules/evaluate_test.go index a4ccc3d0f..fbc88b2e7 100644 --- a/internal/pushrules/evaluate_test.go +++ b/internal/pushrules/evaluate_test.go @@ -13,7 +13,7 @@ func UserIDForSender(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, } func TestRuleSetEvaluatorMatchEvent(t *testing.T) { - ev := mustEventFromJSON(t, `{}`) + ev := mustEventFromJSON(t, `{"room_id":"!room:a"}`) defaultEnabled := &Rule{ RuleID: ".default.enabled", Default: true, @@ -44,8 +44,8 @@ func TestRuleSetEvaluatorMatchEvent(t *testing.T) { {"overrideRoom", RuleSet{Override: []*Rule{userEnabled}, Room: []*Rule{userEnabled2}}, userEnabled, ev}, {"overrideSender", RuleSet{Override: []*Rule{userEnabled}, Sender: []*Rule{userEnabled2}}, userEnabled, ev}, {"overrideUnderride", RuleSet{Override: []*Rule{userEnabled}, Underride: []*Rule{userEnabled2}}, userEnabled, ev}, - {"reactions don't notify", *defaultRuleset, &mRuleReactionDefinition, mustEventFromJSON(t, `{"type":"m.reaction"}`)}, - {"receipts don't notify", *defaultRuleset, nil, mustEventFromJSON(t, `{"type":"m.receipt"}`)}, + {"reactions don't notify", *defaultRuleset, &mRuleReactionDefinition, mustEventFromJSON(t, `{"room_id":"!room:a","type":"m.reaction"}`)}, + {"receipts don't notify", *defaultRuleset, nil, mustEventFromJSON(t, `{"room_id":"!room:a","type":"m.receipt"}`)}, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { @@ -70,28 +70,27 @@ func TestRuleMatches(t *testing.T) { EventJSON string Want bool }{ - {"emptyOverride", OverrideKind, emptyRule, `{}`, true}, - {"emptyContent", ContentKind, emptyRule, `{}`, false}, - {"emptyRoom", RoomKind, emptyRule, `{}`, true}, + {"emptyOverride", OverrideKind, emptyRule, `{"room_id":"!room:example.com"}`, true}, + {"emptyContent", ContentKind, emptyRule, `{"room_id":"!room:example.com"}`, false}, {"emptySender", SenderKind, emptyRule, `{"room_id":"!room:example.com"}`, true}, - {"emptyUnderride", UnderrideKind, emptyRule, `{}`, true}, + {"emptyUnderride", UnderrideKind, emptyRule, `{"room_id":"!room:example.com"}`, true}, - {"disabled", OverrideKind, Rule{}, `{}`, false}, + {"disabled", OverrideKind, Rule{}, `{"room_id":"!room:example.com"}`, false}, - {"overrideConditionMatch", OverrideKind, Rule{Enabled: true}, `{}`, true}, - {"overrideConditionNoMatch", OverrideKind, Rule{Enabled: true, Conditions: []*Condition{{}}}, `{}`, false}, + {"overrideConditionMatch", OverrideKind, Rule{Enabled: true}, `{"room_id":"!room:example.com"}`, true}, + {"overrideConditionNoMatch", OverrideKind, Rule{Enabled: true, Conditions: []*Condition{{}}}, `{"room_id":"!room:example.com"}`, false}, - {"underrideConditionMatch", UnderrideKind, Rule{Enabled: true}, `{}`, true}, - {"underrideConditionNoMatch", UnderrideKind, Rule{Enabled: true, Conditions: []*Condition{{}}}, `{}`, false}, + {"underrideConditionMatch", UnderrideKind, Rule{Enabled: true}, `{"room_id":"!room:example.com"}`, true}, + {"underrideConditionNoMatch", UnderrideKind, Rule{Enabled: true, Conditions: []*Condition{{}}}, `{"room_id":"!room:example.com"}`, false}, - {"contentMatch", ContentKind, Rule{Enabled: true, Pattern: pointer("b")}, `{"content":{"body":"abc"}}`, true}, - {"contentNoMatch", ContentKind, Rule{Enabled: true, Pattern: pointer("d")}, `{"content":{"body":"abc"}}`, false}, + {"contentMatch", ContentKind, Rule{Enabled: true, Pattern: pointer("b")}, `{"room_id":"!room:example.com","content":{"body":"abc"}}`, true}, + {"contentNoMatch", ContentKind, Rule{Enabled: true, Pattern: pointer("d")}, `{"room_id":"!room:example.com","content":{"body":"abc"}}`, false}, {"roomMatch", RoomKind, Rule{Enabled: true, RuleID: "!room:example.com"}, `{"room_id":"!room:example.com"}`, true}, {"roomNoMatch", RoomKind, Rule{Enabled: true, RuleID: "!room:example.com"}, `{"room_id":"!otherroom:example.com"}`, false}, - {"senderMatch", SenderKind, Rule{Enabled: true, RuleID: "@user:example.com"}, `{"sender":"@user:example.com","room_id":"!room:example.com"}`, true}, - {"senderNoMatch", SenderKind, Rule{Enabled: true, RuleID: "@user:example.com"}, `{"sender":"@otheruser:example.com","room_id":"!room:example.com"}`, false}, + {"senderMatch", SenderKind, Rule{Enabled: true, RuleID: "@user:example.com"}, `{"room_id":"!room:example.com","sender":"@user:example.com","room_id":"!room:example.com"}`, true}, + {"senderNoMatch", SenderKind, Rule{Enabled: true, RuleID: "@user:example.com"}, `{"room_id":"!room:example.com","sender":"@otheruser:example.com","room_id":"!room:example.com"}`, false}, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { @@ -114,32 +113,32 @@ func TestConditionMatches(t *testing.T) { WantMatch bool WantErr bool }{ - {Name: "empty", Cond: Condition{}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "empty", Cond: Condition{Kind: "unknownstring"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, + {Name: "empty", Cond: Condition{}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "empty", Cond: Condition{Kind: "unknownstring"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, // Neither of these should match because `content` is not a full string match, // and `content.body` is not a string value. - {Name: "eventMatch", Cond: Condition{Kind: EventMatchCondition, Key: "content", Pattern: pointer("")}, EventJSON: `{"content":{}}`, WantMatch: false, WantErr: false}, - {Name: "eventBodyMatch", Cond: Condition{Kind: EventMatchCondition, Key: "content.body", Is: "3", Pattern: pointer("")}, EventJSON: `{"content":{"body": "3"}}`, WantMatch: false, WantErr: false}, - {Name: "eventBodyMatch matches", Cond: Condition{Kind: EventMatchCondition, Key: "content.body", Pattern: pointer("world")}, EventJSON: `{"content":{"body": "hello world!"}}`, WantMatch: true, WantErr: false}, - {Name: "EventMatch missing pattern", Cond: Condition{Kind: EventMatchCondition, Key: "content.body"}, EventJSON: `{"content":{"body": "hello world!"}}`, WantMatch: false, WantErr: true}, + {Name: "eventMatch", Cond: Condition{Kind: EventMatchCondition, Key: "content", Pattern: pointer("")}, EventJSON: `{"room_id":"!room:example.com","content":{}}`, WantMatch: false, WantErr: false}, + {Name: "eventBodyMatch", Cond: Condition{Kind: EventMatchCondition, Key: "content.body", Is: "3", Pattern: pointer("")}, EventJSON: `{"room_id":"!room:example.com","content":{"body": "3"}}`, WantMatch: false, WantErr: false}, + {Name: "eventBodyMatch matches", Cond: Condition{Kind: EventMatchCondition, Key: "content.body", Pattern: pointer("world")}, EventJSON: `{"room_id":"!room:example.com","content":{"body": "hello world!"}}`, WantMatch: true, WantErr: false}, + {Name: "EventMatch missing pattern", Cond: Condition{Kind: EventMatchCondition, Key: "content.body"}, EventJSON: `{"room_id":"!room:example.com","content":{"body": "hello world!"}}`, WantMatch: false, WantErr: true}, - {Name: "displayNameNoMatch", Cond: Condition{Kind: ContainsDisplayNameCondition}, EventJSON: `{"content":{"body":"something without displayname"}}`, WantMatch: false, WantErr: false}, - {Name: "displayNameMatch", Cond: Condition{Kind: ContainsDisplayNameCondition}, EventJSON: `{"content":{"body":"hello Dear User, how are you?"}}`, WantMatch: true, WantErr: false}, + {Name: "displayNameNoMatch", Cond: Condition{Kind: ContainsDisplayNameCondition}, EventJSON: `{"room_id":"!room:example.com","content":{"body":"something without displayname"}}`, WantMatch: false, WantErr: false}, + {Name: "displayNameMatch", Cond: Condition{Kind: ContainsDisplayNameCondition}, EventJSON: `{"room_id":"!room:example.com","content":{"body":"hello Dear User, how are you?"}}`, WantMatch: true, WantErr: false}, - {Name: "roomMemberCountLessNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<2"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "roomMemberCountLessMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<3"}, EventJSON: `{}`, WantMatch: true, WantErr: false}, - {Name: "roomMemberCountLessEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<=1"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "roomMemberCountLessEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<=2"}, EventJSON: `{}`, WantMatch: true, WantErr: false}, - {Name: "roomMemberCountEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "==1"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "roomMemberCountEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "==2"}, EventJSON: `{}`, WantMatch: true, WantErr: false}, - {Name: "roomMemberCountGreaterEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">=3"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "roomMemberCountGreaterEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">=2"}, EventJSON: `{}`, WantMatch: true, WantErr: false}, - {Name: "roomMemberCountGreaterNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">2"}, EventJSON: `{}`, WantMatch: false, WantErr: false}, - {Name: "roomMemberCountGreaterMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">1"}, EventJSON: `{}`, WantMatch: true, WantErr: false}, + {Name: "roomMemberCountLessNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "roomMemberCountLessMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<3"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false}, + {Name: "roomMemberCountLessEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<=1"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "roomMemberCountLessEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "<=2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false}, + {Name: "roomMemberCountEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "==1"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "roomMemberCountEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: "==2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false}, + {Name: "roomMemberCountGreaterEqualNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">=3"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "roomMemberCountGreaterEqualMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">=2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false}, + {Name: "roomMemberCountGreaterNoMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">2"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "roomMemberCountGreaterMatch", Cond: Condition{Kind: RoomMemberCountCondition, Is: ">1"}, EventJSON: `{"room_id":"!room:example.com"}`, WantMatch: true, WantErr: false}, - {Name: "senderNotificationPermissionMatch", Cond: Condition{Kind: SenderNotificationPermissionCondition, Key: "powerlevel"}, EventJSON: `{"sender":"@poweruser:example.com"}`, WantMatch: true, WantErr: false}, - {Name: "senderNotificationPermissionNoMatch", Cond: Condition{Kind: SenderNotificationPermissionCondition, Key: "powerlevel"}, EventJSON: `{"sender":"@nobody:example.com"}`, WantMatch: false, WantErr: false}, + {Name: "senderNotificationPermissionMatch", Cond: Condition{Kind: SenderNotificationPermissionCondition, Key: "powerlevel"}, EventJSON: `{"room_id":"!room:example.com","sender":"@poweruser:example.com"}`, WantMatch: true, WantErr: false}, + {Name: "senderNotificationPermissionNoMatch", Cond: Condition{Kind: SenderNotificationPermissionCondition, Key: "powerlevel"}, EventJSON: `{"room_id":"!room:example.com","sender":"@nobody:example.com"}`, WantMatch: false, WantErr: false}, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { @@ -170,15 +169,15 @@ func TestPatternMatches(t *testing.T) { EventJSON string Want bool }{ - {"empty", "", "", `{}`, false}, + {"empty", "", "", `{"room_id":"!room:a"}`, false}, - {"patternEmpty", "content", "", `{"content":{}}`, false}, + {"patternEmpty", "content", "", `{"room_id":"!room:a","content":{}}`, false}, - {"literal", "content.creator", "acreator", `{"content":{"creator":"acreator"}}`, true}, - {"substring", "content.creator", "reat", `{"content":{"creator":"acreator"}}`, true}, - {"singlePattern", "content.creator", "acr?ator", `{"content":{"creator":"acreator"}}`, true}, - {"multiPattern", "content.creator", "a*ea*r", `{"content":{"creator":"acreator"}}`, true}, - {"patternNoSubstring", "content.creator", "r*t", `{"content":{"creator":"acreator"}}`, false}, + {"literal", "content.creator", "acreator", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, true}, + {"substring", "content.creator", "reat", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, true}, + {"singlePattern", "content.creator", "acr?ator", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, true}, + {"multiPattern", "content.creator", "a*ea*r", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, true}, + {"patternNoSubstring", "content.creator", "r*t", `{"room_id":"!room:a","content":{"creator":"acreator"}}`, false}, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { diff --git a/internal/pushrules/util.go b/internal/pushrules/util.go index de8fe5cd0..e2821b57a 100644 --- a/internal/pushrules/util.go +++ b/internal/pushrules/util.go @@ -16,10 +16,7 @@ func ActionsToTweaks(as []*Action) (ActionKind, map[string]interface{}, error) { for _, a := range as { switch a.Kind { - case DontNotifyAction: - // Don't bother processing any further - return DontNotifyAction, nil, nil - + case DontNotifyAction: // Ignored case SetTweakAction: if tweaks == nil { tweaks = map[string]interface{}{} diff --git a/internal/pushrules/util_test.go b/internal/pushrules/util_test.go index 89f8243d9..83eee7ede 100644 --- a/internal/pushrules/util_test.go +++ b/internal/pushrules/util_test.go @@ -17,17 +17,16 @@ func TestActionsToTweaks(t *testing.T) { {"empty", nil, UnknownAction, nil}, {"zero", []*Action{{}}, UnknownAction, nil}, {"onlyPrimary", []*Action{{Kind: NotifyAction}}, NotifyAction, nil}, - {"onlyPrimaryDontNotify", []*Action{{Kind: DontNotifyAction}}, DontNotifyAction, nil}, + {"onlyPrimaryDontNotify", []*Action{}, UnknownAction, nil}, {"onlyTweak", []*Action{{Kind: SetTweakAction, Tweak: HighlightTweak}}, UnknownAction, map[string]interface{}{"highlight": nil}}, {"onlyTweakWithValue", []*Action{{Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"}}, UnknownAction, map[string]interface{}{"sound": "default"}}, { "all", []*Action{ - {Kind: CoalesceAction}, {Kind: SetTweakAction, Tweak: HighlightTweak}, {Kind: SetTweakAction, Tweak: SoundTweak, Value: "default"}, }, - CoalesceAction, + UnknownAction, map[string]interface{}{"highlight": nil, "sound": "default"}, }, } diff --git a/internal/pushrules/validate.go b/internal/pushrules/validate.go index b54ec3fb0..4cc479345 100644 --- a/internal/pushrules/validate.go +++ b/internal/pushrules/validate.go @@ -18,7 +18,7 @@ func ValidateRule(kind Kind, rule *Rule) []error { errs = append(errs, fmt.Errorf("invalid rule ID: %s", rule.RuleID)) } - if len(rule.Actions) == 0 { + if rule.Actions == nil { errs = append(errs, fmt.Errorf("missing actions")) } for _, action := range rule.Actions { diff --git a/internal/transactionrequest.go b/internal/transactionrequest.go index 5bf7d819c..0663c2dcb 100644 --- a/internal/transactionrequest.go +++ b/internal/transactionrequest.go @@ -161,7 +161,7 @@ func (t *TxnReq) ProcessTransaction(ctx context.Context) (*fclient.RespSend, *ut if event.Type() == spec.MRoomCreate && event.StateKeyEquals("") { continue } - if api.IsServerBannedFromRoom(ctx, t.rsAPI, event.RoomID(), t.Origin) { + if api.IsServerBannedFromRoom(ctx, t.rsAPI, event.RoomID().String(), t.Origin) { results[event.EventID()] = fclient.PDUResult{ Error: "Forbidden by server ACLs", } diff --git a/internal/validate.go b/internal/validate.go index 99088f240..7f0d8b9e6 100644 --- a/internal/validate.go +++ b/internal/validate.go @@ -25,7 +25,7 @@ import ( ) const ( - maxUsernameLength = 254 // http://matrix.org/speculator/spec/HEAD/intro.html#user-identifiers TODO account for domain + maxUsernameLength = 254 // https://spec.matrix.org/v1.7/appendices/#user-identifiers TODO account for domain minPasswordLength = 8 // http://matrix.org/docs/spec/client_server/r0.2.0.html#password-based maxPasswordLength = 512 // https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161 diff --git a/internal/version.go b/internal/version.go index eedc3327c..55726ddcb 100644 --- a/internal/version.go +++ b/internal/version.go @@ -18,7 +18,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 13 - VersionPatch = 1 + VersionPatch = 4 VersionTag = "" // example: "rc1" gitRevLen = 7 // 7 matches the displayed characters on github.com diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 8fb1b6534..51afa1f9f 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" "io" + "io/fs" "mime" "net/http" "net/url" @@ -126,6 +127,17 @@ func Download( activeRemoteRequests, activeThumbnailGeneration, ) if err != nil { + // If we bubbled up a os.PathError, e.g. no such file or directory, don't send + // it to the client, be more generic. + var perr *fs.PathError + if errors.As(err, &perr) { + dReq.Logger.WithError(err).Error("failed to open file") + dReq.jsonErrorResponse(w, util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.NotFound("File not found"), + }) + return + } // TODO: Handle the fact we might have started writing the response dReq.jsonErrorResponse(w, util.JSONResponse{ Code: http.StatusNotFound, diff --git a/relayapi/storage/postgres/relay_queue_json_table.go b/relayapi/storage/postgres/relay_queue_json_table.go index 74410fc88..94ae41407 100644 --- a/relayapi/storage/postgres/relay_queue_json_table.go +++ b/relayapi/storage/postgres/relay_queue_json_table.go @@ -109,5 +109,5 @@ func (s *relayQueueJSONStatements) SelectQueueJSON( } blobs[nid] = blob } - return blobs, err + return blobs, rows.Err() } diff --git a/relayapi/storage/sqlite3/relay_queue_json_table.go b/relayapi/storage/sqlite3/relay_queue_json_table.go index 502da3b00..a1af82aa0 100644 --- a/relayapi/storage/sqlite3/relay_queue_json_table.go +++ b/relayapi/storage/sqlite3/relay_queue_json_table.go @@ -133,5 +133,5 @@ func (s *relayQueueJSONStatements) SelectQueueJSON( } blobs[nid] = blob } - return blobs, err + return blobs, rows.Err() } diff --git a/roomserver/acls/acls.go b/roomserver/acls/acls.go index b04828b69..601ce9063 100644 --- a/roomserver/acls/acls.go +++ b/roomserver/acls/acls.go @@ -119,7 +119,7 @@ func (s *ServerACLs) OnServerACLUpdate(state gomatrixserverlib.PDU) { }).Debugf("Updating server ACLs for %q", state.RoomID()) s.aclsMutex.Lock() defer s.aclsMutex.Unlock() - s.acls[state.RoomID()] = acls + s.acls[state.RoomID().String()] = acls } func (s *ServerACLs) IsServerBannedFromRoom(serverName spec.ServerName, roomID string) bool { diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go index c091cf6a3..6269d0b04 100644 --- a/roomserver/api/alias.go +++ b/roomserver/api/alias.go @@ -16,26 +16,8 @@ package api import ( "regexp" - - "github.com/matrix-org/gomatrixserverlib/spec" ) -// SetRoomAliasRequest is a request to SetRoomAlias -type SetRoomAliasRequest struct { - // ID of the user setting the alias - UserID string `json:"user_id"` - // New alias for the room - Alias string `json:"alias"` - // The room ID the alias is referring to - RoomID string `json:"room_id"` -} - -// SetRoomAliasResponse is a response to SetRoomAlias -type SetRoomAliasResponse struct { - // Does the alias already refer to a room? - AliasExists bool `json:"alias_exists"` -} - // GetRoomIDForAliasRequest is a request to GetRoomIDForAlias type GetRoomIDForAliasRequest struct { // Alias we want to lookup @@ -63,22 +45,6 @@ type GetAliasesForRoomIDResponse struct { Aliases []string `json:"aliases"` } -// RemoveRoomAliasRequest is a request to RemoveRoomAlias -type RemoveRoomAliasRequest struct { - // ID of the user removing the alias - SenderID spec.SenderID `json:"user_id"` - // The room alias to remove - Alias string `json:"alias"` -} - -// RemoveRoomAliasResponse is a response to RemoveRoomAlias -type RemoveRoomAliasResponse struct { - // Did the alias exist before? - Found bool `json:"found"` - // Did we remove it? - Removed bool `json:"removed"` -} - type AliasEvent struct { Alias string `json:"alias"` AltAliases []string `json:"alt_aliases"` diff --git a/roomserver/api/api.go b/roomserver/api/api.go index 28b381d35..ef5bc3d17 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -55,6 +55,11 @@ type RestrictedJoinAPI interface { LocallyJoinedUsers(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, roomNID types.RoomNID) ([]gomatrixserverlib.PDU, error) } +type DefaultRoomVersionAPI interface { + // Returns the default room version used. + DefaultRoomVersion() gomatrixserverlib.RoomVersion +} + // RoomserverInputAPI is used to write events to the room server. type RoomserverInternalAPI interface { SyncRoomserverAPI @@ -64,6 +69,7 @@ type RoomserverInternalAPI interface { FederationRoomserverAPI QuerySenderIDAPI UserRoomPrivateKeyCreator + DefaultRoomVersionAPI // needed to avoid chicken and egg scenario when setting up the // interdependencies between the roomserver and other input APIs @@ -97,7 +103,7 @@ type InputRoomEventsAPI interface { } type QuerySenderIDAPI interface { - QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (spec.SenderID, error) + QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) } @@ -135,11 +141,28 @@ type QueryRoomHierarchyAPI interface { QueryNextRoomHierarchyPage(ctx context.Context, walker RoomHierarchyWalker, limit int) ([]fclient.RoomHierarchyRoom, *RoomHierarchyWalker, error) } +type QueryMembershipAPI interface { + QueryMembershipForSenderID(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID, res *QueryMembershipForUserResponse) error + QueryMembershipForUser(ctx context.Context, req *QueryMembershipForUserRequest, res *QueryMembershipForUserResponse) error + QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error + QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) + + // QueryMembershipAtEvent queries the memberships at the given events. + // Returns a map from eventID to *types.HeaderedEvent of membership events. + QueryMembershipAtEvent( + ctx context.Context, + roomID spec.RoomID, + eventIDs []string, + senderID spec.SenderID, + ) (map[string]*types.HeaderedEvent, error) +} + // API functions required by the syncapi type SyncRoomserverAPI interface { QueryLatestEventsAndStateAPI QueryBulkStateContentAPI QuerySenderIDAPI + QueryMembershipAPI // QuerySharedUsers returns a list of users who share at least 1 room in common with the given user. QuerySharedUsers(ctx context.Context, req *QuerySharedUsersRequest, res *QuerySharedUsersResponse) error // QueryEventsByID queries a list of events by event ID for one room. If no room is specified, it will try to determine @@ -149,12 +172,6 @@ type SyncRoomserverAPI interface { req *QueryEventsByIDRequest, res *QueryEventsByIDResponse, ) error - // Query the membership event for an user for a room. - QueryMembershipForUser( - ctx context.Context, - req *QueryMembershipForUserRequest, - res *QueryMembershipForUserResponse, - ) error // Query the state after a list of events in a room from the room server. QueryStateAfterEvents( @@ -169,14 +186,6 @@ type SyncRoomserverAPI interface { req *PerformBackfillRequest, res *PerformBackfillResponse, ) error - - // QueryMembershipAtEvent queries the memberships at the given events. - // Returns a map from eventID to a slice of types.HeaderedEvent. - QueryMembershipAtEvent( - ctx context.Context, - request *QueryMembershipAtEventRequest, - response *QueryMembershipAtEventResponse, - ) error } type AppserviceRoomserverAPI interface { @@ -210,9 +219,10 @@ type ClientRoomserverAPI interface { QuerySenderIDAPI UserRoomPrivateKeyCreator QueryRoomHierarchyAPI + DefaultRoomVersionAPI QueryMembershipForUser(ctx context.Context, req *QueryMembershipForUserRequest, res *QueryMembershipForUserResponse) error QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error - QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error + QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) QueryStateAfterEvents(ctx context.Context, req *QueryStateAfterEventsRequest, res *QueryStateAfterEventsResponse) error // QueryKnownUsers returns a list of users that we know about from our joined rooms. QueryKnownUsers(ctx context.Context, req *QueryKnownUsersRequest, res *QueryKnownUsersResponse) error @@ -237,8 +247,19 @@ type ClientRoomserverAPI interface { PerformPublish(ctx context.Context, req *PerformPublishRequest) error // PerformForget forgets a rooms history for a specific user PerformForget(ctx context.Context, req *PerformForgetRequest, resp *PerformForgetResponse) error - SetRoomAlias(ctx context.Context, req *SetRoomAliasRequest, res *SetRoomAliasResponse) error - RemoveRoomAlias(ctx context.Context, req *RemoveRoomAliasRequest, res *RemoveRoomAliasResponse) error + + // Sets a room alias, as provided sender, pointing to the provided room ID. + // + // If err is nil, then the returned boolean indicates if the alias is already in use. + // If true, then the alias has not been set to the provided room, as it already in use. + SetRoomAlias(ctx context.Context, senderID spec.SenderID, roomID spec.RoomID, alias string) (aliasAlreadyExists bool, err error) + + //RemoveRoomAlias(ctx context.Context, req *RemoveRoomAliasRequest, res *RemoveRoomAliasResponse) error + // Removes a room alias, as provided sender. + // + // Returns whether the alias was found, whether it was removed, and an error (if any occurred) + RemoveRoomAlias(ctx context.Context, senderID spec.SenderID, alias string) (aliasFound bool, aliasRemoved bool, err error) + SigningIdentityFor(ctx context.Context, roomID spec.RoomID, senderID spec.UserID) (fclient.SigningIdentity, error) } @@ -260,15 +281,12 @@ type FederationRoomserverAPI interface { QueryBulkStateContentAPI QuerySenderIDAPI QueryRoomHierarchyAPI + QueryMembershipAPI UserRoomPrivateKeyCreator AssignRoomNID(ctx context.Context, roomID spec.RoomID, roomVersion gomatrixserverlib.RoomVersion) (roomNID types.RoomNID, err error) SigningIdentityFor(ctx context.Context, roomID spec.RoomID, senderID spec.UserID) (fclient.SigningIdentity, error) // QueryServerBannedFromRoom returns whether a server is banned from a room by server ACLs. QueryServerBannedFromRoom(ctx context.Context, req *QueryServerBannedFromRoomRequest, res *QueryServerBannedFromRoomResponse) error - QueryMembershipForUser(ctx context.Context, req *QueryMembershipForUserRequest, res *QueryMembershipForUserResponse) error - QueryMembershipForSenderID(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID, res *QueryMembershipForUserResponse) error - QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error - QueryRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) GetRoomIDForAlias(ctx context.Context, req *GetRoomIDForAliasRequest, res *GetRoomIDForAliasResponse) error // QueryEventsByID queries a list of events by event ID for one room. If no room is specified, it will try to determine // which room to use by querying the first events roomID. @@ -282,7 +300,7 @@ type FederationRoomserverAPI interface { QueryMissingEvents(ctx context.Context, req *QueryMissingEventsRequest, res *QueryMissingEventsResponse) error // Query whether a server is allowed to see an event QueryServerAllowedToSeeEvent(ctx context.Context, serverName spec.ServerName, eventID string, roomID string) (allowed bool, err error) - QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error + QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) QueryRestrictedJoinAllowed(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (string, error) PerformInboundPeek(ctx context.Context, req *PerformInboundPeekRequest, res *PerformInboundPeekResponse) error HandleInvite(ctx context.Context, event *types.HeaderedEvent) error diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 57bac2df9..893d5dccf 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -132,6 +132,8 @@ type QueryMembershipForUserResponse struct { // True if the user asked to forget this room. IsRoomForgotten bool `json:"is_room_forgotten"` RoomExists bool `json:"room_exists"` + // The sender ID of the user in the room, if it exists + SenderID *spec.SenderID } // QueryMembershipsForRoomRequest is a request to QueryMembershipsForRoom @@ -289,16 +291,6 @@ type QuerySharedUsersResponse struct { UserIDsToCount map[string]int } -type QueryRoomsForUserRequest struct { - UserID string - // The desired membership of the user. If this is the empty string then no rooms are returned. - WantMembership string -} - -type QueryRoomsForUserResponse struct { - RoomIDs []string -} - type QueryBulkStateContentRequest struct { // Returns state events in these rooms RoomIDs []string @@ -414,22 +406,6 @@ func (r *QueryCurrentStateResponse) UnmarshalJSON(data []byte) error { return nil } -// QueryMembershipAtEventRequest requests the membership event for a user -// for a list of eventIDs. -type QueryMembershipAtEventRequest struct { - RoomID string - EventIDs []string - UserID string -} - -// QueryMembershipAtEventResponse is the response to QueryMembershipAtEventRequest. -type QueryMembershipAtEventResponse struct { - // Membership is a map from eventID to membership event. Events that - // do not have known state will return a nil event, resulting in a "leave" membership - // when calculating history visibility. - Membership map[string]*types.HeaderedEvent `json:"membership"` -} - // QueryLeftUsersRequest is a request to calculate users that we (the server) don't share a // a room with anymore. This is used to cleanup stale device list entries, where we would // otherwise keep on trying to get device lists. diff --git a/roomserver/api/wrapper.go b/roomserver/api/wrapper.go index 2505a993b..0ad5d2013 100644 --- a/roomserver/api/wrapper.go +++ b/roomserver/api/wrapper.go @@ -75,7 +75,7 @@ func SendEventWithState( } logrus.WithContext(ctx).WithFields(logrus.Fields{ - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), "event_id": event.EventID(), "outliers": len(ires), "state_ids": len(stateEventIDs), diff --git a/roomserver/auth/auth.go b/roomserver/auth/auth.go index df95851e3..d5172dab9 100644 --- a/roomserver/auth/auth.go +++ b/roomserver/auth/auth.go @@ -85,11 +85,7 @@ func IsAnyUserOnServerWithMembership(ctx context.Context, querier api.QuerySende continue } - validRoomID, err := spec.NewRoomID(ev.RoomID()) - if err != nil { - continue - } - userID, err := querier.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*stateKey)) + userID, err := querier.QueryUserIDForSender(ctx, ev.RoomID(), spec.SenderID(*stateKey)) if err != nil { continue } diff --git a/roomserver/internal/alias.go b/roomserver/internal/alias.go index b04a56fe8..5ceda7e01 100644 --- a/roomserver/internal/alias.go +++ b/roomserver/internal/alias.go @@ -35,27 +35,27 @@ import ( // SetRoomAlias implements alias.RoomserverInternalAPI func (r *RoomserverInternalAPI) SetRoomAlias( ctx context.Context, - request *api.SetRoomAliasRequest, - response *api.SetRoomAliasResponse, -) error { + senderID spec.SenderID, + roomID spec.RoomID, + alias string, +) (aliasAlreadyUsed bool, err error) { // Check if the alias isn't already referring to a room - roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) + existingRoomID, err := r.DB.GetRoomIDForAlias(ctx, alias) if err != nil { - return err + return false, err } - if len(roomID) > 0 { + + if len(existingRoomID) > 0 { // If the alias already exists, stop the process - response.AliasExists = true - return nil + return true, nil } - response.AliasExists = false // Save the new alias - if err := r.DB.SetRoomAlias(ctx, request.Alias, request.RoomID, request.UserID); err != nil { - return err + if err := r.DB.SetRoomAlias(ctx, alias, roomID.String(), string(senderID)); err != nil { + return false, err } - return nil + return false, nil } // GetRoomIDForAlias implements alias.RoomserverInternalAPI @@ -116,91 +116,80 @@ func (r *RoomserverInternalAPI) GetAliasesForRoomID( // nolint:gocyclo // RemoveRoomAlias implements alias.RoomserverInternalAPI // nolint: gocyclo -func (r *RoomserverInternalAPI) RemoveRoomAlias( - ctx context.Context, - request *api.RemoveRoomAliasRequest, - response *api.RemoveRoomAliasResponse, -) error { - roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) +func (r *RoomserverInternalAPI) RemoveRoomAlias(ctx context.Context, senderID spec.SenderID, alias string) (aliasFound bool, aliasRemoved bool, err error) { + roomID, err := r.DB.GetRoomIDForAlias(ctx, alias) if err != nil { - return fmt.Errorf("r.DB.GetRoomIDForAlias: %w", err) + return false, false, fmt.Errorf("r.DB.GetRoomIDForAlias: %w", err) } if roomID == "" { - response.Found = false - response.Removed = false - return nil + return false, false, nil } validRoomID, err := spec.NewRoomID(roomID) if err != nil { - return err + return true, false, err } - sender, err := r.QueryUserIDForSender(ctx, *validRoomID, request.SenderID) + sender, err := r.QueryUserIDForSender(ctx, *validRoomID, senderID) if err != nil || sender == nil { - return fmt.Errorf("r.QueryUserIDForSender: %w", err) + return true, false, fmt.Errorf("r.QueryUserIDForSender: %w", err) } virtualHost := sender.Domain() - response.Found = true - creatorID, err := r.DB.GetCreatorIDForAlias(ctx, request.Alias) + creatorID, err := r.DB.GetCreatorIDForAlias(ctx, alias) if err != nil { - return fmt.Errorf("r.DB.GetCreatorIDForAlias: %w", err) + return true, false, fmt.Errorf("r.DB.GetCreatorIDForAlias: %w", err) } - if spec.SenderID(creatorID) != request.SenderID { + if spec.SenderID(creatorID) != senderID { var plEvent *types.HeaderedEvent var pls *gomatrixserverlib.PowerLevelContent plEvent, err = r.DB.GetStateEvent(ctx, roomID, spec.MRoomPowerLevels, "") if err != nil { - return fmt.Errorf("r.DB.GetStateEvent: %w", err) + return true, false, fmt.Errorf("r.DB.GetStateEvent: %w", err) } pls, err = plEvent.PowerLevels() if err != nil { - return fmt.Errorf("plEvent.PowerLevels: %w", err) + return true, false, fmt.Errorf("plEvent.PowerLevels: %w", err) } - if pls.UserLevel(request.SenderID) < pls.EventLevel(spec.MRoomCanonicalAlias, true) { - response.Removed = false - return nil + if pls.UserLevel(senderID) < pls.EventLevel(spec.MRoomCanonicalAlias, true) { + return true, false, nil } } ev, err := r.DB.GetStateEvent(ctx, roomID, spec.MRoomCanonicalAlias, "") if err != nil && err != sql.ErrNoRows { - return err + return true, false, err } else if ev != nil { stateAlias := gjson.GetBytes(ev.Content(), "alias").Str // the alias to remove is currently set as the canonical alias, remove it - if stateAlias == request.Alias { + if stateAlias == alias { res, err := sjson.DeleteBytes(ev.Content(), "alias") if err != nil { - return err + return true, false, err } - senderID := request.SenderID - if request.SenderID != ev.SenderID() { - senderID = ev.SenderID() - } - sender, err := r.QueryUserIDForSender(ctx, *validRoomID, senderID) - if err != nil || sender == nil { - return err + canonicalSenderID := ev.SenderID() + canonicalSender, err := r.QueryUserIDForSender(ctx, *validRoomID, canonicalSenderID) + if err != nil || canonicalSender == nil { + return true, false, err } validRoomID, err := spec.NewRoomID(roomID) if err != nil { - return err + return true, false, err } - identity, err := r.SigningIdentityFor(ctx, *validRoomID, *sender) + identity, err := r.SigningIdentityFor(ctx, *validRoomID, *canonicalSender) if err != nil { - return err + return true, false, err } proto := &gomatrixserverlib.ProtoEvent{ - SenderID: string(senderID), - RoomID: ev.RoomID(), + SenderID: string(canonicalSenderID), + RoomID: ev.RoomID().String(), Type: ev.Type(), StateKey: ev.StateKey(), Content: res, @@ -208,34 +197,33 @@ func (r *RoomserverInternalAPI) RemoveRoomAlias( eventsNeeded, err := gomatrixserverlib.StateNeededForProtoEvent(proto) if err != nil { - return fmt.Errorf("gomatrixserverlib.StateNeededForEventBuilder: %w", err) + return true, false, fmt.Errorf("gomatrixserverlib.StateNeededForEventBuilder: %w", err) } if len(eventsNeeded.Tuples()) == 0 { - return errors.New("expecting state tuples for event builder, got none") + return true, false, errors.New("expecting state tuples for event builder, got none") } stateRes := &api.QueryLatestEventsAndStateResponse{} if err = helpers.QueryLatestEventsAndState(ctx, r.DB, r, &api.QueryLatestEventsAndStateRequest{RoomID: roomID, StateToFetch: eventsNeeded.Tuples()}, stateRes); err != nil { - return err + return true, false, err } newEvent, err := eventutil.BuildEvent(ctx, proto, &identity, time.Now(), &eventsNeeded, stateRes) if err != nil { - return err + return true, false, err } err = api.SendEvents(ctx, r, api.KindNew, []*types.HeaderedEvent{newEvent}, virtualHost, r.ServerName, r.ServerName, nil, false) if err != nil { - return err + return true, false, err } } } // Remove the alias from the database - if err := r.DB.RemoveRoomAlias(ctx, request.Alias); err != nil { - return err + if err := r.DB.RemoveRoomAlias(ctx, alias); err != nil { + return true, false, err } - response.Removed = true - return nil + return true, true, nil } diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index e8899a210..1e08f6a3a 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -61,6 +61,7 @@ type RoomserverInternalAPI struct { OutputProducer *producers.RoomEventProducer PerspectiveServerNames []spec.ServerName enableMetrics bool + defaultRoomVersion gomatrixserverlib.RoomVersion } func NewRoomserverAPI( @@ -92,6 +93,7 @@ func NewRoomserverAPI( Durable: dendriteCfg.Global.JetStream.Durable("RoomserverInputConsumer"), ServerACLs: serverACLs, enableMetrics: enableMetrics, + defaultRoomVersion: dendriteCfg.RoomServer.DefaultRoomVersion, // perform-er structs + queryer struct get initialised when we have a federation sender to use } return a @@ -218,6 +220,10 @@ func (r *RoomserverInternalAPI) SetAppserviceAPI(asAPI asAPI.AppServiceInternalA r.asAPI = asAPI } +func (r *RoomserverInternalAPI) DefaultRoomVersion() gomatrixserverlib.RoomVersion { + return r.defaultRoomVersion +} + func (r *RoomserverInternalAPI) IsKnownRoom(ctx context.Context, roomID spec.RoomID) (bool, error) { return r.Inviter.IsKnownRoom(ctx, roomID) } @@ -233,7 +239,7 @@ func (r *RoomserverInternalAPI) HandleInvite( if err != nil { return err } - return r.OutputProducer.ProduceRoomEvents(inviteEvent.RoomID(), outputEvents) + return r.OutputProducer.ProduceRoomEvents(inviteEvent.RoomID().String(), outputEvents) } func (r *RoomserverInternalAPI) PerformCreateRoom( diff --git a/roomserver/internal/helpers/auth.go b/roomserver/internal/helpers/auth.go index 89fae244f..9da751b1a 100644 --- a/roomserver/internal/helpers/auth.go +++ b/roomserver/internal/helpers/auth.go @@ -218,9 +218,9 @@ func loadAuthEvents( roomID := "" for _, ev := range result.events { if roomID == "" { - roomID = ev.RoomID() + roomID = ev.RoomID().String() } - if ev.RoomID() != roomID { + if ev.RoomID().String() != roomID { result.valid = false break } diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go index febabf411..b2e21bf54 100644 --- a/roomserver/internal/helpers/helpers.go +++ b/roomserver/internal/helpers/helpers.go @@ -54,7 +54,7 @@ func UpdateToInviteMembership( Type: api.OutputTypeRetireInviteEvent, RetireInviteEvent: &api.OutputRetireInviteEvent{ EventID: eventID, - RoomID: add.RoomID(), + RoomID: add.RoomID().String(), Membership: spec.Join, RetiredByEventID: add.EventID(), TargetSenderID: spec.SenderID(*add.StateKey()), @@ -396,7 +396,7 @@ BFSLoop: // It's nasty that we have to extract the room ID from an event, but many federation requests // only talk in event IDs, no room IDs at all (!!!) ev := events[0] - isServerInRoom, err = IsServerCurrentlyInRoom(ctx, db, querier, serverName, ev.RoomID()) + isServerInRoom, err = IsServerCurrentlyInRoom(ctx, db, querier, serverName, ev.RoomID().String()) if err != nil { util.GetLogger(ctx).WithError(err).Error("Failed to check if server is currently in room, assuming not.") } @@ -419,7 +419,7 @@ BFSLoop: // hasn't been seen before. if !visited[pre] { visited[pre] = true - allowed, err = CheckServerAllowedToSeeEvent(ctx, db, info, ev.RoomID(), pre, serverName, isServerInRoom, querier) + allowed, err = CheckServerAllowedToSeeEvent(ctx, db, info, ev.RoomID().String(), pre, serverName, isServerInRoom, querier) if err != nil { util.GetLogger(ctx).WithField("server", serverName).WithField("event_id", pre).WithError(err).Error( "Error checking if allowed to see event", diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go index 990563599..404751532 100644 --- a/roomserver/internal/input/input.go +++ b/roomserver/internal/input/input.go @@ -358,7 +358,7 @@ func (r *Inputer) queueInputRoomEvents( // For each event, marshal the input room event and then // send it into the input queue. for _, e := range request.InputRoomEvents { - roomID := e.Event.RoomID() + roomID := e.Event.RoomID().String() subj := r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEventSubj(roomID)) msg := &nats.Msg{ Subject: subj, diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 88049ddf0..77b50d0e2 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -87,7 +87,7 @@ func (r *Inputer) processRoomEvent( } trace, ctx := internal.StartRegion(ctx, "processRoomEvent") - trace.SetTag("room_id", input.Event.RoomID()) + trace.SetTag("room_id", input.Event.RoomID().String()) trace.SetTag("event_id", input.Event.EventID()) defer trace.EndRegion() @@ -96,7 +96,7 @@ func (r *Inputer) processRoomEvent( defer func() { timetaken := time.Since(started) processRoomEventDuration.With(prometheus.Labels{ - "room_id": input.Event.RoomID(), + "room_id": input.Event.RoomID().String(), }).Observe(float64(timetaken.Milliseconds())) }() @@ -105,7 +105,7 @@ func (r *Inputer) processRoomEvent( event := headered.PDU logger := util.GetLogger(ctx).WithFields(logrus.Fields{ "event_id": event.EventID(), - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), "kind": input.Kind, "origin": input.Origin, "type": event.Type(), @@ -120,19 +120,15 @@ func (r *Inputer) processRoomEvent( // Don't waste time processing the event if the room doesn't exist. // A room entry locally will only be created in response to a create // event. - roomInfo, rerr := r.DB.RoomInfo(ctx, event.RoomID()) + roomInfo, rerr := r.DB.RoomInfo(ctx, event.RoomID().String()) if rerr != nil { return fmt.Errorf("r.DB.RoomInfo: %w", rerr) } isCreateEvent := event.Type() == spec.MRoomCreate && event.StateKeyEquals("") if roomInfo == nil && !isCreateEvent { - return fmt.Errorf("room %s does not exist for event %s", event.RoomID(), event.EventID()) + return fmt.Errorf("room %s does not exist for event %s", event.RoomID().String(), event.EventID()) } - validRoomID, err := spec.NewRoomID(event.RoomID()) - if err != nil { - return err - } - sender, err := r.Queryer.QueryUserIDForSender(ctx, *validRoomID, event.SenderID()) + sender, err := r.Queryer.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID()) if err != nil { return fmt.Errorf("failed getting userID for sender %q. %w", event.SenderID(), err) } @@ -179,7 +175,7 @@ func (r *Inputer) processRoomEvent( // If we have missing events (auth or prev), we build a list of servers to ask if missingAuth || missingPrev { serverReq := &fedapi.QueryJoinedHostServerNamesInRoomRequest{ - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), ExcludeSelf: true, ExcludeBlacklisted: true, } @@ -395,12 +391,12 @@ func (r *Inputer) processRoomEvent( // Request the room info again — it's possible that the room has been // created by now if it didn't exist already. - roomInfo, err = r.DB.RoomInfo(ctx, event.RoomID()) + roomInfo, err = r.DB.RoomInfo(ctx, event.RoomID().String()) if err != nil { return fmt.Errorf("updater.RoomInfo: %w", err) } if roomInfo == nil { - return fmt.Errorf("updater.RoomInfo missing for room %s", event.RoomID()) + return fmt.Errorf("updater.RoomInfo missing for room %s", event.RoomID().String()) } if input.HasState || (!missingPrev && stateAtEvent.BeforeStateSnapshotNID == 0) { @@ -459,7 +455,7 @@ func (r *Inputer) processRoomEvent( if userErr != nil { return userErr } - err = r.RSAPI.StoreUserRoomPublicKey(ctx, mapping.MXIDMapping.UserRoomKey, *storeUserID, *validRoomID) + err = r.RSAPI.StoreUserRoomPublicKey(ctx, mapping.MXIDMapping.UserRoomKey, *storeUserID, event.RoomID()) if err != nil { return fmt.Errorf("failed storing user room public key: %w", err) } @@ -481,7 +477,7 @@ func (r *Inputer) processRoomEvent( return fmt.Errorf("r.updateLatestEvents: %w", err) } case api.KindOld: - err = r.OutputProducer.ProduceRoomEvents(event.RoomID(), []api.OutputEvent{ + err = r.OutputProducer.ProduceRoomEvents(event.RoomID().String(), []api.OutputEvent{ { Type: api.OutputTypeOldRoomEvent, OldRoomEvent: &api.OutputOldRoomEvent{ @@ -507,7 +503,7 @@ func (r *Inputer) processRoomEvent( // so notify downstream components to redact this event - they should have it if they've // been tracking our output log. if redactedEventID != "" { - err = r.OutputProducer.ProduceRoomEvents(event.RoomID(), []api.OutputEvent{ + err = r.OutputProducer.ProduceRoomEvents(event.RoomID().String(), []api.OutputEvent{ { Type: api.OutputTypeRedactedEvent, RedactedEvent: &api.OutputRedactedEvent{ @@ -536,7 +532,7 @@ func (r *Inputer) processRoomEvent( // handleRemoteRoomUpgrade updates published rooms and room aliases func (r *Inputer) handleRemoteRoomUpgrade(ctx context.Context, event gomatrixserverlib.PDU) error { - oldRoomID := event.RoomID() + oldRoomID := event.RoomID().String() newRoomID := gjson.GetBytes(event.Content(), "replacement_room").Str return r.DB.UpgradeRoom(ctx, oldRoomID, newRoomID, string(event.SenderID())) } @@ -596,7 +592,7 @@ func (r *Inputer) processStateBefore( StateKey: "", }) stateBeforeReq := &api.QueryStateAfterEventsRequest{ - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), PrevEventIDs: event.PrevEventIDs(), StateToFetch: tuplesNeeded, } @@ -606,7 +602,7 @@ func (r *Inputer) processStateBefore( } switch { case !stateBeforeRes.RoomExists: - rejectionErr = fmt.Errorf("room %q does not exist", event.RoomID()) + rejectionErr = fmt.Errorf("room %q does not exist", event.RoomID().String()) return case !stateBeforeRes.PrevEventsExist: rejectionErr = fmt.Errorf("prev events of %q are not known", event.EventID()) @@ -707,7 +703,7 @@ func (r *Inputer) fetchAuthEvents( // Request the entire auth chain for the event in question. This should // contain all of the auth events — including ones that we already know — // so we'll need to filter through those in the next section. - res, err = r.FSAPI.GetEventAuth(ctx, virtualHost, serverName, event.Version(), event.RoomID(), event.EventID()) + res, err = r.FSAPI.GetEventAuth(ctx, virtualHost, serverName, event.Version(), event.RoomID().String(), event.EventID()) if err != nil { logger.WithError(err).Warnf("Failed to get event auth from federation for %q: %s", event.EventID(), err) continue @@ -866,25 +862,20 @@ func (r *Inputer) kickGuests(ctx context.Context, event gomatrixserverlib.PDU, r inputEvents := make([]api.InputRoomEvent, 0, len(memberEvents)) latestReq := &api.QueryLatestEventsAndStateRequest{ - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), } latestRes := &api.QueryLatestEventsAndStateResponse{} if err = r.Queryer.QueryLatestEventsAndState(ctx, latestReq, latestRes); err != nil { return err } - validRoomID, err := spec.NewRoomID(event.RoomID()) - if err != nil { - return err - } - prevEvents := latestRes.LatestEvents for _, memberEvent := range memberEvents { if memberEvent.StateKey() == nil { continue } - memberUserID, err := r.Queryer.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*memberEvent.StateKey())) + memberUserID, err := r.Queryer.QueryUserIDForSender(ctx, event.RoomID(), spec.SenderID(*memberEvent.StateKey())) if err != nil { continue } @@ -912,7 +903,7 @@ func (r *Inputer) kickGuests(ctx context.Context, event gomatrixserverlib.PDU, r stateKey := *memberEvent.StateKey() fledglingEvent := &gomatrixserverlib.ProtoEvent{ - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), Type: spec.MRoomMember, StateKey: &stateKey, SenderID: stateKey, @@ -928,17 +919,7 @@ func (r *Inputer) kickGuests(ctx context.Context, event gomatrixserverlib.PDU, r return err } - validRoomID, err := spec.NewRoomID(event.RoomID()) - if err != nil { - return err - } - - userID, err := spec.NewUserID(stateKey, true) - if err != nil { - return err - } - - signingIdentity, err := r.SigningIdentity(ctx, *validRoomID, *userID) + signingIdentity, err := r.SigningIdentity(ctx, event.RoomID(), *memberUserID) if err != nil { return err } diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index 940783e03..ec03d6f13 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -197,7 +197,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { // send the event asynchronously but we would need to ensure that 1) the events are written to the log in // the correct order, 2) that pending writes are resent across restarts. In order to avoid writing all the // necessary bookkeeping we'll keep the event sending synchronous for now. - if err = u.api.OutputProducer.ProduceRoomEvents(u.event.RoomID(), updates); err != nil { + if err = u.api.OutputProducer.ProduceRoomEvents(u.event.RoomID().String(), updates); err != nil { return fmt.Errorf("u.api.WriteOutputEvents: %w", err) } @@ -290,7 +290,7 @@ func (u *latestEventsUpdater) latestState() error { if removed := len(u.removed) - len(u.added); !u.rewritesState && removed > 0 { logrus.WithFields(logrus.Fields{ "event_id": u.event.EventID(), - "room_id": u.event.RoomID(), + "room_id": u.event.RoomID().String(), "old_state_nid": u.oldStateNID, "new_state_nid": u.newStateNID, "old_latest": u.oldLatest.EventIDs(), diff --git a/roomserver/internal/input/input_membership.go b/roomserver/internal/input/input_membership.go index c46f8dba1..4cfc2cda9 100644 --- a/roomserver/internal/input/input_membership.go +++ b/roomserver/internal/input/input_membership.go @@ -139,11 +139,7 @@ func (r *Inputer) updateMembership( func (r *Inputer) isLocalTarget(ctx context.Context, event *types.Event) bool { isTargetLocalUser := false if statekey := event.StateKey(); statekey != nil { - validRoomID, err := spec.NewRoomID(event.RoomID()) - if err != nil { - return isTargetLocalUser - } - userID, err := r.Queryer.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*statekey)) + userID, err := r.Queryer.QueryUserIDForSender(ctx, event.RoomID(), spec.SenderID(*statekey)) if err != nil || userID == nil { return isTargetLocalUser } @@ -168,7 +164,7 @@ func updateToJoinMembership( Type: api.OutputTypeRetireInviteEvent, RetireInviteEvent: &api.OutputRetireInviteEvent{ EventID: eventID, - RoomID: add.RoomID(), + RoomID: add.RoomID().String(), Membership: spec.Join, RetiredByEventID: add.EventID(), TargetSenderID: spec.SenderID(*add.StateKey()), @@ -195,7 +191,7 @@ func updateToLeaveMembership( Type: api.OutputTypeRetireInviteEvent, RetireInviteEvent: &api.OutputRetireInviteEvent{ EventID: eventID, - RoomID: add.RoomID(), + RoomID: add.RoomID().String(), Membership: newMembership, RetiredByEventID: add.EventID(), TargetSenderID: spec.SenderID(*add.StateKey()), diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go index 5b4c0727b..21493287e 100644 --- a/roomserver/internal/input/input_missing.go +++ b/roomserver/internal/input/input_missing.go @@ -84,7 +84,7 @@ func (t *missingStateReq) processEventWithMissingState( // need to fallback to /state. t.log = util.GetLogger(ctx).WithFields(map[string]interface{}{ "txn_event": e.EventID(), - "room_id": e.RoomID(), + "room_id": e.RoomID().String(), "txn_prev_events": e.PrevEventIDs(), }) @@ -264,7 +264,7 @@ func (t *missingStateReq) lookupResolvedStateBeforeEvent(ctx context.Context, e // Look up what the state is after the backward extremity. This will either // come from the roomserver, if we know all the required events, or it will // come from a remote server via /state_ids if not. - prevState, trustworthy, err := t.lookupStateAfterEvent(ctx, roomVersion, e.RoomID(), prevEventID) + prevState, trustworthy, err := t.lookupStateAfterEvent(ctx, roomVersion, e.RoomID().String(), prevEventID) switch err2 := err.(type) { case gomatrixserverlib.EventValidationError: if !err2.Persistable { @@ -316,9 +316,9 @@ func (t *missingStateReq) lookupResolvedStateBeforeEvent(ctx context.Context, e } // There's more than one previous state - run them all through state res var err error - t.roomsMu.Lock(e.RoomID()) + t.roomsMu.Lock(e.RoomID().String()) resolvedState, err = t.resolveStatesAndCheck(ctx, roomVersion, respStates, e) - t.roomsMu.Unlock(e.RoomID()) + t.roomsMu.Unlock(e.RoomID().String()) switch err2 := err.(type) { case gomatrixserverlib.EventValidationError: if !err2.Persistable { @@ -498,6 +498,13 @@ func (t *missingStateReq) resolveStatesAndCheck(ctx context.Context, roomVersion roomVersion, gomatrixserverlib.ToPDUs(stateEventList), gomatrixserverlib.ToPDUs(authEventList), func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return t.inputer.Queryer.QueryUserIDForSender(ctx, roomID, senderID) }, + func(eventID string) bool { + isRejected, err := t.db.IsEventRejected(ctx, t.roomInfo.RoomNID, eventID) + if err != nil { + return true + } + return isRejected + }, ) if err != nil { return nil, err @@ -510,7 +517,7 @@ retryAllowedState: }); err != nil { switch missing := err.(type) { case gomatrixserverlib.MissingAuthEventError: - h, err2 := t.lookupEvent(ctx, roomVersion, backwardsExtremity.RoomID(), missing.AuthEventID, true) + h, err2 := t.lookupEvent(ctx, roomVersion, backwardsExtremity.RoomID().String(), missing.AuthEventID, true) switch e := err2.(type) { case gomatrixserverlib.EventValidationError: if !e.Persistable { @@ -546,7 +553,7 @@ func (t *missingStateReq) getMissingEvents(ctx context.Context, e gomatrixserver trace, ctx := internal.StartRegion(ctx, "getMissingEvents") defer trace.EndRegion() - logger := t.log.WithField("event_id", e.EventID()).WithField("room_id", e.RoomID()) + logger := t.log.WithField("event_id", e.EventID()).WithField("room_id", e.RoomID().String()) latest, _, _, err := t.db.LatestEventIDs(ctx, t.roomInfo.RoomNID) if err != nil { return nil, false, false, fmt.Errorf("t.DB.LatestEventIDs: %w", err) @@ -560,7 +567,7 @@ func (t *missingStateReq) getMissingEvents(ctx context.Context, e gomatrixserver var missingResp *fclient.RespMissingEvents for _, server := range t.servers { var m fclient.RespMissingEvents - if m, err = t.federation.LookupMissingEvents(ctx, t.virtualHost, server, e.RoomID(), fclient.MissingEvents{ + if m, err = t.federation.LookupMissingEvents(ctx, t.virtualHost, server, e.RoomID().String(), fclient.MissingEvents{ Limit: 20, // The latest event IDs that the sender already has. These are skipped when retrieving the previous events of latest_events. EarliestEvents: latestEvents, diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go index 12b557f51..ae203854b 100644 --- a/roomserver/internal/perform/perform_admin.go +++ b/roomserver/internal/perform/perform_admin.go @@ -161,12 +161,12 @@ func (r *Admin) PerformAdminEvacuateUser( return nil, fmt.Errorf("can only evacuate local users using this endpoint") } - roomIDs, err := r.DB.GetRoomsByMembership(ctx, userID, spec.Join) + roomIDs, err := r.DB.GetRoomsByMembership(ctx, *fullUserID, spec.Join) if err != nil { return nil, err } - inviteRoomIDs, err := r.DB.GetRoomsByMembership(ctx, userID, spec.Invite) + inviteRoomIDs, err := r.DB.GetRoomsByMembership(ctx, *fullUserID, spec.Invite) if err != nil && err != sql.ErrNoRows { return nil, err } @@ -204,18 +204,6 @@ func (r *Admin) PerformAdminPurgeRoom( return err } - // Evacuate the room before purging it from the database - evacAffected, err := r.PerformAdminEvacuateRoom(ctx, roomID) - if err != nil { - logrus.WithField("room_id", roomID).WithError(err).Warn("Failed to evacuate room before purging") - return err - } - - logrus.WithFields(logrus.Fields{ - "room_id": roomID, - "evacuated_users": len(evacAffected), - }).Warn("Evacuated room, purging room from roomserver now") - logrus.WithField("room_id", roomID).Warn("Purging room from roomserver") if err := r.DB.PurgeRoom(ctx, roomID); err != nil { logrus.WithField("room_id", roomID).WithError(err).Warn("Failed to purge room from roomserver") @@ -304,10 +292,12 @@ func (r *Admin) PerformAdminDownloadState( senderID, err := r.Queryer.QuerySenderIDForUser(ctx, *validRoomID, *fullUserID) if err != nil { return err + } else if senderID == nil { + return fmt.Errorf("sender ID not found for %s in %s", *fullUserID, *validRoomID) } proto := &gomatrixserverlib.ProtoEvent{ Type: "org.matrix.dendrite.state_download", - SenderID: string(senderID), + SenderID: string(*senderID), RoomID: roomID, Content: spec.RawJSON("{}"), } diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go index 33200e819..dafa58736 100644 --- a/roomserver/internal/perform/perform_backfill.go +++ b/roomserver/internal/perform/perform_backfill.go @@ -301,7 +301,7 @@ func (b *backfillRequester) StateIDsBeforeEvent(ctx context.Context, targetEvent return ids, nil } if len(targetEvent.PrevEventIDs()) == 0 && targetEvent.Type() == "m.room.create" && targetEvent.StateKeyEquals("") { - util.GetLogger(ctx).WithField("room_id", targetEvent.RoomID()).Info("Backfilled to the beginning of the room") + util.GetLogger(ctx).WithField("room_id", targetEvent.RoomID().String()).Info("Backfilled to the beginning of the room") b.eventIDToBeforeStateIDs[targetEvent.EventID()] = []string{} return nil, nil } @@ -494,11 +494,7 @@ FindSuccessor: // Store the server names in a temporary map to avoid duplicates. serverSet := make(map[spec.ServerName]bool) for _, event := range memberEvents { - validRoomID, err := spec.NewRoomID(event.RoomID()) - if err != nil { - continue - } - if sender, err := b.querier.QueryUserIDForSender(ctx, *validRoomID, event.SenderID()); err == nil { + if sender, err := b.querier.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID()); err == nil { serverSet[sender.Domain()] = true } } diff --git a/roomserver/internal/perform/perform_create_room.go b/roomserver/internal/perform/perform_create_room.go index 12e756c2e..eb8de7811 100644 --- a/roomserver/internal/perform/perform_create_room.go +++ b/roomserver/internal/perform/perform_create_room.go @@ -90,7 +90,16 @@ func (c *Creator) PerformCreateRoom(ctx context.Context, userID spec.UserID, roo } else { senderID = spec.SenderID(userID.String()) } - createContent["creator"] = senderID + + // TODO: Maybe, at some point, GMSL should return the events to create, so we can define the version + // entirely there. + switch createRequest.RoomVersion { + case gomatrixserverlib.RoomVersionV11: + // RoomVersionV11 removed the creator field from the create content: https://github.com/matrix-org/matrix-spec-proposals/pull/2175 + default: + createContent["creator"] = senderID + } + createContent["room_version"] = createRequest.RoomVersion powerLevelContent := eventutil.InitialPowerLevelsContent(string(senderID)) joinRuleContent := gomatrixserverlib.JoinRuleContent{ @@ -433,23 +442,16 @@ func (c *Creator) PerformCreateRoom(ctx context.Context, userID spec.UserID, roo // from creating the room but still failing due to the alias having already // been taken. if roomAlias != "" { - aliasReq := api.SetRoomAliasRequest{ - Alias: roomAlias, - RoomID: roomID.String(), - UserID: userID.String(), - } - - var aliasResp api.SetRoomAliasResponse - err = c.RSAPI.SetRoomAlias(ctx, &aliasReq, &aliasResp) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("aliasAPI.SetRoomAlias failed") + aliasAlreadyExists, aliasErr := c.RSAPI.SetRoomAlias(ctx, senderID, roomID, roomAlias) + if aliasErr != nil { + util.GetLogger(ctx).WithError(aliasErr).Error("aliasAPI.SetRoomAlias failed") return "", &util.JSONResponse{ Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}, } } - if aliasResp.AliasExists { + if aliasAlreadyExists { return "", &util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.RoomInUse("Room alias already exists."), diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go index 278ddd7d8..3abb69cb9 100644 --- a/roomserver/internal/perform/perform_invite.go +++ b/roomserver/internal/perform/perform_invite.go @@ -100,16 +100,12 @@ func (r *Inviter) ProcessInviteMembership( var outputUpdates []api.OutputEvent var updater *shared.MembershipUpdater - validRoomID, err := spec.NewRoomID(inviteEvent.RoomID()) - if err != nil { - return nil, err - } - userID, err := r.RSAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*inviteEvent.StateKey())) + userID, err := r.RSAPI.QueryUserIDForSender(ctx, inviteEvent.RoomID(), spec.SenderID(*inviteEvent.StateKey())) if err != nil { return nil, api.ErrInvalidID{Err: fmt.Errorf("the user ID %s is invalid", *inviteEvent.StateKey())} } isTargetLocal := r.Cfg.Matrix.IsLocalServerName(userID.Domain()) - if updater, err = r.DB.MembershipUpdater(ctx, inviteEvent.RoomID(), *inviteEvent.StateKey(), isTargetLocal, inviteEvent.Version()); err != nil { + if updater, err = r.DB.MembershipUpdater(ctx, inviteEvent.RoomID().String(), *inviteEvent.StateKey(), isTargetLocal, inviteEvent.Version()); err != nil { return nil, fmt.Errorf("r.DB.MembershipUpdater: %w", err) } outputUpdates, err = helpers.UpdateToInviteMembership(updater, &types.Event{ @@ -133,6 +129,8 @@ func (r *Inviter) PerformInvite( senderID, err := r.RSAPI.QuerySenderIDForUser(ctx, req.InviteInput.RoomID, req.InviteInput.Inviter) if err != nil { return err + } else if senderID == nil { + return fmt.Errorf("sender ID not found for %s in %s", req.InviteInput.Inviter, req.InviteInput.RoomID) } info, err := r.DB.RoomInfo(ctx, req.InviteInput.RoomID.String()) if err != nil { @@ -140,7 +138,7 @@ func (r *Inviter) PerformInvite( } proto := gomatrixserverlib.ProtoEvent{ - SenderID: string(senderID), + SenderID: string(*senderID), RoomID: req.InviteInput.RoomID.String(), Type: "m.room.member", } @@ -187,7 +185,7 @@ func (r *Inviter) PerformInvite( UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return r.RSAPI.QueryUserIDForSender(ctx, roomID, senderID) }, - SenderIDQuerier: func(roomID spec.RoomID, userID spec.UserID) (spec.SenderID, error) { + SenderIDQuerier: func(roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { return r.RSAPI.QuerySenderIDForUser(ctx, roomID, userID) }, SenderIDCreator: func(ctx context.Context, userID spec.UserID, roomID spec.RoomID, roomVersion string) (spec.SenderID, ed25519.PrivateKey, error) { diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go index dfce9cc77..23988e467 100644 --- a/roomserver/internal/perform/perform_join.go +++ b/roomserver/internal/perform/perform_join.go @@ -201,11 +201,11 @@ func (r *Joiner) performJoinRoomByID( if err == nil && info != nil { switch info.RoomVersion { case gomatrixserverlib.RoomVersionPseudoIDs: - senderID, err = r.Queryer.QuerySenderIDForUser(ctx, *roomID, *userID) - if err == nil { + senderIDPtr, queryErr := r.Queryer.QuerySenderIDForUser(ctx, *roomID, *userID) + if queryErr == nil { checkInvitePending = true } - if senderID == "" { + if senderIDPtr == nil { // create user room key if needed key, keyErr := r.RSAPI.GetOrCreateUserRoomPrivateKey(ctx, *userID, *roomID) if keyErr != nil { @@ -213,6 +213,8 @@ func (r *Joiner) performJoinRoomByID( return "", "", fmt.Errorf("GetOrCreateUserRoomPrivateKey failed: %w", keyErr) } senderID = spec.SenderIDFromPseudoIDKey(key) + } else { + senderID = *senderIDPtr } default: checkInvitePending = true diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go index a20896cf7..5bea00445 100644 --- a/roomserver/internal/perform/perform_leave.go +++ b/roomserver/internal/perform/perform_leave.go @@ -73,6 +73,7 @@ func (r *Leaver) PerformLeave( return nil, fmt.Errorf("room ID %q is invalid", req.RoomID) } +// nolint:gocyclo func (r *Leaver) performLeaveRoomByID( ctx context.Context, req *api.PerformLeaveRequest, @@ -83,20 +84,30 @@ func (r *Leaver) performLeaveRoomByID( return nil, err } leaver, err := r.RSAPI.QuerySenderIDForUser(ctx, *roomID, req.Leaver) - if err != nil { + if err != nil || leaver == nil { return nil, fmt.Errorf("leaver %s has no matching senderID in this room", req.Leaver.String()) } // If there's an invite outstanding for the room then respond to // that. - isInvitePending, senderUser, eventID, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, leaver) + isInvitePending, senderUser, eventID, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, *leaver) if err == nil && isInvitePending { sender, serr := r.RSAPI.QueryUserIDForSender(ctx, *roomID, senderUser) - if serr != nil || sender == nil { - return nil, fmt.Errorf("sender %q has no matching userID", senderUser) + if serr != nil { + return nil, fmt.Errorf("failed looking up userID for sender %q: %w", senderUser, serr) } - if !r.Cfg.Matrix.IsLocalServerName(sender.Domain()) { - return r.performFederatedRejectInvite(ctx, req, res, *sender, eventID, leaver) + + var domain spec.ServerName + if sender == nil { + // TODO: Currently a federated invite has no way of knowing the mxid_mapping of the inviter. + // Should we add the inviter's m.room.member event (with mxid_mapping) to invite_room_state to allow + // the invited user to leave via the inviter's server? + domain = roomID.Domain() + } else { + domain = sender.Domain() + } + if !r.Cfg.Matrix.IsLocalServerName(domain) { + return r.performFederatedRejectInvite(ctx, req, res, domain, eventID, *leaver) } // check that this is not a "server notice room" accData := &userapi.QueryAccountDataResponse{} @@ -132,7 +143,7 @@ func (r *Leaver) performLeaveRoomByID( StateToFetch: []gomatrixserverlib.StateKeyTuple{ { EventType: spec.MRoomMember, - StateKey: string(leaver), + StateKey: string(*leaver), }, }, } @@ -157,7 +168,7 @@ func (r *Leaver) performLeaveRoomByID( } // Prepare the template for the leave event. - senderIDString := string(leaver) + senderIDString := string(*leaver) proto := gomatrixserverlib.ProtoEvent{ Type: spec.MRoomMember, SenderID: senderIDString, @@ -218,14 +229,14 @@ func (r *Leaver) performFederatedRejectInvite( ctx context.Context, req *api.PerformLeaveRequest, res *api.PerformLeaveResponse, // nolint:unparam - inviteSender spec.UserID, eventID string, + inviteDomain spec.ServerName, eventID string, leaver spec.SenderID, ) ([]api.OutputEvent, error) { // Ask the federation sender to perform a federated leave for us. leaveReq := fsAPI.PerformLeaveRequest{ RoomID: req.RoomID, UserID: req.Leaver.String(), - ServerNames: []spec.ServerName{inviteSender.Domain()}, + ServerNames: []spec.ServerName{inviteDomain}, } leaveRes := fsAPI.PerformLeaveResponse{} if err := r.FSAPI.PerformLeave(ctx, &leaveReq, &leaveRes); err != nil { diff --git a/roomserver/internal/perform/perform_upgrade.go b/roomserver/internal/perform/perform_upgrade.go index 32f547dc1..9d7c7b567 100644 --- a/roomserver/internal/perform/perform_upgrade.go +++ b/roomserver/internal/perform/perform_upgrade.go @@ -62,10 +62,13 @@ func (r *Upgrader) performRoomUpgrade( if err != nil { util.GetLogger(ctx).WithError(err).Error("Failed getting senderID for user") return "", err + } else if senderID == nil { + util.GetLogger(ctx).WithField("userID", userID).WithField("roomID", *fullRoomID).Error("No senderID for user") + return "", fmt.Errorf("No sender ID for %s in %s", userID, *fullRoomID) } // 1. Check if the user is authorized to actually perform the upgrade (can send m.room.tombstone) - if !r.userIsAuthorized(ctx, senderID, roomID) { + if !r.userIsAuthorized(ctx, *senderID, roomID) { return "", api.ErrNotAllowed{Err: fmt.Errorf("You don't have permission to upgrade the room, power level too low.")} } @@ -83,20 +86,20 @@ func (r *Upgrader) performRoomUpgrade( } // Make the tombstone event - tombstoneEvent, pErr := r.makeTombstoneEvent(ctx, evTime, senderID, userID.Domain(), roomID, newRoomID) + tombstoneEvent, pErr := r.makeTombstoneEvent(ctx, evTime, *senderID, userID.Domain(), roomID, newRoomID) if pErr != nil { return "", pErr } // Generate the initial events we need to send into the new room. This includes copied state events and bans // as well as the power level events needed to set up the room - eventsToMake, pErr := r.generateInitialEvents(ctx, oldRoomRes, senderID, roomID, roomVersion, tombstoneEvent) + eventsToMake, pErr := r.generateInitialEvents(ctx, oldRoomRes, *senderID, roomID, roomVersion, tombstoneEvent) if pErr != nil { return "", pErr } // Send the setup events to the new room - if pErr = r.sendInitialEvents(ctx, evTime, senderID, userID.Domain(), newRoomID, roomVersion, eventsToMake); pErr != nil { + if pErr = r.sendInitialEvents(ctx, evTime, *senderID, userID.Domain(), newRoomID, roomVersion, eventsToMake); pErr != nil { return "", pErr } @@ -111,17 +114,17 @@ func (r *Upgrader) performRoomUpgrade( } // If the old room had a canonical alias event, it should be deleted in the old room - if pErr = r.clearOldCanonicalAliasEvent(ctx, oldRoomRes, evTime, senderID, userID.Domain(), roomID); pErr != nil { + if pErr = r.clearOldCanonicalAliasEvent(ctx, oldRoomRes, evTime, *senderID, userID.Domain(), roomID); pErr != nil { return "", pErr } // 4. Move local aliases to the new room - if pErr = moveLocalAliases(ctx, roomID, newRoomID, senderID, userID, r.URSAPI); pErr != nil { + if pErr = moveLocalAliases(ctx, roomID, newRoomID, *senderID, r.URSAPI); pErr != nil { return "", pErr } // 6. Restrict power levels in the old room - if pErr = r.restrictOldRoomPowerLevels(ctx, evTime, senderID, userID.Domain(), roomID); pErr != nil { + if pErr = r.restrictOldRoomPowerLevels(ctx, evTime, *senderID, userID.Domain(), roomID); pErr != nil { return "", pErr } @@ -171,7 +174,7 @@ func (r *Upgrader) restrictOldRoomPowerLevels(ctx context.Context, evTime time.T } func moveLocalAliases(ctx context.Context, - roomID, newRoomID string, senderID spec.SenderID, userID spec.UserID, + roomID, newRoomID string, senderID spec.SenderID, URSAPI api.RoomserverInternalAPI, ) (err error) { @@ -181,17 +184,27 @@ func moveLocalAliases(ctx context.Context, return fmt.Errorf("Failed to get old room aliases: %w", err) } + // TODO: this should be spec.RoomID further up the call stack + parsedNewRoomID, err := spec.NewRoomID(newRoomID) + if err != nil { + return err + } + for _, alias := range aliasRes.Aliases { - removeAliasReq := api.RemoveRoomAliasRequest{SenderID: senderID, Alias: alias} - removeAliasRes := api.RemoveRoomAliasResponse{} - if err = URSAPI.RemoveRoomAlias(ctx, &removeAliasReq, &removeAliasRes); err != nil { + aliasFound, aliasRemoved, err := URSAPI.RemoveRoomAlias(ctx, senderID, alias) + if err != nil { return fmt.Errorf("Failed to remove old room alias: %w", err) + } else if !aliasFound { + return fmt.Errorf("Failed to remove old room alias: alias not found, possible race") + } else if !aliasRemoved { + return fmt.Errorf("Failed to remove old alias") } - setAliasReq := api.SetRoomAliasRequest{UserID: userID.String(), Alias: alias, RoomID: newRoomID} - setAliasRes := api.SetRoomAliasResponse{} - if err = URSAPI.SetRoomAlias(ctx, &setAliasReq, &setAliasRes); err != nil { + aliasAlreadyExists, err := URSAPI.SetRoomAlias(ctx, senderID, *parsedNewRoomID, alias) + if err != nil { return fmt.Errorf("Failed to set new room alias: %w", err) + } else if aliasAlreadyExists { + return fmt.Errorf("Failed to set new room alias: alias exists when it should have just been removed") } } return nil @@ -355,7 +368,16 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query // in the create event (such as for the room types MSC). newCreateContent := map[string]interface{}{} _ = json.Unmarshal(oldCreateEvent.Content(), &newCreateContent) - newCreateContent["creator"] = string(senderID) + + switch newVersion { + case gomatrixserverlib.RoomVersionV11: + // RoomVersionV11 removed the creator field from the create content: https://github.com/matrix-org/matrix-spec-proposals/pull/2175 + // So if we are upgrading from pre v11, we need to remove the field. + delete(newCreateContent, "creator") + default: + newCreateContent["creator"] = senderID + } + newCreateContent["room_version"] = newVersion newCreateContent["predecessor"] = gomatrixserverlib.PreviousRoom{ EventID: tombstoneEvent.EventID(), diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 11e5564dc..74b010281 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -165,6 +165,13 @@ func (r *Queryer) QueryStateAfterEvents( info.RoomVersion, gomatrixserverlib.ToPDUs(stateEvents), gomatrixserverlib.ToPDUs(authEvents), func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return r.QueryUserIDForSender(ctx, roomID, senderID) }, + func(eventID string) bool { + isRejected, rejectedErr := r.DB.IsEventRejected(ctx, info.RoomNID, eventID) + if rejectedErr != nil { + return true + } + return isRejected + }, ) if err != nil { return fmt.Errorf("state.ResolveConflictsAdhoc: %w", err) @@ -230,6 +237,33 @@ func (r *Queryer) QueryMembershipForSenderID( senderID spec.SenderID, response *api.QueryMembershipForUserResponse, ) error { + return r.queryMembershipForOptionalSenderID(ctx, roomID, &senderID, response) +} + +// QueryMembershipForUser implements api.RoomserverInternalAPI +func (r *Queryer) QueryMembershipForUser( + ctx context.Context, + request *api.QueryMembershipForUserRequest, + response *api.QueryMembershipForUserResponse, +) error { + roomID, err := spec.NewRoomID(request.RoomID) + if err != nil { + return err + } + senderID, err := r.QuerySenderIDForUser(ctx, *roomID, request.UserID) + if err != nil { + return err + } + + return r.queryMembershipForOptionalSenderID(ctx, *roomID, senderID, response) +} + +// Query membership information for provided sender ID and room ID +// +// If sender ID is nil, then act as if the provided sender is not a member of the room. +func (r *Queryer) queryMembershipForOptionalSenderID(ctx context.Context, roomID spec.RoomID, senderID *spec.SenderID, response *api.QueryMembershipForUserResponse) error { + response.SenderID = senderID + info, err := r.DB.RoomInfo(ctx, roomID.String()) if err != nil { return err @@ -240,7 +274,11 @@ func (r *Queryer) QueryMembershipForSenderID( } response.RoomExists = true - membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, senderID) + if senderID == nil { + return nil + } + + membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, *senderID) if err != nil { return err } @@ -268,70 +306,55 @@ func (r *Queryer) QueryMembershipForSenderID( return err } -// QueryMembershipForUser implements api.RoomserverInternalAPI -func (r *Queryer) QueryMembershipForUser( - ctx context.Context, - request *api.QueryMembershipForUserRequest, - response *api.QueryMembershipForUserResponse, -) error { - roomID, err := spec.NewRoomID(request.RoomID) - if err != nil { - return err - } - senderID, err := r.QuerySenderIDForUser(ctx, *roomID, request.UserID) - if err != nil { - return err - } - - return r.QueryMembershipForSenderID(ctx, *roomID, senderID, response) -} - // QueryMembershipAtEvent returns the known memberships at a given event. // If the state before an event is not known, an empty list will be returned // for that event instead. +// +// Returned map from eventID to membership event. Events that +// do not have known state will return a nil event, resulting in a "leave" membership +// when calculating history visibility. func (r *Queryer) QueryMembershipAtEvent( ctx context.Context, - request *api.QueryMembershipAtEventRequest, - response *api.QueryMembershipAtEventResponse, -) error { - response.Membership = make(map[string]*types.HeaderedEvent) - - info, err := r.DB.RoomInfo(ctx, request.RoomID) + roomID spec.RoomID, + eventIDs []string, + senderID spec.SenderID, +) (map[string]*types.HeaderedEvent, error) { + info, err := r.DB.RoomInfo(ctx, roomID.String()) if err != nil { - return fmt.Errorf("unable to get roomInfo: %w", err) + return nil, fmt.Errorf("unable to get roomInfo: %w", err) } if info == nil { - return fmt.Errorf("no roomInfo found") + return nil, fmt.Errorf("no roomInfo found") } // get the users stateKeyNID - stateKeyNIDs, err := r.DB.EventStateKeyNIDs(ctx, []string{request.UserID}) + stateKeyNIDs, err := r.DB.EventStateKeyNIDs(ctx, []string{string(senderID)}) if err != nil { - return fmt.Errorf("unable to get stateKeyNIDs for %s: %w", request.UserID, err) + return nil, fmt.Errorf("unable to get stateKeyNIDs for %s: %w", senderID, err) } - if _, ok := stateKeyNIDs[request.UserID]; !ok { - return fmt.Errorf("requested stateKeyNID for %s was not found", request.UserID) + if _, ok := stateKeyNIDs[string(senderID)]; !ok { + return nil, fmt.Errorf("requested stateKeyNID for %s was not found", senderID) } - response.Membership, err = r.DB.GetMembershipForHistoryVisibility(ctx, stateKeyNIDs[request.UserID], info, request.EventIDs...) + eventIDMembershipMap, err := r.DB.GetMembershipForHistoryVisibility(ctx, stateKeyNIDs[string(senderID)], info, eventIDs...) switch err { case nil: - return nil + return eventIDMembershipMap, nil case tables.OptimisationNotSupportedError: // fallthrough, slow way of getting the membership events for each event default: - return err + return eventIDMembershipMap, err } - response.Membership = make(map[string]*types.HeaderedEvent) - stateEntries, err := helpers.MembershipAtEvent(ctx, r.DB, nil, request.EventIDs, stateKeyNIDs[request.UserID], r) + eventIDMembershipMap = make(map[string]*types.HeaderedEvent) + stateEntries, err := helpers.MembershipAtEvent(ctx, r.DB, nil, eventIDs, stateKeyNIDs[string(senderID)], r) if err != nil { - return fmt.Errorf("unable to get state before event: %w", err) + return eventIDMembershipMap, fmt.Errorf("unable to get state before event: %w", err) } // If we only have one or less state entries, we can short circuit the below // loop and avoid hitting the database allStateEventNIDs := make(map[types.EventNID]types.StateEntry) - for _, eventID := range request.EventIDs { + for _, eventID := range eventIDs { stateEntry := stateEntries[eventID] for _, s := range stateEntry { allStateEventNIDs[s.EventNID] = s @@ -344,10 +367,10 @@ func (r *Queryer) QueryMembershipAtEvent( } var memberships []types.Event - for _, eventID := range request.EventIDs { + for _, eventID := range eventIDs { stateEntry, ok := stateEntries[eventID] if !ok || len(stateEntry) == 0 { - response.Membership[eventID] = nil + eventIDMembershipMap[eventID] = nil continue } @@ -361,7 +384,7 @@ func (r *Queryer) QueryMembershipAtEvent( memberships, err = helpers.GetMembershipsAtState(ctx, r.DB, info, stateEntry, false) } if err != nil { - return fmt.Errorf("unable to get memberships at state: %w", err) + return eventIDMembershipMap, fmt.Errorf("unable to get memberships at state: %w", err) } // Iterate over all membership events we got. Given we only query the membership for @@ -369,13 +392,13 @@ func (r *Queryer) QueryMembershipAtEvent( // a given event, overwrite any other existing membership events. for i := range memberships { ev := memberships[i] - if ev.Type() == spec.MRoomMember && ev.StateKeyEquals(request.UserID) { - response.Membership[eventID] = &types.HeaderedEvent{PDU: ev.PDU} + if ev.Type() == spec.MRoomMember && ev.StateKeyEquals(string(senderID)) { + eventIDMembershipMap[eventID] = &types.HeaderedEvent{PDU: ev.PDU} } } } - return nil + return eventIDMembershipMap, nil } // QueryMembershipsForRoom implements api.RoomserverInternalAPI @@ -660,6 +683,13 @@ func (r *Queryer) QueryStateAndAuthChain( info.RoomVersion, gomatrixserverlib.ToPDUs(stateEvents), gomatrixserverlib.ToPDUs(authEvents), func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return r.QueryUserIDForSender(ctx, roomID, senderID) }, + func(eventID string) bool { + isRejected, rejectedErr := r.DB.IsEventRejected(ctx, info.RoomNID, eventID) + if rejectedErr != nil { + return true + } + return isRejected + }, ) if err != nil { return err @@ -830,13 +860,20 @@ func (r *Queryer) QueryCurrentState(ctx context.Context, req *api.QueryCurrentSt return nil } -func (r *Queryer) QueryRoomsForUser(ctx context.Context, req *api.QueryRoomsForUserRequest, res *api.QueryRoomsForUserResponse) error { - roomIDs, err := r.DB.GetRoomsByMembership(ctx, req.UserID, req.WantMembership) +func (r *Queryer) QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) { + roomIDStrs, err := r.DB.GetRoomsByMembership(ctx, userID, desiredMembership) if err != nil { - return err + return nil, err } - res.RoomIDs = roomIDs - return nil + roomIDs := make([]spec.RoomID, len(roomIDStrs)) + for i, roomIDStr := range roomIDStrs { + roomID, err := spec.NewRoomID(roomIDStr) + if err != nil { + return nil, err + } + roomIDs[i] = *roomID + } + return roomIDs, nil } func (r *Queryer) QueryKnownUsers(ctx context.Context, req *api.QueryKnownUsersRequest, res *api.QueryKnownUsersResponse) error { @@ -879,7 +916,12 @@ func (r *Queryer) QueryLeftUsers(ctx context.Context, req *api.QueryLeftUsersReq } func (r *Queryer) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUsersRequest, res *api.QuerySharedUsersResponse) error { - roomIDs, err := r.DB.GetRoomsByMembership(ctx, req.UserID, "join") + parsedUserID, err := spec.NewUserID(req.UserID, true) + if err != nil { + return err + } + + roomIDs, err := r.DB.GetRoomsByMembership(ctx, *parsedUserID, "join") if err != nil { return err } @@ -1009,21 +1051,26 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, roomID spec.Ro return verImpl.CheckRestrictedJoin(ctx, r.Cfg.Global.ServerName, &api.JoinRoomQuerier{Roomserver: r}, roomID, senderID) } -func (r *Queryer) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (spec.SenderID, error) { +func (r *Queryer) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { version, err := r.DB.GetRoomVersion(ctx, roomID.String()) if err != nil { - return "", err + return nil, err } switch version { case gomatrixserverlib.RoomVersionPseudoIDs: key, err := r.DB.SelectUserRoomPublicKey(ctx, userID, roomID) if err != nil { - return "", err + return nil, err + } else if key == nil { + return nil, nil + } else { + senderID := spec.SenderID(spec.Base64Bytes(key).Encode()) + return &senderID, nil } - return spec.SenderID(spec.Base64Bytes(key).Encode()), nil default: - return spec.SenderID(userID.String()), nil + senderID := spec.SenderID(userID.String()) + return &senderID, nil } } diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go index 7274be520..5f55980f0 100644 --- a/roomserver/internal/query/query_room_hierarchy.go +++ b/roomserver/internal/query/query_room_hierarchy.go @@ -17,6 +17,7 @@ package query import ( "context" "encoding/json" + "errors" "fmt" "sort" @@ -56,6 +57,12 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r break } + // If the context is canceled, we might still have discovered rooms + // return them to the client and let the client know there _may_ be more rooms. + if errors.Is(ctx.Err(), context.Canceled) { + break + } + // pop the stack queuedRoom := unvisited[len(unvisited)-1] unvisited = unvisited[:len(unvisited)-1] @@ -112,6 +119,11 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r pubRoom := publicRoomsChunk(ctx, querier, queuedRoom.RoomID) + if pubRoom == nil { + util.GetLogger(ctx).WithField("room_id", queuedRoom.RoomID).Debug("unable to get publicRoomsChunk") + continue + } + discoveredRooms = append(discoveredRooms, fclient.RoomHierarchyRoom{ PublicRoom: *pubRoom, RoomType: roomType, @@ -513,14 +525,14 @@ func restrictedJoinRuleAllowedRooms(ctx context.Context, joinRuleEv *types.Heade } var jrContent gomatrixserverlib.JoinRuleContent if err := json.Unmarshal(joinRuleEv.Content(), &jrContent); err != nil { - util.GetLogger(ctx).Warnf("failed to check join_rule on room %s: %s", joinRuleEv.RoomID(), err) + util.GetLogger(ctx).Warnf("failed to check join_rule on room %s: %s", joinRuleEv.RoomID().String(), err) return nil } for _, allow := range jrContent.Allow { if allow.Type == spec.MRoomMembership { allowedRoomID, err := spec.NewRoomID(allow.RoomID) if err != nil { - util.GetLogger(ctx).Warnf("invalid room ID '%s' found in join_rule on room %s: %s", allow.RoomID, joinRuleEv.RoomID(), err) + util.GetLogger(ctx).Warnf("invalid room ID '%s' found in join_rule on room %s: %s", allow.RoomID, joinRuleEv.RoomID().String(), err) } else { allows = append(allows, *allowedRoomID) } diff --git a/roomserver/internal/query/query_test.go b/roomserver/internal/query/query_test.go index 619d93030..296960b2f 100644 --- a/roomserver/internal/query/query_test.go +++ b/roomserver/internal/query/query_test.go @@ -49,6 +49,7 @@ func (db *getEventDB) addFakeEvent(eventID string, authIDs []string) error { } builder := map[string]interface{}{ "event_id": eventID, + "room_id": "!room:a", "auth_events": authEvents, } diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index ce0721bea..218a0d8a9 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -7,11 +7,11 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/stretchr/testify/assert" "github.com/tidwall/gjson" @@ -35,6 +35,10 @@ import ( "github.com/matrix-org/dendrite/test/testrig" ) +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + type FakeQuerier struct { api.QuerySenderIDAPI } @@ -59,7 +63,7 @@ func TestUsers(t *testing.T) { }) t.Run("kick users", func(t *testing.T) { - usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) rsAPI.SetUserAPI(usrAPI) testKickUsers(t, rsAPI, usrAPI) }) @@ -227,6 +231,11 @@ func TestPurgeRoom(t *testing.T) { bob := test.NewUser(t) room := test.NewRoom(t, alice, test.RoomPreset(test.PresetTrustedPrivateChat)) + roomID, err := spec.NewRoomID(room.ID) + if err != nil { + t.Fatal(err) + } + // Invite Bob inviteEvent := room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{ "membership": "invite", @@ -254,7 +263,7 @@ func TestPurgeRoom(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, fsAPI.IsBlacklistedOrBackingOff) syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) // Create the room @@ -274,9 +283,7 @@ func TestPurgeRoom(t *testing.T) { if !isPublished { t.Fatalf("room should be published before purging") } - - aliasResp := &api.SetRoomAliasResponse{} - if err = rsAPI.SetRoomAlias(ctx, &api.SetRoomAliasRequest{RoomID: room.ID, Alias: "myalias", UserID: alice.ID}, aliasResp); err != nil { + if _, err = rsAPI.SetRoomAlias(ctx, spec.SenderID(alice.ID), *roomID, "myalias"); err != nil { t.Fatal(err) } // check the alias is actually there @@ -405,7 +412,9 @@ type fledglingEvent struct { RoomID string Redacts string Depth int64 - PrevEvents []interface{} + PrevEvents []any + AuthEvents []any + Content map[string]any } func mustCreateEvent(t *testing.T, ev fledglingEvent) (result *types.HeaderedEvent) { @@ -422,7 +431,13 @@ func mustCreateEvent(t *testing.T, ev fledglingEvent) (result *types.HeaderedEve Depth: ev.Depth, PrevEvents: ev.PrevEvents, }) - err := eb.SetContent(map[string]interface{}{}) + if ev.Content == nil { + ev.Content = map[string]any{} + } + if ev.AuthEvents != nil { + eb.AuthEvents = ev.AuthEvents + } + err := eb.SetContent(ev.Content) if err != nil { t.Fatalf("mustCreateEvent: failed to marshal event content %v", err) } @@ -930,14 +945,17 @@ func TestUpgrade(t *testing.T) { upgradeUser: alice.ID, roomFunc: func(rsAPI api.RoomserverInternalAPI) string { r := test.NewRoom(t, alice) + roomID, err := spec.NewRoomID(r.ID) + if err != nil { + t.Fatal(err) + } if err := api.SendEvents(ctx, rsAPI, api.KindNew, r.Events(), "test", "test", "test", nil, false); err != nil { t.Errorf("failed to send events: %v", err) } - if err := rsAPI.SetRoomAlias(ctx, &api.SetRoomAliasRequest{ - RoomID: r.ID, - Alias: "#myroomalias:test", - }, &api.SetRoomAliasResponse{}); err != nil { + if _, err := rsAPI.SetRoomAlias(ctx, spec.SenderID(alice.ID), + *roomID, + "#myroomalias:test"); err != nil { t.Fatal(err) } @@ -1037,7 +1055,7 @@ func TestUpgrade(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) rsAPI.SetUserAPI(userAPI) for _, tc := range testCases { @@ -1054,7 +1072,7 @@ func TestUpgrade(t *testing.T) { if err != nil { t.Fatalf("upgrade userID is invalid") } - newRoomID, err := rsAPI.PerformRoomUpgrade(processCtx.Context(), roomID, *userID, version.DefaultRoomVersion()) + newRoomID, err := rsAPI.PerformRoomUpgrade(processCtx.Context(), roomID, *userID, rsAPI.DefaultRoomVersion()) if err != nil && tc.wantNewRoom { t.Fatal(err) } @@ -1072,3 +1090,103 @@ func TestUpgrade(t *testing.T) { } }) } + +func TestStateReset(t *testing.T) { + alice := test.NewUser(t) + bob := test.NewUser(t) + charlie := test.NewUser(t) + ctx := context.Background() + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + // Prepare APIs + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + defer close() + + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + natsInstance := jetstream.NATSInstance{} + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + + // create a new room + room := test.NewRoom(t, alice, test.RoomPreset(test.PresetPublicChat)) + + // join with Bob and Charlie + bobJoinEv := room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]any{"membership": "join"}, test.WithStateKey(bob.ID)) + charlieJoinEv := room.CreateAndInsert(t, charlie, spec.MRoomMember, map[string]any{"membership": "join"}, test.WithStateKey(charlie.ID)) + + // Send and create the room + if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // send a message + bobMsg := room.CreateAndInsert(t, bob, "m.room.message", map[string]any{"body": "hello world"}) + charlieMsg := room.CreateAndInsert(t, charlie, "m.room.message", map[string]any{"body": "hello world"}) + + if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{bobMsg, charlieMsg}, "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // Bob changes his name + expectedDisplayname := "Bob!" + bobDisplayname := room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]any{"membership": "join", "displayname": expectedDisplayname}, test.WithStateKey(bob.ID)) + + if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{bobDisplayname}, "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // Change another state event + jrEv := room.CreateAndInsert(t, alice, spec.MRoomJoinRules, gomatrixserverlib.JoinRuleContent{JoinRule: "invite"}, test.WithStateKey("")) + if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{jrEv}, "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // send a message + bobMsg = room.CreateAndInsert(t, bob, "m.room.message", map[string]any{"body": "hello world"}) + charlieMsg = room.CreateAndInsert(t, charlie, "m.room.message", map[string]any{"body": "hello world"}) + + if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{bobMsg, charlieMsg}, "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // Craft the state reset message, which is using Bobs initial join event and the + // last message Charlie sent as the prev_events. This should trigger the recalculation + // of the "current" state, since the message event does not have state and no missing events in the DB. + stateResetMsg := mustCreateEvent(t, fledglingEvent{ + Type: "m.room.message", + SenderID: charlie.ID, + RoomID: room.ID, + Depth: charlieMsg.Depth() + 1, + PrevEvents: []any{ + bobJoinEv.EventID(), + charlieMsg.EventID(), + }, + AuthEvents: []any{ + room.Events()[0].EventID(), // create event + room.Events()[2].EventID(), // PL event + charlieJoinEv.EventID(), // Charlie join event + }, + }) + + // Send the state reset message + if err := api.SendEvents(ctx, rsAPI, api.KindNew, []*types.HeaderedEvent{stateResetMsg}, "test", "test", "test", nil, false); err != nil { + t.Errorf("failed to send events: %v", err) + } + + // Validate that there is a membership event for Bob + bobMembershipEv := api.GetStateEvent(ctx, rsAPI, room.ID, gomatrixserverlib.StateKeyTuple{ + EventType: spec.MRoomMember, + StateKey: bob.ID, + }) + + if bobMembershipEv == nil { + t.Fatalf("Membership event for Bob does not exist. State reset?") + } else { + // Validate it's the correct membership event + if dn := gjson.GetBytes(bobMembershipEv.Content(), "displayname").Str; dn != expectedDisplayname { + t.Fatalf("Expected displayname to be %q, got %q", expectedDisplayname, dn) + } + } + }) +} diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 1e776ff6c..dfd439a22 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -45,6 +45,7 @@ type StateResolutionStorage interface { AddState(ctx context.Context, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, state []types.StateEntry) (types.StateSnapshotNID, error) Events(ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, eventNIDs []types.EventNID) ([]types.Event, error) EventsFromIDs(ctx context.Context, roomInfo *types.RoomInfo, eventIDs []string) ([]types.Event, error) + IsEventRejected(ctx context.Context, roomNID types.RoomNID, eventID string) (bool, error) } type StateResolution struct { @@ -1066,6 +1067,13 @@ func (v *StateResolution) resolveConflictsV2( func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return v.Querier.QueryUserIDForSender(ctx, roomID, senderID) }, + func(eventID string) bool { + isRejected, err := v.db.IsEventRejected(ctx, v.roomInfo.RoomNID, eventID) + if err != nil { + return true + } + return isRejected + }, ) }() diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index e9b4609ec..0638252b2 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -158,7 +158,7 @@ type Database interface { GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*types.HeaderedEvent, error) GetStateEventsWithEventType(ctx context.Context, roomID, evType string) ([]*types.HeaderedEvent, error) // GetRoomsByMembership returns a list of room IDs matching the provided membership and user ID (as state_key). - GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error) + GetRoomsByMembership(ctx context.Context, userID spec.UserID, membership string) ([]string, error) // GetBulkStateContent returns all state events which match a given room ID and a given state key tuple. Both must be satisfied for a match. // If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned. GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error) diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index a00b4b1d7..1c9cd1599 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -249,6 +249,7 @@ func (s *eventStatements) BulkSelectSnapshotsFromEventIDs( if err != nil { return nil, err } + defer internal.CloseAndLogIfError(ctx, rows, "BulkSelectSnapshotsFromEventIDs: rows.close() failed") var eventID string var stateNID types.StateSnapshotNID @@ -563,7 +564,7 @@ func (s *eventStatements) SelectRoomNIDsForEventNIDs( } result[eventNID] = roomNID } - return result, nil + return result, rows.Err() } func eventNIDsAsArray(eventNIDs []types.EventNID) pq.Int64Array { diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index 835a43b2d..1a96e3527 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -363,7 +363,7 @@ func (s *membershipStatements) SelectRoomsWithMembership( } roomNIDs = append(roomNIDs, roomNID) } - return roomNIDs, nil + return roomNIDs, rows.Err() } func (s *membershipStatements) SelectJoinedUsersSetForRooms( diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index c8346733d..bc3820b2c 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -137,7 +137,7 @@ func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.T } roomIDs = append(roomIDs, roomID) } - return roomIDs, nil + return roomIDs, rows.Err() } func (s *roomStatements) InsertRoomNID( ctx context.Context, txn *sql.Tx, @@ -255,7 +255,7 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs( } result[roomNID] = roomVersion } - return result, nil + return result, rows.Err() } func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error) { @@ -277,7 +277,7 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo } roomIDs = append(roomIDs, roomID) } - return roomIDs, nil + return roomIDs, rows.Err() } func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error) { @@ -299,7 +299,7 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro } roomNIDs = append(roomNIDs, roomNID) } - return roomNIDs, nil + return roomNIDs, rows.Err() } func roomNIDsAsArray(roomNIDs []types.RoomNID) pq.Int64Array { diff --git a/roomserver/storage/postgres/user_room_keys_table.go b/roomserver/storage/postgres/user_room_keys_table.go index 202b0abc1..57e8f213b 100644 --- a/roomserver/storage/postgres/user_room_keys_table.go +++ b/roomserver/storage/postgres/user_room_keys_table.go @@ -56,12 +56,15 @@ const selectUserRoomPublicKeySQL = `SELECT pseudo_id_pub_key FROM roomserver_use const selectUserNIDsSQL = `SELECT user_nid, room_nid, pseudo_id_pub_key FROM roomserver_user_room_keys WHERE room_nid = ANY($1) AND pseudo_id_pub_key = ANY($2)` +const selectAllUserRoomPublicKeyForUserSQL = `SELECT room_nid, pseudo_id_pub_key FROM roomserver_user_room_keys WHERE user_nid = $1` + type userRoomKeysStatements struct { - insertUserRoomPrivateKeyStmt *sql.Stmt - insertUserRoomPublicKeyStmt *sql.Stmt - selectUserRoomKeyStmt *sql.Stmt - selectUserRoomPublicKeyStmt *sql.Stmt - selectUserNIDsStmt *sql.Stmt + insertUserRoomPrivateKeyStmt *sql.Stmt + insertUserRoomPublicKeyStmt *sql.Stmt + selectUserRoomKeyStmt *sql.Stmt + selectUserRoomPublicKeyStmt *sql.Stmt + selectUserNIDsStmt *sql.Stmt + selectAllUserRoomPublicKeysForUser *sql.Stmt } func CreateUserRoomKeysTable(db *sql.DB) error { @@ -77,6 +80,7 @@ func PrepareUserRoomKeysTable(db *sql.DB) (tables.UserRoomKeys, error) { {&s.selectUserRoomKeyStmt, selectUserRoomKeySQL}, {&s.selectUserRoomPublicKeyStmt, selectUserRoomPublicKeySQL}, {&s.selectUserNIDsStmt, selectUserNIDsSQL}, + {&s.selectAllUserRoomPublicKeysForUser, selectAllUserRoomPublicKeyForUserSQL}, }.Prepare(db) } @@ -150,3 +154,25 @@ func (s *userRoomKeysStatements) BulkSelectUserNIDs(ctx context.Context, txn *sq } return result, rows.Err() } + +func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID) (map[types.RoomNID]ed25519.PublicKey, error) { + stmt := sqlutil.TxStmtContext(ctx, txn, s.selectAllUserRoomPublicKeysForUser) + + rows, err := stmt.QueryContext(ctx, userNID) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + defer internal.CloseAndLogIfError(ctx, rows, "SelectAllPublicKeysForUser: failed to close rows") + + resultMap := make(map[types.RoomNID]ed25519.PublicKey) + + var roomNID types.RoomNID + var pubkey ed25519.PublicKey + for rows.Next() { + if err = rows.Scan(&roomNID, &pubkey); err != nil { + return nil, err + } + resultMap[roomNID] = pubkey + } + return resultMap, rows.Err() +} diff --git a/roomserver/storage/shared/room_updater.go b/roomserver/storage/shared/room_updater.go index 70672a33e..06284d2e3 100644 --- a/roomserver/storage/shared/room_updater.go +++ b/roomserver/storage/shared/room_updater.go @@ -250,3 +250,7 @@ func (u *RoomUpdater) MarkEventAsSent(eventNID types.EventNID) error { func (u *RoomUpdater) MembershipUpdater(targetUserNID types.EventStateKeyNID, targetLocal bool) (*MembershipUpdater, error) { return u.d.membershipUpdaterTxn(u.ctx, u.txn, u.roomInfo.RoomNID, targetUserNID, targetLocal) } + +func (u *RoomUpdater) IsEventRejected(ctx context.Context, roomNID types.RoomNID, eventID string) (bool, error) { + return u.d.IsEventRejected(ctx, roomNID, eventID) +} diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 3c8b69c32..3331c6029 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -696,8 +696,8 @@ func (d *Database) GetOrCreateRoomInfo(ctx context.Context, event gomatrixserver return nil, fmt.Errorf("extractRoomVersionFromCreateEvent: %w", err) } - roomNID, nidOK := d.Cache.GetRoomServerRoomNID(event.RoomID()) - cachedRoomVersion, versionOK := d.Cache.GetRoomVersion(event.RoomID()) + roomNID, nidOK := d.Cache.GetRoomServerRoomNID(event.RoomID().String()) + cachedRoomVersion, versionOK := d.Cache.GetRoomVersion(event.RoomID().String()) // if we found both, the roomNID and version in our cache, no need to query the database if nidOK && versionOK { return &types.RoomInfo{ @@ -707,14 +707,14 @@ func (d *Database) GetOrCreateRoomInfo(ctx context.Context, event gomatrixserver } err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID(), roomVersion) + roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID().String(), roomVersion) if err != nil { return err } return nil }) if roomVersion != "" { - d.Cache.StoreRoomVersion(event.RoomID(), roomVersion) + d.Cache.StoreRoomVersion(event.RoomID().String(), roomVersion) } return &types.RoomInfo{ RoomVersion: roomVersion, @@ -1026,24 +1026,19 @@ func (d *EventDatabase) MaybeRedactEvent( case validated || redactedEvent == nil || redactionEvent == nil: // we've seen this redaction before or there is nothing to redact return nil - case redactedEvent.RoomID() != redactionEvent.RoomID(): + case redactedEvent.RoomID().String() != redactionEvent.RoomID().String(): // redactions across rooms aren't allowed ignoreRedaction = true return nil } - var validRoomID *spec.RoomID - validRoomID, err = spec.NewRoomID(redactedEvent.RoomID()) - if err != nil { - return err - } sender1Domain := "" - sender1, err1 := querier.QueryUserIDForSender(ctx, *validRoomID, redactedEvent.SenderID()) + sender1, err1 := querier.QueryUserIDForSender(ctx, redactedEvent.RoomID(), redactedEvent.SenderID()) if err1 == nil { sender1Domain = string(sender1.Domain()) } sender2Domain := "" - sender2, err2 := querier.QueryUserIDForSender(ctx, *validRoomID, redactionEvent.SenderID()) + sender2, err2 := querier.QueryUserIDForSender(ctx, redactedEvent.RoomID(), redactionEvent.SenderID()) if err2 == nil { sender2Domain = string(sender2.Domain()) } @@ -1347,7 +1342,7 @@ func (d *Database) GetStateEventsWithEventType(ctx context.Context, roomID, evTy } // GetRoomsByMembership returns a list of room IDs matching the provided membership and user ID (as state_key). -func (d *Database) GetRoomsByMembership(ctx context.Context, userID, membership string) ([]string, error) { +func (d *Database) GetRoomsByMembership(ctx context.Context, userID spec.UserID, membership string) ([]string, error) { var membershipState tables.MembershipState switch membership { case "join": @@ -1361,17 +1356,73 @@ func (d *Database) GetRoomsByMembership(ctx context.Context, userID, membership default: return nil, fmt.Errorf("GetRoomsByMembership: invalid membership %s", membership) } - stateKeyNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, nil, userID) + + // Convert provided user ID to NID + userNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, nil, userID.String()) if err != nil { if err == sql.ErrNoRows { return nil, nil + } else { + return nil, fmt.Errorf("SelectEventStateKeyNID: cannot map user ID to state key NIDs: %w", err) } - return nil, fmt.Errorf("GetRoomsByMembership: cannot map user ID to state key NID: %w", err) } - roomNIDs, err := d.MembershipTable.SelectRoomsWithMembership(ctx, nil, stateKeyNID, membershipState) + + // Use this NID to fetch all associated room keys (for pseudo ID rooms) + roomKeyMap, err := d.UserRoomKeyTable.SelectAllPublicKeysForUser(ctx, nil, userNID) if err != nil { - return nil, fmt.Errorf("GetRoomsByMembership: failed to SelectRoomsWithMembership: %w", err) + if err == sql.ErrNoRows { + roomKeyMap = map[types.RoomNID]ed25519.PublicKey{} + } else { + return nil, fmt.Errorf("SelectAllPublicKeysForUser: could not select user room public keys for user: %w", err) + } } + + var eventStateKeyNIDs []types.EventStateKeyNID + + // If there are room keys (i.e. this user is in pseudo ID rooms), then gather the appropriate NIDs + if len(roomKeyMap) != 0 { + // Convert keys to string representation + userRoomKeys := make([]string, len(roomKeyMap)) + i := 0 + for _, key := range roomKeyMap { + userRoomKeys[i] = spec.Base64Bytes(key).Encode() + i += 1 + } + + // Convert the string representation to its NID + pseudoIDStateKeys, sqlErr := d.EventStateKeysTable.BulkSelectEventStateKeyNID(ctx, nil, userRoomKeys) + if sqlErr != nil { + if sqlErr == sql.ErrNoRows { + pseudoIDStateKeys = map[string]types.EventStateKeyNID{} + } else { + return nil, fmt.Errorf("BulkSelectEventStateKeyNID: could not select state keys for public room keys: %w", err) + } + } + + // Collect all NIDs together + eventStateKeyNIDs = make([]types.EventStateKeyNID, len(pseudoIDStateKeys)+1) + eventStateKeyNIDs[0] = userNID + i = 1 + for _, nid := range pseudoIDStateKeys { + eventStateKeyNIDs[i] = nid + i += 1 + } + } else { + // If there are no room keys (so no pseudo ID rooms), we only need to care about the user ID NID. + eventStateKeyNIDs = []types.EventStateKeyNID{userNID} + } + + // Fetch rooms that match membership for each NID + roomNIDs := []types.RoomNID{} + for _, nid := range eventStateKeyNIDs { + var roomNIDsChunk []types.RoomNID + roomNIDsChunk, err = d.MembershipTable.SelectRoomsWithMembership(ctx, nil, nid, membershipState) + if err != nil { + return nil, fmt.Errorf("GetRoomsByMembership: failed to SelectRoomsWithMembership: %w", err) + } + roomNIDs = append(roomNIDs, roomNIDsChunk...) + } + roomIDs, err := d.RoomsTable.BulkSelectRoomIDs(ctx, nil, roomNIDs) if err != nil { return nil, fmt.Errorf("GetRoomsByMembership: failed to lookup room nids: %w", err) @@ -1466,7 +1517,7 @@ func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tu } result[i] = tables.StrippedEvent{ EventType: ev.Type(), - RoomID: ev.RoomID(), + RoomID: ev.RoomID().String(), StateKey: *ev.StateKey(), ContentValue: tables.ExtractContentValue(&types.HeaderedEvent{PDU: ev}), } diff --git a/roomserver/storage/sqlite3/event_json_table.go b/roomserver/storage/sqlite3/event_json_table.go index dc26885bb..325951c7f 100644 --- a/roomserver/storage/sqlite3/event_json_table.go +++ b/roomserver/storage/sqlite3/event_json_table.go @@ -109,5 +109,5 @@ func (s *eventJSONStatements) BulkSelectEventJSON( } result.EventNID = types.EventNID(eventNID) } - return results[:i], nil + return results[:i], rows.Err() } diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go index 347524a81..a052d69ed 100644 --- a/roomserver/storage/sqlite3/event_state_keys_table.go +++ b/roomserver/storage/sqlite3/event_state_keys_table.go @@ -136,7 +136,7 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKeyNID( } result[stateKey] = types.EventStateKeyNID(stateKeyNID) } - return result, nil + return result, rows.Err() } func (s *eventStateKeyStatements) BulkSelectEventStateKey( @@ -167,5 +167,5 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKey( } result[types.EventStateKeyNID(stateKeyNID)] = stateKey } - return result, nil + return result, rows.Err() } diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go index 0581ec194..c030fffea 100644 --- a/roomserver/storage/sqlite3/event_types_table.go +++ b/roomserver/storage/sqlite3/event_types_table.go @@ -147,5 +147,5 @@ func (s *eventTypeStatements) BulkSelectEventTypeNID( } result[eventType] = types.EventTypeNID(eventTypeNID) } - return result, nil + return result, rows.Err() } diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index c49c6dc38..2c269bced 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -310,6 +310,9 @@ func (s *eventStatements) BulkSelectStateEventByID( } results = append(results, result) } + if err = rows.Err(); err != nil { + return nil, err + } if !excludeRejected && i != len(eventIDs) { // If there are fewer rows returned than IDs then we were asked to lookup event IDs we don't have. // We don't know which ones were missing because we don't return the string IDs in the query. @@ -377,7 +380,7 @@ func (s *eventStatements) BulkSelectStateEventByNID( return nil, err } } - return results[:i], err + return results[:i], rows.Err() } // bulkSelectStateAtEventByID lookups the state at a list of events by event ID. @@ -425,6 +428,9 @@ func (s *eventStatements) BulkSelectStateAtEventByID( ) } } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventIDs) { return nil, types.MissingEventError( fmt.Sprintf("storage: event IDs missing from the database (%d != %d)", i, len(eventIDs)), @@ -507,6 +513,9 @@ func (s *eventStatements) BulkSelectStateAtEventAndReference( result.BeforeStateSnapshotNID = types.StateSnapshotNID(stateSnapshotNID) result.EventID = eventID } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventNIDs) { return nil, fmt.Errorf("storage: event NIDs missing from the database (%d != %d)", i, len(eventNIDs)) } @@ -544,6 +553,9 @@ func (s *eventStatements) BulkSelectEventID(ctx context.Context, txn *sql.Tx, ev } results[types.EventNID(eventNID)] = eventID } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventNIDs) { return nil, fmt.Errorf("storage: event NIDs missing from the database (%d != %d)", i, len(eventNIDs)) } @@ -602,7 +614,7 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, e RoomNID: types.RoomNID(roomNID), } } - return results, nil + return results, rows.Err() } func (s *eventStatements) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) { @@ -652,7 +664,7 @@ func (s *eventStatements) SelectRoomNIDsForEventNIDs( } result[eventNID] = roomNID } - return result, nil + return result, rows.Err() } func eventNIDsAsArray(eventNIDs []types.EventNID) string { diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index ca6e7c511..b678d8add 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -126,6 +126,9 @@ func (s *inviteStatements) UpdateInviteRetired( } eventIDs = append(eventIDs, inviteEventID) } + if err = rows.Err(); err != nil { + return + } // now retire the invites stmt = sqlutil.TxStmt(txn, s.updateInviteRetiredStmt) _, err = stmt.ExecContext(ctx, roomNID, targetUserNID) @@ -157,5 +160,5 @@ func (s *inviteStatements) SelectInviteActiveForUserInRoom( result = append(result, types.EventStateKeyNID(senderUserNID)) eventIDs = append(eventIDs, eventID) } - return result, eventIDs, eventJSON, nil + return result, eventIDs, eventJSON, rows.Err() } diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index 977788d50..1012c074a 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -250,6 +250,7 @@ func (s *membershipStatements) SelectMembershipsFromRoom( } eventNIDs = append(eventNIDs, eNID) } + err = rows.Err() return } @@ -277,6 +278,7 @@ func (s *membershipStatements) SelectMembershipsFromRoomAndMembership( } eventNIDs = append(eventNIDs, eNID) } + err = rows.Err() return } @@ -313,7 +315,7 @@ func (s *membershipStatements) SelectRoomsWithMembership( } roomNIDs = append(roomNIDs, roomNID) } - return roomNIDs, nil + return roomNIDs, rows.Err() } func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, localOnly bool) (map[types.EventStateKeyNID]int, error) { diff --git a/roomserver/storage/sqlite3/room_aliases_table.go b/roomserver/storage/sqlite3/room_aliases_table.go index 3bdbbaa35..815b42a27 100644 --- a/roomserver/storage/sqlite3/room_aliases_table.go +++ b/roomserver/storage/sqlite3/room_aliases_table.go @@ -121,7 +121,7 @@ func (s *roomAliasesStatements) SelectAliasesFromRoomID( aliases = append(aliases, alias) } - + err = rows.Err() return } diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index 7556b3461..22700a710 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -128,7 +128,7 @@ func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.T } roomIDs = append(roomIDs, roomID) } - return roomIDs, nil + return roomIDs, rows.Err() } func (s *roomStatements) SelectRoomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) { @@ -265,7 +265,7 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs( } result[roomNID] = roomVersion } - return result, nil + return result, rows.Err() } func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error) { @@ -293,7 +293,7 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo } roomIDs = append(roomIDs, roomID) } - return roomIDs, nil + return roomIDs, rows.Err() } func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error) { @@ -321,5 +321,5 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro } roomNIDs = append(roomNIDs, roomNID) } - return roomNIDs, nil + return roomNIDs, rows.Err() } diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index 2edff0ba8..dcac0b07c 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -133,13 +133,16 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs( var stateBlockNIDsJSON string for ; rows.Next(); i++ { result := &results[i] - if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil { + if err = rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil { return nil, err } - if err := json.Unmarshal([]byte(stateBlockNIDsJSON), &result.StateBlockNIDs); err != nil { + if err = json.Unmarshal([]byte(stateBlockNIDsJSON), &result.StateBlockNIDs); err != nil { return nil, err } } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(stateNIDs) { return nil, types.MissingStateError(fmt.Sprintf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs))) } diff --git a/roomserver/storage/sqlite3/user_room_keys_table.go b/roomserver/storage/sqlite3/user_room_keys_table.go index 5d6ddc9a8..13906f771 100644 --- a/roomserver/storage/sqlite3/user_room_keys_table.go +++ b/roomserver/storage/sqlite3/user_room_keys_table.go @@ -56,12 +56,15 @@ const selectUserRoomPublicKeySQL = `SELECT pseudo_id_pub_key FROM roomserver_use const selectUserNIDsSQL = `SELECT user_nid, room_nid, pseudo_id_pub_key FROM roomserver_user_room_keys WHERE room_nid IN ($1) AND pseudo_id_pub_key IN ($2)` +const selectAllUserRoomPublicKeyForUserSQL = `SELECT room_nid, pseudo_id_pub_key FROM roomserver_user_room_keys WHERE user_nid = $1` + type userRoomKeysStatements struct { - db *sql.DB - insertUserRoomPrivateKeyStmt *sql.Stmt - insertUserRoomPublicKeyStmt *sql.Stmt - selectUserRoomKeyStmt *sql.Stmt - selectUserRoomPublicKeyStmt *sql.Stmt + db *sql.DB + insertUserRoomPrivateKeyStmt *sql.Stmt + insertUserRoomPublicKeyStmt *sql.Stmt + selectUserRoomKeyStmt *sql.Stmt + selectUserRoomPublicKeyStmt *sql.Stmt + selectAllUserRoomPublicKeysForUser *sql.Stmt //selectUserNIDsStmt *sql.Stmt //prepared at runtime } @@ -77,6 +80,7 @@ func PrepareUserRoomKeysTable(db *sql.DB) (tables.UserRoomKeys, error) { {&s.insertUserRoomPublicKeyStmt, insertUserRoomPublicKeySQL}, {&s.selectUserRoomKeyStmt, selectUserRoomKeySQL}, {&s.selectUserRoomPublicKeyStmt, selectUserRoomPublicKeySQL}, + {&s.selectAllUserRoomPublicKeysForUser, selectAllUserRoomPublicKeyForUserSQL}, //{&s.selectUserNIDsStmt, selectUserNIDsSQL}, //prepared at runtime }.Prepare(db) } @@ -165,3 +169,25 @@ func (s *userRoomKeysStatements) BulkSelectUserNIDs(ctx context.Context, txn *sq } return result, rows.Err() } + +func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID) (map[types.RoomNID]ed25519.PublicKey, error) { + stmt := sqlutil.TxStmtContext(ctx, txn, s.selectAllUserRoomPublicKeysForUser) + + rows, err := stmt.QueryContext(ctx, userNID) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + defer internal.CloseAndLogIfError(ctx, rows, "SelectAllPublicKeysForUser: failed to close rows") + + resultMap := make(map[types.RoomNID]ed25519.PublicKey) + + var roomNID types.RoomNID + var pubkey ed25519.PublicKey + for rows.Next() { + if err = rows.Scan(&roomNID, &pubkey); err != nil { + return nil, err + } + resultMap[roomNID] = pubkey + } + return resultMap, rows.Err() +} diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index 445c1223f..0ae064e6b 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -198,6 +198,8 @@ type UserRoomKeys interface { // BulkSelectUserNIDs selects all userIDs for the requested senderKeys. Returns a map from publicKey -> types.UserRoomKeyPair. // If a senderKey can't be found, it is omitted in the result. BulkSelectUserNIDs(ctx context.Context, txn *sql.Tx, senderKeys map[types.RoomNID][]ed25519.PublicKey) (map[string]types.UserRoomKeyPair, error) + // SelectAllPublicKeysForUser returns all known public keys for a user. Returns a map from room NID -> public key + SelectAllPublicKeysForUser(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID) (map[types.RoomNID]ed25519.PublicKey, error) } // StrippedEvent represents a stripped event for returning extracted content values. diff --git a/roomserver/version/version.go b/roomserver/version/version.go index 270d42897..e8bd890a0 100644 --- a/roomserver/version/version.go +++ b/roomserver/version/version.go @@ -20,12 +20,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// DefaultRoomVersion contains the room version that will, by -// default, be used to create new rooms on this server. -func DefaultRoomVersion() gomatrixserverlib.RoomVersion { - return gomatrixserverlib.RoomVersionV10 -} - // RoomVersions returns a map of all known room versions to this // server. func RoomVersions() map[gomatrixserverlib.RoomVersion]gomatrixserverlib.IRoomVersion { diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go index 44136e2a0..85dfe0beb 100644 --- a/setup/config/config_clientapi.go +++ b/setup/config/config_clientapi.go @@ -93,17 +93,15 @@ func (c *ClientAPI) Verify(configErrs *ConfigErrors) { checkNotEmpty(configErrs, "client_api.recaptcha_sitekey_class", c.RecaptchaSitekeyClass) } // Ensure there is any spam counter measure when enabling registration - if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled { - if !c.RecaptchaEnabled { - configErrs.Add( - "You have tried to enable open registration without any secondary verification methods " + - "(such as reCAPTCHA). By enabling open registration, you are SIGNIFICANTLY " + - "increasing the risk that your server will be used to send spam or abuse, and may result in " + - "your server being banned from some rooms. If you are ABSOLUTELY CERTAIN you want to do this, " + - "start Dendrite with the -really-enable-open-registration command line flag. Otherwise, you " + - "should set the registration_disabled option in your Dendrite config.", - ) - } + if !c.RegistrationDisabled && !c.OpenRegistrationWithoutVerificationEnabled && !c.RecaptchaEnabled { + configErrs.Add( + "You have tried to enable open registration without any secondary verification methods " + + "(such as reCAPTCHA). By enabling open registration, you are SIGNIFICANTLY " + + "increasing the risk that your server will be used to send spam or abuse, and may result in " + + "your server being banned from some rooms. If you are ABSOLUTELY CERTAIN you want to do this, " + + "start Dendrite with the -really-enable-open-registration command line flag. Otherwise, you " + + "should set the registration_disabled option in your Dendrite config.", + ) } } diff --git a/setup/config/config_global.go b/setup/config/config_global.go index 1622bf357..5b4ccf400 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -48,6 +48,10 @@ type Global struct { // The server name to delegate client-server communications to, with optional port WellKnownClientName string `yaml:"well_known_client_name"` + // The server name to delegate sliding sync communications to, with optional port. + // Requires `well_known_client_name` to also be configured. + WellKnownSlidingSyncProxy string `yaml:"well_known_sliding_sync_proxy"` + // Disables federation. Dendrite will not be able to make any outbound HTTP requests // to other servers and the federation API will not be exposed. DisableFederation bool `yaml:"disable_federation"` diff --git a/setup/config/config_roomserver.go b/setup/config/config_roomserver.go index 319c2419c..06e7757fb 100644 --- a/setup/config/config_roomserver.go +++ b/setup/config/config_roomserver.go @@ -1,12 +1,22 @@ package config +import ( + "fmt" + + "github.com/matrix-org/gomatrixserverlib" + log "github.com/sirupsen/logrus" +) + type RoomServer struct { Matrix *Global `yaml:"-"` + DefaultRoomVersion gomatrixserverlib.RoomVersion `yaml:"default_room_version,omitempty"` + Database DatabaseOptions `yaml:"database,omitempty"` } func (c *RoomServer) Defaults(opts DefaultOpts) { + c.DefaultRoomVersion = gomatrixserverlib.RoomVersionV10 if opts.Generate { if !opts.SingleDatabase { c.Database.ConnectionString = "file:roomserver.db" @@ -18,4 +28,10 @@ func (c *RoomServer) Verify(configErrs *ConfigErrors) { if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString)) } + + if !gomatrixserverlib.KnownRoomVersion(c.DefaultRoomVersion) { + configErrs.Add(fmt.Sprintf("invalid value for config key 'room_server.default_room_version': unsupported room version: %q", c.DefaultRoomVersion)) + } else if !gomatrixserverlib.StableRoomVersion(c.DefaultRoomVersion) { + log.Warnf("WARNING: Provided default room version %q is unstable", c.DefaultRoomVersion) + } } diff --git a/setup/config/config_userapi.go b/setup/config/config_userapi.go index e64a3910c..559de72ac 100644 --- a/setup/config/config_userapi.go +++ b/setup/config/config_userapi.go @@ -21,6 +21,10 @@ type UserAPI struct { // Users who register on this homeserver will automatically // be joined to the rooms listed under this option. AutoJoinRooms []string `yaml:"auto_join_rooms"` + + // The number of workers to start for the DeviceListUpdater. Defaults to 8. + // This only needs updating if the "InputDeviceListUpdate" stream keeps growing indefinitely. + WorkerCount int `yaml:"worker_count"` } const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes @@ -28,6 +32,7 @@ const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes func (c *UserAPI) Defaults(opts DefaultOpts) { c.BCryptCost = bcrypt.DefaultCost c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS + c.WorkerCount = 8 if opts.Generate { if !opts.SingleDatabase { c.AccountDatabase.ConnectionString = "file:userapi_accounts.db" diff --git a/setup/mscs/msc2836/msc2836.go b/setup/mscs/msc2836/msc2836.go index 7f8e2de03..15811710d 100644 --- a/setup/mscs/msc2836/msc2836.go +++ b/setup/mscs/msc2836/msc2836.go @@ -271,7 +271,7 @@ func (rc *reqCtx) process() (*MSC2836EventRelationshipsResponse, *util.JSONRespo event = rc.fetchUnknownEvent(rc.req.EventID, rc.req.RoomID) } if rc.req.RoomID == "" && event != nil { - rc.req.RoomID = event.RoomID() + rc.req.RoomID = event.RoomID().String() } if event == nil || !rc.authorisedToSeeEvent(event) { return nil, &util.JSONResponse{ @@ -526,7 +526,7 @@ func (rc *reqCtx) authorisedToSeeEvent(event *types.HeaderedEvent) bool { // make sure the server is in this room var res fs.QueryJoinedHostServerNamesInRoomResponse err := rc.fsAPI.QueryJoinedHostServerNamesInRoom(rc.ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{ - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), }, &res) if err != nil { util.GetLogger(rc.ctx).WithError(err).Error("authorisedToSeeEvent: failed to QueryJoinedHostServerNamesInRoom") @@ -545,7 +545,7 @@ func (rc *reqCtx) authorisedToSeeEvent(event *types.HeaderedEvent) bool { // TODO: This does not honour m.room.create content var queryMembershipRes roomserver.QueryMembershipForUserResponse err := rc.rsAPI.QueryMembershipForUser(rc.ctx, &roomserver.QueryMembershipForUserRequest{ - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), UserID: rc.userID, }, &queryMembershipRes) if err != nil { @@ -612,7 +612,7 @@ func (rc *reqCtx) lookForEvent(eventID string) *types.HeaderedEvent { // inject all the events into the roomserver then return the event in question rc.injectResponseToRoomserver(queryRes) for _, ev := range queryRes.ParsedEvents { - if ev.EventID() == eventID && rc.req.RoomID == ev.RoomID() { + if ev.EventID() == eventID && rc.req.RoomID == ev.RoomID().String() { return &types.HeaderedEvent{PDU: ev} } } @@ -629,7 +629,7 @@ func (rc *reqCtx) lookForEvent(eventID string) *types.HeaderedEvent { } } } - if rc.req.RoomID == event.RoomID() { + if rc.req.RoomID == event.RoomID().String() { return event } return nil diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go index 16fb3efe1..ecbab706f 100644 --- a/setup/mscs/msc2836/msc2836_test.go +++ b/setup/mscs/msc2836/msc2836_test.go @@ -529,8 +529,9 @@ func (r *testRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spe return spec.NewUserID(string(senderID), true) } -func (r *testRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (spec.SenderID, error) { - return spec.SenderID(userID.String()), nil +func (r *testRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + senderID := spec.SenderID(userID.String()) + return &senderID, nil } func (r *testRoomserverAPI) QueryEventsByID(ctx context.Context, req *roomserver.QueryEventsByIDRequest, res *roomserver.QueryEventsByIDResponse) error { diff --git a/setup/mscs/msc2836/storage.go b/setup/mscs/msc2836/storage.go index 73bd6ed4f..696d0b0da 100644 --- a/setup/mscs/msc2836/storage.go +++ b/setup/mscs/msc2836/storage.go @@ -239,7 +239,7 @@ func (p *DB) StoreRelation(ctx context.Context, ev *types.HeaderedEvent) error { return err } util.GetLogger(ctx).Infof("StoreRelation child=%s parent=%s rel_type=%s", child, parent, relType) - _, err = txn.Stmt(p.insertNodeStmt).ExecContext(ctx, ev.EventID(), ev.OriginServerTS(), ev.RoomID(), count, base64.RawStdEncoding.EncodeToString(hash), 0) + _, err = txn.Stmt(p.insertNodeStmt).ExecContext(ctx, ev.EventID(), ev.OriginServerTS(), ev.RoomID().String(), count, base64.RawStdEncoding.EncodeToString(hash), 0) return err }) } @@ -301,7 +301,7 @@ func (p *DB) ChildrenForParent(ctx context.Context, eventID, relType string, rec } children = append(children, evInfo) } - return children, nil + return children, rows.Err() } func (p *DB) ParentForChild(ctx context.Context, eventID, relType string) (*eventInfo, error) { diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index 3ed455e9f..76b447133 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -113,7 +113,7 @@ func (s *OutputClientDataConsumer) Start() error { id = streamPos e := fulltext.IndexElement{ EventID: ev.EventID(), - RoomID: ev.RoomID(), + RoomID: ev.RoomID().String(), StreamPosition: streamPos, } e.SetContentType(ev.Type()) diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 1e87aee99..666f900d7 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -33,6 +33,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/notifier" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/streams" + "github.com/matrix-org/dendrite/syncapi/synctypes" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/nats-io/nats.go" @@ -165,9 +166,9 @@ func (s *OutputRoomEventConsumer) onRedactEvent( return err } - if err = s.db.RedactRelations(ctx, msg.RedactedBecause.RoomID(), msg.RedactedEventID); err != nil { + if err = s.db.RedactRelations(ctx, msg.RedactedBecause.RoomID().String(), msg.RedactedEventID); err != nil { log.WithFields(log.Fields{ - "room_id": msg.RedactedBecause.RoomID(), + "room_id": msg.RedactedBecause.RoomID().String(), "event_id": msg.RedactedBecause.EventID(), "redacted_event_id": msg.RedactedEventID, }).WithError(err).Warn("Failed to redact relations") @@ -221,7 +222,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( // Finally, work out if there are any more events missing. if len(missingEventIDs) > 0 { eventsReq := &api.QueryEventsByIDRequest{ - RoomID: ev.RoomID(), + RoomID: ev.RoomID().String(), EventIDs: missingEventIDs, } eventsRes := &api.QueryEventsByIDResponse{} @@ -256,17 +257,12 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( } if msg.RewritesState { - if err = s.db.PurgeRoomState(ctx, ev.RoomID()); err != nil { + if err = s.db.PurgeRoomState(ctx, ev.RoomID().String()); err != nil { return fmt.Errorf("s.db.PurgeRoom: %w", err) } } - validRoomID, err := spec.NewRoomID(ev.RoomID()) - if err != nil { - return err - } - - userID, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, ev.SenderID()) + userID, err := s.rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), ev.SenderID()) if err != nil { return err } @@ -306,7 +302,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( } s.pduStream.Advance(pduPos) - s.notifier.OnNewEvent(ev, ev.RoomID(), nil, types.StreamingToken{PDUPosition: pduPos}) + s.notifier.OnNewEvent(ev, ev.RoomID().String(), nil, types.StreamingToken{PDUPosition: pduPos}) return nil } @@ -323,12 +319,7 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent( // old events in the sync API, this should at least prevent us // from confusing clients into thinking they've joined/left rooms. - validRoomID, err := spec.NewRoomID(ev.RoomID()) - if err != nil { - return err - } - - userID, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, ev.SenderID()) + userID, err := s.rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), ev.SenderID()) if err != nil { return err } @@ -354,7 +345,7 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent( if err = s.db.UpdateRelations(ctx, ev); err != nil { log.WithFields(log.Fields{ - "room_id": ev.RoomID(), + "room_id": ev.RoomID().String(), "event_id": ev.EventID(), "type": ev.Type(), }).WithError(err).Warn("Failed to update relations") @@ -367,7 +358,7 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent( } s.pduStream.Advance(pduPos) - s.notifier.OnNewEvent(ev, ev.RoomID(), nil, types.StreamingToken{PDUPosition: pduPos}) + s.notifier.OnNewEvent(ev, ev.RoomID().String(), nil, types.StreamingToken{PDUPosition: pduPos}) return nil } @@ -387,11 +378,7 @@ func (s *OutputRoomEventConsumer) notifyJoinedPeeks(ctx context.Context, ev *rst return sp, fmt.Errorf("unexpected nil state_key") } - validRoomID, err := spec.NewRoomID(ev.RoomID()) - if err != nil { - return sp, err - } - userID, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*ev.StateKey())) + userID, err := s.rsAPI.QueryUserIDForSender(ctx, ev.RoomID(), spec.SenderID(*ev.StateKey())) if err != nil || userID == nil { return sp, fmt.Errorf("failed getting userID for sender: %w", err) } @@ -400,7 +387,7 @@ func (s *OutputRoomEventConsumer) notifyJoinedPeeks(ctx context.Context, ev *rst } // cancel any peeks for it - peekSP, peekErr := s.db.DeletePeeks(ctx, ev.RoomID(), *ev.StateKey()) + peekSP, peekErr := s.db.DeletePeeks(ctx, ev.RoomID().String(), *ev.StateKey()) if peekErr != nil { return sp, fmt.Errorf("s.db.DeletePeeks: %w", peekErr) } @@ -418,11 +405,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent( return } - validRoomID, err := spec.NewRoomID(msg.Event.RoomID()) - if err != nil { - return - } - userID, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*msg.Event.StateKey())) + userID, err := s.rsAPI.QueryUserIDForSender(ctx, msg.Event.RoomID(), spec.SenderID(*msg.Event.StateKey())) if err != nil || userID == nil { return } @@ -559,15 +542,10 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event *rstypes.HeaderedEvent) var succeeded bool defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) - validRoomID, err := spec.NewRoomID(event.RoomID()) - if err != nil { - return event, err - } - sKeyUser := "" if stateKey != "" { var sku *spec.UserID - sku, err = s.rsAPI.QueryUserIDForSender(s.ctx, *validRoomID, spec.SenderID(stateKey)) + sku, err = s.rsAPI.QueryUserIDForSender(s.ctx, event.RoomID(), spec.SenderID(stateKey)) if err == nil && sku != nil { sKeyUser = sku.String() event.StateKeyResolved = &sKeyUser @@ -575,13 +553,13 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event *rstypes.HeaderedEvent) } prevEvent, err := snapshot.GetStateEvent( - s.ctx, event.RoomID(), event.Type(), sKeyUser, + s.ctx, event.RoomID().String(), event.Type(), sKeyUser, ) if err != nil { return event, err } - userID, err := s.rsAPI.QueryUserIDForSender(s.ctx, *validRoomID, event.SenderID()) + userID, err := s.rsAPI.QueryUserIDForSender(s.ctx, event.RoomID(), event.SenderID()) if err != nil { return event, err } @@ -592,16 +570,10 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event *rstypes.HeaderedEvent) return event, nil } - prevEventSender := string(prevEvent.SenderID()) - prevUser, err := s.rsAPI.QueryUserIDForSender(s.ctx, *validRoomID, prevEvent.SenderID()) - if err == nil && prevUser != nil { - prevEventSender = prevUser.String() - } - - prev := types.PrevEventRef{ + prev := synctypes.PrevEventRef{ PrevContent: prevEvent.Content(), ReplacesState: prevEvent.EventID(), - PrevSenderID: prevEventSender, + PrevSenderID: string(prevEvent.SenderID()), } event.PDU, err = event.SetUnsigned(prev) @@ -615,7 +587,7 @@ func (s *OutputRoomEventConsumer) writeFTS(ev *rstypes.HeaderedEvent, pduPositio } e := fulltext.IndexElement{ EventID: ev.EventID(), - RoomID: ev.RoomID(), + RoomID: ev.RoomID().String(), StreamPosition: int64(pduPosition), } e.SetContentType(ev.Type()) diff --git a/syncapi/internal/history_visibility.go b/syncapi/internal/history_visibility.go index ce6846ca4..48475327d 100644 --- a/syncapi/internal/history_visibility.go +++ b/syncapi/internal/history_visibility.go @@ -16,6 +16,7 @@ package internal import ( "context" + "fmt" "math" "time" @@ -101,13 +102,15 @@ func (ev eventVisibility) allowed() (allowed bool) { // ApplyHistoryVisibilityFilter applies the room history visibility filter on types.HeaderedEvents. // Returns the filtered events and an error, if any. +// +// This function assumes that all provided events are from the same room. func ApplyHistoryVisibilityFilter( ctx context.Context, syncDB storage.DatabaseTransaction, rsAPI api.SyncRoomserverAPI, events []*types.HeaderedEvent, alwaysIncludeEventIDs map[string]struct{}, - userID, endpoint string, + userID spec.UserID, endpoint string, ) ([]*types.HeaderedEvent, error) { if len(events) == 0 { return events, nil @@ -115,15 +118,26 @@ func ApplyHistoryVisibilityFilter( start := time.Now() // try to get the current membership of the user - membershipCurrent, _, err := syncDB.SelectMembershipForUser(ctx, events[0].RoomID(), userID, math.MaxInt64) + membershipCurrent, _, err := syncDB.SelectMembershipForUser(ctx, events[0].RoomID().String(), userID.String(), math.MaxInt64) if err != nil { return nil, err } // Get the mapping from eventID -> eventVisibility eventsFiltered := make([]*types.HeaderedEvent, 0, len(events)) - visibilities := visibilityForEvents(ctx, rsAPI, events, userID, events[0].RoomID()) + firstEvRoomID := events[0].RoomID() + senderID, err := rsAPI.QuerySenderIDForUser(ctx, firstEvRoomID, userID) + if err != nil { + return nil, err + } + visibilities := visibilityForEvents(ctx, rsAPI, events, senderID, firstEvRoomID) + for _, ev := range events { + // Validate same room assumption + if ev.RoomID().String() != firstEvRoomID.String() { + return nil, fmt.Errorf("events from different rooms supplied to ApplyHistoryVisibilityFilter") + } + evVis := visibilities[ev.EventID()] evVis.membershipCurrent = membershipCurrent // Always include specific state events for /sync responses @@ -133,38 +147,36 @@ func ApplyHistoryVisibilityFilter( continue } } - // NOTSPEC: Always allow user to see their own membership events (spec contains more "rules") - user, err := spec.NewUserID(userID, true) - if err != nil { - return nil, err - } - roomID, err := spec.NewRoomID(ev.RoomID()) - if err != nil { - return nil, err - } - senderID, err := rsAPI.QuerySenderIDForUser(ctx, *roomID, *user) - if err == nil { - if ev.Type() == spec.MRoomMember && ev.StateKeyEquals(string(senderID)) { + // NOTSPEC: Always allow user to see their own membership events (spec contains more "rules") + if senderID != nil { + if ev.Type() == spec.MRoomMember && ev.StateKeyEquals(string(*senderID)) { eventsFiltered = append(eventsFiltered, ev) continue } } + // Always allow history evVis events on boundaries. This is done // by setting the effective evVis to the least restrictive // of the old vs new. // https://spec.matrix.org/v1.3/client-server-api/#server-behaviour-5 - if hisVis, err := ev.HistoryVisibility(); err == nil { - prevHisVis := gjson.GetBytes(ev.Unsigned(), "prev_content.history_visibility").String() - oldPrio, ok := historyVisibilityPriority[gomatrixserverlib.HistoryVisibility(prevHisVis)] - // if we can't get the previous history visibility, default to shared. - if !ok { - oldPrio = historyVisibilityPriority[gomatrixserverlib.HistoryVisibilityShared] - } - // no OK check, since this should have been validated when setting the value - newPrio := historyVisibilityPriority[hisVis] - if oldPrio < newPrio { - evVis.visibility = gomatrixserverlib.HistoryVisibility(prevHisVis) + if ev.Type() == spec.MRoomHistoryVisibility { + hisVis, err := ev.HistoryVisibility() + + if err == nil && hisVis != "" { + prevHisVis := gjson.GetBytes(ev.Unsigned(), "prev_content.history_visibility").String() + oldPrio, ok := historyVisibilityPriority[gomatrixserverlib.HistoryVisibility(prevHisVis)] + // if we can't get the previous history visibility, default to shared. + if !ok { + oldPrio = historyVisibilityPriority[gomatrixserverlib.HistoryVisibilityShared] + } + // no OK check, since this should have been validated when setting the value + newPrio := historyVisibilityPriority[hisVis] + if oldPrio < newPrio { + evVis.visibility = gomatrixserverlib.HistoryVisibility(prevHisVis) + } else { + evVis.visibility = hisVis + } } } // do the actual check @@ -178,13 +190,13 @@ func ApplyHistoryVisibilityFilter( } // visibilityForEvents returns a map from eventID to eventVisibility containing the visibility and the membership -// of `userID` at the given event. +// of `senderID` at the given event. If provided sender ID is nil, assume that membership is Leave // Returns an error if the roomserver can't calculate the memberships. func visibilityForEvents( ctx context.Context, rsAPI api.SyncRoomserverAPI, events []*types.HeaderedEvent, - userID, roomID string, + senderID *spec.SenderID, roomID spec.RoomID, ) map[string]eventVisibility { eventIDs := make([]string, len(events)) for i := range events { @@ -194,15 +206,13 @@ func visibilityForEvents( result := make(map[string]eventVisibility, len(eventIDs)) // get the membership events for all eventIDs - membershipResp := &api.QueryMembershipAtEventResponse{} - - err := rsAPI.QueryMembershipAtEvent(ctx, &api.QueryMembershipAtEventRequest{ - RoomID: roomID, - EventIDs: eventIDs, - UserID: userID, - }, membershipResp) - if err != nil { - logrus.WithError(err).Error("visibilityForEvents: failed to fetch membership at event, defaulting to 'leave'") + var err error + membershipEvents := make(map[string]*types.HeaderedEvent) + if senderID != nil { + membershipEvents, err = rsAPI.QueryMembershipAtEvent(ctx, roomID, eventIDs, *senderID) + if err != nil { + logrus.WithError(err).Error("visibilityForEvents: failed to fetch membership at event, defaulting to 'leave'") + } } // Create a map from eventID -> eventVisibility @@ -212,7 +222,7 @@ func visibilityForEvents( membershipAtEvent: spec.Leave, // default to leave, to not expose events by accident visibility: event.Visibility, } - ev, ok := membershipResp.Membership[eventID] + ev, ok := membershipEvents[eventID] if !ok || ev == nil { result[eventID] = vis continue diff --git a/syncapi/internal/history_visibility_test.go b/syncapi/internal/history_visibility_test.go new file mode 100644 index 000000000..24515bbb2 --- /dev/null +++ b/syncapi/internal/history_visibility_test.go @@ -0,0 +1,214 @@ +package internal + +import ( + "context" + "fmt" + "math" + "testing" + + rsapi "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" + "gotest.tools/v3/assert" +) + +type mockHisVisRoomserverAPI struct { + rsapi.RoomserverInternalAPI + events []*types.HeaderedEvent + roomID string +} + +func (s *mockHisVisRoomserverAPI) QueryMembershipAtEvent(ctx context.Context, roomID spec.RoomID, eventIDs []string, senderID spec.SenderID) (map[string]*types.HeaderedEvent, error) { + if roomID.String() == s.roomID { + membershipMap := map[string]*types.HeaderedEvent{} + + for _, queriedEventID := range eventIDs { + for _, event := range s.events { + if event.EventID() == queriedEventID { + membershipMap[queriedEventID] = event + } + } + } + + return membershipMap, nil + } else { + return nil, fmt.Errorf("room not found: \"%v\"", roomID) + } +} + +func (s *mockHisVisRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + senderID := spec.SenderIDFromUserID(userID) + return &senderID, nil +} + +func (s *mockHisVisRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + userID := senderID.ToUserID() + if userID == nil { + return nil, fmt.Errorf("sender ID not user ID") + } + return userID, nil +} + +type mockDB struct { + storage.DatabaseTransaction + // user ID -> membership (i.e. 'join', 'leave', etc.) + currentMembership map[string]string + roomID string +} + +func (s *mockDB) SelectMembershipForUser(ctx context.Context, roomID string, userID string, pos int64) (string, int64, error) { + if roomID == s.roomID { + membership, ok := s.currentMembership[userID] + if !ok { + return spec.Leave, math.MaxInt64, nil + } + return membership, math.MaxInt64, nil + } + + return "", 0, fmt.Errorf("room not found: \"%v\"", roomID) +} + +// Tests logic around history visibility boundaries +// +// Specifically that if a room's history visibility before or after a particular history visibility event +// allows them to see events (a boundary), then the history visibility event itself should be shown +// ( spec: https://spec.matrix.org/v1.8/client-server-api/#server-behaviour-5 ) +// +// This also aims to emulate "Only see history_visibility changes on bounadries" in sytest/tests/30rooms/30history-visibility.pl +func Test_ApplyHistoryVisbility_Boundaries(t *testing.T) { + ctx := context.Background() + + roomID := "!roomid:domain" + + creatorUserID := spec.NewUserIDOrPanic("@creator:domain", false) + otherUserID := spec.NewUserIDOrPanic("@other:domain", false) + roomVersion := gomatrixserverlib.RoomVersionV10 + roomVerImpl := gomatrixserverlib.MustGetRoomVersion(roomVersion) + + eventsJSON := []struct { + id string + json string + }{ + {id: "$create-event", json: fmt.Sprintf(`{ + "type": "m.room.create", "state_key": "", + "room_id": "%v", "sender": "%v", + "content": {"creator": "%v", "room_version": "%v"} + }`, roomID, creatorUserID.String(), creatorUserID.String(), roomVersion)}, + {id: "$creator-joined", json: fmt.Sprintf(`{ + "type": "m.room.member", "state_key": "%v", + "room_id": "%v", "sender": "%v", + "content": {"membership": "join"} + }`, creatorUserID.String(), roomID, creatorUserID.String())}, + {id: "$hisvis-1", json: fmt.Sprintf(`{ + "type": "m.room.history_visibility", "state_key": "", + "room_id": "%v", "sender": "%v", + "content": {"history_visibility": "shared"} + }`, roomID, creatorUserID.String())}, + {id: "$msg-1", json: fmt.Sprintf(`{ + "type": "m.room.message", + "room_id": "%v", "sender": "%v", + "content": {"body": "1"} + }`, roomID, creatorUserID.String())}, + {id: "$hisvis-2", json: fmt.Sprintf(`{ + "type": "m.room.history_visibility", "state_key": "", + "room_id": "%v", "sender": "%v", + "content": {"history_visibility": "joined"}, + "unsigned": {"prev_content": {"history_visibility": "shared"}} + }`, roomID, creatorUserID.String())}, + {id: "$msg-2", json: fmt.Sprintf(`{ + "type": "m.room.message", + "room_id": "%v", "sender": "%v", + "content": {"body": "1"} + }`, roomID, creatorUserID.String())}, + {id: "$hisvis-3", json: fmt.Sprintf(`{ + "type": "m.room.history_visibility", "state_key": "", + "room_id": "%v", "sender": "%v", + "content": {"history_visibility": "invited"}, + "unsigned": {"prev_content": {"history_visibility": "joined"}} + }`, roomID, creatorUserID.String())}, + {id: "$msg-3", json: fmt.Sprintf(`{ + "type": "m.room.message", + "room_id": "%v", "sender": "%v", + "content": {"body": "2"} + }`, roomID, creatorUserID.String())}, + {id: "$hisvis-4", json: fmt.Sprintf(`{ + "type": "m.room.history_visibility", "state_key": "", + "room_id": "%v", "sender": "%v", + "content": {"history_visibility": "shared"}, + "unsigned": {"prev_content": {"history_visibility": "invited"}} + }`, roomID, creatorUserID.String())}, + {id: "$msg-4", json: fmt.Sprintf(`{ + "type": "m.room.message", + "room_id": "%v", "sender": "%v", + "content": {"body": "3"} + }`, roomID, creatorUserID.String())}, + {id: "$other-joined", json: fmt.Sprintf(`{ + "type": "m.room.member", "state_key": "%v", + "room_id": "%v", "sender": "%v", + "content": {"membership": "join"} + }`, otherUserID.String(), roomID, otherUserID.String())}, + } + + events := make([]*types.HeaderedEvent, len(eventsJSON)) + + hisVis := gomatrixserverlib.HistoryVisibilityShared + + for i, eventJSON := range eventsJSON { + pdu, err := roomVerImpl.NewEventFromTrustedJSONWithEventID(eventJSON.id, []byte(eventJSON.json), false) + if err != nil { + t.Fatalf("failed to prepare event %s for test: %s", eventJSON.id, err.Error()) + } + events[i] = &types.HeaderedEvent{PDU: pdu} + + // 'Visibility' should be the visibility of the room just before this event was sent + // (according to processRoomEvent in roomserver/internal/input/input_events.go) + events[i].Visibility = hisVis + if pdu.Type() == spec.MRoomHistoryVisibility { + newHisVis, err := pdu.HistoryVisibility() + if err != nil { + t.Fatalf("failed to prepare history visibility event: %s", err.Error()) + } + hisVis = newHisVis + } + } + + rsAPI := &mockHisVisRoomserverAPI{ + events: events, + roomID: roomID, + } + syncDB := &mockDB{ + roomID: roomID, + currentMembership: map[string]string{ + creatorUserID.String(): spec.Join, + otherUserID.String(): spec.Join, + }, + } + + filteredEvents, err := ApplyHistoryVisibilityFilter(ctx, syncDB, rsAPI, events, nil, otherUserID, "hisVisTest") + if err != nil { + t.Fatalf("ApplyHistoryVisibility returned non-nil error: %s", err.Error()) + } + + filteredEventIDs := make([]string, len(filteredEvents)) + for i, event := range filteredEvents { + filteredEventIDs[i] = event.EventID() + } + + assert.DeepEqual(t, + []string{ + "$create-event", // Always see m.room.create + "$creator-joined", // Always see membership + "$hisvis-1", // Sets room to shared (technically the room is already shared since shared is default) + "$msg-1", // Room currently 'shared' + "$hisvis-2", // Room changed from 'shared' to 'joined', so boundary event and should be shared + // Other events hidden, as other is not joined yet + // hisvis-3 is also hidden, as it changes from joined to invited, neither of which is visible to other + "$hisvis-4", // Changes from 'invited' to 'shared', so is a boundary event and visible + "$msg-4", // Room is 'shared', so visible + "$other-joined", // other's membership + }, + filteredEventIDs, + ) +} diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go index 3f5e990c4..56954cfa0 100644 --- a/syncapi/internal/keychange_test.go +++ b/syncapi/internal/keychange_test.go @@ -59,22 +59,22 @@ func (k *mockKeyAPI) QueryDeviceMessages(ctx context.Context, req *userapi.Query func (k *mockKeyAPI) QuerySignatures(ctx context.Context, req *userapi.QuerySignaturesRequest, res *userapi.QuerySignaturesResponse) { } -type mockRoomserverAPI struct { +type keyChangeMockRoomserverAPI struct { api.RoomserverInternalAPI roomIDToJoinedMembers map[string][]string } -func (s *mockRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { +func (s *keyChangeMockRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return spec.NewUserID(string(senderID), true) } // QueryRoomsForUser retrieves a list of room IDs matching the given query. -func (s *mockRoomserverAPI) QueryRoomsForUser(ctx context.Context, req *api.QueryRoomsForUserRequest, res *api.QueryRoomsForUserResponse) error { - return nil +func (s *keyChangeMockRoomserverAPI) QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) { + return nil, nil } // QueryBulkStateContent does a bulk query for state event content in the given rooms. -func (s *mockRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *api.QueryBulkStateContentRequest, res *api.QueryBulkStateContentResponse) error { +func (s *keyChangeMockRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *api.QueryBulkStateContentRequest, res *api.QueryBulkStateContentResponse) error { res.Rooms = make(map[string]map[gomatrixserverlib.StateKeyTuple]string) if req.AllowWildcards && len(req.StateTuples) == 1 && req.StateTuples[0].EventType == spec.MRoomMember && req.StateTuples[0].StateKey == "*" { for _, roomID := range req.RoomIDs { @@ -91,7 +91,7 @@ func (s *mockRoomserverAPI) QueryBulkStateContent(ctx context.Context, req *api. } // QuerySharedUsers returns a list of users who share at least 1 room in common with the given user. -func (s *mockRoomserverAPI) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUsersRequest, res *api.QuerySharedUsersResponse) error { +func (s *keyChangeMockRoomserverAPI) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUsersRequest, res *api.QuerySharedUsersResponse) error { roomsToQuery := req.IncludeRoomIDs for roomID, members := range s.roomIDToJoinedMembers { exclude := false @@ -123,7 +123,7 @@ func (s *mockRoomserverAPI) QuerySharedUsers(ctx context.Context, req *api.Query // This is actually a database function, but seeing as we track the state inside the // *mockRoomserverAPI, we'll just comply with the interface here instead. -func (s *mockRoomserverAPI) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) { +func (s *keyChangeMockRoomserverAPI) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) { commonUsers := []string{} for _, members := range s.roomIDToJoinedMembers { for _, member := range members { @@ -211,7 +211,7 @@ func TestKeyChangeCatchupOnJoinShareNewUser(t *testing.T) { syncResponse := types.NewResponse() syncResponse = joinResponseWithRooms(syncResponse, syncingUser, []string{newlyJoinedRoom}) - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ newlyJoinedRoom: {syncingUser, newShareUser}, "!another:room": {syncingUser}, @@ -234,7 +234,7 @@ func TestKeyChangeCatchupOnLeaveShareLeftUser(t *testing.T) { syncResponse := types.NewResponse() syncResponse = leaveResponseWithRooms(syncResponse, syncingUser, []string{newlyLeftRoom}) - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ newlyLeftRoom: {removeUser}, "!another:room": {syncingUser}, @@ -257,7 +257,7 @@ func TestKeyChangeCatchupOnJoinShareNoNewUsers(t *testing.T) { syncResponse := types.NewResponse() syncResponse = joinResponseWithRooms(syncResponse, syncingUser, []string{newlyJoinedRoom}) - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ newlyJoinedRoom: {syncingUser, existingUser}, "!another:room": {syncingUser, existingUser}, @@ -279,7 +279,7 @@ func TestKeyChangeCatchupOnLeaveShareNoUsers(t *testing.T) { syncResponse := types.NewResponse() syncResponse = leaveResponseWithRooms(syncResponse, syncingUser, []string{newlyLeftRoom}) - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ newlyLeftRoom: {existingUser}, "!another:room": {syncingUser, existingUser}, @@ -343,7 +343,7 @@ func TestKeyChangeCatchupNoNewJoinsButMessages(t *testing.T) { jr.Timeline = &types.Timeline{Events: roomTimelineEvents} syncResponse.Rooms.Join[roomID] = jr - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ roomID: {syncingUser, existingUser}, }, @@ -369,7 +369,7 @@ func TestKeyChangeCatchupChangeAndLeft(t *testing.T) { syncResponse = joinResponseWithRooms(syncResponse, syncingUser, []string{newlyJoinedRoom}) syncResponse = leaveResponseWithRooms(syncResponse, syncingUser, []string{newlyLeftRoom}) - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ newlyJoinedRoom: {syncingUser, newShareUser, newShareUser2}, newlyLeftRoom: {newlyLeftUser, newlyLeftUser2}, @@ -459,7 +459,7 @@ func TestKeyChangeCatchupChangeAndLeftSameRoom(t *testing.T) { lr.Timeline = &types.Timeline{Events: roomEvents} syncResponse.Rooms.Leave[roomID] = lr - rsAPI := &mockRoomserverAPI{ + rsAPI := &keyChangeMockRoomserverAPI{ roomIDToJoinedMembers: map[string][]string{ roomID: {newShareUser, newShareUser2}, "!another:room": {syncingUser}, diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go index a8733f6fe..07b80b165 100644 --- a/syncapi/notifier/notifier.go +++ b/syncapi/notifier/notifier.go @@ -101,20 +101,13 @@ func (n *Notifier) OnNewEvent( n._removeEmptyUserStreams() if ev != nil { - validRoomID, err := spec.NewRoomID(ev.RoomID()) - if err != nil { - log.WithError(err).WithField("event_id", ev.EventID()).Errorf( - "Notifier.OnNewEvent: RoomID is invalid", - ) - return - } // Map this event's room_id to a list of joined users, and wake them up. - usersToNotify := n._joinedUsers(ev.RoomID()) + usersToNotify := n._joinedUsers(ev.RoomID().String()) // Map this event's room_id to a list of peeking devices, and wake them up. - peekingDevicesToNotify := n._peekingDevices(ev.RoomID()) + peekingDevicesToNotify := n._peekingDevices(ev.RoomID().String()) // If this is an invite, also add in the invitee to this list. if ev.Type() == "m.room.member" && ev.StateKey() != nil { - targetUserID, err := n.rsAPI.QueryUserIDForSender(context.Background(), *validRoomID, spec.SenderID(*ev.StateKey())) + targetUserID, err := n.rsAPI.QueryUserIDForSender(context.Background(), ev.RoomID(), spec.SenderID(*ev.StateKey())) if err != nil || targetUserID == nil { log.WithError(err).WithField("event_id", ev.EventID()).Errorf( "Notifier.OnNewEvent: Failed to find the userID for this event", @@ -134,11 +127,11 @@ func (n *Notifier) OnNewEvent( // Manually append the new user's ID so they get notified // along all members in the room usersToNotify = append(usersToNotify, targetUserID.String()) - n._addJoinedUser(ev.RoomID(), targetUserID.String()) + n._addJoinedUser(ev.RoomID().String(), targetUserID.String()) case spec.Leave: fallthrough case spec.Ban: - n._removeJoinedUser(ev.RoomID(), targetUserID.String()) + n._removeJoinedUser(ev.RoomID().String(), targetUserID.String()) } } } diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go index 649d77b41..b0c91c40b 100644 --- a/syncapi/routing/context.go +++ b/syncapi/routing/context.go @@ -138,7 +138,7 @@ func Context( // verify the user is allowed to see the context for this room/event startTime := time.Now() - filteredEvents, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, []*rstypes.HeaderedEvent{&requestedEvent}, nil, device.UserID, "context") + filteredEvents, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, []*rstypes.HeaderedEvent{&requestedEvent}, nil, *userID, "context") if err != nil { logrus.WithError(err).Error("unable to apply history visibility filter") return util.JSONResponse{ @@ -176,7 +176,7 @@ func Context( } startTime = time.Now() - eventsBeforeFiltered, eventsAfterFiltered, err := applyHistoryVisibilityOnContextEvents(ctx, snapshot, rsAPI, eventsBefore, eventsAfter, device.UserID) + eventsBeforeFiltered, eventsAfterFiltered, err := applyHistoryVisibilityOnContextEvents(ctx, snapshot, rsAPI, eventsBefore, eventsAfter, *userID) if err != nil { logrus.WithError(err).Error("unable to apply history visibility filter") return util.JSONResponse{ @@ -257,7 +257,7 @@ func Context( func applyHistoryVisibilityOnContextEvents( ctx context.Context, snapshot storage.DatabaseTransaction, rsAPI roomserver.SyncRoomserverAPI, eventsBefore, eventsAfter []*rstypes.HeaderedEvent, - userID string, + userID spec.UserID, ) (filteredBefore, filteredAfter []*rstypes.HeaderedEvent, err error) { eventIDsBefore := make(map[string]struct{}, len(eventsBefore)) eventIDsAfter := make(map[string]struct{}, len(eventsAfter)) diff --git a/syncapi/routing/getevent.go b/syncapi/routing/getevent.go index 09c2aef02..d0227f4ea 100644 --- a/syncapi/routing/getevent.go +++ b/syncapi/routing/getevent.go @@ -37,17 +37,17 @@ import ( func GetEvent( req *http.Request, device *userapi.Device, - roomID string, + rawRoomID string, eventID string, cfg *config.SyncAPI, syncDB storage.Database, rsAPI api.SyncRoomserverAPI, ) util.JSONResponse { ctx := req.Context() - db, err := syncDB.NewDatabaseTransaction(ctx) + db, err := syncDB.NewDatabaseSnapshot(ctx) logger := util.GetLogger(ctx).WithFields(logrus.Fields{ "event_id": eventID, - "room_id": roomID, + "room_id": rawRoomID, }) if err != nil { logger.WithError(err).Error("GetEvent: syncDB.NewDatabaseTransaction failed") @@ -56,6 +56,15 @@ func GetEvent( JSON: spec.InternalServerError{}, } } + defer db.Rollback() // nolint: errcheck + + roomID, err := spec.NewRoomID(rawRoomID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("invalid room ID"), + } + } events, err := db.Events(ctx, []string{eventID}) if err != nil { @@ -76,13 +85,22 @@ func GetEvent( } // If the request is coming from an appservice, get the user from the request - userID := device.UserID + rawUserID := device.UserID if asUserID := req.FormValue("user_id"); device.AppserviceID != "" && asUserID != "" { - userID = asUserID + rawUserID = asUserID + } + + userID, err := spec.NewUserID(rawUserID, true) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("invalid device.UserID") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } } // Apply history visibility to determine if the user is allowed to view the event - events, err = internal.ApplyHistoryVisibilityFilter(ctx, db, rsAPI, events, nil, userID, "event") + events, err = internal.ApplyHistoryVisibilityFilter(ctx, db, rsAPI, events, nil, *userID, "event") if err != nil { logger.WithError(err).Error("GetEvent: internal.ApplyHistoryVisibilityFilter failed") return util.JSONResponse{ @@ -101,36 +119,19 @@ func GetEvent( } } - sender := spec.UserID{} - validRoomID, err := spec.NewRoomID(roomID) + clientEvent, err := synctypes.ToClientEvent(events[0], synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) if err != nil { + util.GetLogger(req.Context()).WithError(err).WithField("senderID", events[0].SenderID()).WithField("roomID", *roomID).Error("Failed converting to ClientEvent") return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.BadJSON("roomID is invalid"), + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), } } - senderUserID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, events[0].SenderID()) - if err == nil && senderUserID != nil { - sender = *senderUserID - } - sk := events[0].StateKey() - if sk != nil && *sk != "" { - evRoomID, err := spec.NewRoomID(events[0].RoomID()) - if err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: spec.BadJSON("roomID is invalid"), - } - } - skUserID, err := rsAPI.QueryUserIDForSender(ctx, *evRoomID, spec.SenderID(*events[0].StateKey())) - if err == nil && skUserID != nil { - skString := skUserID.String() - sk = &skString - } - } return util.JSONResponse{ Code: http.StatusOK, - JSON: synctypes.ToClientEvent(events[0], synctypes.FormatAll, sender, sk), + JSON: *clientEvent, } } diff --git a/syncapi/routing/memberships.go b/syncapi/routing/memberships.go index 5e5d0125f..e849adf6d 100644 --- a/syncapi/routing/memberships.go +++ b/syncapi/routing/memberships.go @@ -152,15 +152,7 @@ func GetMemberships( } } - validRoomID, err := spec.NewRoomID(ev.RoomID()) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("roomID is invalid") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - userID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, ev.SenderID()) + userID, err := rsAPI.QueryUserIDForSender(req.Context(), ev.RoomID(), ev.SenderID()) if err != nil || userID == nil { util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryUserIDForSender failed") return util.JSONResponse{ diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 23a095449..3333cb54d 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -50,6 +50,7 @@ type messagesReq struct { from *types.TopologyToken to *types.TopologyToken device *userapi.Device + deviceUserID spec.UserID wasToProvided bool backwardOrdering bool filter *synctypes.RoomEventFilter @@ -77,6 +78,15 @@ func OnIncomingMessagesRequest( ) util.JSONResponse { var err error + deviceUserID, err := spec.NewUserID(device.UserID, true) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("device.UserID invalid") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + // NewDatabaseTransaction is used here instead of NewDatabaseSnapshot as we // expect to be able to write to the database in response to a /messages // request that requires backfilling from the roomserver or federation. @@ -240,6 +250,7 @@ func OnIncomingMessagesRequest( filter: filter, backwardOrdering: backwardOrdering, device: device, + deviceUserID: *deviceUserID, } clientEvents, start, end, err := mReq.retrieveEvents(req.Context(), rsAPI) @@ -359,7 +370,7 @@ func (r *messagesReq) retrieveEvents(ctx context.Context, rsAPI api.SyncRoomserv // Apply room history visibility filter startTime := time.Now() - filteredEvents, err := internal.ApplyHistoryVisibilityFilter(r.ctx, r.snapshot, r.rsAPI, events, nil, r.device.UserID, "messages") + filteredEvents, err := internal.ApplyHistoryVisibilityFilter(r.ctx, r.snapshot, r.rsAPI, events, nil, r.deviceUserID, "messages") if err != nil { return []synctypes.ClientEvent{}, *r.from, *r.to, nil } diff --git a/syncapi/routing/relations.go b/syncapi/routing/relations.go index 17933b2fb..935ba83b3 100644 --- a/syncapi/routing/relations.go +++ b/syncapi/routing/relations.go @@ -43,9 +43,25 @@ func Relations( req *http.Request, device *userapi.Device, syncDB storage.Database, rsAPI api.SyncRoomserverAPI, - roomID, eventID, relType, eventType string, + rawRoomID, eventID, relType, eventType string, ) util.JSONResponse { - var err error + roomID, err := spec.NewRoomID(rawRoomID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("invalid room ID"), + } + } + + userID, err := spec.NewUserID(device.UserID, true) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("device.UserID invalid") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown("internal server error"), + } + } + var from, to types.StreamPosition var limit int dir := req.URL.Query().Get("dir") @@ -93,7 +109,7 @@ func Relations( } var events []types.StreamEvent events, res.PrevBatch, res.NextBatch, err = snapshot.RelationsFor( - req.Context(), roomID, eventID, relType, eventType, from, to, dir == "b", limit, + req.Context(), roomID.String(), eventID, relType, eventType, from, to, dir == "b", limit, ) if err != nil { return util.ErrorResponse(err) @@ -105,12 +121,7 @@ func Relations( } // Apply history visibility to the result events. - filteredEvents, err := internal.ApplyHistoryVisibilityFilter(req.Context(), snapshot, rsAPI, headeredEvents, nil, device.UserID, "relations") - if err != nil { - return util.ErrorResponse(err) - } - - validRoomID, err := spec.NewRoomID(roomID) + filteredEvents, err := internal.ApplyHistoryVisibilityFilter(req.Context(), snapshot, rsAPI, headeredEvents, nil, *userID, "relations") if err != nil { return util.ErrorResponse(err) } @@ -119,23 +130,16 @@ func Relations( // type if it was specified. res.Chunk = make([]synctypes.ClientEvent, 0, len(filteredEvents)) for _, event := range filteredEvents { - sender := spec.UserID{} - userID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, event.SenderID()) - if err == nil && userID != nil { - sender = *userID - } - - sk := event.StateKey() - if sk != nil && *sk != "" { - skUserID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, spec.SenderID(*event.StateKey())) - if err == nil && skUserID != nil { - skString := skUserID.String() - sk = &skString - } + clientEvent, err := synctypes.ToClientEvent(event.PDU, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(req.Context(), roomID, senderID) + }) + if err != nil { + util.GetLogger(req.Context()).WithError(err).WithField("senderID", events[0].SenderID()).WithField("roomID", *roomID).Error("Failed converting to ClientEvent") + continue } res.Chunk = append( res.Chunk, - synctypes.ToClientEvent(event.PDU, synctypes.FormatAll, sender, sk), + *clientEvent, ) } diff --git a/syncapi/routing/search.go b/syncapi/routing/search.go index d892b604a..4a8be9f49 100644 --- a/syncapi/routing/search.go +++ b/syncapi/routing/search.go @@ -205,12 +205,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts profileInfos := make(map[string]ProfileInfoResponse) for _, ev := range append(eventsBefore, eventsAfter...) { - validRoomID, roomErr := spec.NewRoomID(ev.RoomID()) - if err != nil { - logrus.WithError(roomErr).WithField("room_id", ev.RoomID()).Warn("failed to query userprofile") - continue - } - userID, queryErr := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, ev.SenderID()) + userID, queryErr := rsAPI.QueryUserIDForSender(req.Context(), ev.RoomID(), ev.SenderID()) if queryErr != nil { logrus.WithError(queryErr).WithField("sender_id", ev.SenderID()).Warn("failed to query userprofile") continue @@ -218,7 +213,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts profile, ok := knownUsersProfiles[userID.String()] if !ok { - stateEvent, stateErr := snapshot.GetStateEvent(ctx, ev.RoomID(), spec.MRoomMember, string(ev.SenderID())) + stateEvent, stateErr := snapshot.GetStateEvent(ctx, ev.RoomID().String(), spec.MRoomMember, string(ev.SenderID())) if stateErr != nil { logrus.WithError(stateErr).WithField("sender_id", event.SenderID()).Warn("failed to query userprofile") continue @@ -235,25 +230,14 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts profileInfos[userID.String()] = profile } - sender := spec.UserID{} - validRoomID, roomErr := spec.NewRoomID(event.RoomID()) + clientEvent, err := synctypes.ToClientEvent(event, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) if err != nil { - logrus.WithError(roomErr).WithField("room_id", event.RoomID()).Warn("failed to query userprofile") + util.GetLogger(req.Context()).WithError(err).WithField("senderID", event.SenderID()).Error("Failed converting to ClientEvent") continue } - userID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, event.SenderID()) - if err == nil && userID != nil { - sender = *userID - } - sk := event.StateKey() - if sk != nil && *sk != "" { - skUserID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, spec.SenderID(*event.StateKey())) - if err == nil && skUserID != nil { - skString := skUserID.String() - sk = &skString - } - } results = append(results, Result{ Context: SearchContextResponse{ Start: startToken.String(), @@ -267,14 +251,14 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts ProfileInfo: profileInfos, }, Rank: eventScore[event.EventID()].Score, - Result: synctypes.ToClientEvent(event, synctypes.FormatAll, sender, sk), + Result: *clientEvent, }) - roomGroup := groups[event.RoomID()] + roomGroup := groups[event.RoomID().String()] roomGroup.Results = append(roomGroup.Results, event.EventID()) - groups[event.RoomID()] = roomGroup - if _, ok := stateForRooms[event.RoomID()]; searchReq.SearchCategories.RoomEvents.IncludeState && !ok { + groups[event.RoomID().String()] = roomGroup + if _, ok := stateForRooms[event.RoomID().String()]; searchReq.SearchCategories.RoomEvents.IncludeState && !ok { stateFilter := synctypes.DefaultStateFilter() - state, err := snapshot.CurrentState(ctx, event.RoomID(), &stateFilter, nil) + state, err := snapshot.CurrentState(ctx, event.RoomID().String(), &stateFilter, nil) if err != nil { logrus.WithError(err).Error("unable to get current state") return util.JSONResponse{ @@ -282,7 +266,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts JSON: spec.InternalServerError{}, } } - stateForRooms[event.RoomID()] = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(state), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + stateForRooms[event.RoomID().String()] = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(state), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return rsAPI.QueryUserIDForSender(req.Context(), roomID, senderID) }) } @@ -328,19 +312,19 @@ func contextEvents( roomFilter *synctypes.RoomEventFilter, searchReq SearchRequest, ) ([]*types.HeaderedEvent, []*types.HeaderedEvent, error) { - id, _, err := snapshot.SelectContextEvent(ctx, event.RoomID(), event.EventID()) + id, _, err := snapshot.SelectContextEvent(ctx, event.RoomID().String(), event.EventID()) if err != nil { logrus.WithError(err).Error("failed to query context event") return nil, nil, err } roomFilter.Limit = searchReq.SearchCategories.RoomEvents.EventContext.BeforeLimit - eventsBefore, err := snapshot.SelectContextBeforeEvent(ctx, id, event.RoomID(), roomFilter) + eventsBefore, err := snapshot.SelectContextBeforeEvent(ctx, id, event.RoomID().String(), roomFilter) if err != nil { logrus.WithError(err).Error("failed to query before context event") return nil, nil, err } roomFilter.Limit = searchReq.SearchCategories.RoomEvents.EventContext.AfterLimit - _, eventsAfter, err := snapshot.SelectContextAfterEvent(ctx, id, event.RoomID(), roomFilter) + _, eventsAfter, err := snapshot.SelectContextAfterEvent(ctx, id, event.RoomID().String(), roomFilter) if err != nil { logrus.WithError(err).Error("failed to query after context event") return nil, nil, err diff --git a/syncapi/routing/search_test.go b/syncapi/routing/search_test.go index 905a9a1ac..a983bb7b5 100644 --- a/syncapi/routing/search_test.go +++ b/syncapi/routing/search_test.go @@ -238,7 +238,7 @@ func TestSearch(t *testing.T) { } elements = append(elements, fulltext.IndexElement{ EventID: x.EventID(), - RoomID: x.RoomID(), + RoomID: x.RoomID().String(), Content: string(x.Content()), ContentType: x.Type(), StreamPosition: int64(sp), diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index dca5d1a14..97c781b9b 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -107,7 +107,7 @@ type DatabaseTransaction interface { // SelectMembershipForUser returns the membership of the user before and including the given position. If no membership can be found // returns "leave", the topological position and no error. If an error occurs, other than sql.ErrNoRows, returns that and an empty // string as the membership. - SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) + SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int64, err error) // getUserUnreadNotificationCountsForRooms returns the unread notifications for the given rooms GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, roomIDs map[string]string) (map[string]*eventutil.NotificationData, error) GetPresences(ctx context.Context, userID []string) ([]*types.PresenceInternal, error) diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 112fa9d4a..ec0b27adc 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -340,7 +340,7 @@ func (s *currentRoomStateStatements) UpsertRoomState( stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt) _, err = stmt.ExecContext( ctx, - event.RoomID(), + event.RoomID().String(), event.EventID(), event.Type(), event.UserID.String(), @@ -392,7 +392,7 @@ func currentRoomStateRowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, er }) } - return events, nil + return events, rows.Err() } func rowsToEvents(rows *sql.Rows) ([]*rstypes.HeaderedEvent, error) { diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go index 7b8d2d733..1f46cd09d 100644 --- a/syncapi/storage/postgres/invites_table.go +++ b/syncapi/storage/postgres/invites_table.go @@ -99,7 +99,7 @@ func (s *inviteEventsStatements) InsertInviteEvent( err = sqlutil.TxStmt(txn, s.insertInviteEventStmt).QueryRowContext( ctx, - inviteEvent.RoomID(), + inviteEvent.RoomID().String(), inviteEvent.EventID(), inviteEvent.UserID.String(), headeredJSON, diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go index 09b47432b..e5208b891 100644 --- a/syncapi/storage/postgres/memberships_table.go +++ b/syncapi/storage/postgres/memberships_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "fmt" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -108,7 +109,7 @@ func (s *membershipsStatements) UpsertMembership( } _, err = sqlutil.TxStmt(txn, s.upsertMembershipStmt).ExecContext( ctx, - event.RoomID(), + event.RoomID().String(), event.StateKeyResolved, membership, event.EventID(), @@ -131,7 +132,7 @@ func (s *membershipsStatements) SelectMembershipCount( // string as the membership. func (s *membershipsStatements) SelectMembershipForUser( ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64, -) (membership string, topologyPos int, err error) { +) (membership string, topologyPos int64, err error) { stmt := sqlutil.TxStmt(txn, s.selectMembershipForUserStmt) err = stmt.QueryRowContext(ctx, roomID, userID, pos).Scan(&membership, &topologyPos) if err != nil { @@ -160,6 +161,7 @@ func (s *membershipsStatements) SelectMemberships( if err != nil { return } + defer internal.CloseAndLogIfError(ctx, rows, "SelectMemberships: failed to close rows") var ( eventID string ) diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index b58cf59f0..b2d191111 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -334,7 +334,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, nil, err } - needSet := stateNeeded[ev.RoomID()] + needSet := stateNeeded[ev.RoomID().String()] if needSet == nil { // make set if required needSet = make(map[string]bool) } @@ -344,7 +344,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( for _, id := range addIDs { needSet[id] = true } - stateNeeded[ev.RoomID()] = needSet + stateNeeded[ev.RoomID().String()] = needSet ev.Visibility = historyVisibility eventIDToEvent[eventID] = types.StreamEvent{ @@ -403,7 +403,7 @@ func (s *outputRoomEventsStatements) InsertEvent( stmt := sqlutil.TxStmt(txn, s.insertEventStmt) err = stmt.QueryRowContext( ctx, - event.RoomID(), + event.RoomID().String(), event.EventID(), headeredJSON, event.Type(), diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index b281f3300..2158d99ec 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -107,7 +107,7 @@ func (s *outputRoomEventsTopologyStatements) InsertEventInTopology( ctx context.Context, txn *sql.Tx, event *rstypes.HeaderedEvent, pos types.StreamPosition, ) (topoPos types.StreamPosition, err error) { err = sqlutil.TxStmt(txn, s.insertEventInTopologyStmt).QueryRowContext( - ctx, event.EventID(), event.Depth(), event.RoomID(), pos, + ctx, event.EventID(), event.Depth(), event.RoomID().String(), pos, ).Scan(&topoPos) return } diff --git a/syncapi/storage/postgres/peeks_table.go b/syncapi/storage/postgres/peeks_table.go index 64183073d..1120dce0b 100644 --- a/syncapi/storage/postgres/peeks_table.go +++ b/syncapi/storage/postgres/peeks_table.go @@ -164,7 +164,7 @@ func (s *peekStatements) SelectPeekingDevices( devices = append(devices, types.PeekingDevice{UserID: userID, DeviceID: deviceID}) result[roomID] = devices } - return result, nil + return result, rows.Err() } func (s *peekStatements) SelectMaxPeekID( diff --git a/syncapi/storage/postgres/presence_table.go b/syncapi/storage/postgres/presence_table.go index f37b5331e..53acecce5 100644 --- a/syncapi/storage/postgres/presence_table.go +++ b/syncapi/storage/postgres/presence_table.go @@ -144,7 +144,7 @@ func (p *presenceStatements) GetPresenceForUsers( presence.ClientFields.Presence = presence.Presence.String() result = append(result, presence) } - return result, err + return result, rows.Err() } func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go index 746a324fa..923ead9bd 100644 --- a/syncapi/storage/shared/storage_consumer.go +++ b/syncapi/storage/shared/storage_consumer.go @@ -114,21 +114,14 @@ func (d *Database) StreamEventsToEvents(ctx context.Context, device *userapi.Dev }).WithError(err).Warnf("Failed to add transaction ID to event") continue } - roomID, err := spec.NewRoomID(in[i].RoomID()) - if err != nil { - logrus.WithFields(logrus.Fields{ - "event_id": out[i].EventID(), - }).WithError(err).Warnf("Room ID is invalid") - continue - } - deviceSenderID, err := rsAPI.QuerySenderIDForUser(ctx, *roomID, *userID) - if err != nil { + deviceSenderID, err := rsAPI.QuerySenderIDForUser(ctx, in[i].RoomID(), *userID) + if err != nil || deviceSenderID == nil { logrus.WithFields(logrus.Fields{ "event_id": out[i].EventID(), }).WithError(err).Warnf("Failed to add transaction ID to event") continue } - if deviceSenderID == in[i].SenderID() && device.SessionID == in[i].TransactionID.SessionID { + if *deviceSenderID == in[i].SenderID() && device.SessionID == in[i].TransactionID.SessionID { err := out[i].SetUnsignedField( "transaction_id", in[i].TransactionID.TransactionID, ) @@ -236,7 +229,7 @@ func (d *Database) UpsertAccountData( // to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. // This function should always be called within a sqlutil.Writer for safety in SQLite. func (d *Database) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *rstypes.HeaderedEvent) error { - if err := d.BackwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { + if err := d.BackwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID().String(), ev.EventID()); err != nil { return err } @@ -257,7 +250,7 @@ func (d *Database) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, e // If the event is missing, consider it a backward extremity. if !found { - if err = d.BackwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { + if err = d.BackwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID().String(), ev.EventID(), eID); err != nil { return err } } @@ -426,7 +419,7 @@ func (d *Database) fetchStateEvents( } // we know we got them all otherwise an error would've been returned, so just loop the events for _, ev := range evs { - roomID := ev.RoomID() + roomID := ev.RoomID().String() stateBetween[roomID] = append(stateBetween[roomID], ev) } } @@ -522,16 +515,12 @@ func getMembershipFromEvent(ctx context.Context, ev gomatrixserverlib.PDU, userI if err != nil { return "", "" } - roomID, err := spec.NewRoomID(ev.RoomID()) - if err != nil { - return "", "" - } - senderID, err := rsAPI.QuerySenderIDForUser(ctx, *roomID, *fullUser) - if err != nil { + senderID, err := rsAPI.QuerySenderIDForUser(ctx, ev.RoomID(), *fullUser) + if err != nil || senderID == nil { return "", "" } - if ev.Type() != "m.room.member" || !ev.StateKeyEquals(string(senderID)) { + if ev.Type() != "m.room.member" || !ev.StateKeyEquals(string(*senderID)) { return "", "" } membership, err := ev.Membership() @@ -594,7 +583,7 @@ func (d *Database) GetPresences(ctx context.Context, userIDs []string) ([]*types return d.Presence.GetPresenceForUsers(ctx, nil, userIDs) } -func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) { +func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int64, err error) { return d.Memberships.SelectMembershipForUser(ctx, nil, roomID, userID, pos) } @@ -626,7 +615,7 @@ func (d *Database) UpdateRelations(ctx context.Context, event *rstypes.HeaderedE default: return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { return d.Relations.InsertRelation( - ctx, txn, event.RoomID(), content.Relations.EventID, + ctx, txn, event.RoomID().String(), content.Relations.EventID, event.EventID(), event.Type(), content.Relations.RelationType, ) }) diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 3bd19b367..f430fcc05 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -177,7 +177,7 @@ func (s *currentRoomStateStatements) SelectJoinedUsers( users = append(users, userID) result[roomID] = users } - return result, nil + return result, rows.Err() } // SelectJoinedUsersInRoom returns a map of room ID to a list of joined user IDs for a given room. @@ -236,7 +236,7 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( } result = append(result, roomID) } - return result, nil + return result, rows.Err() } // SelectRoomIDsWithAnyMembership returns a map of all memberships for the given user. @@ -339,7 +339,7 @@ func (s *currentRoomStateStatements) UpsertRoomState( stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt) _, err = stmt.ExecContext( ctx, - event.RoomID(), + event.RoomID().String(), event.EventID(), event.Type(), event.UserID.String(), @@ -419,7 +419,7 @@ func currentRoomStateRowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, er }) } - return events, nil + return events, rows.Err() } func rowsToEvents(rows *sql.Rows) ([]*rstypes.HeaderedEvent, error) { diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index 7e0d895f1..e50b5cbf8 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -106,7 +106,7 @@ func (s *inviteEventsStatements) InsertInviteEvent( _, err = stmt.ExecContext( ctx, streamPos, - inviteEvent.RoomID(), + inviteEvent.RoomID().String(), inviteEvent.EventID(), inviteEvent.UserID.String(), headeredJSON, @@ -176,7 +176,7 @@ func (s *inviteEventsStatements) SelectInviteEventsInRange( if lastPos == 0 { lastPos = r.To } - return result, retired, lastPos, nil + return result, retired, lastPos, rows.Err() } func (s *inviteEventsStatements) SelectMaxInviteID( diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go index a9e880d2a..9e50422e5 100644 --- a/syncapi/storage/sqlite3/memberships_table.go +++ b/syncapi/storage/sqlite3/memberships_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "fmt" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -111,7 +112,7 @@ func (s *membershipsStatements) UpsertMembership( } _, err = sqlutil.TxStmt(txn, s.upsertMembershipStmt).ExecContext( ctx, - event.RoomID(), + event.RoomID().String(), event.StateKeyResolved, membership, event.EventID(), @@ -134,7 +135,7 @@ func (s *membershipsStatements) SelectMembershipCount( // string as the membership. func (s *membershipsStatements) SelectMembershipForUser( ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64, -) (membership string, topologyPos int, err error) { +) (membership string, topologyPos int64, err error) { stmt := sqlutil.TxStmt(txn, s.selectMembershipForUserStmt) err = stmt.QueryRowContext(ctx, roomID, userID, pos).Scan(&membership, &topologyPos) if err != nil { @@ -163,6 +164,7 @@ func (s *membershipsStatements) SelectMemberships( if err != nil { return } + defer internal.CloseAndLogIfError(ctx, rows, "SelectMemberships: failed to close rows") var eventID string for rows.Next() { if err = rows.Scan(&eventID); err != nil { diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 06c65419a..c7b11d3ef 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -254,7 +254,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, nil, err } - needSet := stateNeeded[ev.RoomID()] + needSet := stateNeeded[ev.RoomID().String()] if needSet == nil { // make set if required needSet = make(map[string]bool) } @@ -264,7 +264,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( for _, id := range addIDs { needSet[id] = true } - stateNeeded[ev.RoomID()] = needSet + stateNeeded[ev.RoomID().String()] = needSet ev.Visibility = historyVisibility eventIDToEvent[eventID] = types.StreamEvent{ @@ -274,7 +274,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( } } - return stateNeeded, eventIDToEvent, nil + return stateNeeded, eventIDToEvent, rows.Err() } // MaxID returns the ID of the last inserted event in this table. 'txn' is optional. If it is not supplied, @@ -344,7 +344,7 @@ func (s *outputRoomEventsStatements) InsertEvent( _, err = insertStmt.ExecContext( ctx, streamPos, - event.RoomID(), + event.RoomID().String(), event.EventID(), headeredJSON, event.Type(), @@ -520,7 +520,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { ExcludeFromSync: excludeFromSync, }) } - return result, nil + return result, rows.Err() } func (s *outputRoomEventsStatements) SelectContextEvent( ctx context.Context, txn *sql.Tx, roomID, eventID string, diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index 614e1df9e..c00fb7a79 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -106,7 +107,7 @@ func (s *outputRoomEventsTopologyStatements) InsertEventInTopology( ctx context.Context, txn *sql.Tx, event *rstypes.HeaderedEvent, pos types.StreamPosition, ) (types.StreamPosition, error) { _, err := sqlutil.TxStmt(txn, s.insertEventInTopologyStmt).ExecContext( - ctx, event.EventID(), event.Depth(), event.RoomID(), pos, + ctx, event.EventID(), event.Depth(), event.RoomID().String(), pos, ) return types.StreamPosition(event.Depth()), err } @@ -137,6 +138,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( } else if err != nil { return } + defer internal.CloseAndLogIfError(ctx, rows, "SelectEventIDsInRange: failed to close rows") // Return the IDs. var eventID string @@ -155,7 +157,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( start = tokens[0] end = tokens[len(tokens)-1] } - + err = rows.Err() return } diff --git a/syncapi/storage/sqlite3/peeks_table.go b/syncapi/storage/sqlite3/peeks_table.go index 5d5200abc..d8998e2b8 100644 --- a/syncapi/storage/sqlite3/peeks_table.go +++ b/syncapi/storage/sqlite3/peeks_table.go @@ -184,7 +184,7 @@ func (s *peekStatements) SelectPeekingDevices( devices = append(devices, types.PeekingDevice{UserID: userID, DeviceID: deviceID}) result[roomID] = devices } - return result, nil + return result, rows.Err() } func (s *peekStatements) SelectMaxPeekID( diff --git a/syncapi/storage/sqlite3/presence_table.go b/syncapi/storage/sqlite3/presence_table.go index 573fbad6c..40b57e75d 100644 --- a/syncapi/storage/sqlite3/presence_table.go +++ b/syncapi/storage/sqlite3/presence_table.go @@ -169,7 +169,7 @@ func (p *presenceStatements) GetPresenceForUsers( presence.ClientFields.Presence = presence.Presence.String() result = append(result, presence) } - return result, err + return result, rows.Err() } func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index f5c66c42d..45117d6d3 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -194,7 +194,7 @@ type Receipts interface { type Memberships interface { UpsertMembership(ctx context.Context, txn *sql.Tx, event *rstypes.HeaderedEvent, streamPos, topologicalPos types.StreamPosition) error SelectMembershipCount(ctx context.Context, txn *sql.Tx, roomID, membership string, pos types.StreamPosition) (count int, err error) - SelectMembershipForUser(ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) + SelectMembershipForUser(ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64) (membership string, topologicalPos int64, err error) PurgeMemberships(ctx context.Context, txn *sql.Tx, roomID string) error SelectMemberships( ctx context.Context, txn *sql.Tx, diff --git a/syncapi/storage/tables/memberships_test.go b/syncapi/storage/tables/memberships_test.go index a421a9772..0a36f5887 100644 --- a/syncapi/storage/tables/memberships_test.go +++ b/syncapi/storage/tables/memberships_test.go @@ -124,7 +124,7 @@ func testUpsert(t *testing.T, ctx context.Context, table tables.Memberships, mem if err != nil { t.Fatalf("failed to select membership: %s", err) } - expectedPos := 1 + var expectedPos int64 = 1 if pos != expectedPos { t.Fatalf("expected pos to be %d, got %d", expectedPos, pos) } diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go index 7c29d84ae..a3634c03f 100644 --- a/syncapi/streams/stream_invite.go +++ b/syncapi/streams/stream_invite.go @@ -63,31 +63,27 @@ func (p *InviteStreamProvider) IncrementalSync( return from } + eventFormat := synctypes.FormatSync + if req.Filter.EventFormat == synctypes.EventFormatFederation { + eventFormat = synctypes.FormatSyncFederation + } + for roomID, inviteEvent := range invites { user := spec.UserID{} - validRoomID, err := spec.NewRoomID(inviteEvent.RoomID()) - if err != nil { - continue - } - sender, err := p.rsAPI.QueryUserIDForSender(ctx, *validRoomID, inviteEvent.SenderID()) + sender, err := p.rsAPI.QueryUserIDForSender(ctx, inviteEvent.RoomID(), inviteEvent.SenderID()) if err == nil && sender != nil { user = *sender } - sk := inviteEvent.StateKey() - if sk != nil && *sk != "" { - skUserID, err := p.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*inviteEvent.StateKey())) - if err == nil && skUserID != nil { - skString := skUserID.String() - sk = &skString - } - } - // skip ignored user events if _, ok := req.IgnoredUsers.List[user.String()]; ok { continue } - ir := types.NewInviteResponse(inviteEvent, user, sk) + ir, err := types.NewInviteResponse(ctx, p.rsAPI, inviteEvent, eventFormat) + if err != nil { + req.Log.WithError(err).Error("failed creating invite response") + continue + } req.Response.Rooms.Invite[roomID] = ir } diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 3f6888804..3abb0b3c6 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -3,7 +3,6 @@ package streams import ( "context" "database/sql" - "encoding/json" "fmt" "time" @@ -16,8 +15,6 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib/spec" - "github.com/tidwall/gjson" - "github.com/tidwall/sjson" "github.com/matrix-org/dendrite/syncapi/notifier" "github.com/matrix-org/gomatrixserverlib" @@ -88,6 +85,11 @@ func (p *PDUStreamProvider) CompleteSync( req.Log.WithError(err).Error("unable to update event filter with ignored users") } + eventFormat := synctypes.FormatSync + if req.Filter.EventFormat == synctypes.EventFormatFederation { + eventFormat = synctypes.FormatSyncFederation + } + recentEvents, err := snapshot.RecentEvents(ctx, joinedRoomIDs, r, &eventFilter, true, true) if err != nil { return from @@ -105,7 +107,7 @@ func (p *PDUStreamProvider) CompleteSync( // get the join response for each room jr, jerr := p.getJoinResponseForCompleteSync( ctx, snapshot, roomID, &stateFilter, req.WantFullState, req.Device, false, - events.Events, events.Limited, + events.Events, events.Limited, eventFormat, ) if jerr != nil { req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed") @@ -142,7 +144,7 @@ func (p *PDUStreamProvider) CompleteSync( events := recentEvents[roomID] jr, err = p.getJoinResponseForCompleteSync( ctx, snapshot, roomID, &stateFilter, req.WantFullState, req.Device, true, - events.Events, events.Limited, + events.Events, events.Limited, eventFormat, ) if err != nil { req.Log.WithError(err).Error("p.getJoinResponseForCompleteSync failed") @@ -346,26 +348,14 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( return r.From, fmt.Errorf("p.DB.GetBackwardTopologyPos: %w", err) } + eventFormat := synctypes.FormatSync + if req.Filter.EventFormat == synctypes.EventFormatFederation { + eventFormat = synctypes.FormatSyncFederation + } + // Now that we've filtered the timeline, work out which state events are still // left. Anything that appears in the filtered timeline will be removed from the // "state" section and kept in "timeline". - - // update the powerlevel event for timeline events - for i, ev := range events { - if ev.Version() != gomatrixserverlib.RoomVersionPseudoIDs { - continue - } - if ev.Type() != spec.MRoomPowerLevels || !ev.StateKeyEquals("") { - continue - } - var newEvent gomatrixserverlib.PDU - newEvent, err = p.updatePowerLevelEvent(ctx, ev) - if err != nil { - return r.From, err - } - events[i] = &rstypes.HeaderedEvent{PDU: newEvent} - } - sEvents := gomatrixserverlib.HeaderedReverseTopologicalOrdering( gomatrixserverlib.ToPDUs(removeDuplicates(delta.StateEvents, events)), gomatrixserverlib.TopologicalOrderByAuthEvents, @@ -380,15 +370,6 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( continue } delta.StateEvents[i-skipped] = he - // update the powerlevel event for state events - if ev.Version() == gomatrixserverlib.RoomVersionPseudoIDs && ev.Type() == spec.MRoomPowerLevels && ev.StateKeyEquals("") { - var newEvent gomatrixserverlib.PDU - newEvent, err = p.updatePowerLevelEvent(ctx, he) - if err != nil { - return r.From, err - } - delta.StateEvents[i-skipped] = &rstypes.HeaderedEvent{PDU: newEvent} - } } delta.StateEvents = delta.StateEvents[:len(sEvents)-skipped] @@ -413,13 +394,13 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( } } jr.Timeline.PrevBatch = &prevBatch - jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) }) // If we are limited by the filter AND the history visibility filter // didn't "remove" events, return that the response is limited. jr.Timeline.Limited = (limited && len(events) == len(recentEvents)) || delta.NewlyJoined - jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) }) req.Response.Rooms.Join[delta.RoomID] = jr @@ -428,11 +409,11 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( jr := types.NewJoinResponse() jr.Timeline.PrevBatch = &prevBatch // TODO: Apply history visibility on peeked rooms - jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(recentEvents), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(recentEvents), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) }) jr.Timeline.Limited = limited - jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) }) req.Response.Rooms.Peek[delta.RoomID] = jr @@ -443,13 +424,13 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( case spec.Ban: lr := types.NewLeaveResponse() lr.Timeline.PrevBatch = &prevBatch - lr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + lr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) }) // If we are limited by the filter AND the history visibility filter // didn't "remove" events, return that the response is limited. lr.Timeline.Limited = limited && len(events) == len(recentEvents) - lr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + lr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(delta.StateEvents), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) }) req.Response.Rooms.Leave[delta.RoomID] = lr @@ -458,75 +439,6 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( return latestPosition, nil } -func (p *PDUStreamProvider) updatePowerLevelEvent(ctx context.Context, ev *rstypes.HeaderedEvent) (gomatrixserverlib.PDU, error) { - pls, err := gomatrixserverlib.NewPowerLevelContentFromEvent(ev) - if err != nil { - return nil, err - } - newPls := make(map[string]int64) - var userID *spec.UserID - for user, level := range pls.Users { - validRoomID, _ := spec.NewRoomID(ev.RoomID()) - userID, err = p.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(user)) - if err != nil { - return nil, err - } - newPls[userID.String()] = level - } - var newPlBytes, newEv []byte - newPlBytes, err = json.Marshal(newPls) - if err != nil { - return nil, err - } - newEv, err = sjson.SetRawBytes(ev.JSON(), "content.users", newPlBytes) - if err != nil { - return nil, err - } - - // do the same for prev content - prevContent := gjson.GetBytes(ev.JSON(), "unsigned.prev_content") - if !prevContent.Exists() { - var evNew gomatrixserverlib.PDU - evNew, err = gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON(newEv, false) - if err != nil { - return nil, err - } - - return evNew, err - } - pls = gomatrixserverlib.PowerLevelContent{} - err = json.Unmarshal([]byte(prevContent.Raw), &pls) - if err != nil { - return nil, err - } - - newPls = make(map[string]int64) - for user, level := range pls.Users { - validRoomID, _ := spec.NewRoomID(ev.RoomID()) - userID, err = p.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(user)) - if err != nil { - return nil, err - } - newPls[userID.String()] = level - } - newPlBytes, err = json.Marshal(newPls) - if err != nil { - return nil, err - } - newEv, err = sjson.SetRawBytes(newEv, "unsigned.prev_content.users", newPlBytes) - if err != nil { - return nil, err - } - - var evNew gomatrixserverlib.PDU - evNew, err = gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON(newEv, false) - if err != nil { - return nil, err - } - - return evNew, err -} - // applyHistoryVisibilityFilter gets the current room state and supplies it to ApplyHistoryVisibilityFilter, to make // sure we always return the required events in the timeline. func applyHistoryVisibilityFilter( @@ -562,8 +474,13 @@ func applyHistoryVisibilityFilter( } } + parsedUserID, err := spec.NewUserID(userID, true) + if err != nil { + return nil, err + } + startTime := time.Now() - events, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, recentEvents, alwaysIncludeIDs, userID, "sync") + events, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, recentEvents, alwaysIncludeIDs, *parsedUserID, "sync") if err != nil { return nil, err } @@ -587,6 +504,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( isPeek bool, recentStreamEvents []types.StreamEvent, limited bool, + eventFormat synctypes.ClientEventFormat, ) (jr *types.JoinResponse, err error) { jr = types.NewJoinResponse() // TODO: When filters are added, we may need to call this multiple times to get enough events. @@ -670,43 +588,14 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( prevBatch.Decrement() } - // Update powerlevel events for timeline events - for i, ev := range events { - if ev.Version() != gomatrixserverlib.RoomVersionPseudoIDs { - continue - } - if ev.Type() != spec.MRoomPowerLevels || !ev.StateKeyEquals("") { - continue - } - newEvent, err := p.updatePowerLevelEvent(ctx, ev) - if err != nil { - return nil, err - } - events[i] = &rstypes.HeaderedEvent{PDU: newEvent} - } - // Update powerlevel events for state events - for i, ev := range stateEvents { - if ev.Version() != gomatrixserverlib.RoomVersionPseudoIDs { - continue - } - if ev.Type() != spec.MRoomPowerLevels || !ev.StateKeyEquals("") { - continue - } - newEvent, err := p.updatePowerLevelEvent(ctx, ev) - if err != nil { - return nil, err - } - stateEvents[i] = &rstypes.HeaderedEvent{PDU: newEvent} - } - jr.Timeline.PrevBatch = prevBatch - jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + jr.Timeline.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(events), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) }) // If we are limited by the filter AND the history visibility filter // didn't "remove" events, return that the response is limited. jr.Timeline.Limited = limited && len(events) == len(recentEvents) - jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(stateEvents), synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + jr.State.Events = synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(stateEvents), eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { return p.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) }) return jr, nil diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go index 996b21e90..ac5268511 100644 --- a/syncapi/syncapi_test.go +++ b/syncapi/syncapi_test.go @@ -44,6 +44,11 @@ func (s *syncRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spe return spec.NewUserID(string(senderID), true) } +func (s *syncRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + senderID := spec.SenderID(userID.String()) + return &senderID, nil +} + func (s *syncRoomserverAPI) QueryLatestEventsAndState(ctx context.Context, req *rsapi.QueryLatestEventsAndStateRequest, res *rsapi.QueryLatestEventsAndStateResponse) error { var room *test.Room for _, r := range s.rooms { @@ -74,8 +79,13 @@ func (s *syncRoomserverAPI) QueryMembershipForUser(ctx context.Context, req *rsa return nil } -func (s *syncRoomserverAPI) QueryMembershipAtEvent(ctx context.Context, req *rsapi.QueryMembershipAtEventRequest, res *rsapi.QueryMembershipAtEventResponse) error { - return nil +func (s *syncRoomserverAPI) QueryMembershipAtEvent( + ctx context.Context, + roomID spec.RoomID, + eventIDs []string, + senderID spec.SenderID, +) (map[string]*rstypes.HeaderedEvent, error) { + return map[string]*rstypes.HeaderedEvent{}, nil } type syncUserAPI struct { @@ -199,6 +209,156 @@ func testSyncAccessTokens(t *testing.T, dbType test.DBType) { } } +func TestSyncAPIEventFormatPowerLevels(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + testSyncEventFormatPowerLevels(t, dbType) + }) +} + +func testSyncEventFormatPowerLevels(t *testing.T, dbType test.DBType) { + user := test.NewUser(t) + setRoomVersion := func(t *testing.T, r *test.Room) { r.Version = gomatrixserverlib.RoomVersionPseudoIDs } + room := test.NewRoom(t, user, setRoomVersion) + alice := userapi.Device{ + ID: "ALICEID", + UserID: user.ID, + AccessToken: "ALICE_BEARER_TOKEN", + DisplayName: "Alice", + AccountType: userapi.AccountTypeUser, + } + + room.CreateAndInsert(t, user, spec.MRoomPowerLevels, gomatrixserverlib.PowerLevelContent{ + Users: map[string]int64{ + user.ID: 100, + }, + }, test.WithStateKey("")) + + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + natsInstance := jetstream.NATSInstance{} + defer close() + + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) + msgs := toNATSMsgs(t, cfg, room.Events()...) + AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, caches, caching.DisableMetrics) + testrig.MustPublishMsgs(t, jsctx, msgs...) + + testCases := []struct { + name string + wantCode int + wantJoinedRooms []string + eventFormat synctypes.ClientEventFormat + }{ + { + name: "Client format", + wantCode: 200, + wantJoinedRooms: []string{room.ID}, + eventFormat: synctypes.FormatSync, + }, + { + name: "Federation format", + wantCode: 200, + wantJoinedRooms: []string{room.ID}, + eventFormat: synctypes.FormatSyncFederation, + }, + } + + syncUntil(t, routers, alice.AccessToken, false, func(syncBody string) bool { + // wait for the last sent eventID to come down sync + path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(event_id=="%s")`, room.ID, room.Events()[len(room.Events())-1].EventID()) + return gjson.Get(syncBody, path).Exists() + }) + + for _, tc := range testCases { + format := "" + if tc.eventFormat == synctypes.FormatSyncFederation { + format = "federation" + } + + w := httptest.NewRecorder() + routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + "access_token": alice.AccessToken, + "timeout": "0", + "filter": fmt.Sprintf(`{"event_format":"%s"}`, format), + }))) + if w.Code != tc.wantCode { + t.Fatalf("%s: got HTTP %d want %d", tc.name, w.Code, tc.wantCode) + } + if tc.wantJoinedRooms != nil { + var res types.Response + if err := json.NewDecoder(w.Body).Decode(&res); err != nil { + t.Fatalf("%s: failed to decode response body: %s", tc.name, err) + } + if len(res.Rooms.Join) != len(tc.wantJoinedRooms) { + t.Errorf("%s: got %v joined rooms, want %v.\nResponse: %+v", tc.name, len(res.Rooms.Join), len(tc.wantJoinedRooms), res) + } + t.Logf("res: %+v", res.Rooms.Join[room.ID]) + + gotEventIDs := make([]string, len(res.Rooms.Join[room.ID].Timeline.Events)) + for i, ev := range res.Rooms.Join[room.ID].Timeline.Events { + gotEventIDs[i] = ev.EventID + } + test.AssertEventIDsEqual(t, gotEventIDs, room.Events()) + + event := room.CreateAndInsert(t, user, spec.MRoomPowerLevels, gomatrixserverlib.PowerLevelContent{ + Users: map[string]int64{ + user.ID: 100, + "@otheruser:localhost": 50, + }, + }, test.WithStateKey("")) + + msgs := toNATSMsgs(t, cfg, event) + testrig.MustPublishMsgs(t, jsctx, msgs...) + + syncUntil(t, routers, alice.AccessToken, false, func(syncBody string) bool { + // wait for the last sent eventID to come down sync + path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(event_id=="%s")`, room.ID, room.Events()[len(room.Events())-1].EventID()) + return gjson.Get(syncBody, path).Exists() + }) + + since := res.NextBatch.String() + w := httptest.NewRecorder() + routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ + "access_token": alice.AccessToken, + "timeout": "0", + "filter": fmt.Sprintf(`{"event_format":"%s"}`, format), + "since": since, + }))) + if w.Code != 200 { + t.Errorf("since=%s got HTTP %d want 200", since, w.Code) + } + + res = *types.NewResponse() + if err := json.NewDecoder(w.Body).Decode(&res); err != nil { + t.Errorf("failed to decode response body: %s", err) + } + if len(res.Rooms.Join) != 1 { + t.Fatalf("since=%s got %d joined rooms, want 1", since, len(res.Rooms.Join)) + } + gotEventIDs = make([]string, len(res.Rooms.Join[room.ID].Timeline.Events)) + for j, ev := range res.Rooms.Join[room.ID].Timeline.Events { + gotEventIDs[j] = ev.EventID + if ev.Type == spec.MRoomPowerLevels { + content := gomatrixserverlib.PowerLevelContent{} + err := json.Unmarshal(ev.Content, &content) + if err != nil { + t.Errorf("failed to unmarshal power level content: %s", err) + } + otherUserLevel := content.UserLevel("@otheruser:localhost") + if otherUserLevel != 50 { + t.Errorf("Expected user PL of %d but got %d", 50, otherUserLevel) + } + } + } + events := []*rstypes.HeaderedEvent{room.Events()[len(room.Events())-1]} + test.AssertEventIDsEqual(t, gotEventIDs, events) + } + } +} + // Tests what happens when we create a room and then /sync before all events from /createRoom have // been sent to the syncapi func TestSyncAPICreateRoomSyncEarly(t *testing.T) { @@ -1241,7 +1401,7 @@ func toNATSMsgs(t *testing.T, cfg *config.Dendrite, input ...*rstypes.HeaderedEv if ev.StateKey() != nil { addsStateIDs = append(addsStateIDs, ev.EventID()) } - result[i] = testrig.NewOutputEventMsg(t, cfg, ev.RoomID(), api.OutputEvent{ + result[i] = testrig.NewOutputEventMsg(t, cfg, ev.RoomID().String(), api.OutputEvent{ Type: rsapi.OutputTypeNewRoomEvent, NewRoomEvent: &rsapi.OutputNewRoomEvent{ Event: ev, diff --git a/syncapi/synctypes/clientevent.go b/syncapi/synctypes/clientevent.go index 6f03d9ff0..fe4f6c07f 100644 --- a/syncapi/synctypes/clientevent.go +++ b/syncapi/synctypes/clientevent.go @@ -16,10 +16,23 @@ package synctypes import ( + "encoding/json" + "fmt" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/sirupsen/logrus" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" ) +// PrevEventRef represents a reference to a previous event in a state event upgrade +type PrevEventRef struct { + PrevContent json.RawMessage `json:"prev_content"` + ReplacesState string `json:"replaces_state"` + PrevSenderID string `json:"prev_sender"` +} + type ClientEventFormat int const ( @@ -28,8 +41,21 @@ const ( // FormatSync will include only the event keys required by the /sync API. Notably, this // means the 'room_id' will be missing from the events. FormatSync + // FormatSyncFederation will include all event keys normally included in federated events. + // This allows clients to request federated formatted events via the /sync API. + FormatSyncFederation ) +// ClientFederationFields extends a ClientEvent to contain the additional fields present in a +// federation event. Used when the client requests `event_format` of type `federation`. +type ClientFederationFields struct { + Depth int64 `json:"depth,omitempty"` + PrevEvents []string `json:"prev_events,omitempty"` + AuthEvents []string `json:"auth_events,omitempty"` + Signatures spec.RawJSON `json:"signatures,omitempty"` + Hashes spec.RawJSON `json:"hashes,omitempty"` +} + // ClientEvent is an event which is fit for consumption by clients, in accordance with the specification. type ClientEvent struct { Content spec.RawJSON `json:"content"` @@ -42,6 +68,9 @@ type ClientEvent struct { Type string `json:"type"` Unsigned spec.RawJSON `json:"unsigned,omitempty"` Redacts string `json:"redacts,omitempty"` + + // Only sent to clients when `event_format` == `federation`. + ClientFederationFields } // ToClientEvents converts server events to client events. @@ -51,70 +80,351 @@ func ToClientEvents(serverEvs []gomatrixserverlib.PDU, format ClientEventFormat, if se == nil { continue // TODO: shouldn't happen? } - sender := spec.UserID{} - validRoomID, err := spec.NewRoomID(se.RoomID()) + ev, err := ToClientEvent(se, format, userIDForSender) if err != nil { + logrus.WithError(err).Warn("Failed converting event to ClientEvent") continue } - userID, err := userIDForSender(*validRoomID, se.SenderID()) - if err == nil && userID != nil { - sender = *userID - } - - sk := se.StateKey() - if sk != nil && *sk != "" { - skUserID, err := userIDForSender(*validRoomID, spec.SenderID(*sk)) - if err == nil && skUserID != nil { - skString := skUserID.String() - sk = &skString - } - } - evs = append(evs, ToClientEvent(se, format, sender, sk)) + evs = append(evs, *ev) } return evs } +// ToClientEventDefault converts a single server event to a client event. +// It provides default logic for event.SenderID & event.StateKey -> userID conversions. +func ToClientEventDefault(userIDQuery spec.UserIDForSender, event gomatrixserverlib.PDU) ClientEvent { + ev, err := ToClientEvent(event, FormatAll, userIDQuery) + if err != nil { + return ClientEvent{} + } + return *ev +} + +// If provided state key is a user ID (state keys beginning with @ are reserved for this purpose) +// fetch it's associated sender ID and use that instead. Otherwise returns the same state key back. +// +// # This function either returns the state key that should be used, or an error +// +// TODO: handle failure cases better (e.g. no sender ID) +func FromClientStateKey(roomID spec.RoomID, stateKey string, senderIDQuery spec.SenderIDForUser) (*string, error) { + if len(stateKey) >= 1 && stateKey[0] == '@' { + parsedStateKey, err := spec.NewUserID(stateKey, true) + if err != nil { + // If invalid user ID, then there is no associated state event. + return nil, fmt.Errorf("Provided state key begins with @ but is not a valid user ID: %w", err) + } + senderID, err := senderIDQuery(roomID, *parsedStateKey) + if err != nil { + return nil, fmt.Errorf("Failed to query sender ID: %w", err) + } + if senderID == nil { + // If no sender ID, then there is no associated state event. + return nil, fmt.Errorf("No associated sender ID found.") + } + newStateKey := string(*senderID) + return &newStateKey, nil + } else { + return &stateKey, nil + } +} + // ToClientEvent converts a single server event to a client event. -func ToClientEvent(se gomatrixserverlib.PDU, format ClientEventFormat, sender spec.UserID, stateKey *string) ClientEvent { +func ToClientEvent(se gomatrixserverlib.PDU, format ClientEventFormat, userIDForSender spec.UserIDForSender) (*ClientEvent, error) { ce := ClientEvent{ - Content: spec.RawJSON(se.Content()), - Sender: sender.String(), + Content: se.Content(), + Sender: string(se.SenderID()), Type: se.Type(), - StateKey: stateKey, - Unsigned: spec.RawJSON(se.Unsigned()), + StateKey: se.StateKey(), + Unsigned: se.Unsigned(), OriginServerTS: se.OriginServerTS(), EventID: se.EventID(), Redacts: se.Redacts(), } - if format == FormatAll { - ce.RoomID = se.RoomID() - } - if se.Version() == gomatrixserverlib.RoomVersionPseudoIDs { - ce.SenderKey = se.SenderID() - } - return ce -} -// ToClientEvent converts a single server event to a client event. -// It provides default logic for event.SenderID & event.StateKey -> userID conversions. -func ToClientEventDefault(userIDQuery spec.UserIDForSender, event gomatrixserverlib.PDU) ClientEvent { - sender := spec.UserID{} - validRoomID, err := spec.NewRoomID(event.RoomID()) - if err != nil { - return ClientEvent{} - } - userID, err := userIDQuery(*validRoomID, event.SenderID()) - if err == nil && userID != nil { - sender = *userID + switch format { + case FormatAll: + ce.RoomID = se.RoomID().String() + case FormatSync: + case FormatSyncFederation: + ce.RoomID = se.RoomID().String() + ce.AuthEvents = se.AuthEventIDs() + ce.PrevEvents = se.PrevEventIDs() + ce.Depth = se.Depth() + // TODO: Set Signatures & Hashes fields } - sk := event.StateKey() - if sk != nil && *sk != "" { - skUserID, err := userIDQuery(*validRoomID, spec.SenderID(*event.StateKey())) - if err == nil && skUserID != nil { - skString := skUserID.String() - sk = &skString + if format != FormatSyncFederation && se.Version() == gomatrixserverlib.RoomVersionPseudoIDs { + err := updatePseudoIDs(&ce, se, userIDForSender, format) + if err != nil { + return nil, err } } - return ToClientEvent(event, FormatAll, sender, sk) + + return &ce, nil +} + +func updatePseudoIDs(ce *ClientEvent, se gomatrixserverlib.PDU, userIDForSender spec.UserIDForSender, format ClientEventFormat) error { + ce.SenderKey = se.SenderID() + + userID, err := userIDForSender(se.RoomID(), se.SenderID()) + if err == nil && userID != nil { + ce.Sender = userID.String() + } + + sk := se.StateKey() + if sk != nil && *sk != "" { + skUserID, err := userIDForSender(se.RoomID(), spec.SenderID(*sk)) + if err == nil && skUserID != nil { + skString := skUserID.String() + ce.StateKey = &skString + } + } + + var prev PrevEventRef + if err := json.Unmarshal(se.Unsigned(), &prev); err == nil && prev.PrevSenderID != "" { + prevUserID, err := userIDForSender(se.RoomID(), spec.SenderID(prev.PrevSenderID)) + if err == nil && userID != nil { + prev.PrevSenderID = prevUserID.String() + } else { + errString := "userID unknown" + if err != nil { + errString = err.Error() + } + logrus.Warnf("Failed to find userID for prev_sender in ClientEvent: %s", errString) + // NOTE: Not much can be done here, so leave the previous value in place. + } + ce.Unsigned, err = json.Marshal(prev) + if err != nil { + err = fmt.Errorf("Failed to marshal unsigned content for ClientEvent: %w", err) + return err + } + } + + switch se.Type() { + case spec.MRoomCreate: + updatedContent, err := updateCreateEvent(se.Content(), userIDForSender, se.RoomID()) + if err != nil { + err = fmt.Errorf("Failed to update m.room.create event for ClientEvent: %w", err) + return err + } + ce.Content = updatedContent + case spec.MRoomMember: + updatedEvent, err := updateInviteEvent(userIDForSender, se, format) + if err != nil { + err = fmt.Errorf("Failed to update m.room.member event for ClientEvent: %w", err) + return err + } + if updatedEvent != nil { + ce.Unsigned = updatedEvent.Unsigned() + } + case spec.MRoomPowerLevels: + updatedEvent, err := updatePowerLevelEvent(userIDForSender, se, format) + if err != nil { + err = fmt.Errorf("Failed update m.room.power_levels event for ClientEvent: %w", err) + return err + } + if updatedEvent != nil { + ce.Content = updatedEvent.Content() + ce.Unsigned = updatedEvent.Unsigned() + } + } + + return nil +} + +func updateCreateEvent(content spec.RawJSON, userIDForSender spec.UserIDForSender, roomID spec.RoomID) (spec.RawJSON, error) { + if creator := gjson.GetBytes(content, "creator"); creator.Exists() { + oldCreator := creator.Str + userID, err := userIDForSender(roomID, spec.SenderID(oldCreator)) + if err != nil { + err = fmt.Errorf("Failed to find userID for creator in ClientEvent: %w", err) + return nil, err + } + + if userID != nil { + var newCreatorBytes, newContent []byte + newCreatorBytes, err = json.Marshal(userID.String()) + if err != nil { + err = fmt.Errorf("Failed to marshal new creator for ClientEvent: %w", err) + return nil, err + } + + newContent, err = sjson.SetRawBytes([]byte(content), "creator", newCreatorBytes) + if err != nil { + err = fmt.Errorf("Failed to set new creator for ClientEvent: %w", err) + return nil, err + } + + return newContent, nil + } + } + + return content, nil +} + +func updateInviteEvent(userIDForSender spec.UserIDForSender, ev gomatrixserverlib.PDU, eventFormat ClientEventFormat) (gomatrixserverlib.PDU, error) { + if inviteRoomState := gjson.GetBytes(ev.Unsigned(), "invite_room_state"); inviteRoomState.Exists() { + userID, err := userIDForSender(ev.RoomID(), ev.SenderID()) + if err != nil || userID == nil { + if err != nil { + err = fmt.Errorf("invalid userID found when updating invite_room_state: %w", err) + } + return nil, err + } + + newState, err := GetUpdatedInviteRoomState(userIDForSender, inviteRoomState, ev, ev.RoomID(), eventFormat) + if err != nil { + return nil, err + } + + var newEv []byte + newEv, err = sjson.SetRawBytes(ev.JSON(), "unsigned.invite_room_state", newState) + if err != nil { + return nil, err + } + + return gomatrixserverlib.MustGetRoomVersion(ev.Version()).NewEventFromTrustedJSON(newEv, false) + } + + return ev, nil +} + +type InviteRoomStateEvent struct { + Content spec.RawJSON `json:"content"` + SenderID string `json:"sender"` + StateKey *string `json:"state_key"` + Type string `json:"type"` +} + +func GetUpdatedInviteRoomState(userIDForSender spec.UserIDForSender, inviteRoomState gjson.Result, event gomatrixserverlib.PDU, roomID spec.RoomID, eventFormat ClientEventFormat) (spec.RawJSON, error) { + var res spec.RawJSON + inviteStateEvents := []InviteRoomStateEvent{} + err := json.Unmarshal([]byte(inviteRoomState.Raw), &inviteStateEvents) + if err != nil { + return nil, err + } + + if event.Version() == gomatrixserverlib.RoomVersionPseudoIDs && eventFormat != FormatSyncFederation { + for i, ev := range inviteStateEvents { + userID, userIDErr := userIDForSender(roomID, spec.SenderID(ev.SenderID)) + if userIDErr != nil { + return nil, userIDErr + } + if userID != nil { + inviteStateEvents[i].SenderID = userID.String() + } + + if ev.StateKey != nil && *ev.StateKey != "" { + userID, senderErr := userIDForSender(roomID, spec.SenderID(*ev.StateKey)) + if senderErr != nil { + return nil, senderErr + } + if userID != nil { + user := userID.String() + inviteStateEvents[i].StateKey = &user + } + } + + updatedContent, updateErr := updateCreateEvent(ev.Content, userIDForSender, roomID) + if updateErr != nil { + updateErr = fmt.Errorf("Failed to update m.room.create event for ClientEvent: %w", userIDErr) + return nil, updateErr + } + inviteStateEvents[i].Content = updatedContent + } + } + + res, err = json.Marshal(inviteStateEvents) + if err != nil { + return nil, err + } + + return res, nil +} + +func updatePowerLevelEvent(userIDForSender spec.UserIDForSender, se gomatrixserverlib.PDU, eventFormat ClientEventFormat) (gomatrixserverlib.PDU, error) { + if !se.StateKeyEquals("") { + return se, nil + } + + newEv := se.JSON() + + usersField := gjson.GetBytes(se.JSON(), "content.users") + if usersField.Exists() { + pls, err := gomatrixserverlib.NewPowerLevelContentFromEvent(se) + if err != nil { + return nil, err + } + + newPls := make(map[string]int64) + var userID *spec.UserID + for user, level := range pls.Users { + if eventFormat != FormatSyncFederation { + userID, err = userIDForSender(se.RoomID(), spec.SenderID(user)) + if err != nil { + return nil, err + } + user = userID.String() + } + newPls[user] = level + } + + var newPlBytes []byte + newPlBytes, err = json.Marshal(newPls) + if err != nil { + return nil, err + } + newEv, err = sjson.SetRawBytes(se.JSON(), "content.users", newPlBytes) + if err != nil { + return nil, err + } + } + + // do the same for prev content + prevUsersField := gjson.GetBytes(se.JSON(), "unsigned.prev_content.users") + if prevUsersField.Exists() { + prevContent := gjson.GetBytes(se.JSON(), "unsigned.prev_content") + if !prevContent.Exists() { + evNew, err := gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSON(newEv, false) + if err != nil { + return nil, err + } + + return evNew, err + } + pls := gomatrixserverlib.PowerLevelContent{} + err := json.Unmarshal([]byte(prevContent.Raw), &pls) + if err != nil { + return nil, err + } + + newPls := make(map[string]int64) + for user, level := range pls.Users { + if eventFormat != FormatSyncFederation { + userID, userErr := userIDForSender(se.RoomID(), spec.SenderID(user)) + if userErr != nil { + return nil, userErr + } + user = userID.String() + } + newPls[user] = level + } + + var newPlBytes []byte + newPlBytes, err = json.Marshal(newPls) + if err != nil { + return nil, err + } + newEv, err = sjson.SetRawBytes(newEv, "unsigned.prev_content.users", newPlBytes) + if err != nil { + return nil, err + } + } + + evNew, err := gomatrixserverlib.MustGetRoomVersion(se.Version()).NewEventFromTrustedJSONWithEventID(se.EventID(), newEv, false) + if err != nil { + return nil, err + } + + return evNew, err } diff --git a/syncapi/synctypes/clientevent_test.go b/syncapi/synctypes/clientevent_test.go index 63c65b2af..662f9ea43 100644 --- a/syncapi/synctypes/clientevent_test.go +++ b/syncapi/synctypes/clientevent_test.go @@ -18,12 +18,77 @@ package synctypes import ( "bytes" "encoding/json" + "fmt" + "reflect" "testing" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" ) +func queryUserIDForSender(senderID spec.SenderID) (*spec.UserID, error) { + if senderID == "" { + return nil, nil + } + + return spec.NewUserID(string(senderID), true) +} + +const testSenderID = "testSenderID" +const testUserID = "@test:localhost" + +type EventFieldsToVerify struct { + EventID string + Type string + OriginServerTS spec.Timestamp + StateKey *string + Content spec.RawJSON + Unsigned spec.RawJSON + Sender string + Depth int64 + PrevEvents []string + AuthEvents []string +} + +func verifyEventFields(t *testing.T, got EventFieldsToVerify, want EventFieldsToVerify) { + if got.EventID != want.EventID { + t.Errorf("ClientEvent.EventID: wanted %s, got %s", want.EventID, got.EventID) + } + if got.OriginServerTS != want.OriginServerTS { + t.Errorf("ClientEvent.OriginServerTS: wanted %d, got %d", want.OriginServerTS, got.OriginServerTS) + } + if got.StateKey == nil && want.StateKey != nil { + t.Errorf("ClientEvent.StateKey: no state key present when one was wanted: %s", *want.StateKey) + } + if got.StateKey != nil && want.StateKey == nil { + t.Errorf("ClientEvent.StateKey: state key present when one was not wanted: %s", *got.StateKey) + } + if got.StateKey != nil && want.StateKey != nil && *got.StateKey != *want.StateKey { + t.Errorf("ClientEvent.StateKey: wanted %s, got %s", *want.StateKey, *got.StateKey) + } + if got.Type != want.Type { + t.Errorf("ClientEvent.Type: wanted %s, got %s", want.Type, got.Type) + } + if !bytes.Equal(got.Content, want.Content) { + t.Errorf("ClientEvent.Content: wanted %s, got %s", string(want.Content), string(got.Content)) + } + if !bytes.Equal(got.Unsigned, want.Unsigned) { + t.Errorf("ClientEvent.Unsigned: wanted %s, got %s", string(want.Unsigned), string(got.Unsigned)) + } + if got.Sender != want.Sender { + t.Errorf("ClientEvent.Sender: wanted %s, got %s", want.Sender, got.Sender) + } + if got.Depth != want.Depth { + t.Errorf("ClientEvent.Depth: wanted %d, got %d", want.Depth, got.Depth) + } + if !reflect.DeepEqual(got.PrevEvents, want.PrevEvents) { + t.Errorf("ClientEvent.PrevEvents: wanted %v, got %v", want.PrevEvents, got.PrevEvents) + } + if !reflect.DeepEqual(got.AuthEvents, want.AuthEvents) { + t.Errorf("ClientEvent.AuthEvents: wanted %v, got %v", want.AuthEvents, got.AuthEvents) + } +} + func TestToClientEvent(t *testing.T) { // nolint: gocyclo ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionV1).NewEventFromTrustedJSON([]byte(`{ "type": "m.room.name", @@ -49,28 +114,33 @@ func TestToClientEvent(t *testing.T) { // nolint: gocyclo t.Fatalf("failed to create userID: %s", err) } sk := "" - ce := ToClientEvent(ev, FormatAll, *userID, &sk) - if ce.EventID != ev.EventID() { - t.Errorf("ClientEvent.EventID: wanted %s, got %s", ev.EventID(), ce.EventID) - } - if ce.OriginServerTS != ev.OriginServerTS() { - t.Errorf("ClientEvent.OriginServerTS: wanted %d, got %d", ev.OriginServerTS(), ce.OriginServerTS) - } - if ce.StateKey == nil || *ce.StateKey != "" { - t.Errorf("ClientEvent.StateKey: wanted '', got %v", ce.StateKey) - } - if ce.Type != ev.Type() { - t.Errorf("ClientEvent.Type: wanted %s, got %s", ev.Type(), ce.Type) - } - if !bytes.Equal(ce.Content, ev.Content()) { - t.Errorf("ClientEvent.Content: wanted %s, got %s", string(ev.Content()), string(ce.Content)) - } - if !bytes.Equal(ce.Unsigned, ev.Unsigned()) { - t.Errorf("ClientEvent.Unsigned: wanted %s, got %s", string(ev.Unsigned()), string(ce.Unsigned)) - } - if ce.Sender != userID.String() { - t.Errorf("ClientEvent.Sender: wanted %s, got %s", userID.String(), ce.Sender) + ce, err := ToClientEvent(ev, FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return queryUserIDForSender(senderID) + }) + if err != nil { + t.Fatalf("failed to create ClientEvent: %s", err) } + + verifyEventFields(t, + EventFieldsToVerify{ + EventID: ce.EventID, + Type: ce.Type, + OriginServerTS: ce.OriginServerTS, + StateKey: ce.StateKey, + Content: ce.Content, + Unsigned: ce.Unsigned, + Sender: ce.Sender, + }, + EventFieldsToVerify{ + EventID: ev.EventID(), + Type: ev.Type(), + OriginServerTS: ev.OriginServerTS(), + StateKey: &sk, + Content: ev.Content(), + Unsigned: ev.Unsigned(), + Sender: userID.String(), + }) + j, err := json.Marshal(ce) if err != nil { t.Fatalf("failed to Marshal ClientEvent: %s", err) @@ -104,13 +174,388 @@ func TestToClientFormatSync(t *testing.T) { if err != nil { t.Fatalf("failed to create Event: %s", err) } + ce, err := ToClientEvent(ev, FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return queryUserIDForSender(senderID) + }) + if err != nil { + t.Fatalf("failed to create ClientEvent: %s", err) + } + if ce.RoomID != "" { + t.Errorf("ClientEvent.RoomID: wanted '', got %s", ce.RoomID) + } +} + +func TestToClientEventFormatSyncFederation(t *testing.T) { // nolint: gocyclo + ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionV10).NewEventFromTrustedJSON([]byte(`{ + "type": "m.room.name", + "state_key": "", + "event_id": "$test:localhost", + "room_id": "!test:localhost", + "sender": "@test:localhost", + "content": { + "name": "Hello World" + }, + "origin_server_ts": 123456, + "unsigned": { + "prev_content": { + "name": "Goodbye World" + } + }, + "depth": 8, + "prev_events": [ + "$f597Tp0Mm1PPxEgiprzJc2cZAjVhxCxACOGuwJb33Oo" + ], + "auth_events": [ + "$Bj0ZGgX6VTqAQdqKH4ZG3l6rlbxY3rZlC5D3MeuK1OQ", + "$QsMs6A1PUVUhgSvmHBfpqEYJPgv4DXt96r8P2AK7iXQ", + "$tBteKtlnFiwlmPJsv0wkKTMEuUVWpQH89H7Xskxve1Q" + ] + }`), false) + if err != nil { + t.Fatalf("failed to create Event: %s", err) + } userID, err := spec.NewUserID("@test:localhost", true) if err != nil { t.Fatalf("failed to create userID: %s", err) } sk := "" - ce := ToClientEvent(ev, FormatSync, *userID, &sk) - if ce.RoomID != "" { - t.Errorf("ClientEvent.RoomID: wanted '', got %s", ce.RoomID) + ce, err := ToClientEvent(ev, FormatSyncFederation, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return queryUserIDForSender(senderID) + }) + if err != nil { + t.Fatalf("failed to create ClientEvent: %s", err) } + + verifyEventFields(t, + EventFieldsToVerify{ + EventID: ce.EventID, + Type: ce.Type, + OriginServerTS: ce.OriginServerTS, + StateKey: ce.StateKey, + Content: ce.Content, + Unsigned: ce.Unsigned, + Sender: ce.Sender, + Depth: ce.Depth, + PrevEvents: ce.PrevEvents, + AuthEvents: ce.AuthEvents, + }, + EventFieldsToVerify{ + EventID: ev.EventID(), + Type: ev.Type(), + OriginServerTS: ev.OriginServerTS(), + StateKey: &sk, + Content: ev.Content(), + Unsigned: ev.Unsigned(), + Sender: userID.String(), + Depth: ev.Depth(), + PrevEvents: ev.PrevEventIDs(), + AuthEvents: ev.AuthEventIDs(), + }) +} + +func userIDForSender(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + if senderID == "unknownSenderID" { + return nil, fmt.Errorf("Cannot find userID") + } + return spec.NewUserID(testUserID, true) +} + +func TestToClientEventsFormatSyncFederation(t *testing.T) { // nolint: gocyclo + ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World" + }, + "origin_server_ts": 123456, + "unsigned": { + "prev_content": { + "name": "Goodbye World" + } + }, + "depth": 8, + "prev_events": [ + "$f597Tp0Mm1PPxEgiprzJc2cZAjVhxCxACOGuwJb33Oo" + ], + "auth_events": [ + "$Bj0ZGgX6VTqAQdqKH4ZG3l6rlbxY3rZlC5D3MeuK1OQ", + "$QsMs6A1PUVUhgSvmHBfpqEYJPgv4DXt96r8P2AK7iXQ", + "$tBteKtlnFiwlmPJsv0wkKTMEuUVWpQH89H7Xskxve1Q" + ] + }`), false) + if err != nil { + t.Fatalf("failed to create Event: %s", err) + } + ev2, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test2:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World 2" + }, + "origin_server_ts": 1234567, + "unsigned": { + "prev_content": { + "name": "Goodbye World 2" + }, + "prev_sender": "testSenderID" + }, + "depth": 9, + "prev_events": [ + "$f597Tp0Mm1PPxEgiprzJc2cZAjVhxCxACOGuwJb33Oo" + ], + "auth_events": [ + "$Bj0ZGgX6VTqAQdqKH4ZG3l6rlbxY3rZlC5D3MeuK1OQ", + "$QsMs6A1PUVUhgSvmHBfpqEYJPgv4DXt96r8P2AK7iXQ", + "$tBteKtlnFiwlmPJsv0wkKTMEuUVWpQH89H7Xskxve1Q" + ] + }`), false) + if err != nil { + t.Fatalf("failed to create Event: %s", err) + } + + clientEvents := ToClientEvents([]gomatrixserverlib.PDU{ev, ev2}, FormatSyncFederation, userIDForSender) + ce := clientEvents[0] + sk := testSenderID + verifyEventFields(t, + EventFieldsToVerify{ + EventID: ce.EventID, + Type: ce.Type, + OriginServerTS: ce.OriginServerTS, + StateKey: ce.StateKey, + Content: ce.Content, + Unsigned: ce.Unsigned, + Sender: ce.Sender, + Depth: ce.Depth, + PrevEvents: ce.PrevEvents, + AuthEvents: ce.AuthEvents, + }, + EventFieldsToVerify{ + EventID: ev.EventID(), + Type: ev.Type(), + OriginServerTS: ev.OriginServerTS(), + StateKey: &sk, + Content: ev.Content(), + Unsigned: ev.Unsigned(), + Sender: testSenderID, + Depth: ev.Depth(), + PrevEvents: ev.PrevEventIDs(), + AuthEvents: ev.AuthEventIDs(), + }) + + ce2 := clientEvents[1] + verifyEventFields(t, + EventFieldsToVerify{ + EventID: ce2.EventID, + Type: ce2.Type, + OriginServerTS: ce2.OriginServerTS, + StateKey: ce2.StateKey, + Content: ce2.Content, + Unsigned: ce2.Unsigned, + Sender: ce2.Sender, + Depth: ce2.Depth, + PrevEvents: ce2.PrevEvents, + AuthEvents: ce2.AuthEvents, + }, + EventFieldsToVerify{ + EventID: ev2.EventID(), + Type: ev2.Type(), + OriginServerTS: ev2.OriginServerTS(), + StateKey: &sk, + Content: ev2.Content(), + Unsigned: ev2.Unsigned(), + Sender: testSenderID, + Depth: ev2.Depth(), + PrevEvents: ev2.PrevEventIDs(), + AuthEvents: ev2.AuthEventIDs(), + }) +} + +func TestToClientEventsFormatSync(t *testing.T) { // nolint: gocyclo + ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World" + }, + "origin_server_ts": 123456, + "unsigned": { + "prev_content": { + "name": "Goodbye World" + } + } + }`), false) + if err != nil { + t.Fatalf("failed to create Event: %s", err) + } + ev2, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test2:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World 2" + }, + "origin_server_ts": 1234567, + "unsigned": { + "prev_content": { + "name": "Goodbye World 2" + }, + "prev_sender": "testSenderID" + }, + "depth": 9 + }`), false) + if err != nil { + t.Fatalf("failed to create Event: %s", err) + } + + clientEvents := ToClientEvents([]gomatrixserverlib.PDU{ev, ev2}, FormatSync, userIDForSender) + ce := clientEvents[0] + sk := testUserID + verifyEventFields(t, + EventFieldsToVerify{ + EventID: ce.EventID, + Type: ce.Type, + OriginServerTS: ce.OriginServerTS, + StateKey: ce.StateKey, + Content: ce.Content, + Unsigned: ce.Unsigned, + Sender: ce.Sender, + }, + EventFieldsToVerify{ + EventID: ev.EventID(), + Type: ev.Type(), + OriginServerTS: ev.OriginServerTS(), + StateKey: &sk, + Content: ev.Content(), + Unsigned: ev.Unsigned(), + Sender: testUserID, + }) + + var prev PrevEventRef + prev.PrevContent = []byte(`{"name": "Goodbye World 2"}`) + prev.PrevSenderID = testUserID + expectedUnsigned, _ := json.Marshal(prev) + + ce2 := clientEvents[1] + verifyEventFields(t, + EventFieldsToVerify{ + EventID: ce2.EventID, + Type: ce2.Type, + OriginServerTS: ce2.OriginServerTS, + StateKey: ce2.StateKey, + Content: ce2.Content, + Unsigned: ce2.Unsigned, + Sender: ce2.Sender, + }, + EventFieldsToVerify{ + EventID: ev2.EventID(), + Type: ev2.Type(), + OriginServerTS: ev2.OriginServerTS(), + StateKey: &sk, + Content: ev2.Content(), + Unsigned: expectedUnsigned, + Sender: testUserID, + }) +} + +func TestToClientEventsFormatSyncUnknownPrevSender(t *testing.T) { // nolint: gocyclo + ev, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World" + }, + "origin_server_ts": 123456, + "unsigned": { + "prev_content": { + "name": "Goodbye World" + } + } + }`), false) + if err != nil { + t.Fatalf("failed to create Event: %s", err) + } + ev2, err := gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionPseudoIDs).NewEventFromTrustedJSON([]byte(`{ + "type": "m.room.name", + "state_key": "testSenderID", + "event_id": "$test2:localhost", + "room_id": "!test:localhost", + "sender": "testSenderID", + "content": { + "name": "Hello World 2" + }, + "origin_server_ts": 1234567, + "unsigned": { + "prev_content": { + "name": "Goodbye World 2" + }, + "prev_sender": "unknownSenderID" + }, + "depth": 9 + }`), false) + if err != nil { + t.Fatalf("failed to create Event: %s", err) + } + + clientEvents := ToClientEvents([]gomatrixserverlib.PDU{ev, ev2}, FormatSync, userIDForSender) + ce := clientEvents[0] + sk := testUserID + verifyEventFields(t, + EventFieldsToVerify{ + EventID: ce.EventID, + Type: ce.Type, + OriginServerTS: ce.OriginServerTS, + StateKey: ce.StateKey, + Content: ce.Content, + Unsigned: ce.Unsigned, + Sender: ce.Sender, + }, + EventFieldsToVerify{ + EventID: ev.EventID(), + Type: ev.Type(), + OriginServerTS: ev.OriginServerTS(), + StateKey: &sk, + Content: ev.Content(), + Unsigned: ev.Unsigned(), + Sender: testUserID, + }) + + var prev PrevEventRef + prev.PrevContent = []byte(`{"name": "Goodbye World 2"}`) + prev.PrevSenderID = "unknownSenderID" + expectedUnsigned, _ := json.Marshal(prev) + + ce2 := clientEvents[1] + verifyEventFields(t, + EventFieldsToVerify{ + EventID: ce2.EventID, + Type: ce2.Type, + OriginServerTS: ce2.OriginServerTS, + StateKey: ce2.StateKey, + Content: ce2.Content, + Unsigned: ce2.Unsigned, + Sender: ce2.Sender, + }, + EventFieldsToVerify{ + EventID: ev2.EventID(), + Type: ev2.Type(), + OriginServerTS: ev2.OriginServerTS(), + StateKey: &sk, + Content: ev2.Content(), + Unsigned: expectedUnsigned, + Sender: testUserID, + }) } diff --git a/syncapi/synctypes/filter.go b/syncapi/synctypes/filter.go index c994ddb96..8998d4433 100644 --- a/syncapi/synctypes/filter.go +++ b/syncapi/synctypes/filter.go @@ -78,9 +78,14 @@ type RoomEventFilter struct { ContainsURL *bool `json:"contains_url,omitempty"` } +const ( + EventFormatClient = "client" + EventFormatFederation = "federation" +) + // Validate checks if the filter contains valid property values func (filter *Filter) Validate() error { - if filter.EventFormat != "" && filter.EventFormat != "client" && filter.EventFormat != "federation" { + if filter.EventFormat != "" && filter.EventFormat != EventFormatClient && filter.EventFormat != EventFormatFederation { return errors.New("Bad event_format value. Must be one of [\"client\", \"federation\"]") } return nil diff --git a/syncapi/types/types.go b/syncapi/types/types.go index cb3c362d5..bca11855c 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -15,6 +15,7 @@ package types import ( + "context" "encoding/json" "errors" "fmt" @@ -339,13 +340,6 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) { return token, nil } -// PrevEventRef represents a reference to a previous event in a state event upgrade -type PrevEventRef struct { - PrevContent json.RawMessage `json:"prev_content"` - ReplacesState string `json:"replaces_state"` - PrevSenderID string `json:"prev_sender"` -} - type DeviceLists struct { Changed []string `json:"changed,omitempty"` Left []string `json:"left,omitempty"` @@ -539,7 +533,7 @@ type InviteResponse struct { } // NewInviteResponse creates an empty response with initialised arrays. -func NewInviteResponse(event *types.HeaderedEvent, userID spec.UserID, stateKey *string) *InviteResponse { +func NewInviteResponse(ctx context.Context, rsAPI api.QuerySenderIDAPI, event *types.HeaderedEvent, eventFormat synctypes.ClientEventFormat) (*InviteResponse, error) { res := InviteResponse{} res.InviteState.Events = []json.RawMessage{} @@ -547,18 +541,42 @@ func NewInviteResponse(event *types.HeaderedEvent, userID spec.UserID, stateKey // If there is then unmarshal it into the response. This will contain the // partial room state such as join rules, room name etc. if inviteRoomState := gjson.GetBytes(event.Unsigned(), "invite_room_state"); inviteRoomState.Exists() { - _ = json.Unmarshal([]byte(inviteRoomState.Raw), &res.InviteState.Events) + if event.Version() == gomatrixserverlib.RoomVersionPseudoIDs && eventFormat != synctypes.FormatSyncFederation { + updatedInvite, err := synctypes.GetUpdatedInviteRoomState(func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }, inviteRoomState, event.PDU, event.RoomID(), eventFormat) + if err != nil { + return nil, err + } + _ = json.Unmarshal(updatedInvite, &res.InviteState.Events) + } else { + _ = json.Unmarshal([]byte(inviteRoomState.Raw), &res.InviteState.Events) + } + } + + // Clear unsigned so it doesn't have pseudoIDs converted during ToClientEvent + eventNoUnsigned, err := event.SetUnsigned(nil) + if err != nil { + return nil, err } // Then we'll see if we can create a partial of the invite event itself. // This is needed for clients to work out *who* sent the invite. - inviteEvent := synctypes.ToClientEvent(event.PDU, synctypes.FormatSync, userID, stateKey) + inviteEvent, err := synctypes.ToClientEvent(eventNoUnsigned, eventFormat, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) + if err != nil { + return nil, err + } + + // Ensure unsigned field is empty so it isn't marshalled into the final JSON inviteEvent.Unsigned = nil - if ev, err := json.Marshal(inviteEvent); err == nil { + + if ev, err := json.Marshal(*inviteEvent); err == nil { res.InviteState.Events = append(res.InviteState.Events, ev) } - return &res + return &res, nil } // LeaveResponse represents a /sync response for a room which is under the 'leave' key. diff --git a/syncapi/types/types_test.go b/syncapi/types/types_test.go index c1b7f70bd..35e1882cb 100644 --- a/syncapi/types/types_test.go +++ b/syncapi/types/types_test.go @@ -1,6 +1,7 @@ package types import ( + "context" "encoding/json" "reflect" "testing" @@ -11,8 +12,19 @@ import ( "github.com/matrix-org/gomatrixserverlib/spec" ) -func UserIDForSender(roomID string, senderID string) (*spec.UserID, error) { - return spec.NewUserID(senderID, true) +type FakeRoomserverAPI struct{} + +func (f *FakeRoomserverAPI) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + if senderID == "" { + return nil, nil + } + + return spec.NewUserID(string(senderID), true) +} + +func (f *FakeRoomserverAPI) QuerySenderIDForUser(ctx context.Context, roomID spec.RoomID, userID spec.UserID) (*spec.SenderID, error) { + sender := spec.SenderID(userID.String()) + return &sender, nil } func TestSyncTokens(t *testing.T) { @@ -61,25 +73,18 @@ func TestNewInviteResponse(t *testing.T) { t.Fatal(err) } - sender, err := spec.NewUserID("@neilalexander:matrix.org", true) + rsAPI := FakeRoomserverAPI{} + res, err := NewInviteResponse(context.Background(), &rsAPI, &types.HeaderedEvent{PDU: ev}, synctypes.FormatSync) if err != nil { t.Fatal(err) } - skUserID, err := spec.NewUserID("@neilalexander:dendrite.neilalexander.dev", true) - if err != nil { - t.Fatal(err) - } - skString := skUserID.String() - sk := &skString - - res := NewInviteResponse(&types.HeaderedEvent{PDU: ev}, *sender, sk) j, err := json.Marshal(res) if err != nil { t.Fatal(err) } if string(j) != expected { - t.Fatalf("Invite response didn't contain correct info") + t.Fatalf("Invite response didn't contain correct info, \nexpected: %s \ngot: %s", expected, string(j)) } } diff --git a/sytest-whitelist b/sytest-whitelist index c61e0bc3c..492c756ba 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -783,4 +783,16 @@ Invited user can reject invite for empty room Invited user can reject local invite after originator leaves Guest users can join guest_access rooms Forgotten room messages cannot be paginated -Local device key changes get to remote servers with correct prev_id \ No newline at end of file +Local device key changes get to remote servers with correct prev_id +HS provides query metadata +HS can provide query metadata on a single protocol +Invites over federation are correctly pushed +Invites over federation are correctly pushed with name +User can create and send/receive messages in a room with version 11 +local user can join room with version 11 +User can invite local user to room with version 11 +remote user can join room with version 11 +User can invite remote user to room with version 11 +Remote user can backfill in a room with version 11 +Can reject invites over federation for rooms with version 11 +Can receive redactions from regular users over federation in room version 11 \ No newline at end of file diff --git a/userapi/consumers/devicelistupdate.go b/userapi/consumers/devicelistupdate.go index 3389bb808..b3ccb573b 100644 --- a/userapi/consumers/devicelistupdate.go +++ b/userapi/consumers/devicelistupdate.go @@ -17,6 +17,7 @@ package consumers import ( "context" "encoding/json" + "time" "github.com/matrix-org/dendrite/userapi/internal" "github.com/matrix-org/gomatrixserverlib" @@ -82,7 +83,10 @@ func (t *DeviceListUpdateConsumer) onMessage(ctx context.Context, msgs []*nats.M return true } - err := t.updater.Update(ctx, m) + timeoutCtx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() + + err := t.updater.Update(timeoutCtx, m) if err != nil { logrus.WithFields(logrus.Fields{ "user_id": m.UserID, diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go index 1f866ef4d..fca741298 100644 --- a/userapi/consumers/roomserver.go +++ b/userapi/consumers/roomserver.go @@ -92,30 +92,43 @@ func (s *OutputRoomEventConsumer) Start() error { func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called // Only handle events we care about - if rsapi.OutputType(msg.Header.Get(jetstream.RoomEventType)) != rsapi.OutputTypeNewRoomEvent { - return true - } - var output rsapi.OutputEvent - if err := json.Unmarshal(msg.Data, &output); err != nil { - // If the message was invalid, log it and move on to the next message in the stream - log.WithError(err).Errorf("roomserver output log: message parse failure") - return true - } - event := output.NewRoomEvent.Event - if event == nil { - log.Errorf("userapi consumer: expected event") + + var event *rstypes.HeaderedEvent + var isNewRoomEvent bool + switch rsapi.OutputType(msg.Header.Get(jetstream.RoomEventType)) { + case rsapi.OutputTypeNewRoomEvent: + isNewRoomEvent = true + fallthrough + case rsapi.OutputTypeNewInviteEvent: + var output rsapi.OutputEvent + if err := json.Unmarshal(msg.Data, &output); err != nil { + // If the message was invalid, log it and move on to the next message in the stream + log.WithError(err).Errorf("roomserver output log: message parse failure") + return true + } + if isNewRoomEvent { + event = output.NewRoomEvent.Event + } else { + event = output.NewInviteEvent.Event + } + + if event == nil { + log.Errorf("userapi consumer: expected event") + return true + } + + log.WithFields(log.Fields{ + "event_id": event.EventID(), + "event_type": event.Type(), + }).Tracef("Received message from roomserver: %#v", output) + default: return true } if s.cfg.Matrix.ReportStats.Enabled { - go s.storeMessageStats(ctx, event.Type(), string(event.SenderID()), event.RoomID()) + go s.storeMessageStats(ctx, event.Type(), string(event.SenderID()), event.RoomID().String()) } - log.WithFields(log.Fields{ - "event_id": event.EventID(), - "event_type": event.Type(), - }).Tracef("Received message from roomserver: %#v", output) - metadata, err := msg.Metadata() if err != nil { return true @@ -253,8 +266,8 @@ func (s *OutputRoomEventConsumer) updateMDirect(ctx context.Context, oldRoomID, directChats := gjson.ParseBytes(directChatsRaw) newDirectChats := make(map[string][]string) // iterate over all userID -> roomIDs + var found bool directChats.ForEach(func(userID, roomIDs gjson.Result) bool { - var found bool for _, roomID := range roomIDs.Array() { newDirectChats[userID.Str] = append(newDirectChats[userID.Str], roomID.Str) // add the new roomID to m.direct @@ -263,22 +276,21 @@ func (s *OutputRoomEventConsumer) updateMDirect(ctx context.Context, oldRoomID, newDirectChats[userID.Str] = append(newDirectChats[userID.Str], newRoomID) } } - // Only hit the database if we found the old room as a DM for this user - if found { - var data []byte - data, err = json.Marshal(newDirectChats) - if err != nil { - return true - } - if err = s.db.SaveAccountData(ctx, localpart, serverName, "", "m.direct", data); err != nil { - return true - } - } return true }) - if err != nil { - return fmt.Errorf("failed to update m.direct state") + + // Only hit the database if we found the old room as a DM for this user + if found { + var data []byte + data, err = json.Marshal(newDirectChats) + if err != nil { + return err + } + if err = s.db.SaveAccountData(ctx, localpart, serverName, "", "m.direct", data); err != nil { + return fmt.Errorf("failed to update m.direct state: %w", err) + } } + return nil } @@ -294,36 +306,21 @@ func (s *OutputRoomEventConsumer) copyTags(ctx context.Context, oldRoomID, newRo } func (s *OutputRoomEventConsumer) processMessage(ctx context.Context, event *rstypes.HeaderedEvent, streamPos uint64) error { - members, roomSize, err := s.localRoomMembers(ctx, event.RoomID()) + members, roomSize, err := s.localRoomMembers(ctx, event.RoomID().String()) if err != nil { return fmt.Errorf("s.localRoomMembers: %w", err) } switch { case event.Type() == spec.MRoomMember: - sender := spec.UserID{} - validRoomID, roomErr := spec.NewRoomID(event.RoomID()) - if roomErr != nil { - return roomErr + cevent, clientEvErr := synctypes.ToClientEvent(event, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return s.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) + if clientEvErr != nil { + return clientEvErr } - userID, queryErr := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, event.SenderID()) - if queryErr == nil && userID != nil { - sender = *userID - } - - sk := event.StateKey() - if sk != nil && *sk != "" { - skUserID, queryErr := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*sk)) - if queryErr == nil && skUserID != nil { - skString := skUserID.String() - sk = &skString - } else { - return fmt.Errorf("queryUserIDForSender: userID unknown for %s", *sk) - } - } - cevent := synctypes.ToClientEvent(event, synctypes.FormatAll, sender, sk) var member *localMembership - member, err = newLocalMembership(&cevent) + member, err = newLocalMembership(cevent) if err != nil { return fmt.Errorf("newLocalMembership: %w", err) } @@ -334,7 +331,7 @@ func (s *OutputRoomEventConsumer) processMessage(ctx context.Context, event *rst } case event.Type() == "m.room.tombstone" && event.StateKeyEquals(""): // Handle room upgrades - oldRoomID := event.RoomID() + oldRoomID := event.RoomID().String() newRoomID := gjson.GetBytes(event.Content(), "replacement_room").Str if err = s.handleRoomUpgrade(ctx, oldRoomID, newRoomID, members, roomSize); err != nil { // while inconvenient, this shouldn't stop us from sending push notifications @@ -351,7 +348,7 @@ func (s *OutputRoomEventConsumer) processMessage(ctx context.Context, event *rst log.WithFields(log.Fields{ "event_id": event.EventID(), - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), "num_members": len(members), "room_size": roomSize, }).Tracef("Notifying members") @@ -463,8 +460,21 @@ func (s *OutputRoomEventConsumer) roomName(ctx context.Context, event *rstypes.H } } + // Special case for invites, as we don't store any "current state" for these events, + // we need to make sure that, if present, the m.room.name is sent as well. + if event.Type() == spec.MRoomMember && + gjson.GetBytes(event.Content(), "membership").Str == "invite" { + invState := gjson.GetBytes(event.JSON(), "unsigned.invite_room_state") + for _, ev := range invState.Array() { + if ev.Get("type").Str == spec.MRoomName { + name := ev.Get("content.name").Str + return name, nil + } + } + } + req := &rsapi.QueryCurrentStateRequest{ - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), StateTuples: []gomatrixserverlib.StateKeyTuple{roomNameTuple, canonicalAliasTuple}, } var res rsapi.QueryCurrentStateResponse @@ -528,11 +538,11 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype if err != nil { return fmt.Errorf("pushrules.ActionsToTweaks: %w", err) } - // TODO: support coalescing. - if a != pushrules.NotifyAction && a != pushrules.CoalesceAction { + + if a != pushrules.NotifyAction { log.WithFields(log.Fields{ "event_id": event.EventID(), - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), "localpart": mem.Localpart, }).Tracef("Push rule evaluation rejected the event") return nil @@ -542,44 +552,32 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype if err != nil { return fmt.Errorf("s.localPushDevices: %w", err) } - - sender := spec.UserID{} - validRoomID, err := spec.NewRoomID(event.RoomID()) + clientEvent, err := synctypes.ToClientEvent(event, synctypes.FormatSync, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return s.rsAPI.QueryUserIDForSender(ctx, roomID, senderID) + }) if err != nil { return err } - userID, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, event.SenderID()) - if err == nil && userID != nil { - sender = *userID - } - sk := event.StateKey() - if sk != nil && *sk != "" { - skUserID, queryErr := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*event.StateKey())) - if queryErr == nil && skUserID != nil { - skString := skUserID.String() - sk = &skString - } - } n := &api.Notification{ Actions: actions, // UNSPEC: the spec doesn't say this is a ClientEvent, but the // fields seem to match. room_id should be missing, which // matches the behaviour of FormatSync. - Event: synctypes.ToClientEvent(event, synctypes.FormatSync, sender, sk), + Event: *clientEvent, // TODO: this is per-device, but it's not part of the primary // key. So inserting one notification per profile tag doesn't // make sense. What is this supposed to be? Sytests require it // to "work", but they only use a single device. ProfileTag: profileTag, - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), TS: spec.AsTimestamp(time.Now()), } if err = s.db.InsertNotification(ctx, mem.Localpart, mem.Domain, event.EventID(), streamPos, tweaks, n); err != nil { return fmt.Errorf("s.db.InsertNotification: %w", err) } - if err = s.syncProducer.GetAndSendNotificationData(ctx, mem.UserID, event.RoomID()); err != nil { + if err = s.syncProducer.GetAndSendNotificationData(ctx, mem.UserID, event.RoomID().String()); err != nil { return fmt.Errorf("s.syncProducer.GetAndSendNotificationData: %w", err) } @@ -591,7 +589,7 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype log.WithFields(log.Fields{ "event_id": event.EventID(), - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), "localpart": mem.Localpart, "num_urls": len(devicesByURLAndFormat), "num_unread": userNumUnreadNotifs, @@ -648,11 +646,7 @@ func (s *OutputRoomEventConsumer) notifyLocal(ctx context.Context, event *rstype // user. Returns actions (including dont_notify). func (s *OutputRoomEventConsumer) evaluatePushRules(ctx context.Context, event *rstypes.HeaderedEvent, mem *localMembership, roomSize int) ([]*pushrules.Action, error) { user := "" - validRoomID, err := spec.NewRoomID(event.RoomID()) - if err != nil { - return nil, err - } - sender, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, event.SenderID()) + sender, err := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID()) if err == nil { user = sender.String() } @@ -686,7 +680,7 @@ func (s *OutputRoomEventConsumer) evaluatePushRules(ctx context.Context, event * ctx: ctx, rsAPI: s.rsAPI, mem: mem, - roomID: event.RoomID(), + roomID: event.RoomID().String(), roomSize: roomSize, } eval := pushrules.NewRuleSetEvaluator(ec, &ruleSets.Global) @@ -704,7 +698,7 @@ func (s *OutputRoomEventConsumer) evaluatePushRules(ctx context.Context, event * log.WithFields(log.Fields{ "event_id": event.EventID(), - "room_id": event.RoomID(), + "room_id": event.RoomID().String(), "localpart": mem.Localpart, "rule_id": rule.RuleID, }).Trace("Matched a push rule") @@ -793,16 +787,12 @@ func (s *OutputRoomEventConsumer) notifyHTTP(ctx context.Context, event *rstypes }, Devices: devices, EventID: event.EventID(), - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), }, } default: - validRoomID, err := spec.NewRoomID(event.RoomID()) - if err != nil { - return nil, err - } - sender, err := s.rsAPI.QueryUserIDForSender(ctx, *validRoomID, event.SenderID()) + sender, err := s.rsAPI.QueryUserIDForSender(ctx, event.RoomID(), event.SenderID()) if err != nil { logger.WithError(err).Errorf("Failed to get userID for sender %s", event.SenderID()) return nil, err @@ -816,7 +806,7 @@ func (s *OutputRoomEventConsumer) notifyHTTP(ctx context.Context, event *rstypes Devices: devices, EventID: event.EventID(), ID: event.EventID(), - RoomID: event.RoomID(), + RoomID: event.RoomID().String(), RoomName: roomName, Sender: sender.String(), Type: event.Type(), @@ -830,18 +820,15 @@ func (s *OutputRoomEventConsumer) notifyHTTP(ctx context.Context, event *rstypes logger.WithError(err).Errorf("Failed to convert local user to userID %s", localpart) return nil, err } - roomID, err := spec.NewRoomID(event.RoomID()) + localSender, err := s.rsAPI.QuerySenderIDForUser(ctx, event.RoomID(), *userID) if err != nil { - logger.WithError(err).Errorf("event roomID is invalid %s", event.RoomID()) + logger.WithError(err).Errorf("Failed to get local user senderID for room %s: %s", userID.String(), event.RoomID().String()) return nil, err + } else if localSender == nil { + logger.WithError(err).Errorf("Failed to get local user senderID for room %s: %s", userID.String(), event.RoomID().String()) + return nil, fmt.Errorf("no sender ID for user %s in %s", userID.String(), event.RoomID().String()) } - - localSender, err := s.rsAPI.QuerySenderIDForUser(ctx, *roomID, *userID) - if err != nil { - logger.WithError(err).Errorf("Failed to get local user senderID for room %s: %s", userID.String(), event.RoomID()) - return nil, err - } - if event.StateKey() != nil && *event.StateKey() == string(localSender) { + if event.StateKey() != nil && *event.StateKey() == string(*localSender) { req.Notification.UserIsTarget = true } } diff --git a/userapi/consumers/roomserver_test.go b/userapi/consumers/roomserver_test.go index 49dd5b238..7b7c08618 100644 --- a/userapi/consumers/roomserver_test.go +++ b/userapi/consumers/roomserver_test.go @@ -81,12 +81,8 @@ func Test_evaluatePushRules(t *testing.T) { { name: "m.reaction doesn't notify", eventContent: `{"type":"m.reaction","room_id":"!room:example.com"}`, - wantAction: pushrules.DontNotifyAction, - wantActions: []*pushrules.Action{ - { - Kind: pushrules.DontNotifyAction, - }, - }, + wantAction: pushrules.UnknownAction, + wantActions: []*pushrules.Action{}, }, { name: "m.room.message notifies", @@ -136,7 +132,7 @@ func Test_evaluatePushRules(t *testing.T) { t.Fatalf("expected action to be '%s', got '%s'", tc.wantAction, gotAction) } // this is taken from `notifyLocal` - if tc.wantNotify && gotAction != pushrules.NotifyAction && gotAction != pushrules.CoalesceAction { + if tc.wantNotify && gotAction != pushrules.NotifyAction { t.Fatalf("expected to notify but didn't") } }) diff --git a/userapi/internal/device_list_update.go b/userapi/internal/device_list_update.go index 3fccf56bb..b40635160 100644 --- a/userapi/internal/device_list_update.go +++ b/userapi/internal/device_list_update.go @@ -21,9 +21,11 @@ import ( "fmt" "hash/fnv" "net" + "strconv" "sync" "time" + "github.com/matrix-org/dendrite/federationapi/statistics" rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" @@ -107,6 +109,8 @@ type DeviceListUpdater struct { userIDToChan map[string]chan bool userIDToChanMu *sync.Mutex rsAPI rsapi.KeyserverRoomserverAPI + + isBlacklistedOrBackingOffFn func(s spec.ServerName) (*statistics.ServerStatistics, error) } // DeviceListUpdaterDatabase is the subset of functionality from storage.Database required for the updater. @@ -142,26 +146,52 @@ type KeyChangeProducer interface { ProduceKeyChanges(keys []api.DeviceMessage) error } +var deviceListUpdaterBackpressure = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "dendrite", + Subsystem: "keyserver", + Name: "worker_backpressure", + Help: "How many device list updater requests are queued", + }, + []string{"worker_id"}, +) +var deviceListUpdaterServersRetrying = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "dendrite", + Subsystem: "keyserver", + Name: "worker_servers_retrying", + Help: "How many servers are queued for retry", + }, + []string{"worker_id"}, +) + // NewDeviceListUpdater creates a new updater which fetches fresh device lists when they go stale. func NewDeviceListUpdater( process *process.ProcessContext, db DeviceListUpdaterDatabase, api DeviceListUpdaterAPI, producer KeyChangeProducer, fedClient fedsenderapi.KeyserverFederationAPI, numWorkers int, - rsAPI rsapi.KeyserverRoomserverAPI, thisServer spec.ServerName, + rsAPI rsapi.KeyserverRoomserverAPI, + thisServer spec.ServerName, + enableMetrics bool, + isBlacklistedOrBackingOffFn func(s spec.ServerName) (*statistics.ServerStatistics, error), ) *DeviceListUpdater { + if enableMetrics { + prometheus.MustRegister(deviceListUpdaterBackpressure, deviceListUpdaterServersRetrying) + } return &DeviceListUpdater{ - process: process, - userIDToMutex: make(map[string]*sync.Mutex), - mu: &sync.Mutex{}, - db: db, - api: api, - producer: producer, - fedClient: fedClient, - thisServer: thisServer, - workerChans: make([]chan spec.ServerName, numWorkers), - userIDToChan: make(map[string]chan bool), - userIDToChanMu: &sync.Mutex{}, - rsAPI: rsAPI, + process: process, + userIDToMutex: make(map[string]*sync.Mutex), + mu: &sync.Mutex{}, + db: db, + api: api, + producer: producer, + fedClient: fedClient, + thisServer: thisServer, + workerChans: make([]chan spec.ServerName, numWorkers), + userIDToChan: make(map[string]chan bool), + userIDToChanMu: &sync.Mutex{}, + rsAPI: rsAPI, + isBlacklistedOrBackingOffFn: isBlacklistedOrBackingOffFn, } } @@ -173,18 +203,20 @@ func (u *DeviceListUpdater) Start() error { // to stop (in this transaction) until key requests can be made. ch := make(chan spec.ServerName, 10) u.workerChans[i] = ch - go u.worker(ch) + go u.worker(ch, i) } staleLists, err := u.db.StaleDeviceLists(u.process.Context(), []spec.ServerName{}) if err != nil { return err } + + newStaleLists := dedupeStaleLists(staleLists) offset, step := time.Second*10, time.Second - if max := len(staleLists); max > 120 { + if max := len(newStaleLists); max > 120 { step = (time.Second * 120) / time.Duration(max) } - for _, userID := range staleLists { + for _, userID := range newStaleLists { userID := userID // otherwise we are only sending the last entry time.AfterFunc(offset, func() { u.notifyWorkers(userID) @@ -336,11 +368,22 @@ func (u *DeviceListUpdater) notifyWorkers(userID string) { if err != nil { return } + _, err = u.isBlacklistedOrBackingOffFn(remoteServer) + var federationClientError *fedsenderapi.FederationClientError + if errors.As(err, &federationClientError) { + if federationClientError.Blacklisted { + return + } + } + hash := fnv.New32a() _, _ = hash.Write([]byte(remoteServer)) index := int(int64(hash.Sum32()) % int64(len(u.workerChans))) ch := u.assignChannel(userID) + // Since workerChans are buffered, we only increment here and let the worker + // decrement it once it is done processing. + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(index)}).Inc() u.workerChans[index] <- remoteServer select { case <-ch: @@ -370,27 +413,44 @@ func (u *DeviceListUpdater) clearChannel(userID string) { } } -func (u *DeviceListUpdater) worker(ch chan spec.ServerName) { +func (u *DeviceListUpdater) worker(ch chan spec.ServerName, workerID int) { retries := make(map[spec.ServerName]time.Time) retriesMu := &sync.Mutex{} // restarter goroutine which will inject failed servers into ch when it is time go func() { var serversToRetry []spec.ServerName for { - serversToRetry = serversToRetry[:0] // reuse memory - time.Sleep(time.Second) + // nuke serversToRetry by re-slicing it to be "empty". + // The capacity of the slice is unchanged, which ensures we can reuse the memory. + serversToRetry = serversToRetry[:0] + + deviceListUpdaterServersRetrying.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Set(float64(len(retries))) + time.Sleep(time.Second * 2) + + // -2, so we have space for incoming device list updates over federation + maxServers := (cap(ch) - len(ch)) - 2 + if maxServers <= 0 { + continue + } + retriesMu.Lock() now := time.Now() for srv, retryAt := range retries { if now.After(retryAt) { serversToRetry = append(serversToRetry, srv) + if maxServers == len(serversToRetry) { + break + } } } + for _, srv := range serversToRetry { delete(retries, srv) } retriesMu.Unlock() + for _, srv := range serversToRetry { + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Inc() ch <- srv } } @@ -399,8 +459,18 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName) { retriesMu.Lock() _, exists := retries[serverName] retriesMu.Unlock() - if exists { - // Don't retry a server that we're already waiting for. + + // If the serverName is coming from retries, maybe it was + // blacklisted in the meantime. + _, err := u.isBlacklistedOrBackingOffFn(serverName) + var federationClientError *fedsenderapi.FederationClientError + // unwrap errors and check for FederationClientError, if found, federationClientError will be not nil + errors.As(err, &federationClientError) + isBlacklisted := federationClientError != nil && federationClientError.Blacklisted + + // Don't retry a server that we're already waiting for or is blacklisted by now. + if exists || isBlacklisted { + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Dec() continue } waitTime, shouldRetry := u.processServer(serverName) @@ -411,11 +481,18 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName) { } retriesMu.Unlock() } + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Dec() } } func (u *DeviceListUpdater) processServer(serverName spec.ServerName) (time.Duration, bool) { ctx := u.process.Context() + // If the process.Context is canceled, there is no need to go further. + // This avoids spamming the logs when shutting down + if errors.Is(ctx.Err(), context.Canceled) { + return defaultWaitTime, false + } + logger := util.GetLogger(ctx).WithField("server_name", serverName) deviceListUpdateCount.WithLabelValues(string(serverName)).Inc() @@ -428,13 +505,6 @@ func (u *DeviceListUpdater) processServer(serverName spec.ServerName) (time.Dura return waitTime, true } - defer func() { - for _, userID := range userIDs { - // always clear the channel to unblock Update calls regardless of success/failure - u.clearChannel(userID) - } - }() - for _, userID := range userIDs { userWait, err := u.processServerUser(ctx, serverName, userID) if err != nil { @@ -461,6 +531,11 @@ func (u *DeviceListUpdater) processServer(serverName spec.ServerName) (time.Dura func (u *DeviceListUpdater) processServerUser(ctx context.Context, serverName spec.ServerName, userID string) (time.Duration, error) { ctx, cancel := context.WithTimeout(ctx, requestTimeout) defer cancel() + + // If we are processing more than one user per server, this unblocks further calls to Update + // immediately instead of just after **all** users have been processed. + defer u.clearChannel(userID) + logger := util.GetLogger(ctx).WithFields(logrus.Fields{ "server_name": serverName, "user_id": userID, @@ -579,3 +654,24 @@ func (u *DeviceListUpdater) updateDeviceList(res *fclient.RespUserDevices) error } return nil } + +// dedupeStaleLists de-duplicates the stateList entries using the domain. +// This is used on startup, processServer is getting all users anyway, so +// there is no need to send every user to the workers. +func dedupeStaleLists(staleLists []string) []string { + seenDomains := make(map[spec.ServerName]struct{}) + newStaleLists := make([]string, 0, len(staleLists)) + for _, userID := range staleLists { + _, domain, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + // non-fatal and should not block starting up + continue + } + if _, ok := seenDomains[domain]; ok { + continue + } + newStaleLists = append(newStaleLists, userID) + seenDomains[domain] = struct{}{} + } + return newStaleLists +} diff --git a/userapi/internal/device_list_update_test.go b/userapi/internal/device_list_update_test.go index 10b9c6521..a2f1869d1 100644 --- a/userapi/internal/device_list_update_test.go +++ b/userapi/internal/device_list_update_test.go @@ -27,6 +27,9 @@ import ( "testing" "time" + api2 "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/statistics" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/fclient" @@ -128,6 +131,10 @@ type mockDeviceListUpdaterAPI struct { func (d *mockDeviceListUpdaterAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) { } +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + type roundTripper struct { fn func(*http.Request) (*http.Response, error) } @@ -161,7 +168,7 @@ func TestUpdateHavePrevID(t *testing.T) { } ap := &mockDeviceListUpdaterAPI{} producer := &mockKeyChangeProducer{} - updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, nil, 1, nil, "localhost") + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, nil, 1, nil, "localhost", caching.DisableMetrics, testIsBlacklistedOrBackingOff) event := gomatrixserverlib.DeviceListUpdateEvent{ DeviceDisplayName: "Foo Bar", Deleted: false, @@ -233,7 +240,7 @@ func TestUpdateNoPrevID(t *testing.T) { `)), }, nil }) - updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 2, nil, "example.test") + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 2, nil, "example.test", caching.DisableMetrics, testIsBlacklistedOrBackingOff) if err := updater.Start(); err != nil { t.Fatalf("failed to start updater: %s", err) } @@ -303,7 +310,7 @@ func TestDebounce(t *testing.T) { close(incomingFedReq) return <-fedCh, nil }) - updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 1, nil, "localhost") + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 1, nil, "localhost", caching.DisableMetrics, testIsBlacklistedOrBackingOff) if err := updater.Start(); err != nil { t.Fatalf("failed to start updater: %s", err) } @@ -406,7 +413,7 @@ func TestDeviceListUpdater_CleanUp(t *testing.T) { updater := NewDeviceListUpdater(processCtx, db, nil, nil, nil, - 0, rsAPI, "test") + 0, rsAPI, "test", caching.DisableMetrics, testIsBlacklistedOrBackingOff) if err := updater.CleanUp(); err != nil { t.Error(err) } @@ -428,3 +435,114 @@ func TestDeviceListUpdater_CleanUp(t *testing.T) { } }) } + +func Test_dedupeStateList(t *testing.T) { + alice := "@alice:localhost" + bob := "@bob:localhost" + charlie := "@charlie:notlocalhost" + invalidUserID := "iaminvalid:localhost" + + tests := []struct { + name string + staleLists []string + want []string + }{ + { + name: "empty stateLists", + staleLists: []string{}, + want: []string{}, + }, + { + name: "single entry", + staleLists: []string{alice}, + want: []string{alice}, + }, + { + name: "multiple entries without dupe servers", + staleLists: []string{alice, charlie}, + want: []string{alice, charlie}, + }, + { + name: "multiple entries with dupe servers", + staleLists: []string{alice, bob, charlie}, + want: []string{alice, charlie}, + }, + { + name: "list with invalid userID", + staleLists: []string{alice, bob, invalidUserID}, + want: []string{alice}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := dedupeStaleLists(tt.staleLists); !reflect.DeepEqual(got, tt.want) { + t.Errorf("dedupeStaleLists() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDeviceListUpdaterIgnoreBlacklisted(t *testing.T) { + unreachableServer := spec.ServerName("notlocalhost") + + updater := DeviceListUpdater{ + workerChans: make([]chan spec.ServerName, 1), + isBlacklistedOrBackingOffFn: func(s spec.ServerName) (*statistics.ServerStatistics, error) { + switch s { + case unreachableServer: + return nil, &api2.FederationClientError{Blacklisted: true} + } + return nil, nil + }, + mu: &sync.Mutex{}, + userIDToChanMu: &sync.Mutex{}, + userIDToChan: make(map[string]chan bool), + userIDToMutex: make(map[string]*sync.Mutex), + } + workerCh := make(chan spec.ServerName) + defer close(workerCh) + updater.workerChans[0] = workerCh + + // happy case + alice := "@alice:localhost" + aliceCh := updater.assignChannel(alice) + defer updater.clearChannel(alice) + + // failing case + bob := "@bob:" + unreachableServer + bobCh := updater.assignChannel(string(bob)) + defer updater.clearChannel(string(bob)) + + expectedServers := map[spec.ServerName]struct{}{ + "localhost": {}, + } + unexpectedServers := make(map[spec.ServerName]struct{}) + + go func() { + for serverName := range workerCh { + switch serverName { + case "localhost": + delete(expectedServers, serverName) + aliceCh <- true // unblock notifyWorkers + case unreachableServer: // this should not happen as it is "filtered" away by the blacklist + unexpectedServers[serverName] = struct{}{} + bobCh <- true + default: + unexpectedServers[serverName] = struct{}{} + } + } + }() + + // alice is not blacklisted + updater.notifyWorkers(alice) + // bob is blacklisted + updater.notifyWorkers(string(bob)) + + for server := range expectedServers { + t.Errorf("Server still in expectedServers map: %s", server) + } + + for server := range unexpectedServers { + t.Errorf("unexpected server in result: %s", server) + } +} diff --git a/userapi/internal/user_api.go b/userapi/internal/user_api.go index 4305c13a9..4e3c2671a 100644 --- a/userapi/internal/user_api.go +++ b/userapi/internal/user_api.go @@ -563,12 +563,15 @@ func (a *UserInternalAPI) QueryAccountData(ctx context.Context, req *api.QueryAc func (a *UserInternalAPI) QueryAccessToken(ctx context.Context, req *api.QueryAccessTokenRequest, res *api.QueryAccessTokenResponse) error { if req.AppServiceUserID != "" { appServiceDevice, err := a.queryAppServiceToken(ctx, req.AccessToken, req.AppServiceUserID) - if err != nil { - res.Err = err.Error() - } - res.Device = appServiceDevice + if err != nil || appServiceDevice != nil { + if err != nil { + res.Err = err.Error() + } + res.Device = appServiceDevice - return nil + return nil + } + // If the provided token wasn't an as_token (both err and appServiceDevice are nil), continue with normal auth. } device, err := a.DB.GetDeviceByAccessToken(ctx, req.AccessToken) if err != nil { diff --git a/userapi/storage/postgres/cross_signing_keys_table.go b/userapi/storage/postgres/cross_signing_keys_table.go index 138b629d7..deb355718 100644 --- a/userapi/storage/postgres/cross_signing_keys_table.go +++ b/userapi/storage/postgres/cross_signing_keys_table.go @@ -77,7 +77,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( for rows.Next() { var keyTypeInt int16 var keyData spec.Base64Bytes - if err := rows.Scan(&keyTypeInt, &keyData); err != nil { + if err = rows.Scan(&keyTypeInt, &keyData); err != nil { return nil, err } keyType, ok := types.KeyTypeIntToPurpose[keyTypeInt] @@ -86,6 +86,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( } r[keyType] = keyData } + err = rows.Err() return } diff --git a/userapi/storage/postgres/cross_signing_sigs_table.go b/userapi/storage/postgres/cross_signing_sigs_table.go index 61a381184..cba015e13 100644 --- a/userapi/storage/postgres/cross_signing_sigs_table.go +++ b/userapi/storage/postgres/cross_signing_sigs_table.go @@ -98,7 +98,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( var userID string var keyID gomatrixserverlib.KeyID var signature spec.Base64Bytes - if err := rows.Scan(&userID, &keyID, &signature); err != nil { + if err = rows.Scan(&userID, &keyID, &signature); err != nil { return nil, err } if _, ok := r[userID]; !ok { @@ -106,6 +106,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( } r[userID][keyID] = signature } + err = rows.Err() return } diff --git a/userapi/storage/postgres/key_backup_table.go b/userapi/storage/postgres/key_backup_table.go index 91a34c357..59944a125 100644 --- a/userapi/storage/postgres/key_backup_table.go +++ b/userapi/storage/postgres/key_backup_table.go @@ -162,5 +162,5 @@ func unpackKeys(ctx context.Context, rows *sql.Rows) (map[string]map[string]api. roomData[key.SessionID] = key.KeyBackupSession result[key.RoomID] = roomData } - return result, nil + return result, rows.Err() } diff --git a/userapi/storage/postgres/key_changes_table.go b/userapi/storage/postgres/key_changes_table.go index a00494140..de3a9e9c8 100644 --- a/userapi/storage/postgres/key_changes_table.go +++ b/userapi/storage/postgres/key_changes_table.go @@ -115,7 +115,7 @@ func (s *keyChangesStatements) SelectKeyChanges( for rows.Next() { var userID string var offset int64 - if err := rows.Scan(&userID, &offset); err != nil { + if err = rows.Scan(&userID, &offset); err != nil { return nil, 0, err } if offset > latestOffset { @@ -123,5 +123,6 @@ func (s *keyChangesStatements) SelectKeyChanges( } userIDs = append(userIDs, userID) } + err = rows.Err() return } diff --git a/userapi/storage/postgres/one_time_keys_table.go b/userapi/storage/postgres/one_time_keys_table.go index 972a59147..a00f4d6f6 100644 --- a/userapi/storage/postgres/one_time_keys_table.go +++ b/userapi/storage/postgres/one_time_keys_table.go @@ -134,7 +134,7 @@ func (s *oneTimeKeysStatements) CountOneTimeKeys(ctx context.Context, userID, de } counts.KeyCount[algorithm] = count } - return counts, nil + return counts, rows.Err() } func (s *oneTimeKeysStatements) InsertOneTimeKeys(ctx context.Context, txn *sql.Tx, keys api.OneTimeKeys) (*api.OneTimeKeysCount, error) { diff --git a/userapi/storage/postgres/profile_table.go b/userapi/storage/postgres/profile_table.go index e404c32f2..e4f55ed94 100644 --- a/userapi/storage/postgres/profile_table.go +++ b/userapi/storage/postgres/profile_table.go @@ -165,5 +165,5 @@ func (s *profilesStatements) SelectProfilesBySearch( profiles = append(profiles, profile) } } - return profiles, nil + return profiles, rows.Err() } diff --git a/userapi/storage/postgres/threepid_table.go b/userapi/storage/postgres/threepid_table.go index 15b42a0a6..fc46061dc 100644 --- a/userapi/storage/postgres/threepid_table.go +++ b/userapi/storage/postgres/threepid_table.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/gomatrixserverlib/spec" @@ -94,6 +95,7 @@ func (s *threepidStatements) SelectThreePIDsForLocalpart( if err != nil { return } + defer internal.CloseAndLogIfError(ctx, rows, "SelectThreePIDsForLocalpart: failed to close rows") threepids = []authtypes.ThreePID{} for rows.Next() { @@ -107,7 +109,7 @@ func (s *threepidStatements) SelectThreePIDsForLocalpart( Medium: medium, }) } - + err = rows.Err() return } diff --git a/userapi/storage/sqlite3/account_data_table.go b/userapi/storage/sqlite3/account_data_table.go index 3a6367c45..240647b32 100644 --- a/userapi/storage/sqlite3/account_data_table.go +++ b/userapi/storage/sqlite3/account_data_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "encoding/json" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/gomatrixserverlib/spec" @@ -95,6 +96,7 @@ func (s *accountDataStatements) SelectAccountData( if err != nil { return nil, nil, err } + defer internal.CloseAndLogIfError(ctx, rows, "SelectAccountData: failed to close rows") global := map[string]json.RawMessage{} rooms := map[string]map[string]json.RawMessage{} @@ -118,7 +120,7 @@ func (s *accountDataStatements) SelectAccountData( } } - return global, rooms, nil + return global, rooms, rows.Err() } func (s *accountDataStatements) SelectAccountDataByType( diff --git a/userapi/storage/sqlite3/cross_signing_keys_table.go b/userapi/storage/sqlite3/cross_signing_keys_table.go index 5c2ce7039..65b9ff1af 100644 --- a/userapi/storage/sqlite3/cross_signing_keys_table.go +++ b/userapi/storage/sqlite3/cross_signing_keys_table.go @@ -76,7 +76,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( for rows.Next() { var keyTypeInt int16 var keyData spec.Base64Bytes - if err := rows.Scan(&keyTypeInt, &keyData); err != nil { + if err = rows.Scan(&keyTypeInt, &keyData); err != nil { return nil, err } keyType, ok := types.KeyTypeIntToPurpose[keyTypeInt] @@ -85,6 +85,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( } r[keyType] = keyData } + err = rows.Err() return } diff --git a/userapi/storage/sqlite3/cross_signing_sigs_table.go b/userapi/storage/sqlite3/cross_signing_sigs_table.go index 657264115..bf400a00e 100644 --- a/userapi/storage/sqlite3/cross_signing_sigs_table.go +++ b/userapi/storage/sqlite3/cross_signing_sigs_table.go @@ -96,7 +96,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( var userID string var keyID gomatrixserverlib.KeyID var signature spec.Base64Bytes - if err := rows.Scan(&userID, &keyID, &signature); err != nil { + if err = rows.Scan(&userID, &keyID, &signature); err != nil { return nil, err } if _, ok := r[userID]; !ok { @@ -104,6 +104,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( } r[userID][keyID] = signature } + err = rows.Err() return } diff --git a/userapi/storage/sqlite3/devices_table.go b/userapi/storage/sqlite3/devices_table.go index 23e823116..5ce285c87 100644 --- a/userapi/storage/sqlite3/devices_table.go +++ b/userapi/storage/sqlite3/devices_table.go @@ -296,6 +296,7 @@ func (s *devicesStatements) SelectDevicesByLocalpart( if err != nil { return devices, err } + defer internal.CloseAndLogIfError(ctx, rows, "SelectDevicesByLocalpart: failed to close rows") var dev api.Device var lastseents sql.NullInt64 @@ -325,7 +326,7 @@ func (s *devicesStatements) SelectDevicesByLocalpart( devices = append(devices, dev) } - return devices, nil + return devices, rows.Err() } func (s *devicesStatements) SelectDevicesByID(ctx context.Context, deviceIDs []string) ([]api.Device, error) { diff --git a/userapi/storage/sqlite3/key_backup_table.go b/userapi/storage/sqlite3/key_backup_table.go index ed2746310..1cdaca180 100644 --- a/userapi/storage/sqlite3/key_backup_table.go +++ b/userapi/storage/sqlite3/key_backup_table.go @@ -162,5 +162,5 @@ func unpackKeys(ctx context.Context, rows *sql.Rows) (map[string]map[string]api. roomData[key.SessionID] = key.KeyBackupSession result[key.RoomID] = roomData } - return result, nil + return result, rows.Err() } diff --git a/userapi/storage/sqlite3/key_changes_table.go b/userapi/storage/sqlite3/key_changes_table.go index 923bb57eb..7a4898cfb 100644 --- a/userapi/storage/sqlite3/key_changes_table.go +++ b/userapi/storage/sqlite3/key_changes_table.go @@ -113,7 +113,7 @@ func (s *keyChangesStatements) SelectKeyChanges( for rows.Next() { var userID string var offset int64 - if err := rows.Scan(&userID, &offset); err != nil { + if err = rows.Scan(&userID, &offset); err != nil { return nil, 0, err } if offset > latestOffset { @@ -121,5 +121,6 @@ func (s *keyChangesStatements) SelectKeyChanges( } userIDs = append(userIDs, userID) } + err = rows.Err() return } diff --git a/userapi/storage/sqlite3/one_time_keys_table.go b/userapi/storage/sqlite3/one_time_keys_table.go index a992d399c..2a5b1280b 100644 --- a/userapi/storage/sqlite3/one_time_keys_table.go +++ b/userapi/storage/sqlite3/one_time_keys_table.go @@ -140,7 +140,7 @@ func (s *oneTimeKeysStatements) CountOneTimeKeys(ctx context.Context, userID, de } counts.KeyCount[algorithm] = count } - return counts, nil + return counts, rows.Err() } func (s *oneTimeKeysStatements) InsertOneTimeKeys( diff --git a/userapi/storage/sqlite3/profile_table.go b/userapi/storage/sqlite3/profile_table.go index a20d7e848..7285110bc 100644 --- a/userapi/storage/sqlite3/profile_table.go +++ b/userapi/storage/sqlite3/profile_table.go @@ -173,5 +173,5 @@ func (s *profilesStatements) SelectProfilesBySearch( profiles = append(profiles, profile) } } - return profiles, nil + return profiles, rows.Err() } diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index a46ee9ebb..5a789dfd7 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -1,6 +1,7 @@ package storage_test import ( + "bytes" "context" "encoding/json" "fmt" @@ -758,3 +759,53 @@ func TestDeviceKeysStreamIDGeneration(t *testing.T) { } }) } + +func TestOneTimeKeys(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, clean := mustCreateKeyDatabase(t, dbType) + defer clean() + userID := "@alice:localhost" + deviceID := "alice_device" + otk := api.OneTimeKeys{ + UserID: userID, + DeviceID: deviceID, + KeyJSON: map[string]json.RawMessage{"curve25519:KEY1": []byte(`{"key":"v1"}`)}, + } + + // Add a one time key to the DB + _, err := db.StoreOneTimeKeys(ctx, otk) + MustNotError(t, err) + + // Check the count of one time keys is correct + count, err := db.OneTimeKeysCount(ctx, userID, deviceID) + MustNotError(t, err) + if count.KeyCount["curve25519"] != 1 { + t.Fatalf("Expected 1 key, got %d", count.KeyCount["curve25519"]) + } + + // Check the actual key contents are correct + keysJSON, err := db.ExistingOneTimeKeys(ctx, userID, deviceID, []string{"curve25519:KEY1"}) + MustNotError(t, err) + keyJSON, err := keysJSON["curve25519:KEY1"].MarshalJSON() + MustNotError(t, err) + if !bytes.Equal(keyJSON, []byte(`{"key":"v1"}`)) { + t.Fatalf("Existing keys do not match expected. Got %v", keysJSON["curve25519:KEY1"]) + } + + // Claim a one time key from the database. This should remove it from the database. + claimedKeys, err := db.ClaimKeys(ctx, map[string]map[string]string{userID: {deviceID: "curve25519"}}) + MustNotError(t, err) + + // Check the claimed key contents are correct + if !reflect.DeepEqual(claimedKeys[0], otk) { + t.Fatalf("Expected to claim stored key %v. Got %v", otk, claimedKeys[0]) + } + + // Check the count of one time keys is now zero + count, err = db.OneTimeKeysCount(ctx, userID, deviceID) + MustNotError(t, err) + if count.KeyCount["curve25519"] != 0 { + t.Fatalf("Expected 0 keys, got %d", count.KeyCount["curve25519"]) + } + }) +} diff --git a/userapi/userapi.go b/userapi/userapi.go index 6b6dac884..a1c9b94a9 100644 --- a/userapi/userapi.go +++ b/userapi/userapi.go @@ -18,10 +18,12 @@ import ( "time" fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/pushgateway" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/sirupsen/logrus" rsapi "github.com/matrix-org/dendrite/roomserver/api" @@ -46,6 +48,8 @@ func NewInternalAPI( natsInstance *jetstream.NATSInstance, rsAPI rsapi.UserRoomserverAPI, fedClient fedsenderapi.KeyserverFederationAPI, + enableMetrics bool, + blacklistedOrBackingOffFn func(s spec.ServerName) (*statistics.ServerStatistics, error), ) *internal.UserInternalAPI { js, _ := natsInstance.Prepare(processContext, &dendriteCfg.Global.JetStream) appServices := dendriteCfg.Derived.ApplicationServices @@ -99,7 +103,7 @@ func NewInternalAPI( FedClient: fedClient, } - updater := internal.NewDeviceListUpdater(processContext, keyDB, userAPI, keyChangeProducer, fedClient, 8, rsAPI, dendriteCfg.Global.ServerName) // 8 workers TODO: configurable + updater := internal.NewDeviceListUpdater(processContext, keyDB, userAPI, keyChangeProducer, fedClient, dendriteCfg.UserAPI.WorkerCount, rsAPI, dendriteCfg.Global.ServerName, enableMetrics, blacklistedOrBackingOffFn) userAPI.Updater = updater // Remove users which we don't share a room with anymore if err := updater.CleanUp(); err != nil { diff --git a/userapi/util/notify_test.go b/userapi/util/notify_test.go index 3017069bc..2ea978d69 100644 --- a/userapi/util/notify_test.go +++ b/userapi/util/notify_test.go @@ -23,6 +23,14 @@ import ( userUtil "github.com/matrix-org/dendrite/userapi/util" ) +func queryUserIDForSender(senderID spec.SenderID) (*spec.UserID, error) { + if senderID == "" { + return nil, nil + } + + return spec.NewUserID(string(senderID), true) +} + func TestNotifyUserCountsAsync(t *testing.T) { alice := test.NewUser(t) aliceLocalpart, serverName, err := gomatrixserverlib.SplitID('@', alice.ID) @@ -100,13 +108,14 @@ func TestNotifyUserCountsAsync(t *testing.T) { } // Insert a dummy event - sender, err := spec.NewUserID(alice.ID, true) + ev, err := synctypes.ToClientEvent(dummyEvent, synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { + return queryUserIDForSender(senderID) + }) if err != nil { t.Error(err) } - sk := "" if err := db.InsertNotification(ctx, aliceLocalpart, serverName, dummyEvent.EventID(), 0, nil, &api.Notification{ - Event: synctypes.ToClientEvent(dummyEvent, synctypes.FormatAll, *sender, &sk), + Event: *ev, }); err != nil { t.Error(err) }