diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md index f40c56609..a3b8c0754 100644 --- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -17,7 +17,6 @@ see: https://www.matrix.org/security-disclosure-policy/ ### Background information - **Dendrite version or git SHA**: -- **Monolith or Polylith?**: - **SQLite3 or Postgres?**: - **Running in Docker?**: - **`go version`**: diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 2dc5f74c0..2f615a6a4 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -25,7 +25,7 @@ jobs: - name: Install Go uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: "stable" cache: true - name: Install Node @@ -62,14 +62,14 @@ jobs: - name: Install Go uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: "stable" - name: golangci-lint uses: golangci/golangci-lint-action@v3 # run go test with different go versions test: timeout-minutes: 10 - name: Unit tests (Go ${{ matrix.go }}) + name: Unit tests runs-on: ubuntu-latest # Service containers to run with `container-job` services: @@ -91,25 +91,21 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - strategy: - fail-fast: false - matrix: - go: ["1.19"] steps: - uses: actions/checkout@v3 - name: Setup go uses: actions/setup-go@v3 with: - go-version: ${{ matrix.go }} + go-version: "stable" - uses: actions/cache@v3 # manually set up caches, as they otherwise clash with different steps using setup-go with cache=true with: path: | ~/.cache/go-build ~/go/pkg/mod - key: ${{ runner.os }}-go${{ matrix.go }}-unit-${{ hashFiles('**/go.sum') }} + key: ${{ runner.os }}-go-stable-unit-${{ hashFiles('**/go.sum') }} restore-keys: | - ${{ runner.os }}-go${{ matrix.go }}-unit- + ${{ runner.os }}-go-stable-unit- - name: Set up gotestfmt uses: gotesttools/gotestfmt-action@v2 with: @@ -130,7 +126,6 @@ jobs: strategy: fail-fast: false matrix: - go: ["1.18", "1.19"] goos: ["linux"] goarch: ["amd64", "386"] steps: @@ -138,15 +133,15 @@ jobs: - name: Setup go uses: actions/setup-go@v3 with: - go-version: ${{ matrix.go }} + go-version: "stable" - uses: actions/cache@v3 with: path: | ~/.cache/go-build ~/go/pkg/mod - key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }} + key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }} restore-keys: | - key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}- + key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}- - name: Install dependencies x86 if: ${{ matrix.goarch == '386' }} run: sudo apt update && sudo apt-get install -y gcc-multilib @@ -164,23 +159,22 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go: ["1.18", "1.19"] goos: ["windows"] goarch: ["amd64"] steps: - uses: actions/checkout@v3 - - name: Setup Go ${{ matrix.go }} + - name: Setup Go uses: actions/setup-go@v3 with: - go-version: ${{ matrix.go }} + go-version: "stable" - uses: actions/cache@v3 with: path: | ~/.cache/go-build ~/go/pkg/mod - key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }} + key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }} restore-keys: | - key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}- + key: ${{ runner.os }}-go-stable-${{ matrix.goos }}-${{ matrix.goarch }}- - name: Install dependencies run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc - env: @@ -206,7 +200,7 @@ jobs: integration: timeout-minutes: 20 needs: initial-tests-done - name: Integration tests (Go ${{ matrix.go }}) + name: Integration tests runs-on: ubuntu-latest # Service containers to run with `container-job` services: @@ -228,16 +222,12 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - strategy: - fail-fast: false - matrix: - go: ["1.19"] steps: - uses: actions/checkout@v3 - name: Setup go uses: actions/setup-go@v3 with: - go-version: ${{ matrix.go }} + go-version: "stable" - name: Set up gotestfmt uses: gotesttools/gotestfmt-action@v2 with: @@ -248,9 +238,9 @@ jobs: path: | ~/.cache/go-build ~/go/pkg/mod - key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }} + key: ${{ runner.os }}-go-stable-test-race-${{ hashFiles('**/go.sum') }} restore-keys: | - ${{ runner.os }}-go${{ matrix.go }}-test-race- + ${{ runner.os }}-go-stable-test-race- - run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt env: POSTGRES_HOST: localhost @@ -261,6 +251,7 @@ jobs: uses: codecov/codecov-action@v3 with: flags: unittests + fail_ci_if_error: true # run database upgrade tests upgrade_test: @@ -273,7 +264,7 @@ jobs: - name: Setup go uses: actions/setup-go@v3 with: - go-version: "1.18" + go-version: "stable" cache: true - name: Build upgrade-tests run: go build ./cmd/dendrite-upgrade-tests @@ -293,7 +284,7 @@ jobs: - name: Setup go uses: actions/setup-go@v3 with: - go-version: "1.18" + go-version: "stable" cache: true - name: Build upgrade-tests run: go build ./cmd/dendrite-upgrade-tests @@ -317,19 +308,9 @@ jobs: - label: SQLite Cgo cgo: 1 - - label: SQLite native, full HTTP APIs - api: full-http - - - label: SQLite Cgo, full HTTP APIs - api: full-http - cgo: 1 - - label: PostgreSQL postgres: postgres - - label: PostgreSQL, full HTTP APIs - postgres: postgres - api: full-http container: image: matrixdotorg/sytest-dendrite volumes: @@ -338,7 +319,6 @@ jobs: - /root/.cache/go-mod:/gopath/pkg/mod env: POSTGRES: ${{ matrix.postgres && 1}} - API: ${{ matrix.api && 1 }} SYTEST_BRANCH: ${{ github.head_ref }} CGO_ENABLED: ${{ matrix.cgo && 1 }} steps: @@ -390,22 +370,9 @@ jobs: - label: SQLite Cgo cgo: 1 - - label: SQLite native, full HTTP APIs - api: full-http - cgo: 0 - - - label: SQLite Cgo, full HTTP APIs - api: full-http - cgo: 1 - - label: PostgreSQL postgres: Postgres cgo: 0 - - - label: PostgreSQL, full HTTP APIs - postgres: Postgres - api: full-http - cgo: 0 steps: # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement. # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path @@ -447,7 +414,7 @@ jobs: (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break done # Build initial Dendrite image - - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile . + - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile . working-directory: dendrite env: DOCKER_BUILDKIT: 1 @@ -459,8 +426,7 @@ jobs: shell: bash name: Run Complement Tests env: - COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }} - COMPLEMENT_DENDRITE_API: ${{ matrix.api && 1 }} + COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_ working-directory: complement diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2e17539d8..0c3053a56 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -61,7 +61,6 @@ jobs: cache-to: type=gha,mode=max context: . build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} - target: monolith platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -77,7 +76,6 @@ jobs: cache-to: type=gha,mode=max context: . build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} - target: monolith platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -98,86 +96,6 @@ jobs: with: sarif_file: "trivy-results.sarif" - polylith: - name: Polylith image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - security-events: write # To upload Trivy sarif files - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Get release tag & build flags - if: github.event_name == 'release' # Only for GitHub releases - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ env.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to GitHub Containers - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build main polylith image - if: github.ref_name == 'main' - id: docker_build_polylith - uses: docker/build-push-action@v3 - with: - cache-from: type=gha - cache-to: type=gha,mode=max - context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} - target: polylith - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} - - - name: Build release polylith image - if: github.event_name == 'release' # Only for GitHub releases - id: docker_build_polylith_release - uses: docker/build-push-action@v3 - with: - cache-from: type=gha - cache-to: type=gha,mode=max - context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} - target: polylith - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest - ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }} - - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} - format: "sarif" - output: "trivy-results.sarif" - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" - demo-pinecone: name: Pinecone demo image runs-on: ubuntu-latest diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index fa2304c1f..dff9b34c9 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -24,19 +24,8 @@ jobs: - label: SQLite Cgo cgo: 1 - - label: SQLite native, full HTTP APIs - api: full-http - - - label: SQLite Cgo, full HTTP APIs - api: full-http - cgo: 1 - - label: PostgreSQL postgres: postgres - - - label: PostgreSQL, full HTTP APIs - postgres: postgres - api: full-http container: image: matrixdotorg/sytest-dendrite:latest volumes: @@ -45,7 +34,6 @@ jobs: - /root/.cache/go-mod:/gopath/pkg/mod env: POSTGRES: ${{ matrix.postgres && 1}} - API: ${{ matrix.api && 1 }} SYTEST_BRANCH: ${{ github.head_ref }} RACE_DETECTION: 1 COVER: 1 @@ -93,7 +81,7 @@ jobs: - name: Install Go uses: actions/setup-go@v3 with: - go-version: '>=1.19.0' + go-version: 'stable' cache: true - name: Download all artifacts uses: actions/download-artifact@v3 @@ -125,22 +113,9 @@ jobs: - label: SQLite Cgo cgo: 1 - - label: SQLite native, full HTTP APIs - api: full-http - cgo: 0 - - - label: SQLite Cgo, full HTTP APIs - api: full-http - cgo: 1 - - label: PostgreSQL postgres: Postgres cgo: 0 - - - label: PostgreSQL, full HTTP APIs - postgres: Postgres - api: full-http - cgo: 0 steps: # Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement. # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path @@ -182,7 +157,7 @@ jobs: (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break done # Build initial Dendrite image - - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile . + - run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile . working-directory: dendrite env: DOCKER_BUILDKIT: 1 @@ -203,8 +178,7 @@ jobs: shell: bash name: Run Complement Tests env: - COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }} - COMPLEMENT_DENDRITE_API: ${{ matrix.api && 1 }} + COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.cgo }} COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_ COMPLEMENT_DENDRITE_COVER: 1 COMPLEMENT_POST_TEST_SCRIPT: /tmp/posttest.sh @@ -229,7 +203,7 @@ jobs: - name: Install Go uses: actions/setup-go@v3 with: - go-version: '>=1.19.0' + go-version: 'stable' cache: true - name: Download all artifacts uses: actions/download-artifact@v3 @@ -269,6 +243,7 @@ jobs: - name: Build env: CI_PACKAGE: true + NODE_OPTIONS: "--openssl-legacy-provider" run: yarn build working-directory: ./element-web - name: Edit Test Config diff --git a/CHANGES.md b/CHANGES.md index e1f7affb5..f0cd2b4b9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,21 @@ # Changelog +## Dendrite 0.11.1 (2023-02-10) + +**⚠️ DEPRECATION WARNING: This is the last release to have polylith and HTTP API mode. Future releases are monolith only.** + +### Features + +* Dendrite can now be compiled against Go 1.20 +* Initial store and forward support has been added +* A landing page showing that Dendrite is running has been added (contributed by [LukasLJL](https://github.com/LukasLJL)) + +### Fixes + +- `/sync` is now using significantly less database round trips when using Postgres, resulting in faster initial syncs, allowing larger accounts to login again +- Many under the hood pinecone improvements +- Publishing rooms is now possible again + ## Dendrite 0.11.0 (2023-01-20) The last three missing federation API Sytests have been fixed - bringing us to 100% server-server Synapse parity, with client-server parity at 93% 🎉 diff --git a/Dockerfile b/Dockerfile index bec8af21b..2487ea5bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,15 @@ #syntax=docker/dockerfile:1.2 +# +# base installs required dependencies and runs go mod download to cache dependencies +# +FROM --platform=${BUILDPLATFORM} docker.io/golang:1.20-alpine AS base +RUN apk --update --no-cache add bash build-base curl + # # build creates all needed binaries # -FROM --platform=${BUILDPLATFORM} docker.io/golang:1.19-alpine AS build -RUN apk --update --no-cache add bash build-base curl +FROM --platform=${BUILDPLATFORM} base AS build WORKDIR /src ARG TARGETOS ARG TARGETARCH @@ -18,44 +23,27 @@ RUN --mount=target=. \ CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0") \ go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/... + # -# The dendrite base image +# Builds the Dendrite image containing all required binaries # -FROM alpine:latest AS dendrite-base +FROM alpine:latest RUN apk --update --no-cache add curl +LABEL org.opencontainers.image.title="Dendrite" LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" LABEL org.opencontainers.image.licenses="Apache-2.0" LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/" LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C." -# -# Builds the polylith image and only contains the polylith binary -# -FROM dendrite-base AS polylith -LABEL org.opencontainers.image.title="Dendrite (Polylith)" - -COPY --from=build /out/dendrite-polylith-multi /usr/bin/ - -VOLUME /etc/dendrite -WORKDIR /etc/dendrite - -ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] - -# -# Builds the monolith image and contains all required binaries -# -FROM dendrite-base AS monolith -LABEL org.opencontainers.image.title="Dendrite (Monolith)" - COPY --from=build /out/create-account /usr/bin/create-account COPY --from=build /out/generate-config /usr/bin/generate-config COPY --from=build /out/generate-keys /usr/bin/generate-keys -COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server +COPY --from=build /out/dendrite /usr/bin/dendrite VOLUME /etc/dendrite WORKDIR /etc/dendrite -ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] +ENTRYPOINT ["/usr/bin/dendrite"] EXPOSE 8008 8448 diff --git a/README.md b/README.md index 12560cc04..41792ce17 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,6 @@ This does not mean: - Dendrite is ready for massive homeserver deployments. There is no sharding of microservices (although it is possible to run them on separate machines) and there is no high-availability/clustering support. Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices. -In the future, we will be able to scale up to gigantic servers (equivalent to `matrix.org`) via polylith mode. If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or join us in: @@ -88,7 +87,7 @@ Then point your favourite Matrix client at `http://localhost:8008` or `https://l We use a script called Are We Synapse Yet which checks Sytest compliance rates. Sytest is a black-box homeserver test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it -updates with CI. As of August 2022 we're at around 90% CS API coverage and 95% Federation coverage, though check +updates with CI. As of January 2023, we have 100% server-server parity with Synapse, and the client-server parity is at 93% , though check CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse servers such as matrix.org reasonably well, although there are still some missing features (like SSO and Third-party ID APIs). diff --git a/appservice/appservice.go b/appservice/appservice.go index 753850de7..5b1b93de2 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -21,14 +21,12 @@ import ( "sync" "time" - "github.com/gorilla/mux" "github.com/sirupsen/logrus" "github.com/matrix-org/gomatrixserverlib" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/appservice/consumers" - "github.com/matrix-org/dendrite/appservice/inthttp" "github.com/matrix-org/dendrite/appservice/query" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/base" @@ -36,16 +34,11 @@ import ( userapi "github.com/matrix-org/dendrite/userapi/api" ) -// AddInternalRoutes registers HTTP handlers for internal API calls -func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInternalAPI, enableMetrics bool) { - inthttp.AddRoutes(queryAPI, router, enableMetrics) -} - // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( base *base.BaseDendrite, - userAPI userapi.UserInternalAPI, + userAPI userapi.AppserviceUserAPI, rsAPI roomserverAPI.RoomserverInternalAPI, ) appserviceAPI.AppServiceInternalAPI { client := &http.Client{ diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go index 83c551fea..de9f5aaf1 100644 --- a/appservice/appservice_test.go +++ b/appservice/appservice_test.go @@ -10,12 +10,8 @@ import ( "strings" "testing" - "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/appservice/inthttp" - "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" @@ -129,26 +125,10 @@ func TestAppserviceInternalAPI(t *testing.T) { // Create required internal APIs rsAPI := roomserver.NewInternalAPI(base) - usrAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, nil, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(base, rsAPI, nil) asAPI := appservice.NewInternalAPI(base, usrAPI, rsAPI) - // Finally execute the tests - t.Run("HTTP API", func(t *testing.T) { - router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter() - appservice.AddInternalRoutes(router, asAPI, base.EnableMetrics) - apiURL, cancel := test.ListenAndServe(t, router, false) - defer cancel() - - asHTTPApi, err := inthttp.NewAppserviceClient(apiURL, &http.Client{}) - if err != nil { - t.Fatalf("failed to create HTTP client: %s", err) - } - runCases(t, asHTTPApi) - }) - - t.Run("Monolith", func(t *testing.T) { - runCases(t, asAPI) - }) + runCases(t, asAPI) }) } diff --git a/appservice/inthttp/client.go b/appservice/inthttp/client.go deleted file mode 100644 index f7f164877..000000000 --- a/appservice/inthttp/client.go +++ /dev/null @@ -1,84 +0,0 @@ -package inthttp - -import ( - "context" - "errors" - "net/http" - - "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/internal/httputil" -) - -// HTTP paths for the internal HTTP APIs -const ( - AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists" - AppServiceUserIDExistsPath = "/appservice/UserIDExists" - AppServiceLocationsPath = "/appservice/locations" - AppServiceUserPath = "/appservice/users" - AppServiceProtocolsPath = "/appservice/protocols" -) - -// httpAppServiceQueryAPI contains the URL to an appservice query API and a -// reference to a httpClient used to reach it -type httpAppServiceQueryAPI struct { - appserviceURL string - httpClient *http.Client -} - -// NewAppserviceClient creates a AppServiceQueryAPI implemented by talking -// to a HTTP POST API. -// If httpClient is nil an error is returned -func NewAppserviceClient( - appserviceURL string, - httpClient *http.Client, -) (api.AppServiceInternalAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is ") - } - return &httpAppServiceQueryAPI{appserviceURL, httpClient}, nil -} - -// RoomAliasExists implements AppServiceQueryAPI -func (h *httpAppServiceQueryAPI) RoomAliasExists( - ctx context.Context, - request *api.RoomAliasExistsRequest, - response *api.RoomAliasExistsResponse, -) error { - return httputil.CallInternalRPCAPI( - "RoomAliasExists", h.appserviceURL+AppServiceRoomAliasExistsPath, - h.httpClient, ctx, request, response, - ) -} - -// UserIDExists implements AppServiceQueryAPI -func (h *httpAppServiceQueryAPI) UserIDExists( - ctx context.Context, - request *api.UserIDExistsRequest, - response *api.UserIDExistsResponse, -) error { - return httputil.CallInternalRPCAPI( - "UserIDExists", h.appserviceURL+AppServiceUserIDExistsPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpAppServiceQueryAPI) Locations(ctx context.Context, request *api.LocationRequest, response *api.LocationResponse) error { - return httputil.CallInternalRPCAPI( - "ASLocation", h.appserviceURL+AppServiceLocationsPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpAppServiceQueryAPI) User(ctx context.Context, request *api.UserRequest, response *api.UserResponse) error { - return httputil.CallInternalRPCAPI( - "ASUser", h.appserviceURL+AppServiceUserPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpAppServiceQueryAPI) Protocols(ctx context.Context, request *api.ProtocolRequest, response *api.ProtocolResponse) error { - return httputil.CallInternalRPCAPI( - "ASProtocols", h.appserviceURL+AppServiceProtocolsPath, - h.httpClient, ctx, request, response, - ) -} diff --git a/appservice/inthttp/server.go b/appservice/inthttp/server.go deleted file mode 100644 index b70fad673..000000000 --- a/appservice/inthttp/server.go +++ /dev/null @@ -1,36 +0,0 @@ -package inthttp - -import ( - "github.com/gorilla/mux" - - "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/internal/httputil" -) - -// AddRoutes adds the AppServiceQueryAPI handlers to the http.ServeMux. -func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router, enableMetrics bool) { - internalAPIMux.Handle( - AppServiceRoomAliasExistsPath, - httputil.MakeInternalRPCAPI("AppserviceRoomAliasExists", enableMetrics, a.RoomAliasExists), - ) - - internalAPIMux.Handle( - AppServiceUserIDExistsPath, - httputil.MakeInternalRPCAPI("AppserviceUserIDExists", enableMetrics, a.UserIDExists), - ) - - internalAPIMux.Handle( - AppServiceProtocolsPath, - httputil.MakeInternalRPCAPI("AppserviceProtocols", enableMetrics, a.Protocols), - ) - - internalAPIMux.Handle( - AppServiceLocationsPath, - httputil.MakeInternalRPCAPI("AppserviceLocations", enableMetrics, a.Locations), - ) - - internalAPIMux.Handle( - AppServiceUserPath, - httputil.MakeInternalRPCAPI("AppserviceUser", enableMetrics, a.User), - ) -} diff --git a/build/dendritejs-pinecone/main.go b/build/dendritejs-pinecone/main.go index f44a77488..44e52286f 100644 --- a/build/dendritejs-pinecone/main.go +++ b/build/dendritejs-pinecone/main.go @@ -30,7 +30,6 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/base" @@ -177,19 +176,17 @@ func startup() { if err := cfg.Derive(); err != nil { logrus.Fatalf("Failed to derive values from config: %s", err) } - base := base.NewBaseDendrite(cfg, "Monolith") + base := base.NewBaseDendrite(cfg) defer base.Close() // nolint: errcheck rsAPI := roomserver.NewInternalAPI(base) federation := conn.CreateFederationClient(base, pSessions) - keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation, rsAPI) serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient()) - keyAPI.SetUserAPI(userAPI) + userAPI := userapi.NewInternalAPI(base, rsAPI, federation) asQuery := appservice.NewInternalAPI( base, userAPI, rsAPI, @@ -208,14 +205,12 @@ func startup() { FederationAPI: fedSenderAPI, RoomserverAPI: rsAPI, UserAPI: userAPI, - KeyAPI: keyAPI, //ServerKeyAPI: serverKeyAPI, ExtPublicRoomsProvider: rooms.NewPineconeRoomProvider(pRouter, pSessions, fedSenderAPI, federation), } monolith.AddAllPublicRoutes(base) httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() - httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux) httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux) httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) diff --git a/build/docker/README.md b/build/docker/README.md index cb7f68621..23557af5f 100644 --- a/build/docker/README.md +++ b/build/docker/README.md @@ -5,25 +5,21 @@ These are Docker images for Dendrite! They can be found on Docker Hub: - [matrixdotorg/dendrite-monolith](https://hub.docker.com/r/matrixdotorg/dendrite-monolith) for monolith deployments -- [matrixdotorg/dendrite-polylith](https://hub.docker.com/r/matrixdotorg/dendrite-polylith) for polylith deployments ## Dockerfiles -The `Dockerfile` is a multistage file which can build all four Dendrite -images depending on the supplied `--target`. From the root of the Dendrite +The `Dockerfile` is a multistage file which can build Dendrite. From the root of the Dendrite repository, run: ``` docker build . --target monolith -t matrixdotorg/dendrite-monolith -docker build . --target polylith -t matrixdotorg/dendrite-monolith ``` ## Compose files -There are two sample `docker-compose` files: +There is one sample `docker-compose` files: - `docker-compose.monolith.yml` which runs a monolith Dendrite deployment -- `docker-compose.polylith.yml` which runs a polylith Dendrite deployment ## Configuration @@ -49,7 +45,7 @@ docker run --rm --entrypoint="" \ The key files will now exist in your current working directory, and can be mounted into place. -## Starting Dendrite as a monolith deployment +## Starting Dendrite Create your config based on the [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml) sample configuration file. @@ -59,16 +55,6 @@ Then start the deployment: docker-compose -f docker-compose.monolith.yml up ``` -## Starting Dendrite as a polylith deployment - -Create your config based on the [`dendrite-sample.polylith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) sample configuration file. - -Then start the deployment: - -``` -docker-compose -f docker-compose.polylith.yml up -``` - ## Building the images The `build/docker/images-build.sh` script will build the base image, followed by diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index 0d55d6603..e69de29bb 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -1,147 +0,0 @@ -version: "3.4" - -services: - postgres: - hostname: postgres - image: postgres:15-alpine - restart: always - volumes: - # This will create a docker volume to persist the database files in. - # If you prefer those files to be outside of docker, you'll need to change this. - - dendrite_postgres_data:/var/lib/postgresql/data - environment: - POSTGRES_PASSWORD: itsasecret - POSTGRES_USER: dendrite - POSTGRES_DATABASE: dendrite - healthcheck: - test: ["CMD-SHELL", "pg_isready -U dendrite"] - interval: 5s - timeout: 5s - retries: 5 - networks: - - internal - - jetstream: - hostname: jetstream - image: nats:latest - command: | - --jetstream - --store_dir /var/lib/nats - --cluster_name Dendrite - volumes: - - dendrite_jetstream:/var/lib/nats - networks: - - internal - - # This "component" will exit immediately, it's only purpose is - # to provide the common settings - component: - image: matrixdotorg/dendrite-polylith:latest - restart: "no" - networks: - - internal - volumes: - - ./config:/etc/dendrite - depends_on: - jetstream: - condition: service_started - postgres: - condition: service_healthy - - client_api: - extends: - service: component - hostname: client_api - command: clientapi - restart: unless-stopped - depends_on: - user_api: - condition: service_started - - media_api: - extends: - service: component - hostname: media_api - command: mediaapi - restart: unless-stopped - volumes: - - dendrite_media:/var/dendrite/media - - sync_api: - extends: - service: component - hostname: sync_api - command: syncapi - restart: unless-stopped - volumes: - - dendrite_search_index:/var/dendrite/searchindex - depends_on: - room_server: - condition: service_started - user_api: - condition: service_started - key_server: - condition: service_started - - room_server: - extends: - service: component - hostname: room_server - command: roomserver - restart: unless-stopped - - federation_api: - extends: - service: component - hostname: federation_api - command: federationapi - restart: unless-stopped - depends_on: - room_server: - condition: service_started - - key_server: - extends: - service: component - hostname: key_server - command: keyserver - restart: unless-stopped - depends_on: - room_server: - condition: service_started - federation_api: - condition: service_started - - user_api: - extends: - service: component - hostname: user_api - command: userapi - restart: unless-stopped - depends_on: - room_server: - condition: service_started - key_server: - condition: service_started - - appservice_api: - extends: - service: component - hostname: appservice_api - command: appservice - restart: unless-stopped - depends_on: - room_server: - condition: service_started - user_api: - condition: service_started - -networks: - internal: - attachable: true - -volumes: - dendrite_jetstream: - dendrite_postgres_data: - dendrite_media: - dendrite_search_index: \ No newline at end of file diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index d97a701ed..bf227968f 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -7,6 +7,5 @@ TAG=${1:-latest} echo "Building tag '${TAG}'" docker build . --target monolith -t matrixdotorg/dendrite-monolith:${TAG} -docker build . --target polylith -t matrixdotorg/dendrite-monolith:${TAG} docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone:${TAG} docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil:${TAG} \ No newline at end of file diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh index f3f98ce7c..7772ca747 100755 --- a/build/docker/images-pull.sh +++ b/build/docker/images-pull.sh @@ -5,4 +5,3 @@ TAG=${1:-latest} echo "Pulling tag '${TAG}'" docker pull matrixdotorg/dendrite-monolith:${TAG} -docker pull matrixdotorg/dendrite-polylith:${TAG} \ No newline at end of file diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index 248fdee2b..d166d355a 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -5,4 +5,3 @@ TAG=${1:-latest} echo "Pushing tag '${TAG}'" docker push matrixdotorg/dendrite-monolith:${TAG} -docker push matrixdotorg/dendrite-polylith:${TAG} \ No newline at end of file diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 8c2d0a006..32af611ae 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -20,7 +20,6 @@ import ( "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/base" @@ -127,8 +126,8 @@ func (m *DendriteMonolith) Start() { cfg := &config.Dendrite{} cfg.Defaults(config.DefaultOpts{ - Generate: true, - Monolithic: true, + Generate: true, + SingleDatabase: true, }) cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk)) cfg.Global.PrivateKey = sk @@ -149,7 +148,7 @@ func (m *DendriteMonolith) Start() { panic(err) } - base := base.NewBaseDendrite(cfg, "Monolith") + base := base.NewBaseDendrite(cfg) base.ConfigureAdminEndpoints() m.processContext = base.ProcessContext defer base.Close() // nolint: errcheck @@ -165,9 +164,7 @@ func (m *DendriteMonolith) Start() { base, federation, rsAPI, base.Caches, keyRing, true, ) - keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation, rsAPI) - userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient()) - keyAPI.SetUserAPI(userAPI) + userAPI := userapi.NewInternalAPI(base, rsAPI, federation) asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) @@ -186,7 +183,6 @@ func (m *DendriteMonolith) Start() { FederationAPI: fsAPI, RoomserverAPI: rsAPI, UserAPI: userAPI, - KeyAPI: keyAPI, ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider( ygg, fsAPI, federation, ), @@ -194,7 +190,6 @@ func (m *DendriteMonolith) Start() { monolith.AddAllPublicRoutes(base) httpRouter := mux.NewRouter() - httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux) httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux) httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux) diff --git a/build/scripts/Complement.Dockerfile b/build/scripts/Complement.Dockerfile index 3a00fbdf0..70bbe8f95 100644 --- a/build/scripts/Complement.Dockerfile +++ b/build/scripts/Complement.Dockerfile @@ -16,8 +16,8 @@ RUN --mount=target=. \ --mount=type=cache,target=/root/.cache/go-build \ CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \ CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \ - CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/dendrite-monolith-server && \ - CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-monolith-server-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite-monolith-server && \ + CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \ + CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \ cp build/scripts/complement-cmd.sh /complement-cmd.sh WORKDIR /dendrite diff --git a/build/scripts/ComplementLocal.Dockerfile b/build/scripts/ComplementLocal.Dockerfile index e3fbe1aa8..0b80cfc40 100644 --- a/build/scripts/ComplementLocal.Dockerfile +++ b/build/scripts/ComplementLocal.Dockerfile @@ -19,13 +19,13 @@ WORKDIR /runtime # This script compiles Dendrite for us. RUN echo '\ #!/bin/bash -eux \n\ - if test -f "/runtime/dendrite-monolith-server" && test -f "/runtime/dendrite-monolith-server-cover"; then \n\ + if test -f "/runtime/dendrite" && test -f "/runtime/dendrite-cover"; then \n\ echo "Skipping compilation; binaries exist" \n\ exit 0 \n\ fi \n\ cd /dendrite \n\ - go build -v -o /runtime /dendrite/cmd/dendrite-monolith-server \n\ - go test -c -cover -covermode=atomic -o /runtime/dendrite-monolith-server-cover -coverpkg "github.com/matrix-org/..." /dendrite/cmd/dendrite-monolith-server \n\ + go build -v -o /runtime /dendrite/cmd/dendrite \n\ + go test -c -cover -covermode=atomic -o /runtime/dendrite-cover -coverpkg "github.com/matrix-org/..." /dendrite/cmd/dendrite \n\ ' > compile.sh && chmod +x compile.sh # This script runs Dendrite for us. Must be run in the /runtime directory. @@ -35,8 +35,8 @@ RUN echo '\ ./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\ ./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\ cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\ - [ ${COVER} -eq 1 ] && exec ./dendrite-monolith-server-cover --test.coverprofile=integrationcover.log --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\ - exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\ + [ ${COVER} -eq 1 ] && exec ./dendrite-cover --test.coverprofile=integrationcover.log --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\ + exec ./dendrite --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\ ' > run.sh && chmod +x run.sh diff --git a/build/scripts/ComplementPostgres.Dockerfile b/build/scripts/ComplementPostgres.Dockerfile index 444cb947d..d4b6d3f75 100644 --- a/build/scripts/ComplementPostgres.Dockerfile +++ b/build/scripts/ComplementPostgres.Dockerfile @@ -34,8 +34,8 @@ RUN --mount=target=. \ --mount=type=cache,target=/root/.cache/go-build \ CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \ CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \ - CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/dendrite-monolith-server && \ - CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-monolith-server-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite-monolith-server && \ + CGO_ENABLED=${CGO} go build -o /dendrite/dendrite ./cmd/dendrite && \ + CGO_ENABLED=${CGO} go test -c -cover -covermode=atomic -o /dendrite/dendrite-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite && \ cp build/scripts/complement-cmd.sh /complement-cmd.sh WORKDIR /dendrite diff --git a/build/scripts/complement-cmd.sh b/build/scripts/complement-cmd.sh index def091e27..52b063d01 100755 --- a/build/scripts/complement-cmd.sh +++ b/build/scripts/complement-cmd.sh @@ -4,19 +4,17 @@ if [[ "${COVER}" -eq 1 ]]; then echo "Running with coverage" - exec /dendrite/dendrite-monolith-server-cover \ + exec /dendrite/dendrite-cover \ --really-enable-open-registration \ --tls-cert server.crt \ --tls-key server.key \ --config dendrite.yaml \ - -api=${API:-0} \ --test.coverprofile=complementcover.log else echo "Not running with coverage" - exec /dendrite/dendrite-monolith-server \ + exec /dendrite/dendrite \ --really-enable-open-registration \ --tls-cert server.crt \ --tls-key server.key \ - --config dendrite.yaml \ - -api=${API:-0} + --config dendrite.yaml fi diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index 06922963d..1a0b44987 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -9,7 +9,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/federationapi" - "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" @@ -41,11 +40,9 @@ func TestAdminResetPassword(t *testing.T) { rsAPI := roomserver.NewInternalAPI(base) // Needed for changing the password/login - keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI) - userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil) - keyAPI.SetUserAPI(userAPI) + userAPI := userapi.NewInternalAPI(base, rsAPI, nil) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. - AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, nil) + AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil) // Create the users in the userapi and login accessTokens := map[*test.User]string{ @@ -159,14 +156,12 @@ func TestPurgeRoom(t *testing.T) { fedClient := base.CreateFederationClient() rsAPI := roomserver.NewInternalAPI(base) - keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fedClient, rsAPI) - userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil) + userAPI := userapi.NewInternalAPI(base, rsAPI, nil) // this starts the JetStream consumers - syncapi.AddPublicRoutes(base, userAPI, rsAPI, keyAPI) + syncapi.AddPublicRoutes(base, userAPI, rsAPI) federationapi.NewInternalAPI(base, fedClient, rsAPI, base.Caches, nil, true) rsAPI.SetFederationAPI(nil, nil) - keyAPI.SetUserAPI(userAPI) // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { @@ -174,7 +169,7 @@ func TestPurgeRoom(t *testing.T) { } // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. - AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, nil) + AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil) // Create the users in the userapi and login accessTokens := map[*test.User]string{ diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 2d17e0928..e9985d43f 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -15,6 +15,7 @@ package clientapi import ( + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" @@ -23,11 +24,9 @@ import ( "github.com/matrix-org/dendrite/clientapi/routing" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/transactions" - keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/jetstream" - userapi "github.com/matrix-org/dendrite/userapi/api" ) // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component. @@ -40,7 +39,6 @@ func AddPublicRoutes( fsAPI federationAPI.ClientFederationAPI, userAPI userapi.ClientUserAPI, userDirectoryProvider userapi.QuerySearchProfilesAPI, - keyAPI keyserverAPI.ClientKeyAPI, extRoomsProvider api.ExtraPublicRoomsProvider, ) { cfg := &base.Cfg.ClientAPI @@ -61,7 +59,7 @@ func AddPublicRoutes( base, cfg, rsAPI, asAPI, userAPI, userDirectoryProvider, federation, - syncProducer, transactionsCache, fsAPI, keyAPI, + syncProducer, transactionsCache, fsAPI, extRoomsProvider, mscCfg, natsClient, ) } diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go index 4b4dedfd1..a01f6b944 100644 --- a/clientapi/routing/admin.go +++ b/clientapi/routing/admin.go @@ -16,14 +16,13 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/keyserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" - userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/api" ) -func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { +func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -56,7 +55,7 @@ func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *userapi } } -func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { +func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -99,7 +98,7 @@ func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi } } -func AdminPurgeRoom(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { +func AdminPurgeRoom(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -130,7 +129,7 @@ func AdminPurgeRoom(req *http.Request, cfg *config.ClientAPI, device *userapi.De } } -func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse { +func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.Device, userAPI api.ClientUserAPI) util.JSONResponse { if req.Body == nil { return util.JSONResponse{ Code: http.StatusBadRequest, @@ -150,8 +149,8 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap JSON: jsonerror.InvalidArgumentValue(err.Error()), } } - accAvailableResp := &userapi.QueryAccountAvailabilityResponse{} - if err = userAPI.QueryAccountAvailability(req.Context(), &userapi.QueryAccountAvailabilityRequest{ + accAvailableResp := &api.QueryAccountAvailabilityResponse{} + if err = userAPI.QueryAccountAvailability(req.Context(), &api.QueryAccountAvailabilityRequest{ Localpart: localpart, ServerName: serverName, }, accAvailableResp); err != nil { @@ -186,13 +185,13 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap return *internal.PasswordResponse(err) } - updateReq := &userapi.PerformPasswordUpdateRequest{ + updateReq := &api.PerformPasswordUpdateRequest{ Localpart: localpart, ServerName: serverName, Password: request.Password, LogoutDevices: true, } - updateRes := &userapi.PerformPasswordUpdateResponse{} + updateRes := &api.PerformPasswordUpdateResponse{} if err := userAPI.PerformPasswordUpdate(req.Context(), updateReq, updateRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, @@ -209,7 +208,7 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap } } -func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, natsClient *nats.Conn) util.JSONResponse { +func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *api.Device, natsClient *nats.Conn) util.JSONResponse { _, err := natsClient.RequestMsg(nats.NewMsg(cfg.Matrix.JetStream.Prefixed(jetstream.InputFulltextReindex)), time.Second*10) if err != nil { logrus.WithError(err).Error("failed to publish nats message") @@ -255,7 +254,7 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien } } -func AdminDownloadState(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { +func AdminDownloadState(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) diff --git a/clientapi/routing/auth_fallback_test.go b/clientapi/routing/auth_fallback_test.go index 0d77f9a01..534581bdd 100644 --- a/clientapi/routing/auth_fallback_test.go +++ b/clientapi/routing/auth_fallback_test.go @@ -22,7 +22,7 @@ func Test_AuthFallback(t *testing.T) { for _, wantErr := range []bool{false, true} { t.Run(fmt.Sprintf("useHCaptcha(%v) - recaptchaEnabled(%v) - wantErr(%v)", useHCaptcha, recaptchaEnabled, wantErr), func(t *testing.T) { // Set the defaults for each test - base.Cfg.ClientAPI.Defaults(config.DefaultOpts{Generate: true, Monolithic: true}) + base.Cfg.ClientAPI.Defaults(config.DefaultOpts{Generate: true, SingleDatabase: true}) base.Cfg.ClientAPI.RecaptchaEnabled = recaptchaEnabled base.Cfg.ClientAPI.RecaptchaPublicKey = "pub" base.Cfg.ClientAPI.RecaptchaPrivateKey = "priv" @@ -33,7 +33,7 @@ func Test_AuthFallback(t *testing.T) { base.Cfg.ClientAPI.RecaptchaSitekeyClass = "h-captcha" } cfgErrs := &config.ConfigErrors{} - base.Cfg.ClientAPI.Verify(cfgErrs, true) + base.Cfg.ClientAPI.Verify(cfgErrs) if len(*cfgErrs) > 0 { t.Fatalf("(hCaptcha=%v) unexpected config errors: %s", useHCaptcha, cfgErrs.Error()) } diff --git a/clientapi/routing/joinroom_test.go b/clientapi/routing/joinroom_test.go index 9e8208e6d..1450ef4bd 100644 --- a/clientapi/routing/joinroom_test.go +++ b/clientapi/routing/joinroom_test.go @@ -10,7 +10,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" @@ -29,8 +28,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { defer baseClose() rsAPI := roomserver.NewInternalAPI(base) - keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI) - userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil) + userAPI := userapi.NewInternalAPI(base, rsAPI, nil) asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc diff --git a/clientapi/routing/key_crosssigning.go b/clientapi/routing/key_crosssigning.go index 2570db09c..267ba1dc5 100644 --- a/clientapi/routing/key_crosssigning.go +++ b/clientapi/routing/key_crosssigning.go @@ -21,9 +21,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/setup/config" - userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" ) @@ -34,8 +33,8 @@ type crossSigningRequest struct { func UploadCrossSigningDeviceKeys( req *http.Request, userInteractiveAuth *auth.UserInteractive, - keyserverAPI api.ClientKeyAPI, device *userapi.Device, - accountAPI userapi.ClientUserAPI, cfg *config.ClientAPI, + keyserverAPI api.ClientKeyAPI, device *api.Device, + accountAPI api.ClientUserAPI, cfg *config.ClientAPI, ) util.JSONResponse { uploadReq := &crossSigningRequest{} uploadRes := &api.PerformUploadDeviceKeysResponse{} @@ -107,7 +106,7 @@ func UploadCrossSigningDeviceKeys( } } -func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse { +func UploadCrossSigningDeviceSignatures(req *http.Request, keyserverAPI api.ClientKeyAPI, device *api.Device) util.JSONResponse { uploadReq := &api.PerformUploadDeviceSignaturesRequest{} uploadRes := &api.PerformUploadDeviceSignaturesResponse{} diff --git a/clientapi/routing/keys.go b/clientapi/routing/keys.go index 0c12b1117..3d60fcc3a 100644 --- a/clientapi/routing/keys.go +++ b/clientapi/routing/keys.go @@ -23,8 +23,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/keyserver/api" - userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/api" ) type uploadKeysRequest struct { @@ -32,7 +31,7 @@ type uploadKeysRequest struct { OneTimeKeys map[string]json.RawMessage `json:"one_time_keys"` } -func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse { +func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device) util.JSONResponse { var r uploadKeysRequest resErr := httputil.UnmarshalJSONRequest(req, &r) if resErr != nil { @@ -106,7 +105,7 @@ func (r *queryKeysRequest) GetTimeout() time.Duration { return timeout } -func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse { +func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device) util.JSONResponse { var r queryKeysRequest resErr := httputil.UnmarshalJSONRequest(req, &r) if resErr != nil { diff --git a/clientapi/routing/leaveroom.go b/clientapi/routing/leaveroom.go index 86414afca..a71661851 100644 --- a/clientapi/routing/leaveroom.go +++ b/clientapi/routing/leaveroom.go @@ -18,7 +18,6 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/internal/httputil" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" @@ -45,15 +44,6 @@ func LeaveRoomByID( JSON: jsonerror.LeaveServerNoticeError(), } } - switch e := err.(type) { - case httputil.InternalAPIError: - if e.Message == jsonerror.LeaveServerNoticeError().Error() { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.LeaveServerNoticeError(), - } - } - } return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.Unknown(err.Error()), diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go index d429d7f8c..b72db9d8b 100644 --- a/clientapi/routing/login_test.go +++ b/clientapi/routing/login_test.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" @@ -39,12 +38,10 @@ func TestLogin(t *testing.T) { rsAPI := roomserver.NewInternalAPI(base) // Needed for /login - keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI) - userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil) - keyAPI.SetUserAPI(userAPI) + userAPI := userapi.NewInternalAPI(base, rsAPI, nil) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. - Setup(base, &base.Cfg.ClientAPI, nil, nil, userAPI, nil, nil, nil, nil, nil, keyAPI, nil, &base.Cfg.MSCs, nil) + Setup(base, &base.Cfg.ClientAPI, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, &base.Cfg.MSCs, nil) // Create password password := util.RandomString(8) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index be2b192b2..ff6a0900e 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -31,8 +31,6 @@ import ( "time" "github.com/matrix-org/dendrite/internal" - internalHTTPUtil "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/tidwall/gjson" "github.com/matrix-org/dendrite/internal/eventutil" @@ -861,16 +859,6 @@ func completeRegistration( JSON: jsonerror.UserInUse("Desired user ID is already taken."), } } - switch e := err.(type) { - case internalHTTPUtil.InternalAPIError: - conflictErr := &userapi.ErrorConflict{Message: sqlutil.ErrUserExists.Error()} - if e.Message == conflictErr.Error() { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.UserInUse("Desired user ID is already taken."), - } - } - } return util.JSONResponse{ Code: http.StatusInternalServerError, JSON: jsonerror.Unknown("failed to create account: " + err.Error()), diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index bccc1b79b..651e3d3d6 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -30,7 +30,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" @@ -201,8 +200,8 @@ func TestValidationOfApplicationServices(t *testing.T) { // Set up a config fakeConfig := &config.Dendrite{} fakeConfig.Defaults(config.DefaultOpts{ - Generate: true, - Monolithic: true, + Generate: true, + SingleDatabase: true, }) fakeConfig.Global.ServerName = "localhost" fakeConfig.ClientAPI.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService} @@ -409,9 +408,7 @@ func Test_register(t *testing.T) { defer baseClose() rsAPI := roomserver.NewInternalAPI(base) - keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI) - userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil) - keyAPI.SetUserAPI(userAPI) + userAPI := userapi.NewInternalAPI(base, rsAPI, nil) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -582,9 +579,7 @@ func TestRegisterUserWithDisplayName(t *testing.T) { base.Cfg.Global.ServerName = "server" rsAPI := roomserver.NewInternalAPI(base) - keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI) - userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil) - keyAPI.SetUserAPI(userAPI) + userAPI := userapi.NewInternalAPI(base, rsAPI, nil) deviceName, deviceID := "deviceName", "deviceID" expectedDisplayName := "DisplayName" response := completeRegistration( @@ -623,9 +618,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) { base.Cfg.ClientAPI.RegistrationSharedSecret = sharedSecret rsAPI := roomserver.NewInternalAPI(base) - keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI) - userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil) - keyAPI.SetUserAPI(userAPI) + userAPI := userapi.NewInternalAPI(base, rsAPI, nil) expectedDisplayName := "rabbit" jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 66610c0a0..028d02e97 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -22,6 +22,7 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/setup/base" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/nats-io/nats.go" @@ -37,11 +38,9 @@ import ( federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/transactions" - keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" - userapi "github.com/matrix-org/dendrite/userapi/api" ) // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client @@ -61,7 +60,6 @@ func Setup( syncProducer *producers.SyncAPIProducer, transactionsCache *transactions.Cache, federationSender federationAPI.ClientFederationAPI, - keyAPI keyserverAPI.ClientKeyAPI, extRoomsProvider api.ExtraPublicRoomsProvider, mscCfg *config.MSCs, natsClient *nats.Conn, ) { @@ -192,7 +190,7 @@ func Setup( dendriteAdminRouter.Handle("/admin/refreshDevices/{userID}", httputil.MakeAdminAPI("admin_refresh_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return AdminMarkAsStale(req, cfg, keyAPI) + return AdminMarkAsStale(req, cfg, userAPI) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -1372,11 +1370,11 @@ func Setup( // Cross-signing device keys postDeviceSigningKeys := httputil.MakeAuthAPI("post_device_signing_keys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, keyAPI, device, userAPI, cfg) + return UploadCrossSigningDeviceKeys(req, userInteractiveAuth, userAPI, device, userAPI, cfg) }) postDeviceSigningSignatures := httputil.MakeAuthAPI("post_device_signing_signatures", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return UploadCrossSigningDeviceSignatures(req, keyAPI, device) + return UploadCrossSigningDeviceSignatures(req, userAPI, device) }, httputil.WithAllowGuests()) v3mux.Handle("/keys/device_signing/upload", postDeviceSigningKeys).Methods(http.MethodPost, http.MethodOptions) @@ -1388,22 +1386,22 @@ func Setup( // Supplying a device ID is deprecated. v3mux.Handle("/keys/upload/{deviceID}", httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return UploadKeys(req, keyAPI, device) + return UploadKeys(req, userAPI, device) }, httputil.WithAllowGuests()), ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/keys/upload", httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return UploadKeys(req, keyAPI, device) + return UploadKeys(req, userAPI, device) }, httputil.WithAllowGuests()), ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/keys/query", httputil.MakeAuthAPI("keys_query", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return QueryKeys(req, keyAPI, device) + return QueryKeys(req, userAPI, device) }, httputil.WithAllowGuests()), ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/keys/claim", httputil.MakeAuthAPI("keys_claim", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - return ClaimKeys(req, keyAPI) + return ClaimKeys(req, userAPI) }, httputil.WithAllowGuests()), ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}", diff --git a/cmd/dendrite-demo-pinecone/monolith/monolith.go b/cmd/dendrite-demo-pinecone/monolith/monolith.go index 6f1c69a78..27720369e 100644 --- a/cmd/dendrite-demo-pinecone/monolith/monolith.go +++ b/cmd/dendrite-demo-pinecone/monolith/monolith.go @@ -38,7 +38,6 @@ import ( federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/producers" "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/relayapi" relayAPI "github.com/matrix-org/dendrite/relayapi/api" "github.com/matrix-org/dendrite/roomserver" @@ -80,8 +79,8 @@ type P2PMonolith struct { func GenerateDefaultConfig(sk ed25519.PrivateKey, storageDir string, cacheDir string, dbPrefix string) *config.Dendrite { cfg := config.Dendrite{} cfg.Defaults(config.DefaultOpts{ - Generate: true, - Monolithic: true, + Generate: true, + SingleDatabase: true, }) cfg.Global.PrivateKey = sk cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", filepath.Join(cacheDir, dbPrefix))) @@ -121,9 +120,9 @@ func (p *P2PMonolith) SetupPinecone(sk ed25519.PrivateKey) { func (p *P2PMonolith) SetupDendrite(cfg *config.Dendrite, port int, enableRelaying bool, enableMetrics bool, enableWebsockets bool) { if enableMetrics { - p.BaseDendrite = base.NewBaseDendrite(cfg, "Monolith") + p.BaseDendrite = base.NewBaseDendrite(cfg) } else { - p.BaseDendrite = base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics) + p.BaseDendrite = base.NewBaseDendrite(cfg, base.DisableMetrics) } p.port = port p.BaseDendrite.ConfigureAdminEndpoints() @@ -139,9 +138,7 @@ func (p *P2PMonolith) SetupDendrite(cfg *config.Dendrite, port int, enableRelayi p.BaseDendrite, federation, rsAPI, p.BaseDendrite.Caches, keyRing, true, ) - keyAPI := keyserver.NewInternalAPI(p.BaseDendrite, &p.BaseDendrite.Cfg.KeyServer, fsAPI, rsComponent) - userAPI := userapi.NewInternalAPI(p.BaseDendrite, &cfg.UserAPI, nil, keyAPI, rsAPI, p.BaseDendrite.PushGatewayHTTPClient()) - keyAPI.SetUserAPI(userAPI) + userAPI := userapi.NewInternalAPI(p.BaseDendrite, rsAPI, federation) asAPI := appservice.NewInternalAPI(p.BaseDendrite, userAPI, rsAPI) @@ -175,7 +172,6 @@ func (p *P2PMonolith) SetupDendrite(cfg *config.Dendrite, port int, enableRelayi FederationAPI: fsAPI, RoomserverAPI: rsAPI, UserAPI: userAPI, - KeyAPI: keyAPI, RelayAPI: relayAPI, ExtPublicRoomsProvider: roomProvider, ExtUserDirectoryProvider: userProvider, @@ -236,7 +232,6 @@ func (p *P2PMonolith) Addr() string { func (p *P2PMonolith) setupHttpServers(userProvider *users.PineconeUserProvider, enableWebsockets bool) { p.httpMux = mux.NewRouter().SkipClean(true).UseEncodedPath() - p.httpMux.PathPrefix(httputil.InternalPathPrefix).Handler(p.BaseDendrite.InternalAPIMux) p.httpMux.PathPrefix(httputil.PublicClientPathPrefix).Handler(p.BaseDendrite.PublicClientAPIMux) p.httpMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(p.BaseDendrite.PublicMediaAPIMux) p.httpMux.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(p.BaseDendrite.DendriteAdminMux) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 3ea4a08b0..d759c6a73 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -39,7 +39,6 @@ import ( "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/base" @@ -117,8 +116,8 @@ func main() { cfg = setup.ParseFlags(true) } else { cfg.Defaults(config.DefaultOpts{ - Generate: true, - Monolithic: true, + Generate: true, + SingleDatabase: true, }) cfg.Global.PrivateKey = sk cfg.Global.JetStream.StoragePath = config.Path(filepath.Join(*instanceDir, *instanceName)) @@ -143,7 +142,7 @@ func main() { cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk)) cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) - base := base.NewBaseDendrite(cfg, "Monolith") + base := base.NewBaseDendrite(cfg) base.ConfigureAdminEndpoints() defer base.Close() // nolint: errcheck @@ -157,16 +156,11 @@ func main() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - rsComponent := roomserver.NewInternalAPI( + rsAPI := roomserver.NewInternalAPI( base, ) - keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation, rsComponent) - - rsAPI := rsComponent - - userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient()) - keyAPI.SetUserAPI(userAPI) + userAPI := userapi.NewInternalAPI(base, rsAPI, federation) asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) @@ -174,7 +168,7 @@ func main() { base, federation, rsAPI, base.Caches, keyRing, true, ) - rsComponent.SetFederationAPI(fsAPI, keyRing) + rsAPI.SetFederationAPI(fsAPI, keyRing) monolith := setup.Monolith{ Config: base.Cfg, @@ -186,7 +180,6 @@ func main() { FederationAPI: fsAPI, RoomserverAPI: rsAPI, UserAPI: userAPI, - KeyAPI: keyAPI, ExtPublicRoomsProvider: yggrooms.NewYggdrasilRoomProvider( ygg, fsAPI, federation, ), @@ -197,7 +190,6 @@ func main() { } httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() - httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux) httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux) httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux) httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go deleted file mode 100644 index 6836b6426..000000000 --- a/cmd/dendrite-monolith-server/main.go +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "os" - - "github.com/sirupsen/logrus" - - "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/federationapi" - "github.com/matrix-org/dendrite/keyserver" - "github.com/matrix-org/dendrite/roomserver" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup" - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/setup/mscs" - "github.com/matrix-org/dendrite/userapi" - uapi "github.com/matrix-org/dendrite/userapi/api" -) - -var ( - httpBindAddr = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server") - httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server") - apiBindAddr = flag.String("api-bind-address", "localhost:18008", "The HTTP listening port for the internal HTTP APIs (if -api is enabled)") - certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS") - keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS") - enableHTTPAPIs = flag.Bool("api", false, "Use HTTP APIs instead of short-circuiting (warning: exposes API endpoints!)") - traceInternal = os.Getenv("DENDRITE_TRACE_INTERNAL") == "1" -) - -func main() { - cfg := setup.ParseFlags(true) - httpAddr := config.HTTPAddress("http://" + *httpBindAddr) - httpsAddr := config.HTTPAddress("https://" + *httpsBindAddr) - httpAPIAddr := httpAddr - options := []basepkg.BaseDendriteOptions{} - if *enableHTTPAPIs { - logrus.Warnf("DANGER! The -api option is enabled, exposing internal APIs on %q!", *apiBindAddr) - httpAPIAddr = config.HTTPAddress("http://" + *apiBindAddr) - // If the HTTP APIs are enabled then we need to update the Listen - // statements in the configuration so that we know where to find - // the API endpoints. They'll listen on the same port as the monolith - // itself. - cfg.AppServiceAPI.InternalAPI.Connect = httpAPIAddr - cfg.ClientAPI.InternalAPI.Connect = httpAPIAddr - cfg.FederationAPI.InternalAPI.Connect = httpAPIAddr - cfg.KeyServer.InternalAPI.Connect = httpAPIAddr - cfg.MediaAPI.InternalAPI.Connect = httpAPIAddr - cfg.RoomServer.InternalAPI.Connect = httpAPIAddr - cfg.SyncAPI.InternalAPI.Connect = httpAPIAddr - cfg.UserAPI.InternalAPI.Connect = httpAPIAddr - options = append(options, basepkg.UseHTTPAPIs) - } - - base := basepkg.NewBaseDendrite(cfg, "Monolith", options...) - defer base.Close() // nolint: errcheck - - federation := base.CreateFederationClient() - - rsImpl := roomserver.NewInternalAPI(base) - // call functions directly on the impl unless running in HTTP mode - rsAPI := rsImpl - if base.UseHTTPAPIs { - roomserver.AddInternalRoutes(base.InternalAPIMux, rsImpl, base.EnableMetrics) - rsAPI = base.RoomserverHTTPClient() - } - if traceInternal { - rsAPI = &api.RoomserverInternalAPITrace{ - Impl: rsAPI, - } - } - - fsAPI := federationapi.NewInternalAPI( - base, federation, rsAPI, base.Caches, nil, false, - ) - fsImplAPI := fsAPI - if base.UseHTTPAPIs { - federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI, base.EnableMetrics) - fsAPI = base.FederationAPIHTTPClient() - } - keyRing := fsAPI.KeyRing() - - keyImpl := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI) - keyAPI := keyImpl - if base.UseHTTPAPIs { - keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI, base.EnableMetrics) - keyAPI = base.KeyServerHTTPClient() - } - - pgClient := base.PushGatewayHTTPClient() - userImpl := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, pgClient) - userAPI := userImpl - if base.UseHTTPAPIs { - userapi.AddInternalRoutes(base.InternalAPIMux, userAPI, base.EnableMetrics) - userAPI = base.UserAPIClient() - } - if traceInternal { - userAPI = &uapi.UserInternalAPITrace{ - Impl: userAPI, - } - } - - // TODO: This should use userAPI, not userImpl, but the appservice setup races with - // the listeners and panics at startup if it tries to create appservice accounts - // before the listeners are up. - asAPI := appservice.NewInternalAPI(base, userImpl, rsAPI) - if base.UseHTTPAPIs { - appservice.AddInternalRoutes(base.InternalAPIMux, asAPI, base.EnableMetrics) - asAPI = base.AppserviceHTTPClient() - } - - // The underlying roomserver implementation needs to be able to call the fedsender. - // This is different to rsAPI which can be the http client which doesn't need this - // dependency. Other components also need updating after their dependencies are up. - rsImpl.SetFederationAPI(fsAPI, keyRing) - rsImpl.SetAppserviceAPI(asAPI) - rsImpl.SetUserAPI(userAPI) - keyImpl.SetUserAPI(userAPI) - - monolith := setup.Monolith{ - Config: base.Cfg, - Client: base.CreateClient(), - FedClient: federation, - KeyRing: keyRing, - - AppserviceAPI: asAPI, - // always use the concrete impl here even in -http mode because adding public routes - // must be done on the concrete impl not an HTTP client else fedapi will call itself - FederationAPI: fsImplAPI, - RoomserverAPI: rsAPI, - UserAPI: userAPI, - KeyAPI: keyAPI, - } - monolith.AddAllPublicRoutes(base) - - if len(base.Cfg.MSCs.MSCs) > 0 { - if err := mscs.Enable(base, &monolith); err != nil { - logrus.WithError(err).Fatalf("Failed to enable MSCs") - } - } - - // Expose the matrix APIs directly rather than putting them under a /api path. - go func() { - base.SetupAndServeHTTP( - httpAPIAddr, // internal API - httpAddr, // external API - nil, nil, // TLS settings - ) - }() - // Handle HTTPS if certificate and key are provided - if *certFile != "" && *keyFile != "" { - go func() { - base.SetupAndServeHTTP( - basepkg.NoListener, // internal API - httpsAddr, // external API - certFile, keyFile, // TLS settings - ) - }() - } - - // We want to block forever to let the HTTP and HTTPS handler serve the APIs - base.WaitForShutdown() -} diff --git a/cmd/dendrite-polylith-multi/main.go b/cmd/dendrite-polylith-multi/main.go deleted file mode 100644 index c6a560b19..000000000 --- a/cmd/dendrite-polylith-multi/main.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "os" - "strings" - - "github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities" - "github.com/matrix-org/dendrite/setup" - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" - "github.com/sirupsen/logrus" -) - -type entrypoint func(base *base.BaseDendrite, cfg *config.Dendrite) - -func main() { - cfg := setup.ParseFlags(false) - - component := "" - if flag.NFlag() > 0 { - component = flag.Arg(0) // ./dendrite-polylith-multi --config=... clientapi - } else if len(os.Args) > 1 { - component = os.Args[1] // ./dendrite-polylith-multi clientapi - } - - components := map[string]entrypoint{ - "appservice": personalities.Appservice, - "clientapi": personalities.ClientAPI, - "federationapi": personalities.FederationAPI, - "keyserver": personalities.KeyServer, - "mediaapi": personalities.MediaAPI, - "roomserver": personalities.RoomServer, - "syncapi": personalities.SyncAPI, - "userapi": personalities.UserAPI, - } - - start, ok := components[component] - if !ok { - if component == "" { - logrus.Errorf("No component specified") - logrus.Info("The first argument on the command line must be the name of the component to run") - } else { - logrus.Errorf("Unknown component %q specified", component) - } - - var list []string - for c := range components { - list = append(list, c) - } - logrus.Infof("Valid components: %s", strings.Join(list, ", ")) - - os.Exit(1) - } - - logrus.Infof("Starting %q component", component) - - base := base.NewBaseDendrite(cfg, component, base.PolylithMode) // TODO - defer base.Close() // nolint: errcheck - - go start(base, cfg) - base.WaitForShutdown() -} diff --git a/cmd/dendrite-polylith-multi/personalities/appservice.go b/cmd/dendrite-polylith-multi/personalities/appservice.go deleted file mode 100644 index 0547d57f0..000000000 --- a/cmd/dendrite-polylith-multi/personalities/appservice.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/setup/base" - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -func Appservice(base *base.BaseDendrite, cfg *config.Dendrite) { - userAPI := base.UserAPIClient() - rsAPI := base.RoomserverHTTPClient() - - intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) - appservice.AddInternalRoutes(base.InternalAPIMux, intAPI, base.EnableMetrics) - - base.SetupAndServeHTTP( - base.Cfg.AppServiceAPI.InternalAPI.Listen, // internal listener - basepkg.NoListener, // external listener - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/clientapi.go b/cmd/dendrite-polylith-multi/personalities/clientapi.go deleted file mode 100644 index a5d69d07c..000000000 --- a/cmd/dendrite-polylith-multi/personalities/clientapi.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - "github.com/matrix-org/dendrite/clientapi" - "github.com/matrix-org/dendrite/internal/transactions" - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -func ClientAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - federation := base.CreateFederationClient() - - asQuery := base.AppserviceHTTPClient() - rsAPI := base.RoomserverHTTPClient() - fsAPI := base.FederationAPIHTTPClient() - userAPI := base.UserAPIClient() - keyAPI := base.KeyServerHTTPClient() - - clientapi.AddPublicRoutes( - base, federation, rsAPI, asQuery, - transactions.New(), fsAPI, userAPI, userAPI, - keyAPI, nil, - ) - - base.SetupAndServeHTTP( - base.Cfg.ClientAPI.InternalAPI.Listen, - base.Cfg.ClientAPI.ExternalAPI.Listen, - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/federationapi.go b/cmd/dendrite-polylith-multi/personalities/federationapi.go deleted file mode 100644 index 48da42fbf..000000000 --- a/cmd/dendrite-polylith-multi/personalities/federationapi.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - "github.com/matrix-org/dendrite/federationapi" - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - userAPI := base.UserAPIClient() - federation := base.CreateFederationClient() - rsAPI := base.RoomserverHTTPClient() - keyAPI := base.KeyServerHTTPClient() - fsAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, nil, true) - keyRing := fsAPI.KeyRing() - - federationapi.AddPublicRoutes( - base, - userAPI, federation, keyRing, - rsAPI, fsAPI, keyAPI, nil, - ) - - federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI, base.EnableMetrics) - - base.SetupAndServeHTTP( - base.Cfg.FederationAPI.InternalAPI.Listen, - base.Cfg.FederationAPI.ExternalAPI.Listen, - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/keyserver.go b/cmd/dendrite-polylith-multi/personalities/keyserver.go deleted file mode 100644 index ad0bd0e54..000000000 --- a/cmd/dendrite-polylith-multi/personalities/keyserver.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - "github.com/matrix-org/dendrite/keyserver" - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -func KeyServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - fsAPI := base.FederationAPIHTTPClient() - rsAPI := base.RoomserverHTTPClient() - intAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI) - intAPI.SetUserAPI(base.UserAPIClient()) - - keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI, base.EnableMetrics) - - base.SetupAndServeHTTP( - base.Cfg.KeyServer.InternalAPI.Listen, // internal listener - basepkg.NoListener, // external listener - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/roomserver.go b/cmd/dendrite-polylith-multi/personalities/roomserver.go deleted file mode 100644 index 974559bd2..000000000 --- a/cmd/dendrite-polylith-multi/personalities/roomserver.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - "github.com/matrix-org/dendrite/roomserver" - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -func RoomServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - asAPI := base.AppserviceHTTPClient() - fsAPI := base.FederationAPIHTTPClient() - rsAPI := roomserver.NewInternalAPI(base) - rsAPI.SetFederationAPI(fsAPI, fsAPI.KeyRing()) - rsAPI.SetAppserviceAPI(asAPI) - roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI, base.EnableMetrics) - - base.SetupAndServeHTTP( - base.Cfg.RoomServer.InternalAPI.Listen, // internal listener - basepkg.NoListener, // external listener - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/syncapi.go b/cmd/dendrite-polylith-multi/personalities/syncapi.go deleted file mode 100644 index 41637fe1d..000000000 --- a/cmd/dendrite-polylith-multi/personalities/syncapi.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/syncapi" -) - -func SyncAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - userAPI := base.UserAPIClient() - - rsAPI := base.RoomserverHTTPClient() - - syncapi.AddPublicRoutes( - base, - userAPI, rsAPI, - base.KeyServerHTTPClient(), - ) - - base.SetupAndServeHTTP( - base.Cfg.SyncAPI.InternalAPI.Listen, - base.Cfg.SyncAPI.ExternalAPI.Listen, - nil, nil, - ) -} diff --git a/cmd/dendrite-polylith-multi/personalities/userapi.go b/cmd/dendrite-polylith-multi/personalities/userapi.go deleted file mode 100644 index 1bc88cb5f..000000000 --- a/cmd/dendrite-polylith-multi/personalities/userapi.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package personalities - -import ( - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/userapi" -) - -func UserAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - userAPI := userapi.NewInternalAPI( - base, &cfg.UserAPI, cfg.Derived.ApplicationServices, - base.KeyServerHTTPClient(), base.RoomserverHTTPClient(), - base.PushGatewayHTTPClient(), - ) - - userapi.AddInternalRoutes(base.InternalAPIMux, userAPI, base.EnableMetrics) - - base.SetupAndServeHTTP( - base.Cfg.UserAPI.InternalAPI.Listen, // internal listener - basepkg.NoListener, // external listener - nil, nil, - ) -} diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go index 39b9320cb..174a80a3e 100644 --- a/cmd/dendrite-upgrade-tests/main.go +++ b/cmd/dendrite-upgrade-tests/main.go @@ -45,6 +45,10 @@ var ( const HEAD = "HEAD" +// The binary was renamed after v0.11.1, so everything after that should use the new name +var binaryChangeVersion, _ = semver.NewVersion("v0.11.1") +var latest, _ = semver.NewVersion("v6.6.6") // Dummy version, used as "HEAD" + // Embed the Dockerfile to use when building dendrite versions. // We cannot use the dockerfile associated with the repo with each version sadly due to changes in // Docker versions. Specifically, earlier Dendrite versions are incompatible with newer Docker clients @@ -54,12 +58,13 @@ const HEAD = "HEAD" const DockerfilePostgreSQL = `FROM golang:1.18-stretch as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build +ARG BINARY # Copy the build context to the repo as this is the right dendrite code. This is different to the # Complement Dockerfile which wgets a branch. COPY . . -RUN go build ./cmd/dendrite-monolith-server +RUN go build ./cmd/${BINARY} RUN go build ./cmd/generate-keys RUN go build ./cmd/generate-config RUN go build ./cmd/create-account @@ -88,22 +93,24 @@ done \n\ \n\ sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml \n\ PARAMS="--tls-cert server.crt --tls-key server.key --config dendrite.yaml" \n\ -./dendrite-monolith-server --really-enable-open-registration ${PARAMS} || ./dendrite-monolith-server ${PARAMS} \n\ +./${BINARY} --really-enable-open-registration ${PARAMS} || ./${BINARY} ${PARAMS} \n\ ' > run_dendrite.sh && chmod +x run_dendrite.sh ENV SERVER_NAME=localhost +ENV BINARY=dendrite EXPOSE 8008 8448 -CMD /build/run_dendrite.sh ` +CMD /build/run_dendrite.sh` const DockerfileSQLite = `FROM golang:1.18-stretch as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build +ARG BINARY # Copy the build context to the repo as this is the right dendrite code. This is different to the # Complement Dockerfile which wgets a branch. COPY . . -RUN go build ./cmd/dendrite-monolith-server +RUN go build ./cmd/${BINARY} RUN go build ./cmd/generate-keys RUN go build ./cmd/generate-config RUN go build ./cmd/create-account @@ -118,10 +125,11 @@ RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postg RUN echo '\ sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml \n\ PARAMS="--tls-cert server.crt --tls-key server.key --config dendrite.yaml" \n\ -./dendrite-monolith-server --really-enable-open-registration ${PARAMS} || ./dendrite-monolith-server ${PARAMS} \n\ +./${BINARY} --really-enable-open-registration ${PARAMS} || ./${BINARY} ${PARAMS} \n\ ' > run_dendrite.sh && chmod +x run_dendrite.sh ENV SERVER_NAME=localhost +ENV BINARY=dendrite EXPOSE 8008 8448 CMD /build/run_dendrite.sh ` @@ -182,7 +190,7 @@ func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []b } // buildDendrite builds Dendrite on the branchOrTagName given. Returns the image ID or an error -func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir, branchOrTagName string) (string, error) { +func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir string, branchOrTagName, binary string) (string, error) { var tarball *bytes.Buffer var err error // If a custom HEAD location is given, use that, else pull from github. Mostly useful for CI @@ -216,6 +224,9 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir, log.Printf("%s: Building version %s\n", branchOrTagName, branchOrTagName) res, err := dockerClient.ImageBuild(context.Background(), tarball, types.ImageBuildOptions{ Tags: []string{"dendrite-upgrade"}, + BuildArgs: map[string]*string{ + "BINARY": &binary, + }, }) if err != nil { return "", fmt.Errorf("failed to start building image: %s", err) @@ -272,7 +283,7 @@ func getAndSortVersionsFromGithub(httpClient *http.Client) (semVers []*semver.Ve return semVers, nil } -func calculateVersions(cli *http.Client, from, to string, direct bool) []string { +func calculateVersions(cli *http.Client, from, to string, direct bool) []*semver.Version { semvers, err := getAndSortVersionsFromGithub(cli) if err != nil { log.Fatalf("failed to collect semvers from github: %s", err) @@ -320,28 +331,25 @@ func calculateVersions(cli *http.Client, from, to string, direct bool) []string } semvers = semvers[:i+1] } - var versions []string - for _, sv := range semvers { - versions = append(versions, sv.Original()) - } + if to == HEAD { - versions = append(versions, HEAD) + semvers = append(semvers, latest) } if direct { - versions = []string{versions[0], versions[len(versions)-1]} + semvers = []*semver.Version{semvers[0], semvers[len(semvers)-1]} } - return versions + return semvers } -func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, baseTempDir string, concurrency int, branchOrTagNames []string) map[string]string { +func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, baseTempDir string, concurrency int, versions []*semver.Version) map[string]string { // concurrently build all versions, this can be done in any order. The mutex protects the map branchToImageID := make(map[string]string) var mu sync.Mutex var wg sync.WaitGroup wg.Add(concurrency) - ch := make(chan string, len(branchOrTagNames)) - for _, branchName := range branchOrTagNames { + ch := make(chan *semver.Version, len(versions)) + for _, branchName := range versions { ch <- branchName } close(ch) @@ -349,11 +357,13 @@ func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, b for i := 0; i < concurrency; i++ { go func() { defer wg.Done() - for branchName := range ch { + for version := range ch { + branchName, binary := versionToBranchAndBinary(version) + log.Printf("Building version %s with binary %s", branchName, binary) tmpDir := baseTempDir + alphaNumerics.ReplaceAllString(branchName, "") - imgID, err := buildDendrite(httpClient, dockerClient, tmpDir, branchName) + imgID, err := buildDendrite(httpClient, dockerClient, tmpDir, branchName, binary) if err != nil { - log.Fatalf("%s: failed to build dendrite image: %s", branchName, err) + log.Fatalf("%s: failed to build dendrite image: %s", version, err) } mu.Lock() branchToImageID[branchName] = imgID @@ -365,13 +375,14 @@ func buildDendriteImages(httpClient *http.Client, dockerClient *client.Client, b return branchToImageID } -func runImage(dockerClient *client.Client, volumeName, version, imageID string) (csAPIURL, containerID string, err error) { - log.Printf("%s: running image %s\n", version, imageID) +func runImage(dockerClient *client.Client, volumeName string, branchNameToImageID map[string]string, version *semver.Version) (csAPIURL, containerID string, err error) { + branchName, binary := versionToBranchAndBinary(version) + imageID := branchNameToImageID[branchName] ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) defer cancel() body, err := dockerClient.ContainerCreate(ctx, &container.Config{ Image: imageID, - Env: []string{"SERVER_NAME=hs1"}, + Env: []string{"SERVER_NAME=hs1", fmt.Sprintf("BINARY=%s", binary)}, Labels: map[string]string{ dendriteUpgradeTestLabel: "yes", }, @@ -384,7 +395,7 @@ func runImage(dockerClient *client.Client, volumeName, version, imageID string) Target: "/var/lib/postgresql/9.6/main", }, }, - }, nil, nil, "dendrite_upgrade_test_"+version) + }, nil, nil, "dendrite_upgrade_test_"+branchName) if err != nil { return "", "", fmt.Errorf("failed to ContainerCreate: %s", err) } @@ -451,8 +462,8 @@ func destroyContainer(dockerClient *client.Client, containerID string) { } } -func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchToImageID map[string]string) error { - csAPIURL, containerID, err := runImage(dockerClient, volumeName, v, branchToImageID[v]) +func loadAndRunTests(dockerClient *client.Client, volumeName string, v *semver.Version, branchToImageID map[string]string) error { + csAPIURL, containerID, err := runImage(dockerClient, volumeName, branchToImageID, v) if err != nil { return fmt.Errorf("failed to run container for branch %v: %v", v, err) } @@ -470,9 +481,10 @@ func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchTo } // test that create-account is working -func testCreateAccount(dockerClient *client.Client, v string, containerID string) error { - createUser := strings.ToLower("createaccountuser-" + v) - log.Printf("%s: Creating account %s with create-account\n", v, createUser) +func testCreateAccount(dockerClient *client.Client, version *semver.Version, containerID string) error { + branchName, _ := versionToBranchAndBinary(version) + createUser := strings.ToLower("createaccountuser-" + branchName) + log.Printf("%s: Creating account %s with create-account\n", branchName, createUser) respID, err := dockerClient.ContainerExecCreate(context.Background(), containerID, types.ExecConfig{ AttachStderr: true, @@ -504,9 +516,21 @@ func testCreateAccount(dockerClient *client.Client, v string, containerID string return nil } -func verifyTests(dockerClient *client.Client, volumeName string, versions []string, branchToImageID map[string]string) error { +func versionToBranchAndBinary(version *semver.Version) (branchName, binary string) { + binary = "dendrite-monolith-server" + branchName = version.Original() + if version.GreaterThan(binaryChangeVersion) { + binary = "dendrite" + if version.Equal(latest) { + branchName = HEAD + } + } + return +} + +func verifyTests(dockerClient *client.Client, volumeName string, versions []*semver.Version, branchToImageID map[string]string) error { lastVer := versions[len(versions)-1] - csAPIURL, containerID, err := runImage(dockerClient, volumeName, lastVer, branchToImageID[lastVer]) + csAPIURL, containerID, err := runImage(dockerClient, volumeName, branchToImageID, lastVer) if err != nil { return fmt.Errorf("failed to run container for branch %v: %v", lastVer, err) } diff --git a/cmd/dendrite-upgrade-tests/tests.go b/cmd/dendrite-upgrade-tests/tests.go index 5c9589df2..03438bd4d 100644 --- a/cmd/dendrite-upgrade-tests/tests.go +++ b/cmd/dendrite-upgrade-tests/tests.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/Masterminds/semver/v3" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" ) @@ -22,7 +23,8 @@ type user struct { // - register alice and bob with branch name muxed into the localpart // - create a DM room for the 2 users and exchange messages // - create/join a public #global room and exchange messages -func runTests(baseURL, branchName string) error { +func runTests(baseURL string, v *semver.Version) error { + branchName, _ := versionToBranchAndBinary(v) // register 2 users users := []user{ { @@ -164,15 +166,16 @@ func runTests(baseURL, branchName string) error { } // verifyTestsRan checks that the HS has the right rooms/messages -func verifyTestsRan(baseURL string, branchNames []string) error { +func verifyTestsRan(baseURL string, versions []*semver.Version) error { log.Println("Verifying tests....") // check we can login as all users var resp *gomatrix.RespLogin - for _, branchName := range branchNames { + for _, version := range versions { client, err := gomatrix.NewClient(baseURL, "", "") if err != nil { return err } + branchName, _ := versionToBranchAndBinary(version) userLocalparts := []string{ "alice" + branchName, "bob" + branchName, @@ -224,7 +227,7 @@ func verifyTestsRan(baseURL string, branchNames []string) error { msgCount += 1 } } - wantMsgCount := len(branchNames) * 4 + wantMsgCount := len(versions) * 4 if msgCount != wantMsgCount { return fmt.Errorf("got %d messages in global room, want %d", msgCount, wantMsgCount) } diff --git a/cmd/dendrite/main.go b/cmd/dendrite/main.go new file mode 100644 index 000000000..e8ff0a478 --- /dev/null +++ b/cmd/dendrite/main.go @@ -0,0 +1,103 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + + "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/setup" + basepkg "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/mscs" + "github.com/matrix-org/dendrite/userapi" +) + +var ( + httpBindAddr = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server") + httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server") + certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS") + keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS") +) + +func main() { + cfg := setup.ParseFlags(true) + httpAddr := config.HTTPAddress("http://" + *httpBindAddr) + httpsAddr := config.HTTPAddress("https://" + *httpsBindAddr) + options := []basepkg.BaseDendriteOptions{} + + base := basepkg.NewBaseDendrite(cfg, options...) + defer base.Close() // nolint: errcheck + + federation := base.CreateFederationClient() + + rsAPI := roomserver.NewInternalAPI(base) + + fsAPI := federationapi.NewInternalAPI( + base, federation, rsAPI, base.Caches, nil, false, + ) + + keyRing := fsAPI.KeyRing() + + userAPI := userapi.NewInternalAPI(base, rsAPI, federation) + + asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) + + // The underlying roomserver implementation needs to be able to call the fedsender. + // This is different to rsAPI which can be the http client which doesn't need this + // dependency. Other components also need updating after their dependencies are up. + rsAPI.SetFederationAPI(fsAPI, keyRing) + rsAPI.SetAppserviceAPI(asAPI) + rsAPI.SetUserAPI(userAPI) + + monolith := setup.Monolith{ + Config: base.Cfg, + Client: base.CreateClient(), + FedClient: federation, + KeyRing: keyRing, + + AppserviceAPI: asAPI, + // always use the concrete impl here even in -http mode because adding public routes + // must be done on the concrete impl not an HTTP client else fedapi will call itself + FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + } + monolith.AddAllPublicRoutes(base) + + if len(base.Cfg.MSCs.MSCs) > 0 { + if err := mscs.Enable(base, &monolith); err != nil { + logrus.WithError(err).Fatalf("Failed to enable MSCs") + } + } + + // Expose the matrix APIs directly rather than putting them under a /api path. + go func() { + base.SetupAndServeHTTP(httpAddr, nil, nil) + }() + // Handle HTTPS if certificate and key are provided + if *certFile != "" && *keyFile != "" { + go func() { + base.SetupAndServeHTTP(httpsAddr, certFile, keyFile) + }() + } + + // We want to block forever to let the HTTP and HTTPS handler serve the APIs + base.WaitForShutdown() +} diff --git a/cmd/dendrite-monolith-server/main_test.go b/cmd/dendrite/main_test.go similarity index 100% rename from cmd/dendrite-monolith-server/main_test.go rename to cmd/dendrite/main_test.go diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index 56a145653..b0707be11 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -18,7 +18,6 @@ func main() { dbURI := flag.String("db", "", "The DB URI to use for all components (PostgreSQL only)") dirPath := flag.String("dir", "./", "The folder to use for paths (like SQLite databases, media storage)") normalise := flag.String("normalise", "", "Normalise an existing configuration file by adding new/missing options and defaults") - polylith := flag.Bool("polylith", false, "Generate a config that makes sense for polylith deployments") flag.Parse() var cfg *config.Dendrite @@ -27,14 +26,14 @@ func main() { Version: config.Version, } cfg.Defaults(config.DefaultOpts{ - Generate: true, - Monolithic: !*polylith, + Generate: true, + SingleDatabase: true, }) if *serverName != "" { cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName) } uri := config.DataSource(*dbURI) - if *polylith || uri.IsSQLite() || uri == "" { + if uri.IsSQLite() || uri == "" { for name, db := range map[string]*config.DatabaseOptions{ "federationapi": &cfg.FederationAPI.Database, "keyserver": &cfg.KeyServer.Database, @@ -43,6 +42,7 @@ func main() { "roomserver": &cfg.RoomServer.Database, "syncapi": &cfg.SyncAPI.Database, "userapi": &cfg.UserAPI.AccountDatabase, + "relayapi": &cfg.RelayAPI.Database, } { if uri == "" { path := filepath.Join(*dirPath, fmt.Sprintf("dendrite_%s.db", name)) @@ -96,7 +96,7 @@ func main() { } } else { var err error - if cfg, err = config.Load(*normalise, !*polylith); err != nil { + if cfg, err = config.Load(*normalise); err != nil { panic(err) } } diff --git a/cmd/resolve-state/main.go b/cmd/resolve-state/main.go index f8bb130c7..59c66aed4 100644 --- a/cmd/resolve-state/main.go +++ b/cmd/resolve-state/main.go @@ -40,7 +40,7 @@ func main() { Level: "error", }) cfg.ClientAPI.RegistrationDisabled = true - base := base.NewBaseDendrite(cfg, "ResolveState", base.DisableMetrics) + base := base.NewBaseDendrite(cfg, base.DisableMetrics) args := flag.Args() fmt.Println("Room version", *roomVersion) diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml index d29a8172c..e69de29bb 100644 --- a/dendrite-sample.polylith.yaml +++ b/dendrite-sample.polylith.yaml @@ -1,416 +0,0 @@ -# This is the Dendrite configuration file. -# -# The configuration is split up into sections - each Dendrite component has a -# configuration section, in addition to the "global" section which applies to -# all components. - -# The version of the configuration file. -version: 2 - -# Global Matrix configuration. This configuration applies to all components. -global: - # The domain name of this homeserver. - server_name: localhost - - # The path to the signing private key file, used to sign requests and events. - # Note that this is NOT the same private key as used for TLS! To generate a - # signing key, use "./bin/generate-keys --private-key matrix_key.pem". - private_key: matrix_key.pem - - # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) - # to old signing keys that were formerly in use on this domain name. These - # keys will not be used for federation request or event signing, but will be - # provided to any other homeserver that asks when trying to verify old events. - old_private_keys: - # If the old private key file is available: - # - private_key: old_matrix_key.pem - # expired_at: 1601024554498 - # If only the public key (in base64 format) and key ID are known: - # - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM= - # key_id: ed25519:mykeyid - # expired_at: 1601024554498 - - # How long a remote server can cache our server signing key before requesting it - # again. Increasing this number will reduce the number of requests made by other - # servers for our key but increases the period that a compromised key will be - # considered valid by other homeservers. - key_validity_period: 168h0m0s - - # Configuration for in-memory caches. Caches can often improve performance by - # keeping frequently accessed items (like events, identifiers etc.) in memory - # rather than having to read them from the database. - cache: - # The estimated maximum size for the global cache in bytes, or in terabytes, - # gigabytes, megabytes or kilobytes when the appropriate 'tb', 'gb', 'mb' or - # 'kb' suffix is specified. Note that this is not a hard limit, nor is it a - # memory limit for the entire process. A cache that is too small may ultimately - # provide little or no benefit. - max_size_estimated: 1gb - - # The maximum amount of time that a cache entry can live for in memory before - # it will be evicted and/or refreshed from the database. Lower values result in - # easier admission of new cache entries but may also increase database load in - # comparison to higher values, so adjust conservatively. Higher values may make - # it harder for new items to make it into the cache, e.g. if new rooms suddenly - # become popular. - max_age: 1h - - # The server name to delegate server-server communications to, with optional port - # e.g. localhost:443 - well_known_server_name: "" - - # The base URL to delegate client-server communications to e.g. https://localhost - well_known_client_name: "" - - # Lists of domains that the server will trust as identity servers to verify third - # party identifiers such as phone numbers and email addresses. - trusted_third_party_id_servers: - - matrix.org - - vector.im - - # Disables federation. Dendrite will not be able to communicate with other servers - # in the Matrix federation and the federation API will not be exposed. - disable_federation: false - - # Configures the handling of presence events. Inbound controls whether we receive - # presence events from other servers, outbound controls whether we send presence - # events for our local users to other servers. - presence: - enable_inbound: false - enable_outbound: false - - # Configures phone-home statistics reporting. These statistics contain the server - # name, number of active users and some information on your deployment config. - # We use this information to understand how Dendrite is being used in the wild. - report_stats: - enabled: false - endpoint: https://matrix.org/report-usage-stats/push - - # Server notices allows server admins to send messages to all users on the server. - server_notices: - enabled: false - # The local part, display name and avatar URL (as a mxc:// URL) for the user that - # will send the server notices. These are visible to all users on the deployment. - local_part: "_server" - display_name: "Server Alerts" - avatar_url: "" - # The room name to be used when sending server notices. This room name will - # appear in user clients. - room_name: "Server Alerts" - - # Configuration for NATS JetStream - jetstream: - # A list of NATS Server addresses to connect to. If none are specified, an - # internal NATS server will be started automatically when running Dendrite in - # monolith mode. For polylith deployments, it is required to specify the address - # of at least one NATS Server node. - addresses: - - hostname:4222 - - # Disable the validation of TLS certificates of NATS. This is - # not recommended in production since it may allow NATS traffic - # to be sent to an insecure endpoint. - disable_tls_validation: false - - # The prefix to use for stream names for this homeserver - really only useful - # if you are running more than one Dendrite server on the same NATS deployment. - topic_prefix: Dendrite - - # Configuration for Prometheus metric collection. - metrics: - enabled: false - basic_auth: - username: metrics - password: metrics - - # Optional DNS cache. The DNS cache may reduce the load on DNS servers if there - # is no local caching resolver available for use. - dns_cache: - enabled: false - cache_size: 256 - cache_lifetime: "5m" # 5 minutes; https://pkg.go.dev/time@master#ParseDuration - -# Configuration for the Appservice API. -app_service_api: - internal_api: - listen: http://[::]:7777 # The listen address for incoming API requests - connect: http://app_service_api:7777 # The connect address for other components to use - - # Disable the validation of TLS certificates of appservices. This is - # not recommended in production since it may allow appservice traffic - # to be sent to an insecure endpoint. - disable_tls_validation: false - - # Appservice configuration files to load into this homeserver. - config_files: - # - /path/to/appservice_registration.yaml - -# Configuration for the Client API. -client_api: - internal_api: - listen: http://[::]:7771 # The listen address for incoming API requests - connect: http://client_api:7771 # The connect address for other components to use - external_api: - listen: http://[::]:8071 - - # Prevents new users from being able to register on this homeserver, except when - # using the registration shared secret below. - registration_disabled: true - - # Prevents new guest accounts from being created. Guest registration is also - # disabled implicitly by setting 'registration_disabled' above. - guests_disabled: true - - # If set, allows registration by anyone who knows the shared secret, regardless - # of whether registration is otherwise disabled. - registration_shared_secret: "" - - # Whether to require reCAPTCHA for registration. If you have enabled registration - # then this is HIGHLY RECOMMENDED to reduce the risk of your homeserver being used - # for coordinated spam attacks. - enable_registration_captcha: false - - # Settings for ReCAPTCHA. - recaptcha_public_key: "" - recaptcha_private_key: "" - recaptcha_bypass_secret: "" - - # To use hcaptcha.com instead of ReCAPTCHA, set the following parameters, otherwise just keep them empty. - # recaptcha_siteverify_api: "https://hcaptcha.com/siteverify" - # recaptcha_api_js_url: "https://js.hcaptcha.com/1/api.js" - # recaptcha_form_field: "h-captcha-response" - # recaptcha_sitekey_class: "h-captcha" - - - # TURN server information that this homeserver should send to clients. - turn: - turn_user_lifetime: "5m" - turn_uris: - # - turn:turn.server.org?transport=udp - # - turn:turn.server.org?transport=tcp - turn_shared_secret: "" - # If your TURN server requires static credentials, then you will need to enter - # them here instead of supplying a shared secret. Note that these credentials - # will be visible to clients! - # turn_username: "" - # turn_password: "" - - # Settings for rate-limited endpoints. Rate limiting kicks in after the threshold - # number of "slots" have been taken by requests from a specific host. Each "slot" - # will be released after the cooloff time in milliseconds. Server administrators - # and appservice users are exempt from rate limiting by default. - rate_limiting: - enabled: true - threshold: 20 - cooloff_ms: 500 - exempt_user_ids: - # - "@user:domain.com" - -# Configuration for the Federation API. -federation_api: - internal_api: - listen: http://[::]:7772 # The listen address for incoming API requests - connect: http://federation_api:7772 # The connect address for other components to use - external_api: - listen: http://[::]:8072 - database: - connection_string: postgresql://username:password@hostname/dendrite_federationapi?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - - # How many times we will try to resend a failed transaction to a specific server. The - # backoff is 2**x seconds, so 1 = 2 seconds, 2 = 4 seconds, 3 = 8 seconds etc. Once - # the max retries are exceeded, Dendrite will no longer try to send transactions to - # that server until it comes back to life and connects to us again. - send_max_retries: 16 - - # Disable the validation of TLS certificates of remote federated homeservers. Do not - # enable this option in production as it presents a security risk! - disable_tls_validation: false - - # Disable HTTP keepalives, which also prevents connection reuse. Dendrite will typically - # keep HTTP connections open to remote hosts for 5 minutes as they can be reused much - # more quickly than opening new connections each time. Disabling keepalives will close - # HTTP connections immediately after a successful request but may result in more CPU and - # memory being used on TLS handshakes for each new connection instead. - disable_http_keepalives: false - - # Perspective keyservers to use as a backup when direct key fetches fail. This may - # be required to satisfy key requests for servers that are no longer online when - # joining some rooms. - key_perspectives: - - server_name: matrix.org - keys: - - key_id: ed25519:auto - public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw - - key_id: ed25519:a_RXGa - public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ - - # This option will control whether Dendrite will prefer to look up keys directly - # or whether it should try perspective servers first, using direct fetches as a - # last resort. - prefer_direct_fetch: false - -# Configuration for the Key Server (for end-to-end encryption). -key_server: - internal_api: - listen: http://[::]:7779 # The listen address for incoming API requests - connect: http://key_server:7779 # The connect address for other components to use - database: - connection_string: postgresql://username:password@hostname/dendrite_keyserver?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - -# Configuration for the Media API. -media_api: - internal_api: - listen: http://[::]:7774 # The listen address for incoming API requests - connect: http://media_api:7774 # The connect address for other components to use - external_api: - listen: http://[::]:8074 - database: - connection_string: postgresql://username:password@hostname/dendrite_mediaapi?sslmode=disable - max_open_conns: 5 - max_idle_conns: 2 - conn_max_lifetime: -1 - - # Storage path for uploaded media. May be relative or absolute. - base_path: ./media_store - - # The maximum allowed file size (in bytes) for media uploads to this homeserver - # (0 = unlimited). If using a reverse proxy, ensure it allows requests at least - #this large (e.g. the client_max_body_size setting in nginx). - max_file_size_bytes: 10485760 - - # Whether to dynamically generate thumbnails if needed. - dynamic_thumbnails: false - - # The maximum number of simultaneous thumbnail generators to run. - max_thumbnail_generators: 10 - - # A list of thumbnail sizes to be generated for media content. - thumbnail_sizes: - - width: 32 - height: 32 - method: crop - - width: 96 - height: 96 - method: crop - - width: 640 - height: 480 - method: scale - -# Configuration for enabling experimental MSCs on this homeserver. -mscs: - mscs: - # - msc2836 # (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836) - # - msc2946 # (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946) - database: - connection_string: postgresql://username:password@hostname/dendrite_mscs?sslmode=disable - max_open_conns: 5 - max_idle_conns: 2 - conn_max_lifetime: -1 - -# Configuration for the Room Server. -room_server: - internal_api: - listen: http://[::]:7770 # The listen address for incoming API requests - connect: http://room_server:7770 # The connect address for other components to use - database: - connection_string: postgresql://username:password@hostname/dendrite_roomserver?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - -# Configuration for the Sync API. -sync_api: - internal_api: - listen: http://[::]:7773 # The listen address for incoming API requests - connect: http://sync_api:7773 # The connect address for other components to use - external_api: - listen: http://[::]:8073 - database: - connection_string: postgresql://username:password@hostname/dendrite_syncapi?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - - # Configuration for the full-text search engine. - search: - # Whether or not search is enabled. - enabled: false - - # The path where the search index will be created in. - index_path: "./searchindex" - - # The language most likely to be used on the server - used when indexing, to - # ensure the returned results match expectations. A full list of possible languages - # can be found at https://github.com/blevesearch/bleve/tree/master/analysis/lang - language: "en" - - # This option controls which HTTP header to inspect to find the real remote IP - # address of the client. This is likely required if Dendrite is running behind - # a reverse proxy server. - # real_ip_header: X-Real-IP - -# Configuration for the User API. -user_api: - internal_api: - listen: http://[::]:7781 # The listen address for incoming API requests - connect: http://user_api:7781 # The connect address for other components to use - account_database: - connection_string: postgresql://username:password@hostname/dendrite_userapi?sslmode=disable - max_open_conns: 10 - max_idle_conns: 2 - conn_max_lifetime: -1 - - # The cost when hashing passwords on registration/login. Default: 10. Min: 4, Max: 31 - # See https://pkg.go.dev/golang.org/x/crypto/bcrypt for more information. - # Setting this lower makes registration/login consume less CPU resources at the cost - # of security should the database be compromised. Setting this higher makes registration/login - # consume more CPU resources but makes it harder to brute force password hashes. This value - # can be lowered if performing tests or on embedded Dendrite instances (e.g WASM builds). - bcrypt_cost: 10 - - # The length of time that a token issued for a relying party from - # /_matrix/client/r0/user/{userId}/openid/request_token endpoint - # is considered to be valid in milliseconds. - # The default lifetime is 3600000ms (60 minutes). - # openid_token_lifetime_ms: 3600000 - - # Users who register on this homeserver will automatically be joined to the rooms listed under "auto_join_rooms" option. - # By default, any room aliases included in this list will be created as a publicly joinable room - # when the first user registers for the homeserver. If the room already exists, - # make certain it is a publicly joinable room, i.e. the join rule of the room must be set to 'public'. - # As Spaces are just rooms under the hood, Space aliases may also be used. - auto_join_rooms: - # - "#main:matrix.org" - -# Configuration for Opentracing. -# See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on -# how this works and how to set it up. -tracing: - enabled: false - jaeger: - serviceName: "" - disabled: false - rpc_metrics: false - tags: [] - sampler: null - reporter: null - headers: null - baggage_restrictions: null - throttler: null - -# Logging configuration. The "std" logging type controls the logs being sent to -# stdout. The "file" logging type controls logs being written to a log folder on -# the disk. Supported log levels are "debug", "info", "warn", "error". -logging: - - type: std - level: info - - type: file - level: info - params: - path: ./logs diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.yaml similarity index 96% rename from dendrite-sample.monolith.yaml rename to dendrite-sample.yaml index 453b94e01..fed8bf1cd 100644 --- a/dendrite-sample.monolith.yaml +++ b/dendrite-sample.yaml @@ -38,7 +38,7 @@ global: # Global database connection pool, for PostgreSQL monolith deployments only. If # this section is populated then you can omit the "database" blocks in all other - # sections. For polylith deployments, or monolith deployments using SQLite databases, + # sections. For monolith deployments using SQLite databases, # you must configure the "database" block for each component instead. database: connection_string: postgresql://username:password@hostname/dendrite?sslmode=disable @@ -110,6 +110,17 @@ global: # Configuration for NATS JetStream jetstream: + # A list of NATS Server addresses to connect to. If none are specified, an + # internal NATS server will be started automatically when running Dendrite in + # monolith mode. + addresses: + # - localhost:4222 + + # Disable the validation of TLS certificates of NATS. This is + # not recommended in production since it may allow NATS traffic + # to be sent to an insecure endpoint. + disable_tls_validation: false + # Persistent directory to store JetStream streams in. This directory should be # preserved across Dendrite restarts. storage_path: ./ diff --git a/docs/FAQ.md b/docs/FAQ.md index 4047bfffc..2899aa982 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -35,11 +35,6 @@ possible to migrate an existing Synapse deployment to Dendrite. No, Dendrite has a very different database schema to Synapse and the two are not interchangeable. -## Should I run a monolith or a polylith deployment? - -Monolith deployments are always preferred where possible, and at this time, are far better tested than polylith deployments are. The only reason to consider a polylith deployment is if you wish to run different Dendrite components on separate physical machines, but this is an advanced configuration which we don't -recommend. - ## Can I configure which port Dendrite listens on? Yes, use the cli flag `-http-bind-address`. diff --git a/docs/INSTALL.md b/docs/INSTALL.md new file mode 100644 index 000000000..81f7696c4 --- /dev/null +++ b/docs/INSTALL.md @@ -0,0 +1,13 @@ +# Installation + +Please note that new installation instructions can be found +on the [new documentation site](https://matrix-org.github.io/dendrite/), +or alternatively, in the [installation](installation/) folder: + +1. [Planning your deployment](installation/1_planning.md) +2. [Setting up the domain](installation/2_domainname.md) +3. [Preparing database storage](installation/3_database.md) +4. [Generating signing keys](installation/4_signingkey.md) +5. [Installing Dendrite](installation/5_install_monolith.md) +6. [Populate the configuration](installation/7_configuration.md) +7. [Starting Dendrite](installation/8_starting_monolith.md) diff --git a/docs/development/coverage.md b/docs/development/coverage.md index cd53f2e82..cf56b4ba8 100644 --- a/docs/development/coverage.md +++ b/docs/development/coverage.md @@ -42,7 +42,7 @@ total: (statements) 53.7% ``` The total coverage for this run is the last line at the bottom. However, this value is misleading because Dendrite can run in many different configurations, -which will never be tested in a single test run (e.g sqlite or postgres, monolith or polylith). To get a more accurate value, additional processing is required +which will never be tested in a single test run (e.g sqlite or postgres). To get a more accurate value, additional processing is required to remove packages which will never be tested and extension MSCs: (The following commands use [gocovmerge](https://github.com/wadey/gocovmerge), to merge two or more coverage logs, so make sure you have that installed) @@ -50,17 +50,11 @@ to remove packages which will never be tested and extension MSCs: ```bash # These commands are all similar but change which package paths are _removed_ from the output. -# For Postgres (monolith) -find -name 'integrationcover.log' | xargs gocovmerge | grep -Ev 'relayapi|inthttp|sqlite|setup/mscs|api_trace' > final.cov && go tool cover -func=final.cov +# For Postgres +find -name 'integrationcover.log' | xargs gocovmerge | grep -Ev 'relayapi|sqlite|setup/mscs' > final.cov && go tool cover -func=final.cov -# For Postgres (polylith) -find -name 'integrationcover.log' | xargs gocovmerge | grep -Ev 'relayapi|sqlite|setup/mscs|api_trace' > final.cov && go tool cover -func=final.cov - -# For SQLite (monolith) -find -name 'integrationcover.log' | xargs gocovmerge | grep -Ev 'relayapi|inthttp|postgres|setup/mscs|api_trace' > final.cov && go tool cover -func=final.cov - -# For SQLite (polylith) -find -name 'integrationcover.log' | xargs gocovmerge | grep -Ev 'relayapi|postgres|setup/mscs|api_trace' > final.cov && go tool cover -func=final.cov +# For SQLite +find -name 'integrationcover.log' | xargs gocovmerge | grep -Ev 'relayapi|postgres|setup/mscs' > final.cov && go tool cover -func=final.cov ``` ## Running unit tests with coverage enabled diff --git a/docs/development/tracing/setup.md b/docs/development/tracing/setup.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/installation/1_planning.md b/docs/installation/1_planning.md index 5ae67fb1b..e7febd145 100644 --- a/docs/installation/1_planning.md +++ b/docs/installation/1_planning.md @@ -7,6 +7,22 @@ permalink: /installation/planning # Planning your installation +## Modes + +Dendrite consists of several components, each responsible for a different aspect of the Matrix protocol. +Users can run Dendrite in one of two modes which dictate how these components are executed and communicate. + +* **Monolith mode** runs all components in a single process. Components communicate through an internal NATS + server with generally low overhead. This mode dramatically simplifies deployment complexity and offers the + best balance between performance and resource usage for low-to-mid volume deployments. + +* **Polylith mode** runs all components in isolated processes. Components communicate through an external NATS + server and HTTP APIs, which incur considerable overhead. While this mode allows for more granular control of + resources dedicated toward individual processes, given the additional communications overhead, it is only + necessary for very large deployments. + +Given our current state of development, **we recommend monolith mode** for all deployments. + ## Databases Dendrite can run with either a PostgreSQL or a SQLite backend. There are considerable tradeoffs @@ -67,11 +83,17 @@ you should check (by running `go version`) that you are using a suitable version If using the PostgreSQL database engine, you should install PostgreSQL 12 or later. +### NATS Server + +Dendrite comes with a built-in [NATS Server](https://github.com/nats-io/nats-server) and +therefore does not need this to be manually installed. If you are planning a monolith installation, you +do not need to do anything. + + ### Reverse proxy A reverse proxy such as [Caddy](https://caddyserver.com), [NGINX](https://www.nginx.com) or -[HAProxy](http://www.haproxy.org) is recommended for monolith deployments. -Configuring those not covered in this documentation, although sample configurations +[HAProxy](http://www.haproxy.org) is useful for deployments. Configuring those is not covered in this documentation, although sample configurations for [Caddy](https://github.com/matrix-org/dendrite/blob/main/docs/caddy) and [NGINX](https://github.com/matrix-org/dendrite/blob/main/docs/nginx) are provided. diff --git a/docs/installation/manual/3_configuration.md b/docs/installation/manual/3_configuration.md index 728892016..06841f865 100644 --- a/docs/installation/manual/3_configuration.md +++ b/docs/installation/manual/3_configuration.md @@ -8,11 +8,10 @@ permalink: /installation/manual/configuration # Configuring Dendrite -A YAML configuration file is used to configure Dendrite. Sample configuration files are +A YAML configuration file is used to configure Dendrite. A sample configuration file is present in the top level of the Dendrite repository: * [`dendrite-sample.monolith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.monolith.yaml) -* [`dendrite-sample.polylith.yaml`](https://github.com/matrix-org/dendrite/blob/main/dendrite-sample.polylith.yaml) You will need to duplicate the sample, calling it `dendrite.yaml` for example, and then tailor it to your installation. At a minimum, you will need to populate the following @@ -44,17 +43,42 @@ global: private_key: /path/to/matrix_key.pem ``` +## JetStream configuration + +Dendrite deployments can use the built-in NATS Server rather than running a standalone +server. If you want to use a standalone NATS Server anyway, you can also configure that too. + ### Built-in NATS Server -In the `global` section, under the `jetstream` key, set a `storage_path` to a persistent folder on the filesystem: +In the `global` section, under the `jetstream` key, ensure that no server addresses are +configured and set a `storage_path` to a persistent folder on the filesystem: ```yaml global: # ... jetstream: storage_path: /path/to/storage/folder + topic_prefix: Dendrite ``` +### Standalone NATS Server + +To use a standalone NATS Server instance, you will need to configure `addresses` field +to point to the port that your NATS Server is listening on: + +```yaml +global: + # ... + jetstream: + addresses: + - localhost:4222 + topic_prefix: Dendrite +``` + +You do not need to configure the `storage_path` when using a standalone NATS Server instance. +In the case that you are connecting to a multi-node NATS cluster, you can configure more than +one address in the `addresses` field. + ## Database connections Configuring database connections varies based on the [database configuration](database) diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go index 601257d4b..7d9df3d78 100644 --- a/federationapi/consumers/keychange.go +++ b/federationapi/consumers/keychange.go @@ -26,11 +26,11 @@ import ( "github.com/matrix-org/dendrite/federationapi/queue" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/federationapi/types" - "github.com/matrix-org/dendrite/keyserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/dendrite/userapi/api" ) // KeyChangeConsumer consumes events that originate in key server. diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index ed9a545d6..ec482659a 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -17,20 +17,17 @@ package federationapi import ( "time" - "github.com/gorilla/mux" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/federationapi/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/consumers" "github.com/matrix-org/dendrite/federationapi/internal" - "github.com/matrix-org/dendrite/federationapi/inthttp" "github.com/matrix-org/dendrite/federationapi/producers" "github.com/matrix-org/dendrite/federationapi/queue" "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/internal/caching" - keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/jetstream" @@ -41,21 +38,14 @@ import ( "github.com/matrix-org/dendrite/federationapi/routing" ) -// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions -// on the given input API. -func AddInternalRoutes(router *mux.Router, intAPI api.FederationInternalAPI, enableMetrics bool) { - inthttp.AddRoutes(intAPI, router, enableMetrics) -} - // AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component. func AddPublicRoutes( base *base.BaseDendrite, - userAPI userapi.UserInternalAPI, + userAPI userapi.FederationUserAPI, federation *gomatrixserverlib.FederationClient, keyRing gomatrixserverlib.JSONVerifier, rsAPI roomserverAPI.FederationRoomserverAPI, fedAPI federationAPI.FederationInternalAPI, - keyAPI keyserverAPI.FederationKeyAPI, servers federationAPI.ServersInRoomProvider, ) { cfg := &base.Cfg.FederationAPI @@ -87,7 +77,7 @@ func AddPublicRoutes( routing.Setup( base, rsAPI, f, keyRing, - federation, userAPI, keyAPI, mscCfg, + federation, userAPI, mscCfg, servers, producer, ) } diff --git a/federationapi/federationapi_keys_test.go b/federationapi/federationapi_keys_test.go index cc03cdece..bb6ee8935 100644 --- a/federationapi/federationapi_keys_test.go +++ b/federationapi/federationapi_keys_test.go @@ -77,8 +77,8 @@ func TestMain(m *testing.M) { // API to work. cfg := &config.Dendrite{} cfg.Defaults(config.DefaultOpts{ - Generate: true, - Monolithic: true, + Generate: true, + SingleDatabase: false, }) cfg.Global.ServerName = gomatrixserverlib.ServerName(s.name) cfg.Global.PrivateKey = testPriv @@ -109,7 +109,7 @@ func TestMain(m *testing.M) { ) // Finally, build the server key APIs. - sbase := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics) + sbase := base.NewBaseDendrite(cfg, base.DisableMetrics) s.api = NewInternalAPI(sbase, s.fedclient, nil, s.cache, nil, true) } diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index 7009230cc..57d4b9644 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -17,13 +17,13 @@ import ( "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/internal" - keyapi "github.com/matrix-org/dendrite/keyserver/api" rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" + userapi "github.com/matrix-org/dendrite/userapi/api" ) type fedRoomserverAPI struct { @@ -230,9 +230,9 @@ func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) { // Inject a keyserver key change event and ensure we try to send it out. If we don't, then the // federationapi is incorrectly waiting for an output room event to arrive to update the joined // hosts table. - key := keyapi.DeviceMessage{ - Type: keyapi.TypeDeviceKeyUpdate, - DeviceKeys: &keyapi.DeviceKeys{ + key := userapi.DeviceMessage{ + Type: userapi.TypeDeviceKeyUpdate, + DeviceKeys: &userapi.DeviceKeys{ UserID: joiningUser.ID, DeviceID: "MY_DEVICE", DisplayName: "BLARGLE", @@ -266,18 +266,18 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) { _, privKey, _ := ed25519.GenerateKey(nil) cfg := &config.Dendrite{} cfg.Defaults(config.DefaultOpts{ - Generate: true, - Monolithic: true, + Generate: true, + SingleDatabase: false, }) cfg.Global.KeyID = gomatrixserverlib.KeyID("ed25519:auto") cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost") cfg.Global.PrivateKey = privKey cfg.Global.JetStream.InMemory = true - b := base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics) + b := base.NewBaseDendrite(cfg, base.DisableMetrics) keyRing := &test.NopJSONVerifier{} // TODO: This is pretty fragile, as if anything calls anything on these nils this test will break. // Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing. - federationapi.AddPublicRoutes(b, nil, nil, keyRing, nil, &internal.FederationInternalAPI{}, nil, nil) + federationapi.AddPublicRoutes(b, nil, nil, keyRing, nil, &internal.FederationInternalAPI{}, nil) baseURL, cancel := test.ListenAndServe(t, b.PublicFederationAPIMux, true) defer cancel() serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://")) diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index b9684f767..dadb2b2b3 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -121,8 +121,6 @@ func (r *FederationInternalAPI) PerformJoin( var httpErr gomatrix.HTTPError if ok := errors.As(lastErr, &httpErr); ok { httpErr.Message = string(httpErr.Contents) - // Clear the wrapped error, else serialising to JSON (in polylith mode) will fail - httpErr.WrappedError = nil response.LastError = &httpErr } else { response.LastError = &gomatrix.HTTPError{ @@ -391,8 +389,6 @@ func (r *FederationInternalAPI) PerformOutboundPeek( var httpErr gomatrix.HTTPError if ok := errors.As(lastErr, &httpErr); ok { httpErr.Message = string(httpErr.Contents) - // Clear the wrapped error, else serialising to JSON (in polylith mode) will fail - httpErr.WrappedError = nil response.LastError = &httpErr } else { response.LastError = &gomatrix.HTTPError{ diff --git a/federationapi/inthttp/client.go b/federationapi/inthttp/client.go deleted file mode 100644 index 00e069d1e..000000000 --- a/federationapi/inthttp/client.go +++ /dev/null @@ -1,548 +0,0 @@ -package inthttp - -import ( - "context" - "errors" - "net/http" - - "github.com/matrix-org/dendrite/federationapi/api" - "github.com/matrix-org/dendrite/internal/caching" - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/gomatrix" - "github.com/matrix-org/gomatrixserverlib" -) - -// HTTP paths for the internal HTTP API -const ( - FederationAPIQueryJoinedHostServerNamesInRoomPath = "/federationapi/queryJoinedHostServerNamesInRoom" - FederationAPIQueryServerKeysPath = "/federationapi/queryServerKeys" - - FederationAPIPerformDirectoryLookupRequestPath = "/federationapi/performDirectoryLookup" - FederationAPIPerformJoinRequestPath = "/federationapi/performJoinRequest" - FederationAPIPerformLeaveRequestPath = "/federationapi/performLeaveRequest" - FederationAPIPerformInviteRequestPath = "/federationapi/performInviteRequest" - FederationAPIPerformOutboundPeekRequestPath = "/federationapi/performOutboundPeekRequest" - FederationAPIPerformBroadcastEDUPath = "/federationapi/performBroadcastEDU" - FederationAPIPerformWakeupServers = "/federationapi/performWakeupServers" - FederationAPIQueryRelayServers = "/federationapi/queryRelayServers" - FederationAPIAddRelayServers = "/federationapi/addRelayServers" - FederationAPIRemoveRelayServers = "/federationapi/removeRelayServers" - - FederationAPIGetUserDevicesPath = "/federationapi/client/getUserDevices" - FederationAPIClaimKeysPath = "/federationapi/client/claimKeys" - FederationAPIQueryKeysPath = "/federationapi/client/queryKeys" - FederationAPIBackfillPath = "/federationapi/client/backfill" - FederationAPILookupStatePath = "/federationapi/client/lookupState" - FederationAPILookupStateIDsPath = "/federationapi/client/lookupStateIDs" - FederationAPILookupMissingEventsPath = "/federationapi/client/lookupMissingEvents" - FederationAPIGetEventPath = "/federationapi/client/getEvent" - FederationAPILookupServerKeysPath = "/federationapi/client/lookupServerKeys" - FederationAPIEventRelationshipsPath = "/federationapi/client/msc2836eventRelationships" - FederationAPISpacesSummaryPath = "/federationapi/client/msc2946spacesSummary" - FederationAPIGetEventAuthPath = "/federationapi/client/getEventAuth" - - FederationAPIInputPublicKeyPath = "/federationapi/inputPublicKey" - FederationAPIQueryPublicKeyPath = "/federationapi/queryPublicKey" -) - -// NewFederationAPIClient creates a FederationInternalAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewFederationAPIClient(federationSenderURL string, httpClient *http.Client, cache caching.ServerKeyCache) (api.FederationInternalAPI, error) { - if httpClient == nil { - return nil, errors.New("NewFederationInternalAPIHTTP: httpClient is ") - } - return &httpFederationInternalAPI{ - federationAPIURL: federationSenderURL, - httpClient: httpClient, - cache: cache, - }, nil -} - -type httpFederationInternalAPI struct { - federationAPIURL string - httpClient *http.Client - cache caching.ServerKeyCache -} - -// Handle an instruction to make_leave & send_leave with a remote server. -func (h *httpFederationInternalAPI) PerformLeave( - ctx context.Context, - request *api.PerformLeaveRequest, - response *api.PerformLeaveResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformLeave", h.federationAPIURL+FederationAPIPerformLeaveRequestPath, - h.httpClient, ctx, request, response, - ) -} - -// Handle sending an invite to a remote server. -func (h *httpFederationInternalAPI) PerformInvite( - ctx context.Context, - request *api.PerformInviteRequest, - response *api.PerformInviteResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformInvite", h.federationAPIURL+FederationAPIPerformInviteRequestPath, - h.httpClient, ctx, request, response, - ) -} - -// Handle starting a peek on a remote server. -func (h *httpFederationInternalAPI) PerformOutboundPeek( - ctx context.Context, - request *api.PerformOutboundPeekRequest, - response *api.PerformOutboundPeekResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformOutboundPeek", h.federationAPIURL+FederationAPIPerformOutboundPeekRequestPath, - h.httpClient, ctx, request, response, - ) -} - -// QueryJoinedHostServerNamesInRoom implements FederationInternalAPI -func (h *httpFederationInternalAPI) QueryJoinedHostServerNamesInRoom( - ctx context.Context, - request *api.QueryJoinedHostServerNamesInRoomRequest, - response *api.QueryJoinedHostServerNamesInRoomResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryJoinedHostServerNamesInRoom", h.federationAPIURL+FederationAPIQueryJoinedHostServerNamesInRoomPath, - h.httpClient, ctx, request, response, - ) -} - -// Handle an instruction to make_join & send_join with a remote server. -func (h *httpFederationInternalAPI) PerformJoin( - ctx context.Context, - request *api.PerformJoinRequest, - response *api.PerformJoinResponse, -) { - if err := httputil.CallInternalRPCAPI( - "PerformJoinRequest", h.federationAPIURL+FederationAPIPerformJoinRequestPath, - h.httpClient, ctx, request, response, - ); err != nil { - response.LastError = &gomatrix.HTTPError{ - Message: err.Error(), - Code: 0, - WrappedError: err, - } - } -} - -// Handle an instruction to make_join & send_join with a remote server. -func (h *httpFederationInternalAPI) PerformDirectoryLookup( - ctx context.Context, - request *api.PerformDirectoryLookupRequest, - response *api.PerformDirectoryLookupResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformDirectoryLookup", h.federationAPIURL+FederationAPIPerformDirectoryLookupRequestPath, - h.httpClient, ctx, request, response, - ) -} - -// Handle an instruction to broadcast an EDU to all servers in rooms we are joined to. -func (h *httpFederationInternalAPI) PerformBroadcastEDU( - ctx context.Context, - request *api.PerformBroadcastEDURequest, - response *api.PerformBroadcastEDUResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformBroadcastEDU", h.federationAPIURL+FederationAPIPerformBroadcastEDUPath, - h.httpClient, ctx, request, response, - ) -} - -// Handle an instruction to remove the respective servers from being blacklisted. -func (h *httpFederationInternalAPI) PerformWakeupServers( - ctx context.Context, - request *api.PerformWakeupServersRequest, - response *api.PerformWakeupServersResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformWakeupServers", h.federationAPIURL+FederationAPIPerformWakeupServers, - h.httpClient, ctx, request, response, - ) -} - -type getUserDevices struct { - S gomatrixserverlib.ServerName - Origin gomatrixserverlib.ServerName - UserID string -} - -func (h *httpFederationInternalAPI) GetUserDevices( - ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string, -) (gomatrixserverlib.RespUserDevices, error) { - return httputil.CallInternalProxyAPI[getUserDevices, gomatrixserverlib.RespUserDevices, *api.FederationClientError]( - "GetUserDevices", h.federationAPIURL+FederationAPIGetUserDevicesPath, h.httpClient, - ctx, &getUserDevices{ - S: s, - Origin: origin, - UserID: userID, - }, - ) -} - -type claimKeys struct { - S gomatrixserverlib.ServerName - Origin gomatrixserverlib.ServerName - OneTimeKeys map[string]map[string]string -} - -func (h *httpFederationInternalAPI) ClaimKeys( - ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string, -) (gomatrixserverlib.RespClaimKeys, error) { - return httputil.CallInternalProxyAPI[claimKeys, gomatrixserverlib.RespClaimKeys, *api.FederationClientError]( - "ClaimKeys", h.federationAPIURL+FederationAPIClaimKeysPath, h.httpClient, - ctx, &claimKeys{ - S: s, - Origin: origin, - OneTimeKeys: oneTimeKeys, - }, - ) -} - -type queryKeys struct { - S gomatrixserverlib.ServerName - Origin gomatrixserverlib.ServerName - Keys map[string][]string -} - -func (h *httpFederationInternalAPI) QueryKeys( - ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string, -) (gomatrixserverlib.RespQueryKeys, error) { - return httputil.CallInternalProxyAPI[queryKeys, gomatrixserverlib.RespQueryKeys, *api.FederationClientError]( - "QueryKeys", h.federationAPIURL+FederationAPIQueryKeysPath, h.httpClient, - ctx, &queryKeys{ - S: s, - Origin: origin, - Keys: keys, - }, - ) -} - -type backfill struct { - S gomatrixserverlib.ServerName - Origin gomatrixserverlib.ServerName - RoomID string - Limit int - EventIDs []string -} - -func (h *httpFederationInternalAPI) Backfill( - ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string, -) (gomatrixserverlib.Transaction, error) { - return httputil.CallInternalProxyAPI[backfill, gomatrixserverlib.Transaction, *api.FederationClientError]( - "Backfill", h.federationAPIURL+FederationAPIBackfillPath, h.httpClient, - ctx, &backfill{ - S: s, - Origin: origin, - RoomID: roomID, - Limit: limit, - EventIDs: eventIDs, - }, - ) -} - -type lookupState struct { - S gomatrixserverlib.ServerName - Origin gomatrixserverlib.ServerName - RoomID string - EventID string - RoomVersion gomatrixserverlib.RoomVersion -} - -func (h *httpFederationInternalAPI) LookupState( - ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion, -) (gomatrixserverlib.RespState, error) { - return httputil.CallInternalProxyAPI[lookupState, gomatrixserverlib.RespState, *api.FederationClientError]( - "LookupState", h.federationAPIURL+FederationAPILookupStatePath, h.httpClient, - ctx, &lookupState{ - S: s, - Origin: origin, - RoomID: roomID, - EventID: eventID, - RoomVersion: roomVersion, - }, - ) -} - -type lookupStateIDs struct { - S gomatrixserverlib.ServerName - Origin gomatrixserverlib.ServerName - RoomID string - EventID string -} - -func (h *httpFederationInternalAPI) LookupStateIDs( - ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, eventID string, -) (gomatrixserverlib.RespStateIDs, error) { - return httputil.CallInternalProxyAPI[lookupStateIDs, gomatrixserverlib.RespStateIDs, *api.FederationClientError]( - "LookupStateIDs", h.federationAPIURL+FederationAPILookupStateIDsPath, h.httpClient, - ctx, &lookupStateIDs{ - S: s, - Origin: origin, - RoomID: roomID, - EventID: eventID, - }, - ) -} - -type lookupMissingEvents struct { - S gomatrixserverlib.ServerName - Origin gomatrixserverlib.ServerName - RoomID string - Missing gomatrixserverlib.MissingEvents - RoomVersion gomatrixserverlib.RoomVersion -} - -func (h *httpFederationInternalAPI) LookupMissingEvents( - ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, - missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion, -) (res gomatrixserverlib.RespMissingEvents, err error) { - return httputil.CallInternalProxyAPI[lookupMissingEvents, gomatrixserverlib.RespMissingEvents, *api.FederationClientError]( - "LookupMissingEvents", h.federationAPIURL+FederationAPILookupMissingEventsPath, h.httpClient, - ctx, &lookupMissingEvents{ - S: s, - Origin: origin, - RoomID: roomID, - Missing: missing, - RoomVersion: roomVersion, - }, - ) -} - -type getEvent struct { - S gomatrixserverlib.ServerName - Origin gomatrixserverlib.ServerName - EventID string -} - -func (h *httpFederationInternalAPI) GetEvent( - ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string, -) (gomatrixserverlib.Transaction, error) { - return httputil.CallInternalProxyAPI[getEvent, gomatrixserverlib.Transaction, *api.FederationClientError]( - "GetEvent", h.federationAPIURL+FederationAPIGetEventPath, h.httpClient, - ctx, &getEvent{ - S: s, - Origin: origin, - EventID: eventID, - }, - ) -} - -type getEventAuth struct { - S gomatrixserverlib.ServerName - Origin gomatrixserverlib.ServerName - RoomVersion gomatrixserverlib.RoomVersion - RoomID string - EventID string -} - -func (h *httpFederationInternalAPI) GetEventAuth( - ctx context.Context, origin, s gomatrixserverlib.ServerName, - roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string, -) (gomatrixserverlib.RespEventAuth, error) { - return httputil.CallInternalProxyAPI[getEventAuth, gomatrixserverlib.RespEventAuth, *api.FederationClientError]( - "GetEventAuth", h.federationAPIURL+FederationAPIGetEventAuthPath, h.httpClient, - ctx, &getEventAuth{ - S: s, - Origin: origin, - RoomVersion: roomVersion, - RoomID: roomID, - EventID: eventID, - }, - ) -} - -func (h *httpFederationInternalAPI) QueryServerKeys( - ctx context.Context, req *api.QueryServerKeysRequest, res *api.QueryServerKeysResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryServerKeys", h.federationAPIURL+FederationAPIQueryServerKeysPath, - h.httpClient, ctx, req, res, - ) -} - -type lookupServerKeys struct { - S gomatrixserverlib.ServerName - KeyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp -} - -func (h *httpFederationInternalAPI) LookupServerKeys( - ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, -) ([]gomatrixserverlib.ServerKeys, error) { - return httputil.CallInternalProxyAPI[lookupServerKeys, []gomatrixserverlib.ServerKeys, *api.FederationClientError]( - "LookupServerKeys", h.federationAPIURL+FederationAPILookupServerKeysPath, h.httpClient, - ctx, &lookupServerKeys{ - S: s, - KeyRequests: keyRequests, - }, - ) -} - -type eventRelationships struct { - S gomatrixserverlib.ServerName - Origin gomatrixserverlib.ServerName - Req gomatrixserverlib.MSC2836EventRelationshipsRequest - RoomVer gomatrixserverlib.RoomVersion -} - -func (h *httpFederationInternalAPI) MSC2836EventRelationships( - ctx context.Context, origin, s gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, - roomVersion gomatrixserverlib.RoomVersion, -) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) { - return httputil.CallInternalProxyAPI[eventRelationships, gomatrixserverlib.MSC2836EventRelationshipsResponse, *api.FederationClientError]( - "MSC2836EventRelationships", h.federationAPIURL+FederationAPIEventRelationshipsPath, h.httpClient, - ctx, &eventRelationships{ - S: s, - Origin: origin, - Req: r, - RoomVer: roomVersion, - }, - ) -} - -type spacesReq struct { - S gomatrixserverlib.ServerName - Origin gomatrixserverlib.ServerName - SuggestedOnly bool - RoomID string -} - -func (h *httpFederationInternalAPI) MSC2946Spaces( - ctx context.Context, origin, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool, -) (res gomatrixserverlib.MSC2946SpacesResponse, err error) { - return httputil.CallInternalProxyAPI[spacesReq, gomatrixserverlib.MSC2946SpacesResponse, *api.FederationClientError]( - "MSC2836EventRelationships", h.federationAPIURL+FederationAPISpacesSummaryPath, h.httpClient, - ctx, &spacesReq{ - S: dst, - Origin: origin, - SuggestedOnly: suggestedOnly, - RoomID: roomID, - }, - ) -} - -func (s *httpFederationInternalAPI) KeyRing() *gomatrixserverlib.KeyRing { - // This is a bit of a cheat - we tell gomatrixserverlib that this API is - // both the key database and the key fetcher. While this does have the - // rather unfortunate effect of preventing gomatrixserverlib from handling - // key fetchers directly, we can at least reimplement this behaviour on - // the other end of the API. - return &gomatrixserverlib.KeyRing{ - KeyDatabase: s, - KeyFetchers: []gomatrixserverlib.KeyFetcher{}, - } -} - -func (s *httpFederationInternalAPI) FetcherName() string { - return "httpServerKeyInternalAPI" -} - -func (s *httpFederationInternalAPI) StoreKeys( - _ context.Context, - results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, -) error { - // Run in a background context - we don't want to stop this work just - // because the caller gives up waiting. - ctx := context.Background() - request := api.InputPublicKeysRequest{ - Keys: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult), - } - response := api.InputPublicKeysResponse{} - for req, res := range results { - request.Keys[req] = res - s.cache.StoreServerKey(req, res) - } - return s.InputPublicKeys(ctx, &request, &response) -} - -func (s *httpFederationInternalAPI) FetchKeys( - _ context.Context, - requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, -) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { - // Run in a background context - we don't want to stop this work just - // because the caller gives up waiting. - ctx := context.Background() - result := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) - request := api.QueryPublicKeysRequest{ - Requests: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp), - } - response := api.QueryPublicKeysResponse{ - Results: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult), - } - for req, ts := range requests { - if res, ok := s.cache.GetServerKey(req, ts); ok { - result[req] = res - continue - } - request.Requests[req] = ts - } - err := s.QueryPublicKeys(ctx, &request, &response) - if err != nil { - return nil, err - } - for req, res := range response.Results { - result[req] = res - s.cache.StoreServerKey(req, res) - } - return result, nil -} - -func (h *httpFederationInternalAPI) InputPublicKeys( - ctx context.Context, - request *api.InputPublicKeysRequest, - response *api.InputPublicKeysResponse, -) error { - return httputil.CallInternalRPCAPI( - "InputPublicKey", h.federationAPIURL+FederationAPIInputPublicKeyPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpFederationInternalAPI) QueryPublicKeys( - ctx context.Context, - request *api.QueryPublicKeysRequest, - response *api.QueryPublicKeysResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryPublicKeys", h.federationAPIURL+FederationAPIQueryPublicKeyPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpFederationInternalAPI) P2PQueryRelayServers( - ctx context.Context, - request *api.P2PQueryRelayServersRequest, - response *api.P2PQueryRelayServersResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryRelayServers", h.federationAPIURL+FederationAPIQueryRelayServers, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpFederationInternalAPI) P2PAddRelayServers( - ctx context.Context, - request *api.P2PAddRelayServersRequest, - response *api.P2PAddRelayServersResponse, -) error { - return httputil.CallInternalRPCAPI( - "AddRelayServers", h.federationAPIURL+FederationAPIAddRelayServers, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpFederationInternalAPI) P2PRemoveRelayServers( - ctx context.Context, - request *api.P2PRemoveRelayServersRequest, - response *api.P2PRemoveRelayServersResponse, -) error { - return httputil.CallInternalRPCAPI( - "RemoveRelayServers", h.federationAPIURL+FederationAPIRemoveRelayServers, - h.httpClient, ctx, request, response, - ) -} diff --git a/federationapi/inthttp/server.go b/federationapi/inthttp/server.go deleted file mode 100644 index 9068dc400..000000000 --- a/federationapi/inthttp/server.go +++ /dev/null @@ -1,257 +0,0 @@ -package inthttp - -import ( - "context" - "encoding/json" - "net/http" - "net/url" - - "github.com/gorilla/mux" - "github.com/matrix-org/gomatrix" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - - "github.com/matrix-org/dendrite/federationapi/api" - "github.com/matrix-org/dendrite/internal/httputil" -) - -// AddRoutes adds the FederationInternalAPI handlers to the http.ServeMux. -// nolint:gocyclo -func AddRoutes(intAPI api.FederationInternalAPI, internalAPIMux *mux.Router, enableMetrics bool) { - internalAPIMux.Handle( - FederationAPIQueryJoinedHostServerNamesInRoomPath, - httputil.MakeInternalRPCAPI("FederationAPIQueryJoinedHostServerNamesInRoom", enableMetrics, intAPI.QueryJoinedHostServerNamesInRoom), - ) - - internalAPIMux.Handle( - FederationAPIPerformInviteRequestPath, - httputil.MakeInternalRPCAPI("FederationAPIPerformInvite", enableMetrics, intAPI.PerformInvite), - ) - - internalAPIMux.Handle( - FederationAPIPerformLeaveRequestPath, - httputil.MakeInternalRPCAPI("FederationAPIPerformLeave", enableMetrics, intAPI.PerformLeave), - ) - - internalAPIMux.Handle( - FederationAPIPerformDirectoryLookupRequestPath, - httputil.MakeInternalRPCAPI("FederationAPIPerformDirectoryLookupRequest", enableMetrics, intAPI.PerformDirectoryLookup), - ) - - internalAPIMux.Handle( - FederationAPIPerformBroadcastEDUPath, - httputil.MakeInternalRPCAPI("FederationAPIPerformBroadcastEDU", enableMetrics, intAPI.PerformBroadcastEDU), - ) - - internalAPIMux.Handle( - FederationAPIPerformWakeupServers, - httputil.MakeInternalRPCAPI("FederationAPIPerformWakeupServers", enableMetrics, intAPI.PerformWakeupServers), - ) - - internalAPIMux.Handle( - FederationAPIPerformJoinRequestPath, - httputil.MakeInternalRPCAPI( - "FederationAPIPerformJoinRequest", enableMetrics, - func(ctx context.Context, req *api.PerformJoinRequest, res *api.PerformJoinResponse) error { - intAPI.PerformJoin(ctx, req, res) - return nil - }, - ), - ) - - internalAPIMux.Handle( - FederationAPIGetUserDevicesPath, - httputil.MakeInternalProxyAPI( - "FederationAPIGetUserDevices", enableMetrics, - func(ctx context.Context, req *getUserDevices) (*gomatrixserverlib.RespUserDevices, error) { - res, err := intAPI.GetUserDevices(ctx, req.Origin, req.S, req.UserID) - return &res, federationClientError(err) - }, - ), - ) - - internalAPIMux.Handle( - FederationAPIClaimKeysPath, - httputil.MakeInternalProxyAPI( - "FederationAPIClaimKeys", enableMetrics, - func(ctx context.Context, req *claimKeys) (*gomatrixserverlib.RespClaimKeys, error) { - res, err := intAPI.ClaimKeys(ctx, req.Origin, req.S, req.OneTimeKeys) - return &res, federationClientError(err) - }, - ), - ) - - internalAPIMux.Handle( - FederationAPIQueryKeysPath, - httputil.MakeInternalProxyAPI( - "FederationAPIQueryKeys", enableMetrics, - func(ctx context.Context, req *queryKeys) (*gomatrixserverlib.RespQueryKeys, error) { - res, err := intAPI.QueryKeys(ctx, req.Origin, req.S, req.Keys) - return &res, federationClientError(err) - }, - ), - ) - - internalAPIMux.Handle( - FederationAPIBackfillPath, - httputil.MakeInternalProxyAPI( - "FederationAPIBackfill", enableMetrics, - func(ctx context.Context, req *backfill) (*gomatrixserverlib.Transaction, error) { - res, err := intAPI.Backfill(ctx, req.Origin, req.S, req.RoomID, req.Limit, req.EventIDs) - return &res, federationClientError(err) - }, - ), - ) - - internalAPIMux.Handle( - FederationAPILookupStatePath, - httputil.MakeInternalProxyAPI( - "FederationAPILookupState", enableMetrics, - func(ctx context.Context, req *lookupState) (*gomatrixserverlib.RespState, error) { - res, err := intAPI.LookupState(ctx, req.Origin, req.S, req.RoomID, req.EventID, req.RoomVersion) - return &res, federationClientError(err) - }, - ), - ) - - internalAPIMux.Handle( - FederationAPILookupStateIDsPath, - httputil.MakeInternalProxyAPI( - "FederationAPILookupStateIDs", enableMetrics, - func(ctx context.Context, req *lookupStateIDs) (*gomatrixserverlib.RespStateIDs, error) { - res, err := intAPI.LookupStateIDs(ctx, req.Origin, req.S, req.RoomID, req.EventID) - return &res, federationClientError(err) - }, - ), - ) - - internalAPIMux.Handle( - FederationAPILookupMissingEventsPath, - httputil.MakeInternalProxyAPI( - "FederationAPILookupMissingEvents", enableMetrics, - func(ctx context.Context, req *lookupMissingEvents) (*gomatrixserverlib.RespMissingEvents, error) { - res, err := intAPI.LookupMissingEvents(ctx, req.Origin, req.S, req.RoomID, req.Missing, req.RoomVersion) - return &res, federationClientError(err) - }, - ), - ) - - internalAPIMux.Handle( - FederationAPIGetEventPath, - httputil.MakeInternalProxyAPI( - "FederationAPIGetEvent", enableMetrics, - func(ctx context.Context, req *getEvent) (*gomatrixserverlib.Transaction, error) { - res, err := intAPI.GetEvent(ctx, req.Origin, req.S, req.EventID) - return &res, federationClientError(err) - }, - ), - ) - - internalAPIMux.Handle( - FederationAPIGetEventAuthPath, - httputil.MakeInternalProxyAPI( - "FederationAPIGetEventAuth", enableMetrics, - func(ctx context.Context, req *getEventAuth) (*gomatrixserverlib.RespEventAuth, error) { - res, err := intAPI.GetEventAuth(ctx, req.Origin, req.S, req.RoomVersion, req.RoomID, req.EventID) - return &res, federationClientError(err) - }, - ), - ) - - internalAPIMux.Handle( - FederationAPIQueryServerKeysPath, - httputil.MakeInternalRPCAPI("FederationAPIQueryServerKeys", enableMetrics, intAPI.QueryServerKeys), - ) - - internalAPIMux.Handle( - FederationAPILookupServerKeysPath, - httputil.MakeInternalProxyAPI( - "FederationAPILookupServerKeys", enableMetrics, - func(ctx context.Context, req *lookupServerKeys) (*[]gomatrixserverlib.ServerKeys, error) { - res, err := intAPI.LookupServerKeys(ctx, req.S, req.KeyRequests) - return &res, federationClientError(err) - }, - ), - ) - - internalAPIMux.Handle( - FederationAPIEventRelationshipsPath, - httputil.MakeInternalProxyAPI( - "FederationAPIMSC2836EventRelationships", enableMetrics, - func(ctx context.Context, req *eventRelationships) (*gomatrixserverlib.MSC2836EventRelationshipsResponse, error) { - res, err := intAPI.MSC2836EventRelationships(ctx, req.Origin, req.S, req.Req, req.RoomVer) - return &res, federationClientError(err) - }, - ), - ) - - internalAPIMux.Handle( - FederationAPISpacesSummaryPath, - httputil.MakeInternalProxyAPI( - "FederationAPIMSC2946SpacesSummary", enableMetrics, - func(ctx context.Context, req *spacesReq) (*gomatrixserverlib.MSC2946SpacesResponse, error) { - res, err := intAPI.MSC2946Spaces(ctx, req.Origin, req.S, req.RoomID, req.SuggestedOnly) - return &res, federationClientError(err) - }, - ), - ) - - // TODO: Look at this shape - internalAPIMux.Handle(FederationAPIQueryPublicKeyPath, - httputil.MakeInternalAPI("FederationAPIQueryPublicKeys", enableMetrics, func(req *http.Request) util.JSONResponse { - request := api.QueryPublicKeysRequest{} - response := api.QueryPublicKeysResponse{} - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - keys, err := intAPI.FetchKeys(req.Context(), request.Requests) - if err != nil { - return util.ErrorResponse(err) - } - response.Results = keys - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - - // TODO: Look at this shape - internalAPIMux.Handle(FederationAPIInputPublicKeyPath, - httputil.MakeInternalAPI("FederationAPIInputPublicKeys", enableMetrics, func(req *http.Request) util.JSONResponse { - request := api.InputPublicKeysRequest{} - response := api.InputPublicKeysResponse{} - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := intAPI.StoreKeys(req.Context(), request.Keys); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} - -func federationClientError(err error) error { - switch ferr := err.(type) { - case nil: - return nil - case api.FederationClientError: - return &ferr - case *api.FederationClientError: - return ferr - case gomatrix.HTTPError: - return &api.FederationClientError{ - Code: ferr.Code, - } - case *url.Error: // e.g. certificate error, unable to connect - return &api.FederationClientError{ - Err: ferr.Error(), - Code: 400, - } - default: - // We don't know what exactly failed, but we probably don't - // want to retry the request immediately in the device list updater - return &api.FederationClientError{ - Err: err.Error(), - Code: 400, - } - } -} diff --git a/federationapi/producers/syncapi.go b/federationapi/producers/syncapi.go index 7cce13a7d..6bcfafa39 100644 --- a/federationapi/producers/syncapi.go +++ b/federationapi/producers/syncapi.go @@ -41,7 +41,7 @@ type SyncAPIProducer struct { TopicSigningKeyUpdate string JetStream nats.JetStreamContext Config *config.FederationAPI - UserAPI userapi.UserInternalAPI + UserAPI userapi.FederationUserAPI } func (p *SyncAPIProducer) SendReceipt( diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go index ce8b06b70..871d26cd4 100644 --- a/federationapi/routing/devices.go +++ b/federationapi/routing/devices.go @@ -17,7 +17,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/jsonerror" - keyapi "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/tidwall/gjson" @@ -26,11 +26,11 @@ import ( // GetUserDevices for the given user id func GetUserDevices( req *http.Request, - keyAPI keyapi.FederationKeyAPI, + keyAPI api.FederationKeyAPI, userID string, ) util.JSONResponse { - var res keyapi.QueryDeviceMessagesResponse - if err := keyAPI.QueryDeviceMessages(req.Context(), &keyapi.QueryDeviceMessagesRequest{ + var res api.QueryDeviceMessagesResponse + if err := keyAPI.QueryDeviceMessages(req.Context(), &api.QueryDeviceMessagesRequest{ UserID: userID, }, &res); err != nil { return util.ErrorResponse(err) @@ -40,12 +40,12 @@ func GetUserDevices( return jsonerror.InternalServerError() } - sigReq := &keyapi.QuerySignaturesRequest{ + sigReq := &api.QuerySignaturesRequest{ TargetIDs: map[string][]gomatrixserverlib.KeyID{ userID: {}, }, } - sigRes := &keyapi.QuerySignaturesResponse{} + sigRes := &api.QuerySignaturesResponse{} for _, dev := range res.Devices { sigReq.TargetIDs[userID] = append(sigReq.TargetIDs[userID], gomatrixserverlib.KeyID(dev.DeviceID)) } diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go index dc262cfde..2885cc916 100644 --- a/federationapi/routing/keys.go +++ b/federationapi/routing/keys.go @@ -22,8 +22,8 @@ import ( clienthttputil "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" federationAPI "github.com/matrix-org/dendrite/federationapi/api" - "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/sirupsen/logrus" diff --git a/federationapi/routing/profile_test.go b/federationapi/routing/profile_test.go index 763656081..3b9d576bf 100644 --- a/federationapi/routing/profile_test.go +++ b/federationapi/routing/profile_test.go @@ -62,7 +62,7 @@ func TestHandleQueryProfile(t *testing.T) { if !ok { panic("This is a programming error.") } - routing.Setup(base, nil, r, keyRing, &fedClient, &userapi, nil, &base.Cfg.MSCs, nil, nil) + routing.Setup(base, nil, r, keyRing, &fedClient, &userapi, &base.Cfg.MSCs, nil, nil) handler := fedMux.Get(routing.QueryProfileRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/federationapi/routing/query_test.go b/federationapi/routing/query_test.go index 21f35bf0c..d839a16b8 100644 --- a/federationapi/routing/query_test.go +++ b/federationapi/routing/query_test.go @@ -62,7 +62,7 @@ func TestHandleQueryDirectory(t *testing.T) { if !ok { panic("This is a programming error.") } - routing.Setup(base, nil, r, keyRing, &fedClient, &userapi, nil, &base.Cfg.MSCs, nil, nil) + routing.Setup(base, nil, r, keyRing, &fedClient, &userapi, &base.Cfg.MSCs, nil, nil) handler := fedMux.Get(routing.QueryDirectoryRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 5eb30c6ec..324740ddc 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -29,7 +29,6 @@ import ( "github.com/matrix-org/dendrite/federationapi/producers" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/httputil" - keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/base" @@ -62,7 +61,6 @@ func Setup( keys gomatrixserverlib.JSONVerifier, federation federationAPI.FederationClient, userAPI userapi.FederationUserAPI, - keyAPI keyserverAPI.FederationKeyAPI, mscCfg *config.MSCs, servers federationAPI.ServersInRoomProvider, producer *producers.SyncAPIProducer, @@ -141,7 +139,7 @@ func Setup( func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return Send( httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]), - cfg, rsAPI, keyAPI, keys, federation, mu, servers, producer, + cfg, rsAPI, userAPI, keys, federation, mu, servers, producer, ) }, )).Methods(http.MethodPut, http.MethodOptions).Name(SendRouteName) @@ -269,7 +267,7 @@ func Setup( "federation_user_devices", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetUserDevices( - httpReq, keyAPI, vars["userID"], + httpReq, userAPI, vars["userID"], ) }, )).Methods(http.MethodGet) @@ -494,14 +492,14 @@ func Setup( v1fedmux.Handle("/user/keys/claim", MakeFedAPI( "federation_keys_claim", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { - return ClaimOneTimeKeys(httpReq, request, keyAPI, cfg.Matrix.ServerName) + return ClaimOneTimeKeys(httpReq, request, userAPI, cfg.Matrix.ServerName) }, )).Methods(http.MethodPost) v1fedmux.Handle("/user/keys/query", MakeFedAPI( "federation_keys_query", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { - return QueryDeviceKeys(httpReq, request, keyAPI, cfg.Matrix.ServerName) + return QueryDeviceKeys(httpReq, request, userAPI, cfg.Matrix.ServerName) }, )).Methods(http.MethodPost) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 67b513c90..82651719f 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -28,9 +28,9 @@ import ( federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/producers" "github.com/matrix-org/dendrite/internal" - keyapi "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" + userAPI "github.com/matrix-org/dendrite/userapi/api" ) const ( @@ -59,7 +59,7 @@ func Send( txnID gomatrixserverlib.TransactionID, cfg *config.FederationAPI, rsAPI api.FederationRoomserverAPI, - keyAPI keyapi.FederationKeyAPI, + keyAPI userAPI.FederationUserAPI, keys gomatrixserverlib.JSONVerifier, federation federationAPI.FederationClient, mu *internal.MutexByRoom, diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index d7feee0e5..eed4e7e69 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -58,7 +58,7 @@ func TestHandleSend(t *testing.T) { if !ok { panic("This is a programming error.") } - routing.Setup(base, nil, r, keyRing, nil, nil, nil, &base.Cfg.MSCs, nil, nil) + routing.Setup(base, nil, r, keyRing, nil, nil, &base.Cfg.MSCs, nil, nil) handler := fedMux.Get(routing.SendRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/go.mod b/go.mod index 25f022707..6857290f3 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 github.com/matrix-org/gomatrixserverlib v0.0.0-20230131183213-122f1e0e3fa1 - github.com/matrix-org/pinecone v0.11.1-0.20230111184901-61850f0e63cb + github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 github.com/mattn/go-sqlite3 v1.14.15 github.com/nats-io/nats-server/v2 v2.9.8 @@ -44,10 +44,9 @@ require ( github.com/yggdrasil-network/yggdrasil-go v0.4.6 go.uber.org/atomic v1.10.0 golang.org/x/crypto v0.5.0 - golang.org/x/image v0.1.0 + golang.org/x/image v0.5.0 golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e - golang.org/x/net v0.5.0 - golang.org/x/term v0.4.0 + golang.org/x/term v0.5.0 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 gotest.tools/v3 v3.4.0 @@ -95,9 +94,6 @@ require ( github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.15.11 // indirect github.com/kr/pretty v0.3.1 // indirect - github.com/lucas-clemente/quic-go v0.30.0 // indirect - github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect - github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect github.com/mattn/go-isatty v0.0.16 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/minio/highwayhash v1.0.2 // indirect @@ -117,14 +113,19 @@ require ( github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect + github.com/quic-go/qtls-go1-18 v0.2.0 // indirect + github.com/quic-go/qtls-go1-19 v0.2.0 // indirect + github.com/quic-go/qtls-go1-20 v0.1.0 // indirect + github.com/quic-go/quic-go v0.32.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect go.etcd.io/bbolt v1.3.6 // indirect - golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 // indirect + golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect golang.org/x/mod v0.6.0 // indirect - golang.org/x/sys v0.4.0 // indirect - golang.org/x/text v0.6.0 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect golang.org/x/time v0.1.0 // indirect golang.org/x/tools v0.2.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/go.sum b/go.sum index 38122e63c..2272e5404 100644 --- a/go.sum +++ b/go.sum @@ -315,12 +315,6 @@ github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgx github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lucas-clemente/quic-go v0.30.0 h1:nwLW0h8ahVQ5EPTIM7uhl/stHqQDea15oRlYKZmw2O0= -github.com/lucas-clemente/quic-go v0.30.0/go.mod h1:ssOrRsOmdxa768Wr78vnh2B8JozgLsMzG/g+0qEC7uk= -github.com/marten-seemann/qtls-go1-18 v0.1.3 h1:R4H2Ks8P6pAtUagjFty2p7BVHn3XiwDAl7TTQf5h7TI= -github.com/marten-seemann/qtls-go1-18 v0.1.3/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= -github.com/marten-seemann/qtls-go1-19 v0.1.1 h1:mnbxeq3oEyQxQXwI4ReCgW9DPoPR94sNlqWoDZnjRIE= -github.com/marten-seemann/qtls-go1-19 v0.1.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e h1:DP5RC0Z3XdyBEW5dKt8YPeN6vZbm6OzVaGVp7f1BQRM= github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw2QV3YD/fRrzEDPNGgTlJlvXY0EHHnT87wF3OA= @@ -329,8 +323,8 @@ github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrixserverlib v0.0.0-20230131183213-122f1e0e3fa1 h1:JSw0nmjMrgBmoM2aQsa78LTpI5BnuD9+vOiEQ4Qo0qw= github.com/matrix-org/gomatrixserverlib v0.0.0-20230131183213-122f1e0e3fa1/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= -github.com/matrix-org/pinecone v0.11.1-0.20230111184901-61850f0e63cb h1:2L+ltfNKab56FoBBqAvbBLjoAbxwwoZie+B8d+Mp3JI= -github.com/matrix-org/pinecone v0.11.1-0.20230111184901-61850f0e63cb/go.mod h1:F3GHppRuHCTDeoOmmgjZMeJdbql91+RSGGsATWfC7oc= +github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a h1:awrPDf9LEFySxTLKYBMCiObelNx/cBuv/wzllvCCH3A= +github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a/go.mod h1:HchJX9oKMXaT2xYFs0Ha/6Zs06mxLU8k6F1ODnrGkeQ= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66/go.mod h1:iBI1foelCqA09JJgPV0FYz4qA5dUXYOxMi57FxKBdd4= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -424,6 +418,14 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/quic-go/qtls-go1-18 v0.2.0 h1:5ViXqBZ90wpUcZS0ge79rf029yx0dYB0McyPJwqqj7U= +github.com/quic-go/qtls-go1-18 v0.2.0/go.mod h1:moGulGHK7o6O8lSPSZNoOwcLvJKJ85vVNc7oJFD65bc= +github.com/quic-go/qtls-go1-19 v0.2.0 h1:Cvn2WdhyViFUHoOqK52i51k4nDX8EwIh5VJiVM4nttk= +github.com/quic-go/qtls-go1-19 v0.2.0/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= +github.com/quic-go/qtls-go1-20 v0.1.0 h1:d1PK3ErFy9t7zxKsG3NXBJXZjp/kMLoIb3y/kV54oAI= +github.com/quic-go/qtls-go1-20 v0.1.0/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= +github.com/quic-go/quic-go v0.32.0 h1:lY02md31s1JgPiiyfqJijpu/UX/Iun304FI3yUqX7tA= +github.com/quic-go/quic-go v0.32.0/go.mod h1:/fCsKANhQIeD5l76c2JFU+07gVE3KaA0FP+0zMWwfwo= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa h1:tEkEyxYeZ43TR55QU/hsIt9aRGBxbgGuz9CGykjvogY= github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= @@ -517,13 +519,13 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE= -golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20221205204356-47842c84f3db h1:D/cFflL63o2KSLJIwjlcIt8PR064j/xsmdEJL/YvY/o= +golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.1.0 h1:r8Oj8ZA2Xy12/b5KZYj3tuv7NG/fBz3TwQVvpJ9l8Rk= -golang.org/x/image v0.1.0/go.mod h1:iyPr49SD/G/TBxYVB/9RRtGUT5eNbo2u4NamWeQcD5c= +golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= +golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -583,8 +585,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -656,12 +658,12 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -669,9 +671,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml index 702e6c392..3944a76db 100644 --- a/helm/dendrite/Chart.yaml +++ b/helm/dendrite/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: dendrite version: "0.11.1" -appVersion: "0.11.0" +appVersion: "0.11.1" description: Dendrite Matrix Homeserver type: application keywords: diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md index d58f7e67d..d6ffed653 100644 --- a/helm/dendrite/README.md +++ b/helm/dendrite/README.md @@ -1,6 +1,6 @@ # dendrite -![Version: 0.11.1](https://img.shields.io/badge/Version-0.11.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.11.0](https://img.shields.io/badge/AppVersion-0.11.0-informational?style=flat-square) +![Version: 0.11.1](https://img.shields.io/badge/Version-0.11.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.11.1](https://img.shields.io/badge/AppVersion-0.11.1-informational?style=flat-square) Dendrite Matrix Homeserver Status: **NOT PRODUCTION READY** diff --git a/internal/httputil/http.go b/internal/httputil/http.go deleted file mode 100644 index ad26de512..000000000 --- a/internal/httputil/http.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package httputil - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strings" - - "github.com/opentracing/opentracing-go" - "github.com/opentracing/opentracing-go/ext" -) - -// PostJSON performs a POST request with JSON on an internal HTTP API. -// The error will match the errtype if returned from the remote API, or -// will be a different type if there was a problem reaching the API. -func PostJSON[reqtype, restype any, errtype error]( - ctx context.Context, span opentracing.Span, httpClient *http.Client, - apiURL string, request *reqtype, response *restype, -) error { - jsonBytes, err := json.Marshal(request) - if err != nil { - return err - } - - parsedAPIURL, err := url.Parse(apiURL) - if err != nil { - return err - } - - parsedAPIURL.Path = InternalPathPrefix + strings.TrimLeft(parsedAPIURL.Path, "/") - apiURL = parsedAPIURL.String() - - req, err := http.NewRequest(http.MethodPost, apiURL, bytes.NewReader(jsonBytes)) - if err != nil { - return err - } - - // Mark the span as being an RPC client. - ext.SpanKindRPCClient.Set(span) - carrier := opentracing.HTTPHeadersCarrier(req.Header) - tracer := opentracing.GlobalTracer() - - if err = tracer.Inject(span.Context(), opentracing.HTTPHeaders, carrier); err != nil { - return err - } - - req.Header.Set("Content-Type", "application/json") - - res, err := httpClient.Do(req.WithContext(ctx)) - if res != nil { - defer (func() { err = res.Body.Close() })() - } - if err != nil { - return err - } - var body []byte - body, err = io.ReadAll(res.Body) - if err != nil { - return err - } - if res.StatusCode != http.StatusOK { - if len(body) == 0 { - return fmt.Errorf("HTTP %d from %s (no response body)", res.StatusCode, apiURL) - } - var reserr errtype - if err = json.Unmarshal(body, &reserr); err != nil { - return fmt.Errorf("HTTP %d from %s - %w", res.StatusCode, apiURL, err) - } - return reserr - } - if err = json.Unmarshal(body, response); err != nil { - return fmt.Errorf("json.Unmarshal: %w", err) - } - return nil -} diff --git a/internal/httputil/internalapi.go b/internal/httputil/internalapi.go deleted file mode 100644 index 22f436e38..000000000 --- a/internal/httputil/internalapi.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2022 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package httputil - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "reflect" - - "github.com/matrix-org/util" - "github.com/opentracing/opentracing-go" -) - -type InternalAPIError struct { - Type string - Message string -} - -func (e InternalAPIError) Error() string { - return fmt.Sprintf("internal API returned %q error: %s", e.Type, e.Message) -} - -func MakeInternalRPCAPI[reqtype, restype any](metricsName string, enableMetrics bool, f func(context.Context, *reqtype, *restype) error) http.Handler { - return MakeInternalAPI(metricsName, enableMetrics, func(req *http.Request) util.JSONResponse { - var request reqtype - var response restype - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := f(req.Context(), &request, &response); err != nil { - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: &InternalAPIError{ - Type: reflect.TypeOf(err).String(), - Message: fmt.Sprintf("%s", err), - }, - } - } - return util.JSONResponse{ - Code: http.StatusOK, - JSON: &response, - } - }) -} - -func MakeInternalProxyAPI[reqtype, restype any](metricsName string, enableMetrics bool, f func(context.Context, *reqtype) (*restype, error)) http.Handler { - return MakeInternalAPI(metricsName, enableMetrics, func(req *http.Request) util.JSONResponse { - var request reqtype - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - response, err := f(req.Context(), &request) - if err != nil { - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: err, - } - } - return util.JSONResponse{ - Code: http.StatusOK, - JSON: response, - } - }) -} - -func CallInternalRPCAPI[reqtype, restype any](name, url string, client *http.Client, ctx context.Context, request *reqtype, response *restype) error { - span, ctx := opentracing.StartSpanFromContext(ctx, name) - defer span.Finish() - - return PostJSON[reqtype, restype, InternalAPIError](ctx, span, client, url, request, response) -} - -func CallInternalProxyAPI[reqtype, restype any, errtype error](name, url string, client *http.Client, ctx context.Context, request *reqtype) (restype, error) { - span, ctx := opentracing.StartSpanFromContext(ctx, name) - defer span.Finish() - - var response restype - return response, PostJSON[reqtype, restype, errtype](ctx, span, client, url, request, &response) -} diff --git a/internal/httputil/paths.go b/internal/httputil/paths.go index 62eff0415..d06875428 100644 --- a/internal/httputil/paths.go +++ b/internal/httputil/paths.go @@ -21,7 +21,6 @@ const ( PublicMediaPathPrefix = "/_matrix/media/" PublicStaticPath = "/_matrix/static/" PublicWellKnownPrefix = "/.well-known/matrix/" - InternalPathPrefix = "/api/" DendriteAdminPathPrefix = "/_dendrite/" SynapseAdminPathPrefix = "/_synapse/" ) diff --git a/internal/log.go b/internal/log.go index 9e8656c5b..8fe98f20c 100644 --- a/internal/log.go +++ b/internal/log.go @@ -129,9 +129,9 @@ func checkFileHookParams(params map[string]interface{}) { } // Add a new FSHook to the logger. Each component will log in its own file -func setupFileHook(hook config.LogrusHook, level logrus.Level, componentName string) { +func setupFileHook(hook config.LogrusHook, level logrus.Level) { dirPath := (hook.Params["path"]).(string) - fullPath := filepath.Join(dirPath, componentName+".log") + fullPath := filepath.Join(dirPath, "dendrite.log") if err := os.MkdirAll(path.Dir(fullPath), os.ModePerm); err != nil { logrus.Fatalf("Couldn't create directory %s: %q", path.Dir(fullPath), err) diff --git a/internal/log_unix.go b/internal/log_unix.go index 859427041..3f15063d1 100644 --- a/internal/log_unix.go +++ b/internal/log_unix.go @@ -31,7 +31,7 @@ import ( // SetupHookLogging configures the logging hooks defined in the configuration. // If something fails here it means that the logging was improperly configured, // so we just exit with the error -func SetupHookLogging(hooks []config.LogrusHook, componentName string) { +func SetupHookLogging(hooks []config.LogrusHook) { levelLogAddedMu.Lock() defer levelLogAddedMu.Unlock() for _, hook := range hooks { @@ -50,10 +50,10 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) { switch hook.Type { case "file": checkFileHookParams(hook.Params) - setupFileHook(hook, level, componentName) + setupFileHook(hook, level) case "syslog": checkSyslogHookParams(hook.Params) - setupSyslogHook(hook, level, componentName) + setupSyslogHook(hook, level) case "std": setupStdLogHook(level) default: @@ -94,8 +94,8 @@ func setupStdLogHook(level logrus.Level) { stdLevelLogAdded[level] = true } -func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) { - syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, componentName) +func setupSyslogHook(hook config.LogrusHook, level logrus.Level) { + syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, "dendrite") if err == nil { logrus.AddHook(&logLevelHook{level, syslogHook}) } diff --git a/internal/log_windows.go b/internal/log_windows.go index 39562328c..e1f0098a1 100644 --- a/internal/log_windows.go +++ b/internal/log_windows.go @@ -22,7 +22,7 @@ import ( // SetupHookLogging configures the logging hooks defined in the configuration. // If something fails here it means that the logging was improperly configured, // so we just exit with the error -func SetupHookLogging(hooks []config.LogrusHook, componentName string) { +func SetupHookLogging(hooks []config.LogrusHook) { logrus.SetReportCaller(true) for _, hook := range hooks { // Check we received a proper logging level @@ -40,7 +40,7 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) { switch hook.Type { case "file": checkFileHookParams(hook.Params) - setupFileHook(hook, level, componentName) + setupFileHook(hook, level) default: logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type) } diff --git a/internal/transactionrequest.go b/internal/transactionrequest.go index 95673fc14..13b00af50 100644 --- a/internal/transactionrequest.go +++ b/internal/transactionrequest.go @@ -24,9 +24,9 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/federationapi/producers" "github.com/matrix-org/dendrite/federationapi/types" - keyapi "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/roomserver/api" syncTypes "github.com/matrix-org/dendrite/syncapi/types" + userAPI "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" @@ -56,7 +56,7 @@ var ( type TxnReq struct { gomatrixserverlib.Transaction rsAPI api.FederationRoomserverAPI - keyAPI keyapi.FederationKeyAPI + userAPI userAPI.FederationUserAPI ourServerName gomatrixserverlib.ServerName keys gomatrixserverlib.JSONVerifier roomsMu *MutexByRoom @@ -66,7 +66,7 @@ type TxnReq struct { func NewTxnReq( rsAPI api.FederationRoomserverAPI, - keyAPI keyapi.FederationKeyAPI, + userAPI userAPI.FederationUserAPI, ourServerName gomatrixserverlib.ServerName, keys gomatrixserverlib.JSONVerifier, roomsMu *MutexByRoom, @@ -80,7 +80,7 @@ func NewTxnReq( ) TxnReq { t := TxnReq{ rsAPI: rsAPI, - keyAPI: keyAPI, + userAPI: userAPI, ourServerName: ourServerName, keys: keys, roomsMu: roomsMu, diff --git a/internal/transactionrequest_test.go b/internal/transactionrequest_test.go index dd1bd3502..8597ae24b 100644 --- a/internal/transactionrequest_test.go +++ b/internal/transactionrequest_test.go @@ -23,13 +23,13 @@ import ( "time" "github.com/matrix-org/dendrite/federationapi/producers" - keyAPI "github.com/matrix-org/dendrite/keyserver/api" rsAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/test" + keyAPI "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/nats-io/nats.go" "github.com/stretchr/testify/assert" @@ -198,8 +198,8 @@ func TestProcessTransactionRequestPDUSendFail(t *testing.T) { func createTransactionWithEDU(ctx *process.ProcessContext, edus []gomatrixserverlib.EDU) (TxnReq, nats.JetStreamContext, *config.Dendrite) { cfg := &config.Dendrite{} cfg.Defaults(config.DefaultOpts{ - Generate: true, - Monolithic: true, + Generate: true, + SingleDatabase: true, }) cfg.Global.JetStream.InMemory = true natsInstance := &jetstream.NATSInstance{} @@ -647,7 +647,7 @@ func init() { } type testRoomserverAPI struct { - rsAPI.RoomserverInternalAPITrace + rsAPI.RoomserverInternalAPI inputRoomEvents []rsAPI.InputRoomEvent queryStateAfterEvents func(*rsAPI.QueryStateAfterEventsRequest) rsAPI.QueryStateAfterEventsResponse queryEventsByID func(req *rsAPI.QueryEventsByIDRequest) rsAPI.QueryEventsByIDResponse diff --git a/internal/version.go b/internal/version.go index fbe4a01b0..e2655cb63 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 11 - VersionPatch = 0 + VersionPatch = 1 VersionTag = "" // example: "rc1" ) diff --git a/keyserver/README.md b/keyserver/README.md deleted file mode 100644 index fd9f37d27..000000000 --- a/keyserver/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## Key Server - -This is an internal component which manages E2E keys from clients. It handles all the [Key Management APIs](https://matrix.org/docs/spec/client_server/r0.6.1#key-management-api) with the exception of `/keys/changes` which is handled by Sync API. This component is designed to shard by user ID. - -Keys are uploaded and stored in this component, and key changes are emitted to a Kafka topic for downstream components such as Sync API. - -### Internal APIs -- `PerformUploadKeys` stores identity keys and one-time public keys for given user(s). -- `PerformClaimKeys` acquires one-time public keys for given user(s). This may involve outbound federation calls. -- `QueryKeys` returns identity keys for given user(s). This may involve outbound federation calls. This component may then cache federated identity keys to avoid repeatedly hitting remote servers. -- A topic which emits identity keys every time there is a change (addition or deletion). - -### Endpoint mappings -- Client API maps `/keys/upload` to `PerformUploadKeys`. -- Client API maps `/keys/query` to `QueryKeys`. -- Client API maps `/keys/claim` to `PerformClaimKeys`. -- Federation API maps `/user/keys/query` to `QueryKeys`. -- Federation API maps `/user/keys/claim` to `PerformClaimKeys`. -- Sync API maps `/keys/changes` to consuming from the Kafka topic. diff --git a/keyserver/api/api.go b/keyserver/api/api.go deleted file mode 100644 index 14fced3e8..000000000 --- a/keyserver/api/api.go +++ /dev/null @@ -1,346 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "bytes" - "context" - "encoding/json" - "strings" - "time" - - "github.com/matrix-org/gomatrixserverlib" - - "github.com/matrix-org/dendrite/keyserver/types" - userapi "github.com/matrix-org/dendrite/userapi/api" -) - -type KeyInternalAPI interface { - SyncKeyAPI - ClientKeyAPI - FederationKeyAPI - UserKeyAPI - - // SetUserAPI assigns a user API to query when extracting device names. - SetUserAPI(i userapi.KeyserverUserAPI) -} - -// API functions required by the clientapi -type ClientKeyAPI interface { - QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse) error - PerformUploadKeys(ctx context.Context, req *PerformUploadKeysRequest, res *PerformUploadKeysResponse) error - PerformUploadDeviceKeys(ctx context.Context, req *PerformUploadDeviceKeysRequest, res *PerformUploadDeviceKeysResponse) error - PerformUploadDeviceSignatures(ctx context.Context, req *PerformUploadDeviceSignaturesRequest, res *PerformUploadDeviceSignaturesResponse) error - // PerformClaimKeys claims one-time keys for use in pre-key messages - PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse) error - PerformMarkAsStaleIfNeeded(ctx context.Context, req *PerformMarkAsStaleRequest, res *struct{}) error -} - -// API functions required by the userapi -type UserKeyAPI interface { - PerformUploadKeys(ctx context.Context, req *PerformUploadKeysRequest, res *PerformUploadKeysResponse) error - PerformDeleteKeys(ctx context.Context, req *PerformDeleteKeysRequest, res *PerformDeleteKeysResponse) error -} - -// API functions required by the syncapi -type SyncKeyAPI interface { - QueryKeyChanges(ctx context.Context, req *QueryKeyChangesRequest, res *QueryKeyChangesResponse) error - QueryOneTimeKeys(ctx context.Context, req *QueryOneTimeKeysRequest, res *QueryOneTimeKeysResponse) error - PerformMarkAsStaleIfNeeded(ctx context.Context, req *PerformMarkAsStaleRequest, res *struct{}) error -} - -type FederationKeyAPI interface { - QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse) error - QuerySignatures(ctx context.Context, req *QuerySignaturesRequest, res *QuerySignaturesResponse) error - QueryDeviceMessages(ctx context.Context, req *QueryDeviceMessagesRequest, res *QueryDeviceMessagesResponse) error - PerformUploadDeviceKeys(ctx context.Context, req *PerformUploadDeviceKeysRequest, res *PerformUploadDeviceKeysResponse) error - PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse) error -} - -// KeyError is returned if there was a problem performing/querying the server -type KeyError struct { - Err string `json:"error"` - IsInvalidSignature bool `json:"is_invalid_signature,omitempty"` // M_INVALID_SIGNATURE - IsMissingParam bool `json:"is_missing_param,omitempty"` // M_MISSING_PARAM - IsInvalidParam bool `json:"is_invalid_param,omitempty"` // M_INVALID_PARAM -} - -func (k *KeyError) Error() string { - return k.Err -} - -type DeviceMessageType int - -const ( - TypeDeviceKeyUpdate DeviceMessageType = iota - TypeCrossSigningUpdate -) - -// DeviceMessage represents the message produced into Kafka by the key server. -type DeviceMessage struct { - Type DeviceMessageType `json:"Type,omitempty"` - *DeviceKeys `json:"DeviceKeys,omitempty"` - *OutputCrossSigningKeyUpdate `json:"CrossSigningKeyUpdate,omitempty"` - // A monotonically increasing number which represents device changes for this user. - StreamID int64 - DeviceChangeID int64 -} - -// OutputCrossSigningKeyUpdate is an entry in the signing key update output kafka log -type OutputCrossSigningKeyUpdate struct { - CrossSigningKeyUpdate `json:"signing_keys"` -} - -type CrossSigningKeyUpdate struct { - MasterKey *gomatrixserverlib.CrossSigningKey `json:"master_key,omitempty"` - SelfSigningKey *gomatrixserverlib.CrossSigningKey `json:"self_signing_key,omitempty"` - UserID string `json:"user_id"` -} - -// DeviceKeysEqual returns true if the device keys updates contain the -// same display name and key JSON. This will return false if either of -// the updates is not a device keys update, or if the user ID/device ID -// differ between the two. -func (m1 *DeviceMessage) DeviceKeysEqual(m2 *DeviceMessage) bool { - if m1.DeviceKeys == nil || m2.DeviceKeys == nil { - return false - } - if m1.UserID != m2.UserID || m1.DeviceID != m2.DeviceID { - return false - } - if m1.DisplayName != m2.DisplayName { - return false // different display names - } - if len(m1.KeyJSON) == 0 || len(m2.KeyJSON) == 0 { - return false // either is empty - } - return bytes.Equal(m1.KeyJSON, m2.KeyJSON) -} - -// DeviceKeys represents a set of device keys for a single device -// https://matrix.org/docs/spec/client_server/r0.6.1#post-matrix-client-r0-keys-upload -type DeviceKeys struct { - // The user who owns this device - UserID string - // The device ID of this device - DeviceID string - // The device display name - DisplayName string - // The raw device key JSON - KeyJSON []byte -} - -// WithStreamID returns a copy of this device message with the given stream ID -func (k *DeviceKeys) WithStreamID(streamID int64) DeviceMessage { - return DeviceMessage{ - DeviceKeys: k, - StreamID: streamID, - } -} - -// OneTimeKeys represents a set of one-time keys for a single device -// https://matrix.org/docs/spec/client_server/r0.6.1#post-matrix-client-r0-keys-upload -type OneTimeKeys struct { - // The user who owns this device - UserID string - // The device ID of this device - DeviceID string - // A map of algorithm:key_id => key JSON - KeyJSON map[string]json.RawMessage -} - -// Split a key in KeyJSON into algorithm and key ID -func (k *OneTimeKeys) Split(keyIDWithAlgo string) (algo string, keyID string) { - segments := strings.Split(keyIDWithAlgo, ":") - return segments[0], segments[1] -} - -// OneTimeKeysCount represents the counts of one-time keys for a single device -type OneTimeKeysCount struct { - // The user who owns this device - UserID string - // The device ID of this device - DeviceID string - // algorithm to count e.g: - // { - // "curve25519": 10, - // "signed_curve25519": 20 - // } - KeyCount map[string]int -} - -// PerformUploadKeysRequest is the request to PerformUploadKeys -type PerformUploadKeysRequest struct { - UserID string // Required - User performing the request - DeviceID string // Optional - Device performing the request, for fetching OTK count - DeviceKeys []DeviceKeys - OneTimeKeys []OneTimeKeys - // OnlyDisplayNameUpdates should be `true` if ALL the DeviceKeys are present to update - // the display name for their respective device, and NOT to modify the keys. The key - // itself doesn't change but it's easier to pretend upload new keys and reuse the same code paths. - // Without this flag, requests to modify device display names would delete device keys. - OnlyDisplayNameUpdates bool -} - -// PerformUploadKeysResponse is the response to PerformUploadKeys -type PerformUploadKeysResponse struct { - // A fatal error when processing e.g database failures - Error *KeyError - // A map of user_id -> device_id -> Error for tracking failures. - KeyErrors map[string]map[string]*KeyError - OneTimeKeyCounts []OneTimeKeysCount -} - -// PerformDeleteKeysRequest asks the keyserver to forget about certain -// keys, and signatures related to those keys. -type PerformDeleteKeysRequest struct { - UserID string - KeyIDs []gomatrixserverlib.KeyID -} - -// PerformDeleteKeysResponse is the response to PerformDeleteKeysRequest. -type PerformDeleteKeysResponse struct { - Error *KeyError -} - -// KeyError sets a key error field on KeyErrors -func (r *PerformUploadKeysResponse) KeyError(userID, deviceID string, err *KeyError) { - if r.KeyErrors[userID] == nil { - r.KeyErrors[userID] = make(map[string]*KeyError) - } - r.KeyErrors[userID][deviceID] = err -} - -type PerformClaimKeysRequest struct { - // Map of user_id to device_id to algorithm name - OneTimeKeys map[string]map[string]string - Timeout time.Duration -} - -type PerformClaimKeysResponse struct { - // Map of user_id to device_id to algorithm:key_id to key JSON - OneTimeKeys map[string]map[string]map[string]json.RawMessage - // Map of remote server domain to error JSON - Failures map[string]interface{} - // Set if there was a fatal error processing this action - Error *KeyError -} - -type PerformUploadDeviceKeysRequest struct { - gomatrixserverlib.CrossSigningKeys - // The user that uploaded the key, should be populated by the clientapi. - UserID string -} - -type PerformUploadDeviceKeysResponse struct { - Error *KeyError -} - -type PerformUploadDeviceSignaturesRequest struct { - Signatures map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice - // The user that uploaded the sig, should be populated by the clientapi. - UserID string -} - -type PerformUploadDeviceSignaturesResponse struct { - Error *KeyError -} - -type QueryKeysRequest struct { - // The user ID asking for the keys, e.g. if from a client API request. - // Will not be populated if the key request came from federation. - UserID string - // Maps user IDs to a list of devices - UserToDevices map[string][]string - Timeout time.Duration -} - -type QueryKeysResponse struct { - // Map of remote server domain to error JSON - Failures map[string]interface{} - // Map of user_id to device_id to device_key - DeviceKeys map[string]map[string]json.RawMessage - // Maps of user_id to cross signing key - MasterKeys map[string]gomatrixserverlib.CrossSigningKey - SelfSigningKeys map[string]gomatrixserverlib.CrossSigningKey - UserSigningKeys map[string]gomatrixserverlib.CrossSigningKey - // Set if there was a fatal error processing this query - Error *KeyError -} - -type QueryKeyChangesRequest struct { - // The offset of the last received key event, or sarama.OffsetOldest if this is from the beginning - Offset int64 - // The inclusive offset where to track key changes up to. Messages with this offset are included in the response. - // Use types.OffsetNewest if the offset is unknown (then check the response Offset to avoid racing). - ToOffset int64 -} - -type QueryKeyChangesResponse struct { - // The set of users who have had their keys change. - UserIDs []string - // The latest offset represented in this response. - Offset int64 - // Set if there was a problem handling the request. - Error *KeyError -} - -type QueryOneTimeKeysRequest struct { - // The local user to query OTK counts for - UserID string - // The device to query OTK counts for - DeviceID string -} - -type QueryOneTimeKeysResponse struct { - // OTK key counts, in the extended /sync form described by https://matrix.org/docs/spec/client_server/r0.6.1#id84 - Count OneTimeKeysCount - Error *KeyError -} - -type QueryDeviceMessagesRequest struct { - UserID string -} - -type QueryDeviceMessagesResponse struct { - // The latest stream ID - StreamID int64 - Devices []DeviceMessage - Error *KeyError -} - -type QuerySignaturesRequest struct { - // A map of target user ID -> target key/device IDs to retrieve signatures for - TargetIDs map[string][]gomatrixserverlib.KeyID `json:"target_ids"` -} - -type QuerySignaturesResponse struct { - // A map of target user ID -> target key/device ID -> origin user ID -> origin key/device ID -> signatures - Signatures map[string]map[gomatrixserverlib.KeyID]types.CrossSigningSigMap - // A map of target user ID -> cross-signing master key - MasterKeys map[string]gomatrixserverlib.CrossSigningKey - // A map of target user ID -> cross-signing self-signing key - SelfSigningKeys map[string]gomatrixserverlib.CrossSigningKey - // A map of target user ID -> cross-signing user-signing key - UserSigningKeys map[string]gomatrixserverlib.CrossSigningKey - // The request error, if any - Error *KeyError -} - -type PerformMarkAsStaleRequest struct { - UserID string - Domain gomatrixserverlib.ServerName - DeviceID string -} diff --git a/keyserver/inthttp/client.go b/keyserver/inthttp/client.go deleted file mode 100644 index 75d537d9c..000000000 --- a/keyserver/inthttp/client.go +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package inthttp - -import ( - "context" - "errors" - "net/http" - - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/keyserver/api" - userapi "github.com/matrix-org/dendrite/userapi/api" -) - -// HTTP paths for the internal HTTP APIs -const ( - InputDeviceListUpdatePath = "/keyserver/inputDeviceListUpdate" - PerformUploadKeysPath = "/keyserver/performUploadKeys" - PerformClaimKeysPath = "/keyserver/performClaimKeys" - PerformDeleteKeysPath = "/keyserver/performDeleteKeys" - PerformUploadDeviceKeysPath = "/keyserver/performUploadDeviceKeys" - PerformUploadDeviceSignaturesPath = "/keyserver/performUploadDeviceSignatures" - QueryKeysPath = "/keyserver/queryKeys" - QueryKeyChangesPath = "/keyserver/queryKeyChanges" - QueryOneTimeKeysPath = "/keyserver/queryOneTimeKeys" - QueryDeviceMessagesPath = "/keyserver/queryDeviceMessages" - QuerySignaturesPath = "/keyserver/querySignatures" - PerformMarkAsStalePath = "/keyserver/markAsStale" -) - -// NewKeyServerClient creates a KeyInternalAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewKeyServerClient( - apiURL string, - httpClient *http.Client, -) (api.KeyInternalAPI, error) { - if httpClient == nil { - return nil, errors.New("NewKeyServerClient: httpClient is ") - } - return &httpKeyInternalAPI{ - apiURL: apiURL, - httpClient: httpClient, - }, nil -} - -type httpKeyInternalAPI struct { - apiURL string - httpClient *http.Client -} - -func (h *httpKeyInternalAPI) SetUserAPI(i userapi.KeyserverUserAPI) { - // no-op: doesn't need it -} - -func (h *httpKeyInternalAPI) PerformClaimKeys( - ctx context.Context, - request *api.PerformClaimKeysRequest, - response *api.PerformClaimKeysResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformClaimKeys", h.apiURL+PerformClaimKeysPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpKeyInternalAPI) PerformDeleteKeys( - ctx context.Context, - request *api.PerformDeleteKeysRequest, - response *api.PerformDeleteKeysResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformDeleteKeys", h.apiURL+PerformDeleteKeysPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpKeyInternalAPI) PerformUploadKeys( - ctx context.Context, - request *api.PerformUploadKeysRequest, - response *api.PerformUploadKeysResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformUploadKeys", h.apiURL+PerformUploadKeysPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpKeyInternalAPI) QueryKeys( - ctx context.Context, - request *api.QueryKeysRequest, - response *api.QueryKeysResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryKeys", h.apiURL+QueryKeysPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpKeyInternalAPI) QueryOneTimeKeys( - ctx context.Context, - request *api.QueryOneTimeKeysRequest, - response *api.QueryOneTimeKeysResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryOneTimeKeys", h.apiURL+QueryOneTimeKeysPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpKeyInternalAPI) QueryDeviceMessages( - ctx context.Context, - request *api.QueryDeviceMessagesRequest, - response *api.QueryDeviceMessagesResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryDeviceMessages", h.apiURL+QueryDeviceMessagesPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpKeyInternalAPI) QueryKeyChanges( - ctx context.Context, - request *api.QueryKeyChangesRequest, - response *api.QueryKeyChangesResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryKeyChanges", h.apiURL+QueryKeyChangesPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpKeyInternalAPI) PerformUploadDeviceKeys( - ctx context.Context, - request *api.PerformUploadDeviceKeysRequest, - response *api.PerformUploadDeviceKeysResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformUploadDeviceKeys", h.apiURL+PerformUploadDeviceKeysPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpKeyInternalAPI) PerformUploadDeviceSignatures( - ctx context.Context, - request *api.PerformUploadDeviceSignaturesRequest, - response *api.PerformUploadDeviceSignaturesResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformUploadDeviceSignatures", h.apiURL+PerformUploadDeviceSignaturesPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpKeyInternalAPI) QuerySignatures( - ctx context.Context, - request *api.QuerySignaturesRequest, - response *api.QuerySignaturesResponse, -) error { - return httputil.CallInternalRPCAPI( - "QuerySignatures", h.apiURL+QuerySignaturesPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpKeyInternalAPI) PerformMarkAsStaleIfNeeded( - ctx context.Context, - request *api.PerformMarkAsStaleRequest, - response *struct{}, -) error { - return httputil.CallInternalRPCAPI( - "MarkAsStale", h.apiURL+PerformMarkAsStalePath, - h.httpClient, ctx, request, response, - ) -} diff --git a/keyserver/inthttp/server.go b/keyserver/inthttp/server.go deleted file mode 100644 index 443269f73..000000000 --- a/keyserver/inthttp/server.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package inthttp - -import ( - "github.com/gorilla/mux" - - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/keyserver/api" -) - -func AddRoutes(internalAPIMux *mux.Router, s api.KeyInternalAPI, enableMetrics bool) { - internalAPIMux.Handle( - PerformClaimKeysPath, - httputil.MakeInternalRPCAPI("KeyserverPerformClaimKeys", enableMetrics, s.PerformClaimKeys), - ) - - internalAPIMux.Handle( - PerformDeleteKeysPath, - httputil.MakeInternalRPCAPI("KeyserverPerformDeleteKeys", enableMetrics, s.PerformDeleteKeys), - ) - - internalAPIMux.Handle( - PerformUploadKeysPath, - httputil.MakeInternalRPCAPI("KeyserverPerformUploadKeys", enableMetrics, s.PerformUploadKeys), - ) - - internalAPIMux.Handle( - PerformUploadDeviceKeysPath, - httputil.MakeInternalRPCAPI("KeyserverPerformUploadDeviceKeys", enableMetrics, s.PerformUploadDeviceKeys), - ) - - internalAPIMux.Handle( - PerformUploadDeviceSignaturesPath, - httputil.MakeInternalRPCAPI("KeyserverPerformUploadDeviceSignatures", enableMetrics, s.PerformUploadDeviceSignatures), - ) - - internalAPIMux.Handle( - QueryKeysPath, - httputil.MakeInternalRPCAPI("KeyserverQueryKeys", enableMetrics, s.QueryKeys), - ) - - internalAPIMux.Handle( - QueryOneTimeKeysPath, - httputil.MakeInternalRPCAPI("KeyserverQueryOneTimeKeys", enableMetrics, s.QueryOneTimeKeys), - ) - - internalAPIMux.Handle( - QueryDeviceMessagesPath, - httputil.MakeInternalRPCAPI("KeyserverQueryDeviceMessages", enableMetrics, s.QueryDeviceMessages), - ) - - internalAPIMux.Handle( - QueryKeyChangesPath, - httputil.MakeInternalRPCAPI("KeyserverQueryKeyChanges", enableMetrics, s.QueryKeyChanges), - ) - - internalAPIMux.Handle( - QuerySignaturesPath, - httputil.MakeInternalRPCAPI("KeyserverQuerySignatures", enableMetrics, s.QuerySignatures), - ) - - internalAPIMux.Handle( - PerformMarkAsStalePath, - httputil.MakeInternalRPCAPI("KeyserverMarkAsStale", enableMetrics, s.PerformMarkAsStaleIfNeeded), - ) -} diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go deleted file mode 100644 index 275576773..000000000 --- a/keyserver/keyserver.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package keyserver - -import ( - "github.com/gorilla/mux" - "github.com/sirupsen/logrus" - - rsapi "github.com/matrix-org/dendrite/roomserver/api" - - fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" - "github.com/matrix-org/dendrite/keyserver/api" - "github.com/matrix-org/dendrite/keyserver/consumers" - "github.com/matrix-org/dendrite/keyserver/internal" - "github.com/matrix-org/dendrite/keyserver/inthttp" - "github.com/matrix-org/dendrite/keyserver/producers" - "github.com/matrix-org/dendrite/keyserver/storage" - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/setup/jetstream" -) - -// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions -// on the given input API. -func AddInternalRoutes(router *mux.Router, intAPI api.KeyInternalAPI, enableMetrics bool) { - inthttp.AddRoutes(router, intAPI, enableMetrics) -} - -// NewInternalAPI returns a concerete implementation of the internal API. Callers -// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. -func NewInternalAPI( - base *base.BaseDendrite, cfg *config.KeyServer, fedClient fedsenderapi.KeyserverFederationAPI, - rsAPI rsapi.KeyserverRoomserverAPI, -) api.KeyInternalAPI { - js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream) - - db, err := storage.NewDatabase(base, &cfg.Database) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to key server database") - } - - keyChangeProducer := &producers.KeyChange{ - Topic: string(cfg.Matrix.JetStream.Prefixed(jetstream.OutputKeyChangeEvent)), - JetStream: js, - DB: db, - } - ap := &internal.KeyInternalAPI{ - DB: db, - Cfg: cfg, - FedClient: fedClient, - Producer: keyChangeProducer, - } - updater := internal.NewDeviceListUpdater(base.ProcessContext, db, ap, keyChangeProducer, fedClient, 8, rsAPI, cfg.Matrix.ServerName) // 8 workers TODO: configurable - ap.Updater = updater - - // Remove users which we don't share a room with anymore - if err := updater.CleanUp(); err != nil { - logrus.WithError(err).Error("failed to cleanup stale device lists") - } - - go func() { - if err := updater.Start(); err != nil { - logrus.WithError(err).Panicf("failed to start device list updater") - } - }() - - dlConsumer := consumers.NewDeviceListUpdateConsumer( - base.ProcessContext, cfg, js, updater, - ) - if err := dlConsumer.Start(); err != nil { - logrus.WithError(err).Panic("failed to start device list consumer") - } - - sigConsumer := consumers.NewSigningKeyUpdateConsumer( - base.ProcessContext, cfg, js, ap, - ) - if err := sigConsumer.Start(); err != nil { - logrus.WithError(err).Panic("failed to start signing key consumer") - } - - return ap -} diff --git a/keyserver/keyserver_test.go b/keyserver/keyserver_test.go deleted file mode 100644 index 159b280f5..000000000 --- a/keyserver/keyserver_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package keyserver - -import ( - "context" - "testing" - - roomserver "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/test" - "github.com/matrix-org/dendrite/test/testrig" -) - -type mockKeyserverRoomserverAPI struct { - leftUsers []string -} - -func (m *mockKeyserverRoomserverAPI) QueryLeftUsers(ctx context.Context, req *roomserver.QueryLeftUsersRequest, res *roomserver.QueryLeftUsersResponse) error { - res.LeftUsers = m.leftUsers - return nil -} - -// Merely tests that we can create an internal keyserver API -func Test_NewInternalAPI(t *testing.T) { - rsAPI := &mockKeyserverRoomserverAPI{} - test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, closeBase := testrig.CreateBaseDendrite(t, dbType) - defer closeBase() - _ = NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI) - }) -} diff --git a/keyserver/storage/interface.go b/keyserver/storage/interface.go deleted file mode 100644 index c6a8f44cd..000000000 --- a/keyserver/storage/interface.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "context" - "encoding/json" - - "github.com/matrix-org/dendrite/keyserver/api" - "github.com/matrix-org/dendrite/keyserver/types" - "github.com/matrix-org/gomatrixserverlib" -) - -type Database interface { - // ExistingOneTimeKeys returns a map of keyIDWithAlgorithm to key JSON for the given parameters. If no keys exist with this combination - // of user/device/key/algorithm 4-uple then it is omitted from the map. Returns an error when failing to communicate with the database. - ExistingOneTimeKeys(ctx context.Context, userID, deviceID string, keyIDsWithAlgorithms []string) (map[string]json.RawMessage, error) - - // StoreOneTimeKeys persists the given one-time keys. - StoreOneTimeKeys(ctx context.Context, keys api.OneTimeKeys) (*api.OneTimeKeysCount, error) - - // OneTimeKeysCount returns a count of all OTKs for this device. - OneTimeKeysCount(ctx context.Context, userID, deviceID string) (*api.OneTimeKeysCount, error) - - // DeviceKeysJSON populates the KeyJSON for the given keys. If any proided `keys` have a `KeyJSON` or `StreamID` already then it will be replaced. - DeviceKeysJSON(ctx context.Context, keys []api.DeviceMessage) error - - // StoreLocalDeviceKeys persists the given keys. Keys with the same user ID and device ID will be replaced. An empty KeyJSON removes the key - // for this (user, device). - // The `StreamID` for each message is set on successful insertion. In the event the key already exists, the existing StreamID is set. - // Returns an error if there was a problem storing the keys. - StoreLocalDeviceKeys(ctx context.Context, keys []api.DeviceMessage) error - - // StoreRemoteDeviceKeys persists the given keys. Keys with the same user ID and device ID will be replaced. An empty KeyJSON removes the key - // for this (user, device). Does not modify the stream ID for keys. User IDs in `clearUserIDs` will have all their device keys deleted prior - // to insertion - use this when you have a complete snapshot of a user's keys in order to track device deletions correctly. - StoreRemoteDeviceKeys(ctx context.Context, keys []api.DeviceMessage, clearUserIDs []string) error - - // PrevIDsExists returns true if all prev IDs exist for this user. - PrevIDsExists(ctx context.Context, userID string, prevIDs []int64) (bool, error) - - // DeviceKeysForUser returns the device keys for the device IDs given. If the length of deviceIDs is 0, all devices are selected. - // If there are some missing keys, they are omitted from the returned slice. There is no ordering on the returned slice. - DeviceKeysForUser(ctx context.Context, userID string, deviceIDs []string, includeEmpty bool) ([]api.DeviceMessage, error) - - // DeleteDeviceKeys removes the device keys for a given user/device, and any accompanying - // cross-signing signatures relating to that device. - DeleteDeviceKeys(ctx context.Context, userID string, deviceIDs []gomatrixserverlib.KeyID) error - - // ClaimKeys based on the 3-uple of user_id, device_id and algorithm name. Returns the keys claimed. Returns no error if a key - // cannot be claimed or if none exist for this (user, device, algorithm), instead it is omitted from the returned slice. - ClaimKeys(ctx context.Context, userToDeviceToAlgorithm map[string]map[string]string) ([]api.OneTimeKeys, error) - - // StoreKeyChange stores key change metadata and returns the device change ID which represents the position in the /sync stream for this device change. - // `userID` is the the user who has changed their keys in some way. - StoreKeyChange(ctx context.Context, userID string) (int64, error) - - // KeyChanges returns a list of user IDs who have modified their keys from the offset given (exclusive) to the offset given (inclusive). - // A to offset of types.OffsetNewest means no upper limit. - // Returns the offset of the latest key change. - KeyChanges(ctx context.Context, fromOffset, toOffset int64) (userIDs []string, latestOffset int64, err error) - - // StaleDeviceLists returns a list of user IDs ending with the domains provided who have stale device lists. - // If no domains are given, all user IDs with stale device lists are returned. - StaleDeviceLists(ctx context.Context, domains []gomatrixserverlib.ServerName) ([]string, error) - - // MarkDeviceListStale sets the stale bit for this user to isStale. - MarkDeviceListStale(ctx context.Context, userID string, isStale bool) error - - CrossSigningKeysForUser(ctx context.Context, userID string) (map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey, error) - CrossSigningKeysDataForUser(ctx context.Context, userID string) (types.CrossSigningKeyMap, error) - CrossSigningSigsForTarget(ctx context.Context, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error) - - StoreCrossSigningKeysForUser(ctx context.Context, userID string, keyMap types.CrossSigningKeyMap) error - StoreCrossSigningSigsForTarget(ctx context.Context, originUserID string, originKeyID gomatrixserverlib.KeyID, targetUserID string, targetKeyID gomatrixserverlib.KeyID, signature gomatrixserverlib.Base64Bytes) error - - DeleteStaleDeviceLists( - ctx context.Context, - userIDs []string, - ) error -} diff --git a/keyserver/storage/postgres/storage.go b/keyserver/storage/postgres/storage.go deleted file mode 100644 index 35e630559..000000000 --- a/keyserver/storage/postgres/storage.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/keyserver/storage/shared" - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -// NewDatabase creates a new sync server database -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*shared.Database, error) { - var err error - db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()) - if err != nil { - return nil, err - } - otk, err := NewPostgresOneTimeKeysTable(db) - if err != nil { - return nil, err - } - dk, err := NewPostgresDeviceKeysTable(db) - if err != nil { - return nil, err - } - kc, err := NewPostgresKeyChangesTable(db) - if err != nil { - return nil, err - } - sdl, err := NewPostgresStaleDeviceListsTable(db) - if err != nil { - return nil, err - } - csk, err := NewPostgresCrossSigningKeysTable(db) - if err != nil { - return nil, err - } - css, err := NewPostgresCrossSigningSigsTable(db) - if err != nil { - return nil, err - } - if err = kc.Prepare(); err != nil { - return nil, err - } - d := &shared.Database{ - DB: db, - Writer: writer, - OneTimeKeysTable: otk, - DeviceKeysTable: dk, - KeyChangesTable: kc, - StaleDeviceListsTable: sdl, - CrossSigningKeysTable: csk, - CrossSigningSigsTable: css, - } - return d, nil -} diff --git a/keyserver/storage/shared/storage.go b/keyserver/storage/shared/storage.go deleted file mode 100644 index 54dd6ddc9..000000000 --- a/keyserver/storage/shared/storage.go +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package shared - -import ( - "context" - "database/sql" - "encoding/json" - "fmt" - - "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/keyserver/api" - "github.com/matrix-org/dendrite/keyserver/storage/tables" - "github.com/matrix-org/dendrite/keyserver/types" - "github.com/matrix-org/gomatrixserverlib" -) - -type Database struct { - DB *sql.DB - Writer sqlutil.Writer - OneTimeKeysTable tables.OneTimeKeys - DeviceKeysTable tables.DeviceKeys - KeyChangesTable tables.KeyChanges - StaleDeviceListsTable tables.StaleDeviceLists - CrossSigningKeysTable tables.CrossSigningKeys - CrossSigningSigsTable tables.CrossSigningSigs -} - -func (d *Database) ExistingOneTimeKeys(ctx context.Context, userID, deviceID string, keyIDsWithAlgorithms []string) (map[string]json.RawMessage, error) { - return d.OneTimeKeysTable.SelectOneTimeKeys(ctx, userID, deviceID, keyIDsWithAlgorithms) -} - -func (d *Database) StoreOneTimeKeys(ctx context.Context, keys api.OneTimeKeys) (counts *api.OneTimeKeysCount, err error) { - _ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - counts, err = d.OneTimeKeysTable.InsertOneTimeKeys(ctx, txn, keys) - return err - }) - return -} - -func (d *Database) OneTimeKeysCount(ctx context.Context, userID, deviceID string) (*api.OneTimeKeysCount, error) { - return d.OneTimeKeysTable.CountOneTimeKeys(ctx, userID, deviceID) -} - -func (d *Database) DeviceKeysJSON(ctx context.Context, keys []api.DeviceMessage) error { - return d.DeviceKeysTable.SelectDeviceKeysJSON(ctx, keys) -} - -func (d *Database) PrevIDsExists(ctx context.Context, userID string, prevIDs []int64) (bool, error) { - count, err := d.DeviceKeysTable.CountStreamIDsForUser(ctx, userID, prevIDs) - if err != nil { - return false, err - } - return count == len(prevIDs), nil -} - -func (d *Database) StoreRemoteDeviceKeys(ctx context.Context, keys []api.DeviceMessage, clearUserIDs []string) error { - return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - for _, userID := range clearUserIDs { - err := d.DeviceKeysTable.DeleteAllDeviceKeys(ctx, txn, userID) - if err != nil { - return err - } - } - return d.DeviceKeysTable.InsertDeviceKeys(ctx, txn, keys) - }) -} - -func (d *Database) StoreLocalDeviceKeys(ctx context.Context, keys []api.DeviceMessage) error { - // work out the latest stream IDs for each user - userIDToStreamID := make(map[string]int64) - for _, k := range keys { - userIDToStreamID[k.UserID] = 0 - } - return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - for userID := range userIDToStreamID { - streamID, err := d.DeviceKeysTable.SelectMaxStreamIDForUser(ctx, txn, userID) - if err != nil { - return err - } - userIDToStreamID[userID] = streamID - } - // set the stream IDs for each key - for i := range keys { - k := keys[i] - userIDToStreamID[k.UserID]++ // start stream from 1 - k.StreamID = userIDToStreamID[k.UserID] - keys[i] = k - } - return d.DeviceKeysTable.InsertDeviceKeys(ctx, txn, keys) - }) -} - -func (d *Database) DeviceKeysForUser(ctx context.Context, userID string, deviceIDs []string, includeEmpty bool) ([]api.DeviceMessage, error) { - return d.DeviceKeysTable.SelectBatchDeviceKeys(ctx, userID, deviceIDs, includeEmpty) -} - -func (d *Database) ClaimKeys(ctx context.Context, userToDeviceToAlgorithm map[string]map[string]string) ([]api.OneTimeKeys, error) { - var result []api.OneTimeKeys - err := d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - for userID, deviceToAlgo := range userToDeviceToAlgorithm { - for deviceID, algo := range deviceToAlgo { - keyJSON, err := d.OneTimeKeysTable.SelectAndDeleteOneTimeKey(ctx, txn, userID, deviceID, algo) - if err != nil { - return err - } - if keyJSON != nil { - result = append(result, api.OneTimeKeys{ - UserID: userID, - DeviceID: deviceID, - KeyJSON: keyJSON, - }) - } - } - } - return nil - }) - return result, err -} - -func (d *Database) StoreKeyChange(ctx context.Context, userID string) (id int64, err error) { - err = d.Writer.Do(nil, nil, func(_ *sql.Tx) error { - id, err = d.KeyChangesTable.InsertKeyChange(ctx, userID) - return err - }) - return -} - -func (d *Database) KeyChanges(ctx context.Context, fromOffset, toOffset int64) (userIDs []string, latestOffset int64, err error) { - return d.KeyChangesTable.SelectKeyChanges(ctx, fromOffset, toOffset) -} - -// StaleDeviceLists returns a list of user IDs ending with the domains provided who have stale device lists. -// If no domains are given, all user IDs with stale device lists are returned. -func (d *Database) StaleDeviceLists(ctx context.Context, domains []gomatrixserverlib.ServerName) ([]string, error) { - return d.StaleDeviceListsTable.SelectUserIDsWithStaleDeviceLists(ctx, domains) -} - -// MarkDeviceListStale sets the stale bit for this user to isStale. -func (d *Database) MarkDeviceListStale(ctx context.Context, userID string, isStale bool) error { - return d.Writer.Do(nil, nil, func(_ *sql.Tx) error { - return d.StaleDeviceListsTable.InsertStaleDeviceList(ctx, userID, isStale) - }) -} - -// DeleteDeviceKeys removes the device keys for a given user/device, and any accompanying -// cross-signing signatures relating to that device. -func (d *Database) DeleteDeviceKeys(ctx context.Context, userID string, deviceIDs []gomatrixserverlib.KeyID) error { - return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - for _, deviceID := range deviceIDs { - if err := d.CrossSigningSigsTable.DeleteCrossSigningSigsForTarget(ctx, txn, userID, deviceID); err != nil && err != sql.ErrNoRows { - return fmt.Errorf("d.CrossSigningSigsTable.DeleteCrossSigningSigsForTarget: %w", err) - } - if err := d.DeviceKeysTable.DeleteDeviceKeys(ctx, txn, userID, string(deviceID)); err != nil && err != sql.ErrNoRows { - return fmt.Errorf("d.DeviceKeysTable.DeleteDeviceKeys: %w", err) - } - if err := d.OneTimeKeysTable.DeleteOneTimeKeys(ctx, txn, userID, string(deviceID)); err != nil && err != sql.ErrNoRows { - return fmt.Errorf("d.OneTimeKeysTable.DeleteOneTimeKeys: %w", err) - } - } - return nil - }) -} - -// CrossSigningKeysForUser returns the latest known cross-signing keys for a user, if any. -func (d *Database) CrossSigningKeysForUser(ctx context.Context, userID string) (map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey, error) { - keyMap, err := d.CrossSigningKeysTable.SelectCrossSigningKeysForUser(ctx, nil, userID) - if err != nil { - return nil, fmt.Errorf("d.CrossSigningKeysTable.SelectCrossSigningKeysForUser: %w", err) - } - results := map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey{} - for purpose, key := range keyMap { - keyID := gomatrixserverlib.KeyID("ed25519:" + key.Encode()) - result := gomatrixserverlib.CrossSigningKey{ - UserID: userID, - Usage: []gomatrixserverlib.CrossSigningKeyPurpose{purpose}, - Keys: map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{ - keyID: key, - }, - } - sigMap, err := d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, userID, userID, keyID) - if err != nil { - continue - } - for sigUserID, forSigUserID := range sigMap { - if userID != sigUserID { - continue - } - if result.Signatures == nil { - result.Signatures = map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{} - } - if _, ok := result.Signatures[sigUserID]; !ok { - result.Signatures[sigUserID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{} - } - for sigKeyID, sigBytes := range forSigUserID { - result.Signatures[sigUserID][sigKeyID] = sigBytes - } - } - results[purpose] = result - } - return results, nil -} - -// CrossSigningKeysForUser returns the latest known cross-signing keys for a user, if any. -func (d *Database) CrossSigningKeysDataForUser(ctx context.Context, userID string) (types.CrossSigningKeyMap, error) { - return d.CrossSigningKeysTable.SelectCrossSigningKeysForUser(ctx, nil, userID) -} - -// CrossSigningSigsForTarget returns the signatures for a given user's key ID, if any. -func (d *Database) CrossSigningSigsForTarget(ctx context.Context, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error) { - return d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, originUserID, targetUserID, targetKeyID) -} - -// StoreCrossSigningKeysForUser stores the latest known cross-signing keys for a user. -func (d *Database) StoreCrossSigningKeysForUser(ctx context.Context, userID string, keyMap types.CrossSigningKeyMap) error { - return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - for keyType, keyData := range keyMap { - if err := d.CrossSigningKeysTable.UpsertCrossSigningKeysForUser(ctx, txn, userID, keyType, keyData); err != nil { - return fmt.Errorf("d.CrossSigningKeysTable.InsertCrossSigningKeysForUser: %w", err) - } - } - return nil - }) -} - -// StoreCrossSigningSigsForTarget stores a signature for a target user ID and key/dvice. -func (d *Database) StoreCrossSigningSigsForTarget( - ctx context.Context, - originUserID string, originKeyID gomatrixserverlib.KeyID, - targetUserID string, targetKeyID gomatrixserverlib.KeyID, - signature gomatrixserverlib.Base64Bytes, -) error { - return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - if err := d.CrossSigningSigsTable.UpsertCrossSigningSigsForTarget(ctx, nil, originUserID, originKeyID, targetUserID, targetKeyID, signature); err != nil { - return fmt.Errorf("d.CrossSigningSigsTable.InsertCrossSigningSigsForTarget: %w", err) - } - return nil - }) -} - -// DeleteStaleDeviceLists deletes stale device list entries for users we don't share a room with anymore. -func (d *Database) DeleteStaleDeviceLists( - ctx context.Context, - userIDs []string, -) error { - return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.StaleDeviceListsTable.DeleteStaleDeviceLists(ctx, txn, userIDs) - }) -} diff --git a/keyserver/storage/sqlite3/storage.go b/keyserver/storage/sqlite3/storage.go deleted file mode 100644 index 873fe3e24..000000000 --- a/keyserver/storage/sqlite3/storage.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/keyserver/storage/shared" - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*shared.Database, error) { - db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()) - if err != nil { - return nil, err - } - otk, err := NewSqliteOneTimeKeysTable(db) - if err != nil { - return nil, err - } - dk, err := NewSqliteDeviceKeysTable(db) - if err != nil { - return nil, err - } - kc, err := NewSqliteKeyChangesTable(db) - if err != nil { - return nil, err - } - sdl, err := NewSqliteStaleDeviceListsTable(db) - if err != nil { - return nil, err - } - csk, err := NewSqliteCrossSigningKeysTable(db) - if err != nil { - return nil, err - } - css, err := NewSqliteCrossSigningSigsTable(db) - if err != nil { - return nil, err - } - - if err = kc.Prepare(); err != nil { - return nil, err - } - d := &shared.Database{ - DB: db, - Writer: writer, - OneTimeKeysTable: otk, - DeviceKeysTable: dk, - KeyChangesTable: kc, - StaleDeviceListsTable: sdl, - CrossSigningKeysTable: csk, - CrossSigningSigsTable: css, - } - return d, nil -} diff --git a/keyserver/storage/storage.go b/keyserver/storage/storage.go deleted file mode 100644 index ab6a35401..000000000 --- a/keyserver/storage/storage.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build !wasm -// +build !wasm - -package storage - -import ( - "fmt" - - "github.com/matrix-org/dendrite/keyserver/storage/postgres" - "github.com/matrix-org/dendrite/keyserver/storage/sqlite3" - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) -// and sets postgres connection parameters -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) { - switch { - case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(base, dbProperties) - case dbProperties.ConnectionString.IsPostgres(): - return postgres.NewDatabase(base, dbProperties) - default: - return nil, fmt.Errorf("unexpected database type") - } -} diff --git a/keyserver/storage/storage_test.go b/keyserver/storage/storage_test.go deleted file mode 100644 index e7a2af7c2..000000000 --- a/keyserver/storage/storage_test.go +++ /dev/null @@ -1,197 +0,0 @@ -package storage_test - -import ( - "context" - "reflect" - "sync" - "testing" - - "github.com/matrix-org/dendrite/keyserver/api" - "github.com/matrix-org/dendrite/keyserver/storage" - "github.com/matrix-org/dendrite/keyserver/types" - "github.com/matrix-org/dendrite/test" - "github.com/matrix-org/dendrite/test/testrig" -) - -var ctx = context.Background() - -func MustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { - base, close := testrig.CreateBaseDendrite(t, dbType) - db, err := storage.NewDatabase(base, &base.Cfg.KeyServer.Database) - if err != nil { - t.Fatalf("failed to create new database: %v", err) - } - return db, close -} - -func MustNotError(t *testing.T, err error) { - t.Helper() - if err == nil { - return - } - t.Fatalf("operation failed: %s", err) -} - -func TestKeyChanges(t *testing.T) { - test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, clean := MustCreateDatabase(t, dbType) - defer clean() - _, err := db.StoreKeyChange(ctx, "@alice:localhost") - MustNotError(t, err) - deviceChangeIDB, err := db.StoreKeyChange(ctx, "@bob:localhost") - MustNotError(t, err) - deviceChangeIDC, err := db.StoreKeyChange(ctx, "@charlie:localhost") - MustNotError(t, err) - userIDs, latest, err := db.KeyChanges(ctx, deviceChangeIDB, types.OffsetNewest) - if err != nil { - t.Fatalf("Failed to KeyChanges: %s", err) - } - if latest != deviceChangeIDC { - t.Fatalf("KeyChanges: got latest=%d want %d", latest, deviceChangeIDC) - } - if !reflect.DeepEqual(userIDs, []string{"@charlie:localhost"}) { - t.Fatalf("KeyChanges: wrong user_ids: %v", userIDs) - } - }) -} - -func TestKeyChangesNoDupes(t *testing.T) { - test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, clean := MustCreateDatabase(t, dbType) - defer clean() - deviceChangeIDA, err := db.StoreKeyChange(ctx, "@alice:localhost") - MustNotError(t, err) - deviceChangeIDB, err := db.StoreKeyChange(ctx, "@alice:localhost") - MustNotError(t, err) - if deviceChangeIDA == deviceChangeIDB { - t.Fatalf("Expected change ID to be different even when inserting key change for the same user, got %d for both changes", deviceChangeIDA) - } - deviceChangeID, err := db.StoreKeyChange(ctx, "@alice:localhost") - MustNotError(t, err) - userIDs, latest, err := db.KeyChanges(ctx, 0, types.OffsetNewest) - if err != nil { - t.Fatalf("Failed to KeyChanges: %s", err) - } - if latest != deviceChangeID { - t.Fatalf("KeyChanges: got latest=%d want %d", latest, deviceChangeID) - } - if !reflect.DeepEqual(userIDs, []string{"@alice:localhost"}) { - t.Fatalf("KeyChanges: wrong user_ids: %v", userIDs) - } - }) -} - -func TestKeyChangesUpperLimit(t *testing.T) { - test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, clean := MustCreateDatabase(t, dbType) - defer clean() - deviceChangeIDA, err := db.StoreKeyChange(ctx, "@alice:localhost") - MustNotError(t, err) - deviceChangeIDB, err := db.StoreKeyChange(ctx, "@bob:localhost") - MustNotError(t, err) - _, err = db.StoreKeyChange(ctx, "@charlie:localhost") - MustNotError(t, err) - userIDs, latest, err := db.KeyChanges(ctx, deviceChangeIDA, deviceChangeIDB) - if err != nil { - t.Fatalf("Failed to KeyChanges: %s", err) - } - if latest != deviceChangeIDB { - t.Fatalf("KeyChanges: got latest=%d want %d", latest, deviceChangeIDB) - } - if !reflect.DeepEqual(userIDs, []string{"@bob:localhost"}) { - t.Fatalf("KeyChanges: wrong user_ids: %v", userIDs) - } - }) -} - -var dbLock sync.Mutex -var deviceArray = []string{"AAA", "another_device"} - -// The purpose of this test is to make sure that the storage layer is generating sequential stream IDs per user, -// and that they are returned correctly when querying for device keys. -func TestDeviceKeysStreamIDGeneration(t *testing.T) { - var err error - test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, clean := MustCreateDatabase(t, dbType) - defer clean() - alice := "@alice:TestDeviceKeysStreamIDGeneration" - bob := "@bob:TestDeviceKeysStreamIDGeneration" - msgs := []api.DeviceMessage{ - { - Type: api.TypeDeviceKeyUpdate, - DeviceKeys: &api.DeviceKeys{ - DeviceID: "AAA", - UserID: alice, - KeyJSON: []byte(`{"key":"v1"}`), - }, - // StreamID: 1 - }, - { - Type: api.TypeDeviceKeyUpdate, - DeviceKeys: &api.DeviceKeys{ - DeviceID: "AAA", - UserID: bob, - KeyJSON: []byte(`{"key":"v1"}`), - }, - // StreamID: 1 as this is a different user - }, - { - Type: api.TypeDeviceKeyUpdate, - DeviceKeys: &api.DeviceKeys{ - DeviceID: "another_device", - UserID: alice, - KeyJSON: []byte(`{"key":"v1"}`), - }, - // StreamID: 2 as this is a 2nd device key - }, - } - MustNotError(t, db.StoreLocalDeviceKeys(ctx, msgs)) - if msgs[0].StreamID != 1 { - t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=1 but got %d", msgs[0].StreamID) - } - if msgs[1].StreamID != 1 { - t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=1 (different user) but got %d", msgs[1].StreamID) - } - if msgs[2].StreamID != 2 { - t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=2 (another device) but got %d", msgs[2].StreamID) - } - - // updating a device sets the next stream ID for that user - msgs = []api.DeviceMessage{ - { - Type: api.TypeDeviceKeyUpdate, - DeviceKeys: &api.DeviceKeys{ - DeviceID: "AAA", - UserID: alice, - KeyJSON: []byte(`{"key":"v2"}`), - }, - // StreamID: 3 - }, - } - MustNotError(t, db.StoreLocalDeviceKeys(ctx, msgs)) - if msgs[0].StreamID != 3 { - t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=3 (new key same device) but got %d", msgs[0].StreamID) - } - - dbLock.Lock() - defer dbLock.Unlock() - // Querying for device keys returns the latest stream IDs - msgs, err = db.DeviceKeysForUser(ctx, alice, deviceArray, false) - - if err != nil { - t.Fatalf("DeviceKeysForUser returned error: %s", err) - } - wantStreamIDs := map[string]int64{ - "AAA": 3, - "another_device": 2, - } - if len(msgs) != len(wantStreamIDs) { - t.Fatalf("DeviceKeysForUser: wrong number of devices, got %d want %d", len(msgs), len(wantStreamIDs)) - } - for _, m := range msgs { - if m.StreamID != wantStreamIDs[m.DeviceID] { - t.Errorf("DeviceKeysForUser: wrong returned stream ID for key, got %d want %d", m.StreamID, wantStreamIDs[m.DeviceID]) - } - } - }) -} diff --git a/keyserver/storage/storage_wasm.go b/keyserver/storage/storage_wasm.go deleted file mode 100644 index 75c9053e8..000000000 --- a/keyserver/storage/storage_wasm.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "fmt" - - "github.com/matrix-org/dendrite/keyserver/storage/sqlite3" - "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" -) - -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) { - switch { - case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(base, dbProperties) - case dbProperties.ConnectionString.IsPostgres(): - return nil, fmt.Errorf("can't use Postgres implementation") - default: - return nil, fmt.Errorf("unexpected database type") - } -} diff --git a/keyserver/storage/tables/interface.go b/keyserver/storage/tables/interface.go deleted file mode 100644 index 24da1125e..000000000 --- a/keyserver/storage/tables/interface.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tables - -import ( - "context" - "database/sql" - "encoding/json" - - "github.com/matrix-org/dendrite/keyserver/api" - "github.com/matrix-org/dendrite/keyserver/types" - "github.com/matrix-org/gomatrixserverlib" -) - -type OneTimeKeys interface { - SelectOneTimeKeys(ctx context.Context, userID, deviceID string, keyIDsWithAlgorithms []string) (map[string]json.RawMessage, error) - CountOneTimeKeys(ctx context.Context, userID, deviceID string) (*api.OneTimeKeysCount, error) - InsertOneTimeKeys(ctx context.Context, txn *sql.Tx, keys api.OneTimeKeys) (*api.OneTimeKeysCount, error) - // SelectAndDeleteOneTimeKey selects a single one time key matching the user/device/algorithm specified and returns the algo:key_id => JSON. - // Returns an empty map if the key does not exist. - SelectAndDeleteOneTimeKey(ctx context.Context, txn *sql.Tx, userID, deviceID, algorithm string) (map[string]json.RawMessage, error) - DeleteOneTimeKeys(ctx context.Context, txn *sql.Tx, userID, deviceID string) error -} - -type DeviceKeys interface { - SelectDeviceKeysJSON(ctx context.Context, keys []api.DeviceMessage) error - InsertDeviceKeys(ctx context.Context, txn *sql.Tx, keys []api.DeviceMessage) error - SelectMaxStreamIDForUser(ctx context.Context, txn *sql.Tx, userID string) (streamID int64, err error) - CountStreamIDsForUser(ctx context.Context, userID string, streamIDs []int64) (int, error) - SelectBatchDeviceKeys(ctx context.Context, userID string, deviceIDs []string, includeEmpty bool) ([]api.DeviceMessage, error) - DeleteDeviceKeys(ctx context.Context, txn *sql.Tx, userID, deviceID string) error - DeleteAllDeviceKeys(ctx context.Context, txn *sql.Tx, userID string) error -} - -type KeyChanges interface { - InsertKeyChange(ctx context.Context, userID string) (int64, error) - // SelectKeyChanges returns the set (de-duplicated) of users who have changed their keys between the two offsets. - // Results are exclusive of fromOffset and inclusive of toOffset. A toOffset of types.OffsetNewest means no upper offset. - SelectKeyChanges(ctx context.Context, fromOffset, toOffset int64) (userIDs []string, latestOffset int64, err error) - - Prepare() error -} - -type StaleDeviceLists interface { - InsertStaleDeviceList(ctx context.Context, userID string, isStale bool) error - SelectUserIDsWithStaleDeviceLists(ctx context.Context, domains []gomatrixserverlib.ServerName) ([]string, error) - DeleteStaleDeviceLists(ctx context.Context, txn *sql.Tx, userIDs []string) error -} - -type CrossSigningKeys interface { - SelectCrossSigningKeysForUser(ctx context.Context, txn *sql.Tx, userID string) (r types.CrossSigningKeyMap, err error) - UpsertCrossSigningKeysForUser(ctx context.Context, txn *sql.Tx, userID string, keyType gomatrixserverlib.CrossSigningKeyPurpose, keyData gomatrixserverlib.Base64Bytes) error -} - -type CrossSigningSigs interface { - SelectCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (r types.CrossSigningSigMap, err error) - UpsertCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, originUserID string, originKeyID gomatrixserverlib.KeyID, targetUserID string, targetKeyID gomatrixserverlib.KeyID, signature gomatrixserverlib.Base64Bytes) error - DeleteCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID) error -} diff --git a/roomserver/api/api.go b/roomserver/api/api.go index a8228ae81..73732ae32 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -17,7 +17,6 @@ type RoomserverInternalAPI interface { ClientRoomserverAPI UserRoomserverAPI FederationRoomserverAPI - KeyserverRoomserverAPI // needed to avoid chicken and egg scenario when setting up the // interdependencies between the roomserver and other input APIs @@ -167,6 +166,7 @@ type ClientRoomserverAPI interface { type UserRoomserverAPI interface { QueryLatestEventsAndStateAPI + KeyserverRoomserverAPI QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error PerformAdminEvacuateUser(ctx context.Context, req *PerformAdminEvacuateUserRequest, res *PerformAdminEvacuateUserResponse) error diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go deleted file mode 100644 index 166b651a2..000000000 --- a/roomserver/api/api_trace.go +++ /dev/null @@ -1,427 +0,0 @@ -package api - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - - asAPI "github.com/matrix-org/dendrite/appservice/api" - fsAPI "github.com/matrix-org/dendrite/federationapi/api" - userapi "github.com/matrix-org/dendrite/userapi/api" -) - -// RoomserverInternalAPITrace wraps a RoomserverInternalAPI and logs the -// complete request/response/error -type RoomserverInternalAPITrace struct { - Impl RoomserverInternalAPI -} - -func (t *RoomserverInternalAPITrace) QueryLeftUsers(ctx context.Context, req *QueryLeftUsersRequest, res *QueryLeftUsersResponse) error { - err := t.Impl.QueryLeftUsers(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryLeftUsers req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) SetFederationAPI(fsAPI fsAPI.RoomserverFederationAPI, keyRing *gomatrixserverlib.KeyRing) { - t.Impl.SetFederationAPI(fsAPI, keyRing) -} - -func (t *RoomserverInternalAPITrace) SetAppserviceAPI(asAPI asAPI.AppServiceInternalAPI) { - t.Impl.SetAppserviceAPI(asAPI) -} - -func (t *RoomserverInternalAPITrace) SetUserAPI(userAPI userapi.RoomserverUserAPI) { - t.Impl.SetUserAPI(userAPI) -} - -func (t *RoomserverInternalAPITrace) InputRoomEvents( - ctx context.Context, - req *InputRoomEventsRequest, - res *InputRoomEventsResponse, -) error { - err := t.Impl.InputRoomEvents(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("InputRoomEvents req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) PerformInvite( - ctx context.Context, - req *PerformInviteRequest, - res *PerformInviteResponse, -) error { - err := t.Impl.PerformInvite(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("PerformInvite req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) PerformPeek( - ctx context.Context, - req *PerformPeekRequest, - res *PerformPeekResponse, -) error { - err := t.Impl.PerformPeek(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("PerformPeek req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) PerformUnpeek( - ctx context.Context, - req *PerformUnpeekRequest, - res *PerformUnpeekResponse, -) error { - err := t.Impl.PerformUnpeek(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("PerformUnpeek req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) PerformRoomUpgrade( - ctx context.Context, - req *PerformRoomUpgradeRequest, - res *PerformRoomUpgradeResponse, -) error { - err := t.Impl.PerformRoomUpgrade(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("PerformRoomUpgrade req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) PerformJoin( - ctx context.Context, - req *PerformJoinRequest, - res *PerformJoinResponse, -) error { - err := t.Impl.PerformJoin(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("PerformJoin req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) PerformLeave( - ctx context.Context, - req *PerformLeaveRequest, - res *PerformLeaveResponse, -) error { - err := t.Impl.PerformLeave(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("PerformLeave req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) PerformPublish( - ctx context.Context, - req *PerformPublishRequest, - res *PerformPublishResponse, -) error { - err := t.Impl.PerformPublish(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("PerformPublish req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) PerformAdminEvacuateRoom( - ctx context.Context, - req *PerformAdminEvacuateRoomRequest, - res *PerformAdminEvacuateRoomResponse, -) error { - err := t.Impl.PerformAdminEvacuateRoom(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("PerformAdminEvacuateRoom req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) PerformAdminEvacuateUser( - ctx context.Context, - req *PerformAdminEvacuateUserRequest, - res *PerformAdminEvacuateUserResponse, -) error { - err := t.Impl.PerformAdminEvacuateUser(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("PerformAdminEvacuateUser req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) PerformAdminPurgeRoom( - ctx context.Context, - req *PerformAdminPurgeRoomRequest, - res *PerformAdminPurgeRoomResponse, -) error { - err := t.Impl.PerformAdminPurgeRoom(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("PerformAdminPurgeRoom req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) PerformAdminDownloadState( - ctx context.Context, - req *PerformAdminDownloadStateRequest, - res *PerformAdminDownloadStateResponse, -) error { - err := t.Impl.PerformAdminDownloadState(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("PerformAdminDownloadState req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) PerformInboundPeek( - ctx context.Context, - req *PerformInboundPeekRequest, - res *PerformInboundPeekResponse, -) error { - err := t.Impl.PerformInboundPeek(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("PerformInboundPeek req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) QueryPublishedRooms( - ctx context.Context, - req *QueryPublishedRoomsRequest, - res *QueryPublishedRoomsResponse, -) error { - err := t.Impl.QueryPublishedRooms(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryPublishedRooms req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) QueryLatestEventsAndState( - ctx context.Context, - req *QueryLatestEventsAndStateRequest, - res *QueryLatestEventsAndStateResponse, -) error { - err := t.Impl.QueryLatestEventsAndState(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryLatestEventsAndState req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) QueryStateAfterEvents( - ctx context.Context, - req *QueryStateAfterEventsRequest, - res *QueryStateAfterEventsResponse, -) error { - err := t.Impl.QueryStateAfterEvents(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryStateAfterEvents req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) QueryEventsByID( - ctx context.Context, - req *QueryEventsByIDRequest, - res *QueryEventsByIDResponse, -) error { - err := t.Impl.QueryEventsByID(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryEventsByID req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) QueryMembershipForUser( - ctx context.Context, - req *QueryMembershipForUserRequest, - res *QueryMembershipForUserResponse, -) error { - err := t.Impl.QueryMembershipForUser(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryMembershipForUser req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) QueryMembershipsForRoom( - ctx context.Context, - req *QueryMembershipsForRoomRequest, - res *QueryMembershipsForRoomResponse, -) error { - err := t.Impl.QueryMembershipsForRoom(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryMembershipsForRoom req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) QueryServerJoinedToRoom( - ctx context.Context, - req *QueryServerJoinedToRoomRequest, - res *QueryServerJoinedToRoomResponse, -) error { - err := t.Impl.QueryServerJoinedToRoom(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryServerJoinedToRoom req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) QueryServerAllowedToSeeEvent( - ctx context.Context, - req *QueryServerAllowedToSeeEventRequest, - res *QueryServerAllowedToSeeEventResponse, -) error { - err := t.Impl.QueryServerAllowedToSeeEvent(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryServerAllowedToSeeEvent req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) QueryMissingEvents( - ctx context.Context, - req *QueryMissingEventsRequest, - res *QueryMissingEventsResponse, -) error { - err := t.Impl.QueryMissingEvents(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryMissingEvents req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) QueryStateAndAuthChain( - ctx context.Context, - req *QueryStateAndAuthChainRequest, - res *QueryStateAndAuthChainResponse, -) error { - err := t.Impl.QueryStateAndAuthChain(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryStateAndAuthChain req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) PerformBackfill( - ctx context.Context, - req *PerformBackfillRequest, - res *PerformBackfillResponse, -) error { - err := t.Impl.PerformBackfill(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("PerformBackfill req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) PerformForget( - ctx context.Context, - req *PerformForgetRequest, - res *PerformForgetResponse, -) error { - err := t.Impl.PerformForget(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("PerformForget req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) QueryRoomVersionCapabilities( - ctx context.Context, - req *QueryRoomVersionCapabilitiesRequest, - res *QueryRoomVersionCapabilitiesResponse, -) error { - err := t.Impl.QueryRoomVersionCapabilities(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryRoomVersionCapabilities req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) QueryRoomVersionForRoom( - ctx context.Context, - req *QueryRoomVersionForRoomRequest, - res *QueryRoomVersionForRoomResponse, -) error { - err := t.Impl.QueryRoomVersionForRoom(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryRoomVersionForRoom req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) SetRoomAlias( - ctx context.Context, - req *SetRoomAliasRequest, - res *SetRoomAliasResponse, -) error { - err := t.Impl.SetRoomAlias(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("SetRoomAlias req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) GetRoomIDForAlias( - ctx context.Context, - req *GetRoomIDForAliasRequest, - res *GetRoomIDForAliasResponse, -) error { - err := t.Impl.GetRoomIDForAlias(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("GetRoomIDForAlias req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) GetAliasesForRoomID( - ctx context.Context, - req *GetAliasesForRoomIDRequest, - res *GetAliasesForRoomIDResponse, -) error { - err := t.Impl.GetAliasesForRoomID(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("GetAliasesForRoomID req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) RemoveRoomAlias( - ctx context.Context, - req *RemoveRoomAliasRequest, - res *RemoveRoomAliasResponse, -) error { - err := t.Impl.RemoveRoomAlias(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("RemoveRoomAlias req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error { - err := t.Impl.QueryCurrentState(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryCurrentState req=%+v res=%+v", js(req), js(res)) - return err -} - -// QueryRoomsForUser retrieves a list of room IDs matching the given query. -func (t *RoomserverInternalAPITrace) QueryRoomsForUser(ctx context.Context, req *QueryRoomsForUserRequest, res *QueryRoomsForUserResponse) error { - err := t.Impl.QueryRoomsForUser(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryRoomsForUser req=%+v res=%+v", js(req), js(res)) - return err -} - -// QueryBulkStateContent does a bulk query for state event content in the given rooms. -func (t *RoomserverInternalAPITrace) QueryBulkStateContent(ctx context.Context, req *QueryBulkStateContentRequest, res *QueryBulkStateContentResponse) error { - err := t.Impl.QueryBulkStateContent(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryBulkStateContent req=%+v res=%+v", js(req), js(res)) - return err -} - -// QuerySharedUsers returns a list of users who share at least 1 room in common with the given user. -func (t *RoomserverInternalAPITrace) QuerySharedUsers(ctx context.Context, req *QuerySharedUsersRequest, res *QuerySharedUsersResponse) error { - err := t.Impl.QuerySharedUsers(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QuerySharedUsers req=%+v res=%+v", js(req), js(res)) - return err -} - -// QueryKnownUsers returns a list of users that we know about from our joined rooms. -func (t *RoomserverInternalAPITrace) QueryKnownUsers(ctx context.Context, req *QueryKnownUsersRequest, res *QueryKnownUsersResponse) error { - err := t.Impl.QueryKnownUsers(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryKnownUsers req=%+v res=%+v", js(req), js(res)) - return err -} - -// QueryServerBannedFromRoom returns whether a server is banned from a room by server ACLs. -func (t *RoomserverInternalAPITrace) QueryServerBannedFromRoom(ctx context.Context, req *QueryServerBannedFromRoomRequest, res *QueryServerBannedFromRoomResponse) error { - err := t.Impl.QueryServerBannedFromRoom(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("QueryServerBannedFromRoom req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *RoomserverInternalAPITrace) QueryAuthChain( - ctx context.Context, - request *QueryAuthChainRequest, - response *QueryAuthChainResponse, -) error { - err := t.Impl.QueryAuthChain(ctx, request, response) - util.GetLogger(ctx).WithError(err).Infof("QueryAuthChain req=%+v res=%+v", js(request), js(response)) - return err -} - -func (t *RoomserverInternalAPITrace) QueryRestrictedJoinAllowed( - ctx context.Context, - request *QueryRestrictedJoinAllowedRequest, - response *QueryRestrictedJoinAllowedResponse, -) error { - err := t.Impl.QueryRestrictedJoinAllowed(ctx, request, response) - util.GetLogger(ctx).WithError(err).Infof("QueryRestrictedJoinAllowed req=%+v res=%+v", js(request), js(response)) - return err -} - -func (t *RoomserverInternalAPITrace) QueryMembershipAtEvent( - ctx context.Context, - request *QueryMembershipAtEventRequest, - response *QueryMembershipAtEventResponse, -) error { - err := t.Impl.QueryMembershipAtEvent(ctx, request, response) - util.GetLogger(ctx).WithError(err).Infof("QueryMembershipAtEvent req=%+v res=%+v", js(request), js(response)) - return err -} - -func js(thing interface{}) string { - b, err := json.Marshal(thing) - if err != nil { - return fmt.Sprintf("Marshal error:%s", err) - } - return string(b) -} diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 76f8298ca..4ef548e19 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -433,7 +433,7 @@ func (r *QueryCurrentStateResponse) UnmarshalJSON(data []byte) error { return nil } -// QueryMembershipAtEventRequest requests the membership events for a user +// QueryMembershipAtEventRequest requests the membership event for a user // for a list of eventIDs. type QueryMembershipAtEventRequest struct { RoomID string @@ -443,9 +443,10 @@ type QueryMembershipAtEventRequest struct { // QueryMembershipAtEventResponse is the response to QueryMembershipAtEventRequest. type QueryMembershipAtEventResponse struct { - // Memberships is a map from eventID to a list of events (if any). Events that - // do not have known state will return an empty array here. - Memberships map[string][]*gomatrixserverlib.HeaderedEvent `json:"memberships"` + // Membership is a map from eventID to membership event. Events that + // do not have known state will return a nil event, resulting in a "leave" membership + // when calculating history visibility. + Membership map[string]*gomatrixserverlib.HeaderedEvent `json:"membership"` } // QueryLeftUsersRequest is a request to calculate users that we (the server) don't share a diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 69d841dda..1083bb237 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/sirupsen/logrus" @@ -216,7 +217,8 @@ func (r *Queryer) QueryMembershipAtEvent( request *api.QueryMembershipAtEventRequest, response *api.QueryMembershipAtEventResponse, ) error { - response.Memberships = make(map[string][]*gomatrixserverlib.HeaderedEvent) + response.Membership = make(map[string]*gomatrixserverlib.HeaderedEvent) + info, err := r.DB.RoomInfo(ctx, request.RoomID) if err != nil { return fmt.Errorf("unable to get roomInfo: %w", err) @@ -234,7 +236,17 @@ func (r *Queryer) QueryMembershipAtEvent( return fmt.Errorf("requested stateKeyNID for %s was not found", request.UserID) } - stateEntries, err := helpers.MembershipAtEvent(ctx, r.DB, info, request.EventIDs, stateKeyNIDs[request.UserID]) + response.Membership, err = r.DB.GetMembershipForHistoryVisibility(ctx, stateKeyNIDs[request.UserID], info, request.EventIDs...) + switch err { + case nil: + return nil + case tables.OptimisationNotSupportedError: // fallthrough, slow way of getting the membership events for each event + default: + return err + } + + response.Membership = make(map[string]*gomatrixserverlib.HeaderedEvent) + stateEntries, err := helpers.MembershipAtEvent(ctx, r.DB, nil, request.EventIDs, stateKeyNIDs[request.UserID]) if err != nil { return fmt.Errorf("unable to get state before event: %w", err) } @@ -258,7 +270,7 @@ func (r *Queryer) QueryMembershipAtEvent( for _, eventID := range request.EventIDs { stateEntry, ok := stateEntries[eventID] if !ok || len(stateEntry) == 0 { - response.Memberships[eventID] = []*gomatrixserverlib.HeaderedEvent{} + response.Membership[eventID] = nil continue } @@ -275,15 +287,15 @@ func (r *Queryer) QueryMembershipAtEvent( return fmt.Errorf("unable to get memberships at state: %w", err) } - res := make([]*gomatrixserverlib.HeaderedEvent, 0, len(memberships)) - + // Iterate over all membership events we got. Given we only query the membership for + // one user and assuming this user only ever has one membership event associated to + // a given event, overwrite any other existing membership events. for i := range memberships { ev := memberships[i] if ev.Type() == gomatrixserverlib.MRoomMember && ev.StateKeyEquals(request.UserID) { - res = append(res, ev.Headered(info.RoomVersion)) + response.Membership[eventID] = ev.Event.Headered(info.RoomVersion) } } - response.Memberships[eventID] = res } return nil diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go deleted file mode 100644 index 556a137ba..000000000 --- a/roomserver/inthttp/client.go +++ /dev/null @@ -1,575 +0,0 @@ -package inthttp - -import ( - "context" - "errors" - "net/http" - - "github.com/matrix-org/gomatrixserverlib" - - asAPI "github.com/matrix-org/dendrite/appservice/api" - fsInputAPI "github.com/matrix-org/dendrite/federationapi/api" - "github.com/matrix-org/dendrite/internal/caching" - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/roomserver/api" - userapi "github.com/matrix-org/dendrite/userapi/api" -) - -const ( - // Alias operations - RoomserverSetRoomAliasPath = "/roomserver/setRoomAlias" - RoomserverGetRoomIDForAliasPath = "/roomserver/GetRoomIDForAlias" - RoomserverGetAliasesForRoomIDPath = "/roomserver/GetAliasesForRoomID" - RoomserverGetCreatorIDForAliasPath = "/roomserver/GetCreatorIDForAlias" - RoomserverRemoveRoomAliasPath = "/roomserver/removeRoomAlias" - - // Input operations - RoomserverInputRoomEventsPath = "/roomserver/inputRoomEvents" - - // Perform operations - RoomserverPerformInvitePath = "/roomserver/performInvite" - RoomserverPerformPeekPath = "/roomserver/performPeek" - RoomserverPerformUnpeekPath = "/roomserver/performUnpeek" - RoomserverPerformRoomUpgradePath = "/roomserver/performRoomUpgrade" - RoomserverPerformJoinPath = "/roomserver/performJoin" - RoomserverPerformLeavePath = "/roomserver/performLeave" - RoomserverPerformBackfillPath = "/roomserver/performBackfill" - RoomserverPerformPublishPath = "/roomserver/performPublish" - RoomserverPerformInboundPeekPath = "/roomserver/performInboundPeek" - RoomserverPerformForgetPath = "/roomserver/performForget" - RoomserverPerformAdminEvacuateRoomPath = "/roomserver/performAdminEvacuateRoom" - RoomserverPerformAdminEvacuateUserPath = "/roomserver/performAdminEvacuateUser" - RoomserverPerformAdminDownloadStatePath = "/roomserver/performAdminDownloadState" - RoomserverPerformAdminPurgeRoomPath = "/roomserver/performAdminPurgeRoom" - - // Query operations - RoomserverQueryLatestEventsAndStatePath = "/roomserver/queryLatestEventsAndState" - RoomserverQueryStateAfterEventsPath = "/roomserver/queryStateAfterEvents" - RoomserverQueryEventsByIDPath = "/roomserver/queryEventsByID" - RoomserverQueryMembershipForUserPath = "/roomserver/queryMembershipForUser" - RoomserverQueryMembershipsForRoomPath = "/roomserver/queryMembershipsForRoom" - RoomserverQueryServerJoinedToRoomPath = "/roomserver/queryServerJoinedToRoomPath" - RoomserverQueryServerAllowedToSeeEventPath = "/roomserver/queryServerAllowedToSeeEvent" - RoomserverQueryMissingEventsPath = "/roomserver/queryMissingEvents" - RoomserverQueryStateAndAuthChainPath = "/roomserver/queryStateAndAuthChain" - RoomserverQueryRoomVersionCapabilitiesPath = "/roomserver/queryRoomVersionCapabilities" - RoomserverQueryRoomVersionForRoomPath = "/roomserver/queryRoomVersionForRoom" - RoomserverQueryPublishedRoomsPath = "/roomserver/queryPublishedRooms" - RoomserverQueryCurrentStatePath = "/roomserver/queryCurrentState" - RoomserverQueryRoomsForUserPath = "/roomserver/queryRoomsForUser" - RoomserverQueryBulkStateContentPath = "/roomserver/queryBulkStateContent" - RoomserverQuerySharedUsersPath = "/roomserver/querySharedUsers" - RoomserverQueryKnownUsersPath = "/roomserver/queryKnownUsers" - RoomserverQueryServerBannedFromRoomPath = "/roomserver/queryServerBannedFromRoom" - RoomserverQueryAuthChainPath = "/roomserver/queryAuthChain" - RoomserverQueryRestrictedJoinAllowed = "/roomserver/queryRestrictedJoinAllowed" - RoomserverQueryMembershipAtEventPath = "/roomserver/queryMembershipAtEvent" - RoomserverQueryLeftMembersPath = "/roomserver/queryLeftMembers" -) - -type httpRoomserverInternalAPI struct { - roomserverURL string - httpClient *http.Client - cache caching.RoomVersionCache -} - -// NewRoomserverClient creates a RoomserverInputAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewRoomserverClient( - roomserverURL string, - httpClient *http.Client, - cache caching.RoomVersionCache, -) (api.RoomserverInternalAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverInternalAPIHTTP: httpClient is ") - } - return &httpRoomserverInternalAPI{ - roomserverURL: roomserverURL, - httpClient: httpClient, - cache: cache, - }, nil -} - -// SetFederationInputAPI no-ops in HTTP client mode as there is no chicken/egg scenario -func (h *httpRoomserverInternalAPI) SetFederationAPI(fsAPI fsInputAPI.RoomserverFederationAPI, keyRing *gomatrixserverlib.KeyRing) { -} - -// SetAppserviceAPI no-ops in HTTP client mode as there is no chicken/egg scenario -func (h *httpRoomserverInternalAPI) SetAppserviceAPI(asAPI asAPI.AppServiceInternalAPI) { -} - -// SetUserAPI no-ops in HTTP client mode as there is no chicken/egg scenario -func (h *httpRoomserverInternalAPI) SetUserAPI(userAPI userapi.RoomserverUserAPI) { -} - -// SetRoomAlias implements RoomserverAliasAPI -func (h *httpRoomserverInternalAPI) SetRoomAlias( - ctx context.Context, - request *api.SetRoomAliasRequest, - response *api.SetRoomAliasResponse, -) error { - return httputil.CallInternalRPCAPI( - "SetRoomAlias", h.roomserverURL+RoomserverSetRoomAliasPath, - h.httpClient, ctx, request, response, - ) -} - -// GetRoomIDForAlias implements RoomserverAliasAPI -func (h *httpRoomserverInternalAPI) GetRoomIDForAlias( - ctx context.Context, - request *api.GetRoomIDForAliasRequest, - response *api.GetRoomIDForAliasResponse, -) error { - return httputil.CallInternalRPCAPI( - "GetRoomIDForAlias", h.roomserverURL+RoomserverGetRoomIDForAliasPath, - h.httpClient, ctx, request, response, - ) -} - -// GetAliasesForRoomID implements RoomserverAliasAPI -func (h *httpRoomserverInternalAPI) GetAliasesForRoomID( - ctx context.Context, - request *api.GetAliasesForRoomIDRequest, - response *api.GetAliasesForRoomIDResponse, -) error { - return httputil.CallInternalRPCAPI( - "GetAliasesForRoomID", h.roomserverURL+RoomserverGetAliasesForRoomIDPath, - h.httpClient, ctx, request, response, - ) -} - -// RemoveRoomAlias implements RoomserverAliasAPI -func (h *httpRoomserverInternalAPI) RemoveRoomAlias( - ctx context.Context, - request *api.RemoveRoomAliasRequest, - response *api.RemoveRoomAliasResponse, -) error { - return httputil.CallInternalRPCAPI( - "RemoveRoomAlias", h.roomserverURL+RoomserverRemoveRoomAliasPath, - h.httpClient, ctx, request, response, - ) -} - -// InputRoomEvents implements RoomserverInputAPI -func (h *httpRoomserverInternalAPI) InputRoomEvents( - ctx context.Context, - request *api.InputRoomEventsRequest, - response *api.InputRoomEventsResponse, -) error { - if err := httputil.CallInternalRPCAPI( - "InputRoomEvents", h.roomserverURL+RoomserverInputRoomEventsPath, - h.httpClient, ctx, request, response, - ); err != nil { - response.ErrMsg = err.Error() - } - return nil -} - -func (h *httpRoomserverInternalAPI) PerformInvite( - ctx context.Context, - request *api.PerformInviteRequest, - response *api.PerformInviteResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformInvite", h.roomserverURL+RoomserverPerformInvitePath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) PerformJoin( - ctx context.Context, - request *api.PerformJoinRequest, - response *api.PerformJoinResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformJoin", h.roomserverURL+RoomserverPerformJoinPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) PerformPeek( - ctx context.Context, - request *api.PerformPeekRequest, - response *api.PerformPeekResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformPeek", h.roomserverURL+RoomserverPerformPeekPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) PerformInboundPeek( - ctx context.Context, - request *api.PerformInboundPeekRequest, - response *api.PerformInboundPeekResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformInboundPeek", h.roomserverURL+RoomserverPerformInboundPeekPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) PerformUnpeek( - ctx context.Context, - request *api.PerformUnpeekRequest, - response *api.PerformUnpeekResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformUnpeek", h.roomserverURL+RoomserverPerformUnpeekPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) PerformRoomUpgrade( - ctx context.Context, - request *api.PerformRoomUpgradeRequest, - response *api.PerformRoomUpgradeResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformRoomUpgrade", h.roomserverURL+RoomserverPerformRoomUpgradePath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) PerformLeave( - ctx context.Context, - request *api.PerformLeaveRequest, - response *api.PerformLeaveResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformLeave", h.roomserverURL+RoomserverPerformLeavePath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) PerformPublish( - ctx context.Context, - request *api.PerformPublishRequest, - response *api.PerformPublishResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformPublish", h.roomserverURL+RoomserverPerformPublishPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) PerformAdminEvacuateRoom( - ctx context.Context, - request *api.PerformAdminEvacuateRoomRequest, - response *api.PerformAdminEvacuateRoomResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformAdminEvacuateRoom", h.roomserverURL+RoomserverPerformAdminEvacuateRoomPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) PerformAdminDownloadState( - ctx context.Context, - request *api.PerformAdminDownloadStateRequest, - response *api.PerformAdminDownloadStateResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformAdminDownloadState", h.roomserverURL+RoomserverPerformAdminDownloadStatePath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) PerformAdminEvacuateUser( - ctx context.Context, - request *api.PerformAdminEvacuateUserRequest, - response *api.PerformAdminEvacuateUserResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformAdminEvacuateUser", h.roomserverURL+RoomserverPerformAdminEvacuateUserPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) PerformAdminPurgeRoom( - ctx context.Context, - request *api.PerformAdminPurgeRoomRequest, - response *api.PerformAdminPurgeRoomResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformAdminPurgeRoom", h.roomserverURL+RoomserverPerformAdminPurgeRoomPath, - h.httpClient, ctx, request, response, - ) -} - -// QueryLatestEventsAndState implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState( - ctx context.Context, - request *api.QueryLatestEventsAndStateRequest, - response *api.QueryLatestEventsAndStateResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryLatestEventsAndState", h.roomserverURL+RoomserverQueryLatestEventsAndStatePath, - h.httpClient, ctx, request, response, - ) -} - -// QueryStateAfterEvents implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryStateAfterEvents( - ctx context.Context, - request *api.QueryStateAfterEventsRequest, - response *api.QueryStateAfterEventsResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryStateAfterEvents", h.roomserverURL+RoomserverQueryStateAfterEventsPath, - h.httpClient, ctx, request, response, - ) -} - -// QueryEventsByID implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryEventsByID( - ctx context.Context, - request *api.QueryEventsByIDRequest, - response *api.QueryEventsByIDResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryEventsByID", h.roomserverURL+RoomserverQueryEventsByIDPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) QueryPublishedRooms( - ctx context.Context, - request *api.QueryPublishedRoomsRequest, - response *api.QueryPublishedRoomsResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryPublishedRooms", h.roomserverURL+RoomserverQueryPublishedRoomsPath, - h.httpClient, ctx, request, response, - ) -} - -// QueryMembershipForUser implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryMembershipForUser( - ctx context.Context, - request *api.QueryMembershipForUserRequest, - response *api.QueryMembershipForUserResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryMembershipForUser", h.roomserverURL+RoomserverQueryMembershipForUserPath, - h.httpClient, ctx, request, response, - ) -} - -// QueryMembershipsForRoom implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryMembershipsForRoom( - ctx context.Context, - request *api.QueryMembershipsForRoomRequest, - response *api.QueryMembershipsForRoomResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryMembershipsForRoom", h.roomserverURL+RoomserverQueryMembershipsForRoomPath, - h.httpClient, ctx, request, response, - ) -} - -// QueryMembershipsForRoom implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryServerJoinedToRoom( - ctx context.Context, - request *api.QueryServerJoinedToRoomRequest, - response *api.QueryServerJoinedToRoomResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryServerJoinedToRoom", h.roomserverURL+RoomserverQueryServerJoinedToRoomPath, - h.httpClient, ctx, request, response, - ) -} - -// QueryServerAllowedToSeeEvent implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryServerAllowedToSeeEvent( - ctx context.Context, - request *api.QueryServerAllowedToSeeEventRequest, - response *api.QueryServerAllowedToSeeEventResponse, -) (err error) { - return httputil.CallInternalRPCAPI( - "QueryServerAllowedToSeeEvent", h.roomserverURL+RoomserverQueryServerAllowedToSeeEventPath, - h.httpClient, ctx, request, response, - ) -} - -// QueryMissingEvents implements RoomServerQueryAPI -func (h *httpRoomserverInternalAPI) QueryMissingEvents( - ctx context.Context, - request *api.QueryMissingEventsRequest, - response *api.QueryMissingEventsResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryMissingEvents", h.roomserverURL+RoomserverQueryMissingEventsPath, - h.httpClient, ctx, request, response, - ) -} - -// QueryStateAndAuthChain implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryStateAndAuthChain( - ctx context.Context, - request *api.QueryStateAndAuthChainRequest, - response *api.QueryStateAndAuthChainResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryStateAndAuthChain", h.roomserverURL+RoomserverQueryStateAndAuthChainPath, - h.httpClient, ctx, request, response, - ) -} - -// PerformBackfill implements RoomServerQueryAPI -func (h *httpRoomserverInternalAPI) PerformBackfill( - ctx context.Context, - request *api.PerformBackfillRequest, - response *api.PerformBackfillResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformBackfill", h.roomserverURL+RoomserverPerformBackfillPath, - h.httpClient, ctx, request, response, - ) -} - -// QueryRoomVersionCapabilities implements RoomServerQueryAPI -func (h *httpRoomserverInternalAPI) QueryRoomVersionCapabilities( - ctx context.Context, - request *api.QueryRoomVersionCapabilitiesRequest, - response *api.QueryRoomVersionCapabilitiesResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryRoomVersionCapabilities", h.roomserverURL+RoomserverQueryRoomVersionCapabilitiesPath, - h.httpClient, ctx, request, response, - ) -} - -// QueryRoomVersionForRoom implements RoomServerQueryAPI -func (h *httpRoomserverInternalAPI) QueryRoomVersionForRoom( - ctx context.Context, - request *api.QueryRoomVersionForRoomRequest, - response *api.QueryRoomVersionForRoomResponse, -) error { - if roomVersion, ok := h.cache.GetRoomVersion(request.RoomID); ok { - response.RoomVersion = roomVersion - return nil - } - err := httputil.CallInternalRPCAPI( - "QueryRoomVersionForRoom", h.roomserverURL+RoomserverQueryRoomVersionForRoomPath, - h.httpClient, ctx, request, response, - ) - if err == nil { - h.cache.StoreRoomVersion(request.RoomID, response.RoomVersion) - } - return err -} - -func (h *httpRoomserverInternalAPI) QueryCurrentState( - ctx context.Context, - request *api.QueryCurrentStateRequest, - response *api.QueryCurrentStateResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryCurrentState", h.roomserverURL+RoomserverQueryCurrentStatePath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) QueryRoomsForUser( - ctx context.Context, - request *api.QueryRoomsForUserRequest, - response *api.QueryRoomsForUserResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryRoomsForUser", h.roomserverURL+RoomserverQueryRoomsForUserPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) QueryBulkStateContent( - ctx context.Context, - request *api.QueryBulkStateContentRequest, - response *api.QueryBulkStateContentResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryBulkStateContent", h.roomserverURL+RoomserverQueryBulkStateContentPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) QuerySharedUsers( - ctx context.Context, - request *api.QuerySharedUsersRequest, - response *api.QuerySharedUsersResponse, -) error { - return httputil.CallInternalRPCAPI( - "QuerySharedUsers", h.roomserverURL+RoomserverQuerySharedUsersPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) QueryKnownUsers( - ctx context.Context, - request *api.QueryKnownUsersRequest, - response *api.QueryKnownUsersResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryKnownUsers", h.roomserverURL+RoomserverQueryKnownUsersPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) QueryAuthChain( - ctx context.Context, - request *api.QueryAuthChainRequest, - response *api.QueryAuthChainResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryAuthChain", h.roomserverURL+RoomserverQueryAuthChainPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) QueryServerBannedFromRoom( - ctx context.Context, - request *api.QueryServerBannedFromRoomRequest, - response *api.QueryServerBannedFromRoomResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryServerBannedFromRoom", h.roomserverURL+RoomserverQueryServerBannedFromRoomPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) QueryRestrictedJoinAllowed( - ctx context.Context, - request *api.QueryRestrictedJoinAllowedRequest, - response *api.QueryRestrictedJoinAllowedResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryRestrictedJoinAllowed", h.roomserverURL+RoomserverQueryRestrictedJoinAllowed, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) PerformForget( - ctx context.Context, - request *api.PerformForgetRequest, - response *api.PerformForgetResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformForget", h.roomserverURL+RoomserverPerformForgetPath, - h.httpClient, ctx, request, response, - ) - -} - -func (h *httpRoomserverInternalAPI) QueryMembershipAtEvent(ctx context.Context, request *api.QueryMembershipAtEventRequest, response *api.QueryMembershipAtEventResponse) error { - return httputil.CallInternalRPCAPI( - "QueryMembershiptAtEvent", h.roomserverURL+RoomserverQueryMembershipAtEventPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpRoomserverInternalAPI) QueryLeftUsers(ctx context.Context, request *api.QueryLeftUsersRequest, response *api.QueryLeftUsersResponse) error { - return httputil.CallInternalRPCAPI( - "RoomserverQueryLeftMembers", h.roomserverURL+RoomserverQueryLeftMembersPath, - h.httpClient, ctx, request, response, - ) -} diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go deleted file mode 100644 index f3a51b0b1..000000000 --- a/roomserver/inthttp/server.go +++ /dev/null @@ -1,216 +0,0 @@ -package inthttp - -import ( - "github.com/gorilla/mux" - - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/roomserver/api" -) - -// AddRoutes adds the RoomserverInternalAPI handlers to the http.ServeMux. -// nolint: gocyclo -func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router, enableMetrics bool) { - internalAPIMux.Handle( - RoomserverInputRoomEventsPath, - httputil.MakeInternalRPCAPI("RoomserverInputRoomEvents", enableMetrics, r.InputRoomEvents), - ) - - internalAPIMux.Handle( - RoomserverPerformInvitePath, - httputil.MakeInternalRPCAPI("RoomserverPerformInvite", enableMetrics, r.PerformInvite), - ) - - internalAPIMux.Handle( - RoomserverPerformJoinPath, - httputil.MakeInternalRPCAPI("RoomserverPerformJoin", enableMetrics, r.PerformJoin), - ) - - internalAPIMux.Handle( - RoomserverPerformLeavePath, - httputil.MakeInternalRPCAPI("RoomserverPerformLeave", enableMetrics, r.PerformLeave), - ) - - internalAPIMux.Handle( - RoomserverPerformPeekPath, - httputil.MakeInternalRPCAPI("RoomserverPerformPeek", enableMetrics, r.PerformPeek), - ) - - internalAPIMux.Handle( - RoomserverPerformInboundPeekPath, - httputil.MakeInternalRPCAPI("RoomserverPerformInboundPeek", enableMetrics, r.PerformInboundPeek), - ) - - internalAPIMux.Handle( - RoomserverPerformUnpeekPath, - httputil.MakeInternalRPCAPI("RoomserverPerformUnpeek", enableMetrics, r.PerformUnpeek), - ) - - internalAPIMux.Handle( - RoomserverPerformRoomUpgradePath, - httputil.MakeInternalRPCAPI("RoomserverPerformRoomUpgrade", enableMetrics, r.PerformRoomUpgrade), - ) - - internalAPIMux.Handle( - RoomserverPerformPublishPath, - httputil.MakeInternalRPCAPI("RoomserverPerformPublish", enableMetrics, r.PerformPublish), - ) - - internalAPIMux.Handle( - RoomserverPerformAdminEvacuateRoomPath, - httputil.MakeInternalRPCAPI("RoomserverPerformAdminEvacuateRoom", enableMetrics, r.PerformAdminEvacuateRoom), - ) - - internalAPIMux.Handle( - RoomserverPerformAdminEvacuateUserPath, - httputil.MakeInternalRPCAPI("RoomserverPerformAdminEvacuateUser", enableMetrics, r.PerformAdminEvacuateUser), - ) - - internalAPIMux.Handle( - RoomserverPerformAdminPurgeRoomPath, - httputil.MakeInternalRPCAPI("RoomserverPerformAdminPurgeRoom", enableMetrics, r.PerformAdminPurgeRoom), - ) - - internalAPIMux.Handle( - RoomserverPerformAdminDownloadStatePath, - httputil.MakeInternalRPCAPI("RoomserverPerformAdminDownloadState", enableMetrics, r.PerformAdminDownloadState), - ) - - internalAPIMux.Handle( - RoomserverQueryPublishedRoomsPath, - httputil.MakeInternalRPCAPI("RoomserverQueryPublishedRooms", enableMetrics, r.QueryPublishedRooms), - ) - - internalAPIMux.Handle( - RoomserverQueryLatestEventsAndStatePath, - httputil.MakeInternalRPCAPI("RoomserverQueryLatestEventsAndState", enableMetrics, r.QueryLatestEventsAndState), - ) - - internalAPIMux.Handle( - RoomserverQueryStateAfterEventsPath, - httputil.MakeInternalRPCAPI("RoomserverQueryStateAfterEvents", enableMetrics, r.QueryStateAfterEvents), - ) - - internalAPIMux.Handle( - RoomserverQueryEventsByIDPath, - httputil.MakeInternalRPCAPI("RoomserverQueryEventsByID", enableMetrics, r.QueryEventsByID), - ) - - internalAPIMux.Handle( - RoomserverQueryMembershipForUserPath, - httputil.MakeInternalRPCAPI("RoomserverQueryMembershipForUser", enableMetrics, r.QueryMembershipForUser), - ) - - internalAPIMux.Handle( - RoomserverQueryMembershipsForRoomPath, - httputil.MakeInternalRPCAPI("RoomserverQueryMembershipsForRoom", enableMetrics, r.QueryMembershipsForRoom), - ) - - internalAPIMux.Handle( - RoomserverQueryServerJoinedToRoomPath, - httputil.MakeInternalRPCAPI("RoomserverQueryServerJoinedToRoom", enableMetrics, r.QueryServerJoinedToRoom), - ) - - internalAPIMux.Handle( - RoomserverQueryServerAllowedToSeeEventPath, - httputil.MakeInternalRPCAPI("RoomserverQueryServerAllowedToSeeEvent", enableMetrics, r.QueryServerAllowedToSeeEvent), - ) - - internalAPIMux.Handle( - RoomserverQueryMissingEventsPath, - httputil.MakeInternalRPCAPI("RoomserverQueryMissingEvents", enableMetrics, r.QueryMissingEvents), - ) - - internalAPIMux.Handle( - RoomserverQueryStateAndAuthChainPath, - httputil.MakeInternalRPCAPI("RoomserverQueryStateAndAuthChain", enableMetrics, r.QueryStateAndAuthChain), - ) - - internalAPIMux.Handle( - RoomserverPerformBackfillPath, - httputil.MakeInternalRPCAPI("RoomserverPerformBackfill", enableMetrics, r.PerformBackfill), - ) - - internalAPIMux.Handle( - RoomserverPerformForgetPath, - httputil.MakeInternalRPCAPI("RoomserverPerformForget", enableMetrics, r.PerformForget), - ) - - internalAPIMux.Handle( - RoomserverQueryRoomVersionCapabilitiesPath, - httputil.MakeInternalRPCAPI("RoomserverQueryRoomVersionCapabilities", enableMetrics, r.QueryRoomVersionCapabilities), - ) - - internalAPIMux.Handle( - RoomserverQueryRoomVersionForRoomPath, - httputil.MakeInternalRPCAPI("RoomserverQueryRoomVersionForRoom", enableMetrics, r.QueryRoomVersionForRoom), - ) - - internalAPIMux.Handle( - RoomserverSetRoomAliasPath, - httputil.MakeInternalRPCAPI("RoomserverSetRoomAlias", enableMetrics, r.SetRoomAlias), - ) - - internalAPIMux.Handle( - RoomserverGetRoomIDForAliasPath, - httputil.MakeInternalRPCAPI("RoomserverGetRoomIDForAlias", enableMetrics, r.GetRoomIDForAlias), - ) - - internalAPIMux.Handle( - RoomserverGetAliasesForRoomIDPath, - httputil.MakeInternalRPCAPI("RoomserverGetAliasesForRoomID", enableMetrics, r.GetAliasesForRoomID), - ) - - internalAPIMux.Handle( - RoomserverRemoveRoomAliasPath, - httputil.MakeInternalRPCAPI("RoomserverRemoveRoomAlias", enableMetrics, r.RemoveRoomAlias), - ) - - internalAPIMux.Handle( - RoomserverQueryCurrentStatePath, - httputil.MakeInternalRPCAPI("RoomserverQueryCurrentState", enableMetrics, r.QueryCurrentState), - ) - - internalAPIMux.Handle( - RoomserverQueryRoomsForUserPath, - httputil.MakeInternalRPCAPI("RoomserverQueryRoomsForUser", enableMetrics, r.QueryRoomsForUser), - ) - - internalAPIMux.Handle( - RoomserverQueryBulkStateContentPath, - httputil.MakeInternalRPCAPI("RoomserverQueryBulkStateContent", enableMetrics, r.QueryBulkStateContent), - ) - - internalAPIMux.Handle( - RoomserverQuerySharedUsersPath, - httputil.MakeInternalRPCAPI("RoomserverQuerySharedUsers", enableMetrics, r.QuerySharedUsers), - ) - - internalAPIMux.Handle( - RoomserverQueryKnownUsersPath, - httputil.MakeInternalRPCAPI("RoomserverQueryKnownUsers", enableMetrics, r.QueryKnownUsers), - ) - - internalAPIMux.Handle( - RoomserverQueryServerBannedFromRoomPath, - httputil.MakeInternalRPCAPI("RoomserverQueryServerBannedFromRoom", enableMetrics, r.QueryServerBannedFromRoom), - ) - - internalAPIMux.Handle( - RoomserverQueryAuthChainPath, - httputil.MakeInternalRPCAPI("RoomserverQueryAuthChain", enableMetrics, r.QueryAuthChain), - ) - - internalAPIMux.Handle( - RoomserverQueryRestrictedJoinAllowed, - httputil.MakeInternalRPCAPI("RoomserverQueryRestrictedJoinAllowed", enableMetrics, r.QueryRestrictedJoinAllowed), - ) - internalAPIMux.Handle( - RoomserverQueryMembershipAtEventPath, - httputil.MakeInternalRPCAPI("RoomserverQueryMembershipAtEventPath", enableMetrics, r.QueryMembershipAtEvent), - ) - - internalAPIMux.Handle( - RoomserverQueryLeftMembersPath, - httputil.MakeInternalRPCAPI("RoomserverQueryLeftMembersPath", enableMetrics, r.QueryLeftUsers), - ) -} diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 0f6b48bf9..c5c51b255 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -15,22 +15,14 @@ package roomserver import ( - "github.com/gorilla/mux" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/internal" - "github.com/matrix-org/dendrite/roomserver/inthttp" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/setup/base" ) -// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions -// on the given input API. -func AddInternalRoutes(router *mux.Router, intAPI api.RoomserverInternalAPI, enableMetrics bool) { - inthttp.AddRoutes(intAPI, router, enableMetrics) -} - // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index 3ec2560d6..304311c4c 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -2,27 +2,22 @@ package roomserver_test import ( "context" - "net/http" "reflect" "testing" "time" - "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/userapi" userAPI "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/federationapi" - "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/syncapi" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/roomserver/inthttp" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" @@ -51,7 +46,7 @@ func TestUsers(t *testing.T) { }) t.Run("kick users", func(t *testing.T) { - usrAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, nil, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(base, rsAPI, nil) rsAPI.SetUserAPI(usrAPI) testKickUsers(t, rsAPI, usrAPI) }) @@ -207,24 +202,7 @@ func Test_QueryLeftUsers(t *testing.T) { } } - t.Run("HTTP API", func(t *testing.T) { - router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter() - roomserver.AddInternalRoutes(router, rsAPI, false) - apiURL, cancel := test.ListenAndServe(t, router, false) - defer cancel() - httpAPI, err := inthttp.NewRoomserverClient(apiURL, &http.Client{Timeout: time.Second * 5}, nil) - if err != nil { - t.Fatalf("failed to create HTTP client") - } - testCase(httpAPI) - }) - t.Run("Monolith", func(t *testing.T) { - testCase(rsAPI) - // also test tracing - traceAPI := &api.RoomserverInternalAPITrace{Impl: rsAPI} - testCase(traceAPI) - }) - + testCase(rsAPI) }) } @@ -249,11 +227,10 @@ func TestPurgeRoom(t *testing.T) { fedClient := base.CreateFederationClient() rsAPI := roomserver.NewInternalAPI(base) - keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fedClient, rsAPI) - userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil) + userAPI := userapi.NewInternalAPI(base, rsAPI, nil) // this starts the JetStream consumers - syncapi.AddPublicRoutes(base, userAPI, rsAPI, keyAPI) + syncapi.AddPublicRoutes(base, userAPI, rsAPI) federationapi.NewInternalAPI(base, fedClient, rsAPI, base.Caches, nil, true) rsAPI.SetFederationAPI(nil, nil) diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index e0b9c56b3..998915122 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -175,4 +175,11 @@ type Database interface { GetLeftUsers(ctx context.Context, userIDs []string) ([]string, error) PurgeRoom(ctx context.Context, roomID string) error UpgradeRoom(ctx context.Context, oldRoomID, newRoomID, eventSender string) error + + // GetMembershipForHistoryVisibility queries the membership events for the given eventIDs. + // Returns a map from (input) eventID -> membership event. If no membership event is found, returns an empty event, resulting in + // a membership of "leave" when calculating history visibility. + GetMembershipForHistoryVisibility( + ctx context.Context, userNID types.EventStateKeyNID, info *types.RoomInfo, eventIDs ...string, + ) (map[string]*gomatrixserverlib.HeaderedEvent, error) } diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index 9b5ed6eda..f4a21c8a7 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -69,6 +69,9 @@ CREATE TABLE IF NOT EXISTS roomserver_events ( auth_event_nids BIGINT[] NOT NULL, is_rejected BOOLEAN NOT NULL DEFAULT FALSE ); + +-- Create an index which helps in resolving membership events (event_type_nid = 5) - (used for history visibility) +CREATE INDEX IF NOT EXISTS roomserver_events_memberships_idx ON roomserver_events (room_nid, event_state_key_nid) WHERE (event_type_nid = 5); ` const insertEventSQL = "" + diff --git a/roomserver/storage/postgres/state_snapshot_table.go b/roomserver/storage/postgres/state_snapshot_table.go index a00c026f4..0e83cfc25 100644 --- a/roomserver/storage/postgres/state_snapshot_table.go +++ b/roomserver/storage/postgres/state_snapshot_table.go @@ -21,10 +21,10 @@ import ( "fmt" "github.com/lib/pq" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -99,10 +99,26 @@ const bulkSelectStateForHistoryVisibilitySQL = ` AND (event_type_nid = 7 OR event_state_key LIKE '%:' || $2); ` +// bulkSelectMembershipForHistoryVisibilitySQL is an optimization to get membership events for a specific user for defined set of events. +// Returns the event_id of the event we want the membership event for, the event_id of the membership event and the membership event JSON. +const bulkSelectMembershipForHistoryVisibilitySQL = ` +SELECT re.event_id, re2.event_id, rej.event_json +FROM roomserver_events re +LEFT JOIN roomserver_state_snapshots rss on re.state_snapshot_nid = rss.state_snapshot_nid +CROSS JOIN unnest(rss.state_block_nids) AS blocks(block_nid) +LEFT JOIN roomserver_state_block rsb ON rsb.state_block_nid = blocks.block_nid +CROSS JOIN unnest(rsb.event_nids) AS rsb2(event_nid) +JOIN roomserver_events re2 ON re2.room_nid = $3 AND re2.event_type_nid = 5 AND re2.event_nid = rsb2.event_nid AND re2.event_state_key_nid = $1 +LEFT JOIN roomserver_event_json rej ON rej.event_nid = re2.event_nid +WHERE re.event_id = ANY($2) + +` + type stateSnapshotStatements struct { - insertStateStmt *sql.Stmt - bulkSelectStateBlockNIDsStmt *sql.Stmt - bulkSelectStateForHistoryVisibilityStmt *sql.Stmt + insertStateStmt *sql.Stmt + bulkSelectStateBlockNIDsStmt *sql.Stmt + bulkSelectStateForHistoryVisibilityStmt *sql.Stmt + bulktSelectMembershipForHistoryVisibilityStmt *sql.Stmt } func CreateStateSnapshotTable(db *sql.DB) error { @@ -110,13 +126,14 @@ func CreateStateSnapshotTable(db *sql.DB) error { return err } -func PrepareStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) { +func PrepareStateSnapshotTable(db *sql.DB) (*stateSnapshotStatements, error) { s := &stateSnapshotStatements{} return s, sqlutil.StatementList{ {&s.insertStateStmt, insertStateSQL}, {&s.bulkSelectStateBlockNIDsStmt, bulkSelectStateBlockNIDsSQL}, {&s.bulkSelectStateForHistoryVisibilityStmt, bulkSelectStateForHistoryVisibilitySQL}, + {&s.bulktSelectMembershipForHistoryVisibilityStmt, bulkSelectMembershipForHistoryVisibilitySQL}, }.Prepare(db) } @@ -185,3 +202,45 @@ func (s *stateSnapshotStatements) BulkSelectStateForHistoryVisibility( } return results, rows.Err() } + +func (s *stateSnapshotStatements) BulkSelectMembershipForHistoryVisibility( + ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID, roomInfo *types.RoomInfo, eventIDs ...string, +) (map[string]*gomatrixserverlib.HeaderedEvent, error) { + stmt := sqlutil.TxStmt(txn, s.bulktSelectMembershipForHistoryVisibilityStmt) + rows, err := stmt.QueryContext(ctx, userNID, pq.Array(eventIDs), roomInfo.RoomNID) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + result := make(map[string]*gomatrixserverlib.HeaderedEvent, len(eventIDs)) + var evJson []byte + var eventID string + var membershipEventID string + + knownEvents := make(map[string]*gomatrixserverlib.HeaderedEvent, len(eventIDs)) + + for rows.Next() { + if err = rows.Scan(&eventID, &membershipEventID, &evJson); err != nil { + return nil, err + } + if len(evJson) == 0 { + result[eventID] = &gomatrixserverlib.HeaderedEvent{} + continue + } + // If we already know this event, don't try to marshal the json again + if ev, ok := knownEvents[membershipEventID]; ok { + result[eventID] = ev + continue + } + event, err := gomatrixserverlib.NewEventFromTrustedJSON(evJson, false, roomInfo.RoomVersion) + if err != nil { + result[eventID] = &gomatrixserverlib.HeaderedEvent{} + // not fatal + continue + } + he := event.Headered(roomInfo.RoomVersion) + result[eventID] = he + knownEvents[membershipEventID] = he + } + return result, rows.Err() +} diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 654b078d2..f8672496b 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -51,6 +51,12 @@ func (d *Database) SupportsConcurrentRoomInputs() bool { return true } +func (d *Database) GetMembershipForHistoryVisibility( + ctx context.Context, userNID types.EventStateKeyNID, roomInfo *types.RoomInfo, eventIDs ...string, +) (map[string]*gomatrixserverlib.HeaderedEvent, error) { + return d.StateSnapshotTable.BulkSelectMembershipForHistoryVisibility(ctx, nil, userNID, roomInfo, eventIDs...) +} + func (d *Database) EventTypeNIDs( ctx context.Context, eventTypes []string, ) (map[string]types.EventTypeNID, error) { diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index 930ad14dd..e57e1a4bf 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -152,6 +153,10 @@ func (s *stateSnapshotStatements) BulkSelectStateForHistoryVisibility( return nil, tables.OptimisationNotSupportedError } +func (s *stateSnapshotStatements) BulkSelectMembershipForHistoryVisibility(ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID, roomInfo *types.RoomInfo, eventIDs ...string) (map[string]*gomatrixserverlib.HeaderedEvent, error) { + return nil, tables.OptimisationNotSupportedError +} + func (s *stateSnapshotStatements) selectStateBlockNIDsForRoomNID( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, ) ([]types.StateBlockNID, error) { diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index 64145f83d..c7f1064db 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -91,6 +91,10 @@ type StateSnapshot interface { // which users are in a room faster than having to load the entire room state. In the // case of SQLite, this will return tables.OptimisationNotSupportedError. BulkSelectStateForHistoryVisibility(ctx context.Context, txn *sql.Tx, stateSnapshotNID types.StateSnapshotNID, domain string) ([]types.EventNID, error) + + BulkSelectMembershipForHistoryVisibility( + ctx context.Context, txn *sql.Tx, userNID types.EventStateKeyNID, roomInfo *types.RoomInfo, eventIDs ...string, + ) (map[string]*gomatrixserverlib.HeaderedEvent, error) } type StateBlock interface { diff --git a/roomserver/storage/tables/state_snapshot_table_test.go b/roomserver/storage/tables/state_snapshot_table_test.go index b2e59377d..c7c991b20 100644 --- a/roomserver/storage/tables/state_snapshot_table_test.go +++ b/roomserver/storage/tables/state_snapshot_table_test.go @@ -29,6 +29,8 @@ func mustCreateStateSnapshotTable(t *testing.T, dbType test.DBType) (tab tables. assert.NoError(t, err) err = postgres.CreateEventsTable(db) assert.NoError(t, err) + err = postgres.CreateEventJSONTable(db) + assert.NoError(t, err) err = postgres.CreateStateBlockTable(db) assert.NoError(t, err) // ... and then the snapshot table itself diff --git a/setup/base/base.go b/setup/base/base.go index 6ea68119d..aabdd7937 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -17,7 +17,6 @@ package base import ( "bytes" "context" - "crypto/tls" "database/sql" "embed" "encoding/json" @@ -38,8 +37,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/prometheus/client_golang/prometheus/promhttp" "go.uber.org/atomic" - "golang.org/x/net/http2" - "golang.org/x/net/http2/h2c" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/caching" @@ -53,19 +50,9 @@ import ( "github.com/sirupsen/logrus" - appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - asinthttp "github.com/matrix-org/dendrite/appservice/inthttp" - federationAPI "github.com/matrix-org/dendrite/federationapi/api" - federationIntHTTP "github.com/matrix-org/dendrite/federationapi/inthttp" - keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" - keyinthttp "github.com/matrix-org/dendrite/keyserver/inthttp" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - rsinthttp "github.com/matrix-org/dendrite/roomserver/inthttp" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" - userapi "github.com/matrix-org/dendrite/userapi/api" - userapiinthttp "github.com/matrix-org/dendrite/userapi/inthttp" ) //go:embed static/*.gotmpl @@ -78,7 +65,6 @@ var staticContent embed.FS // Must be closed when shutting down. type BaseDendrite struct { *process.ProcessContext - componentName string tracerCloser io.Closer PublicClientAPIMux *mux.Router PublicFederationAPIMux *mux.Router @@ -86,12 +72,9 @@ type BaseDendrite struct { PublicMediaAPIMux *mux.Router PublicWellKnownAPIMux *mux.Router PublicStaticMux *mux.Router - InternalAPIMux *mux.Router DendriteAdminMux *mux.Router SynapseAdminMux *mux.Router NATS *jetstream.NATSInstance - UseHTTPAPIs bool - apiHttpClient *http.Client Cfg *config.Dendrite Caches *caching.Caches DNSCache *gomatrixserverlib.DNSCache @@ -105,38 +88,26 @@ type BaseDendrite struct { const NoListener = "" const HTTPServerTimeout = time.Minute * 5 -const HTTPClientTimeout = time.Second * 30 type BaseDendriteOptions int const ( DisableMetrics BaseDendriteOptions = iota - UseHTTPAPIs - PolylithMode ) // NewBaseDendrite creates a new instance to be used by a component. -// The componentName is used for logging purposes, and should be a friendly name -// of the compontent running, e.g. "SyncAPI" -func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...BaseDendriteOptions) *BaseDendrite { +func NewBaseDendrite(cfg *config.Dendrite, options ...BaseDendriteOptions) *BaseDendrite { platformSanityChecks() - useHTTPAPIs := false enableMetrics := true - isMonolith := true for _, opt := range options { switch opt { case DisableMetrics: enableMetrics = false - case UseHTTPAPIs: - useHTTPAPIs = true - case PolylithMode: - isMonolith = false - useHTTPAPIs = true } } configErrors := &config.ConfigErrors{} - cfg.Verify(configErrors, isMonolith) + cfg.Verify(configErrors) if len(*configErrors) > 0 { for _, err := range *configErrors { logrus.Errorf("Configuration error: %s", err) @@ -145,7 +116,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base } internal.SetupStdLogging() - internal.SetupHookLogging(cfg.Logging, componentName) + internal.SetupHookLogging(cfg.Logging) internal.SetupPprof() logrus.Infof("Dendrite version %s", internal.VersionString()) @@ -154,14 +125,13 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base logrus.Warn("Open registration is enabled") } - closer, err := cfg.SetupTracing("Dendrite" + componentName) + closer, err := cfg.SetupTracing() if err != nil { logrus.WithError(err).Panicf("failed to start opentracing") } var fts *fulltext.Search - isSyncOrMonolith := componentName == "syncapi" || isMonolith - if cfg.SyncAPI.Fulltext.Enabled && isSyncOrMonolith { + if cfg.SyncAPI.Fulltext.Enabled { fts, err = fulltext.New(cfg.SyncAPI.Fulltext) if err != nil { logrus.WithError(err).Panicf("failed to create full text") @@ -196,32 +166,12 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base ) } - apiClient := http.Client{ - Timeout: time.Minute * 10, - Transport: &http2.Transport{ - AllowHTTP: true, - DialTLS: func(network, addr string, _ *tls.Config) (net.Conn, error) { - // Ordinarily HTTP/2 would expect TLS, but the remote listener is - // H2C-enabled (HTTP/2 without encryption). Overriding the DialTLS - // function with a plain Dial allows us to trick the HTTP client - // into establishing a HTTP/2 connection without TLS. - // TODO: Eventually we will want to look at authenticating and - // encrypting these internal HTTP APIs, at which point we will have - // to reconsider H2C and change all this anyway. - return net.Dial(network, addr) - }, - }, - } - // If we're in monolith mode, we'll set up a global pool of database // connections. A component is welcome to use this pool if they don't // have a separate database config of their own. var db *sql.DB var writer sqlutil.Writer if cfg.Global.DatabaseOptions.ConnectionString != "" { - if !isMonolith { - logrus.Panic("Using a global database connection pool is not supported in polylith deployments") - } if cfg.Global.DatabaseOptions.ConnectionString.IsSQLite() { logrus.Panic("Using a global database connection pool is not supported with SQLite databases") } @@ -246,8 +196,6 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base return &BaseDendrite{ ProcessContext: process.NewProcessContext(), - componentName: componentName, - UseHTTPAPIs: useHTTPAPIs, tracerCloser: closer, Cfg: cfg, Caches: caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, enableMetrics), @@ -258,11 +206,9 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base PublicMediaAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicMediaPathPrefix).Subrouter().UseEncodedPath(), PublicWellKnownAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicWellKnownPrefix).Subrouter().UseEncodedPath(), PublicStaticMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicStaticPath).Subrouter().UseEncodedPath(), - InternalAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(), DendriteAdminMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.DendriteAdminPathPrefix).Subrouter().UseEncodedPath(), SynapseAdminMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.SynapseAdminPathPrefix).Subrouter().UseEncodedPath(), NATS: &jetstream.NATSInstance{}, - apiHttpClient: &apiClient, Database: db, // set if monolith with global connection pool only DatabaseWriter: writer, // set if monolith with global connection pool only EnableMetrics: enableMetrics, @@ -300,52 +246,6 @@ func (b *BaseDendrite) DatabaseConnection(dbProperties *config.DatabaseOptions, return nil, nil, fmt.Errorf("no database connections configured") } -// AppserviceHTTPClient returns the AppServiceInternalAPI for hitting the appservice component over HTTP. -func (b *BaseDendrite) AppserviceHTTPClient() appserviceAPI.AppServiceInternalAPI { - a, err := asinthttp.NewAppserviceClient(b.Cfg.AppServiceURL(), b.apiHttpClient) - if err != nil { - logrus.WithError(err).Panic("CreateHTTPAppServiceAPIs failed") - } - return a -} - -// RoomserverHTTPClient returns RoomserverInternalAPI for hitting the roomserver over HTTP. -func (b *BaseDendrite) RoomserverHTTPClient() roomserverAPI.RoomserverInternalAPI { - rsAPI, err := rsinthttp.NewRoomserverClient(b.Cfg.RoomServerURL(), b.apiHttpClient, b.Caches) - if err != nil { - logrus.WithError(err).Panic("RoomserverHTTPClient failed", b.apiHttpClient) - } - return rsAPI -} - -// UserAPIClient returns UserInternalAPI for hitting the userapi over HTTP. -func (b *BaseDendrite) UserAPIClient() userapi.UserInternalAPI { - userAPI, err := userapiinthttp.NewUserAPIClient(b.Cfg.UserAPIURL(), b.apiHttpClient) - if err != nil { - logrus.WithError(err).Panic("UserAPIClient failed", b.apiHttpClient) - } - return userAPI -} - -// FederationAPIHTTPClient returns FederationInternalAPI for hitting -// the federation API server over HTTP -func (b *BaseDendrite) FederationAPIHTTPClient() federationAPI.FederationInternalAPI { - f, err := federationIntHTTP.NewFederationAPIClient(b.Cfg.FederationAPIURL(), b.apiHttpClient, b.Caches) - if err != nil { - logrus.WithError(err).Panic("FederationAPIHTTPClient failed", b.apiHttpClient) - } - return f -} - -// KeyServerHTTPClient returns KeyInternalAPI for hitting the key server over HTTP -func (b *BaseDendrite) KeyServerHTTPClient() keyserverAPI.KeyInternalAPI { - f, err := keyinthttp.NewKeyServerClient(b.Cfg.KeyServerURL(), b.apiHttpClient) - if err != nil { - logrus.WithError(err).Panic("KeyServerHTTPClient failed", b.apiHttpClient) - } - return f -} - // PushGatewayHTTPClient returns a new client for interacting with (external) Push Gateways. func (b *BaseDendrite) PushGatewayHTTPClient() pushgateway.Client { return pushgateway.NewHTTPClient(b.Cfg.UserAPI.PushGatewayDisableTLSValidation) @@ -442,20 +342,18 @@ func (b *BaseDendrite) ConfigureAdminEndpoints() { }) } -// SetupAndServeHTTP sets up the HTTP server to serve endpoints registered on -// ApiMux under /api/ and adds a prometheus handler under /metrics. +// SetupAndServeHTTP sets up the HTTP server to serve client & federation APIs +// and adds a prometheus handler under /_dendrite/metrics. func (b *BaseDendrite) SetupAndServeHTTP( - internalHTTPAddr, externalHTTPAddr config.HTTPAddress, + externalHTTPAddr config.HTTPAddress, certFile, keyFile *string, ) { // Manually unlocked right before actually serving requests, // as we don't return from this method (defer doesn't work). b.startupLock.Lock() - internalAddr, _ := internalHTTPAddr.Address() externalAddr, _ := externalHTTPAddr.Address() externalRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() - internalRouter := externalRouter externalServ := &http.Server{ Addr: string(externalAddr), @@ -465,25 +363,6 @@ func (b *BaseDendrite) SetupAndServeHTTP( return b.ProcessContext.Context() }, } - internalServ := externalServ - - if internalAddr != NoListener && externalAddr != internalAddr { - // H2C allows us to accept HTTP/2 connections without TLS - // encryption. Since we don't currently require any form of - // authentication or encryption on these internal HTTP APIs, - // H2C gives us all of the advantages of HTTP/2 (such as - // stream multiplexing and avoiding head-of-line blocking) - // without enabling TLS. - internalH2S := &http2.Server{} - internalRouter = mux.NewRouter().SkipClean(true).UseEncodedPath() - internalServ = &http.Server{ - Addr: string(internalAddr), - Handler: h2c.NewHandler(internalRouter, internalH2S), - BaseContext: func(_ net.Listener) context.Context { - return b.ProcessContext.Context() - }, - } - } b.configureHTTPErrors() @@ -492,9 +371,8 @@ func (b *BaseDendrite) SetupAndServeHTTP( http.Redirect(w, r, httputil.PublicStaticPath, http.StatusFound) }) - internalRouter.PathPrefix(httputil.InternalPathPrefix).Handler(b.InternalAPIMux) if b.Cfg.Global.Metrics.Enabled { - internalRouter.Handle("/metrics", httputil.WrapHandlerInBasicAuth(promhttp.Handler(), b.Cfg.Global.Metrics.BasicAuth)) + externalRouter.Handle("/metrics", httputil.WrapHandlerInBasicAuth(promhttp.Handler(), b.Cfg.Global.Metrics.BasicAuth)) } b.ConfigureAdminEndpoints() @@ -528,7 +406,7 @@ func (b *BaseDendrite) SetupAndServeHTTP( }) federationHandler = sentryHandler.Handle(b.PublicFederationAPIMux) } - internalRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(b.DendriteAdminMux) + externalRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(b.DendriteAdminMux) externalRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(clientHandler) if !b.Cfg.Global.DisableFederation { externalRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(b.PublicKeyAPIMux) @@ -540,38 +418,11 @@ func (b *BaseDendrite) SetupAndServeHTTP( externalRouter.PathPrefix(httputil.PublicStaticPath).Handler(b.PublicStaticMux) b.startupLock.Unlock() - if internalAddr != NoListener && internalAddr != externalAddr { - go func() { - var internalShutdown atomic.Bool // RegisterOnShutdown can be called more than once - logrus.Infof("Starting internal %s listener on %s", b.componentName, internalServ.Addr) - b.ProcessContext.ComponentStarted() - internalServ.RegisterOnShutdown(func() { - if internalShutdown.CompareAndSwap(false, true) { - b.ProcessContext.ComponentFinished() - logrus.Infof("Stopped internal HTTP listener") - } - }) - if certFile != nil && keyFile != nil { - if err := internalServ.ListenAndServeTLS(*certFile, *keyFile); err != nil { - if err != http.ErrServerClosed { - logrus.WithError(err).Fatal("failed to serve HTTPS") - } - } - } else { - if err := internalServ.ListenAndServe(); err != nil { - if err != http.ErrServerClosed { - logrus.WithError(err).Fatal("failed to serve HTTP") - } - } - } - logrus.Infof("Stopped internal %s listener on %s", b.componentName, internalServ.Addr) - }() - } if externalAddr != NoListener { go func() { var externalShutdown atomic.Bool // RegisterOnShutdown can be called more than once - logrus.Infof("Starting external %s listener on %s", b.componentName, externalServ.Addr) + logrus.Infof("Starting external listener on %s", externalServ.Addr) b.ProcessContext.ComponentStarted() externalServ.RegisterOnShutdown(func() { if externalShutdown.CompareAndSwap(false, true) { @@ -592,7 +443,7 @@ func (b *BaseDendrite) SetupAndServeHTTP( } } } - logrus.Infof("Stopped external %s listener on %s", b.componentName, externalServ.Addr) + logrus.Infof("Stopped external listener on %s", externalServ.Addr) }() } @@ -600,7 +451,6 @@ func (b *BaseDendrite) SetupAndServeHTTP( <-b.ProcessContext.WaitForShutdown() logrus.Infof("Stopping HTTP listeners") - _ = internalServ.Shutdown(context.Background()) _ = externalServ.Shutdown(context.Background()) logrus.Infof("Stopped HTTP listeners") } diff --git a/setup/base/base_test.go b/setup/base/base_test.go index 61cb530a9..d906294c0 100644 --- a/setup/base/base_test.go +++ b/setup/base/base_test.go @@ -35,7 +35,7 @@ func TestLandingPage(t *testing.T) { s.Close() // start base with the listener and wait for it to be started - go b.SetupAndServeHTTP("", config.HTTPAddress(s.URL), nil, nil) + go b.SetupAndServeHTTP(config.HTTPAddress(s.URL), nil, nil) time.Sleep(time.Millisecond * 10) // When hitting /, we should be redirected to /_matrix/static, which should contain the landing page diff --git a/setup/config/config.go b/setup/config/config.go index 2b38cd512..848766162 100644 --- a/setup/config/config.go +++ b/setup/config/config.go @@ -79,8 +79,6 @@ type Dendrite struct { // Any information derived from the configuration options for later use. Derived Derived `yaml:"-"` - - IsMonolith bool `yaml:"-"` } // TODO: Kill Derived @@ -114,15 +112,6 @@ type Derived struct { // servers from creating RoomIDs in exclusive application service namespaces } -type InternalAPIOptions struct { - Listen HTTPAddress `yaml:"listen"` - Connect HTTPAddress `yaml:"connect"` -} - -type ExternalAPIOptions struct { - Listen HTTPAddress `yaml:"listen"` -} - // A Path on the filesystem. type Path string @@ -191,7 +180,7 @@ type ConfigErrors []string // Load a yaml config file for a server run as multiple processes or as a monolith. // Checks the config to ensure that it is valid. -func Load(configPath string, monolith bool) (*Dendrite, error) { +func Load(configPath string) (*Dendrite, error) { configData, err := os.ReadFile(configPath) if err != nil { return nil, err @@ -202,28 +191,26 @@ func Load(configPath string, monolith bool) (*Dendrite, error) { } // Pass the current working directory and os.ReadFile so that they can // be mocked in the tests - return loadConfig(basePath, configData, os.ReadFile, monolith) + return loadConfig(basePath, configData, os.ReadFile) } func loadConfig( basePath string, configData []byte, readFile func(string) ([]byte, error), - monolithic bool, ) (*Dendrite, error) { var c Dendrite c.Defaults(DefaultOpts{ - Generate: false, - Monolithic: monolithic, + Generate: false, + SingleDatabase: true, }) - c.IsMonolith = monolithic var err error if err = yaml.Unmarshal(configData, &c); err != nil { return nil, err } - if err = c.check(monolithic); err != nil { + if err = c.check(); err != nil { return nil, err } @@ -333,8 +320,8 @@ func (config *Dendrite) Derive() error { } type DefaultOpts struct { - Generate bool - Monolithic bool + Generate bool + SingleDatabase bool } // SetDefaults sets default config values if they are not explicitly set. @@ -355,9 +342,9 @@ func (c *Dendrite) Defaults(opts DefaultOpts) { c.Wiring() } -func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) { +func (c *Dendrite) Verify(configErrs *ConfigErrors) { type verifiable interface { - Verify(configErrs *ConfigErrors, isMonolith bool) + Verify(configErrs *ConfigErrors) } for _, c := range []verifiable{ &c.Global, &c.ClientAPI, &c.FederationAPI, @@ -365,7 +352,7 @@ func (c *Dendrite) Verify(configErrs *ConfigErrors, isMonolith bool) { &c.SyncAPI, &c.UserAPI, &c.AppServiceAPI, &c.RelayAPI, &c.MSCs, } { - c.Verify(configErrs, isMonolith) + c.Verify(configErrs) } } @@ -415,14 +402,6 @@ func checkNotEmpty(configErrs *ConfigErrors, key, value string) { } } -// checkNotZero verifies the given value is not zero in the configuration. -// If it is, adds an error to the list. -func checkNotZero(configErrs *ConfigErrors, key string, value int64) { - if value == 0 { - configErrs.Add(fmt.Sprintf("missing config key %q", key)) - } -} - // checkPositive verifies the given value is positive (zero included) // in the configuration. If it is not, adds an error to the list. func checkPositive(configErrs *ConfigErrors, key string, value int64) { @@ -431,26 +410,6 @@ func checkPositive(configErrs *ConfigErrors, key string, value int64) { } } -// checkURL verifies that the parameter is a valid URL -func checkURL(configErrs *ConfigErrors, key, value string) { - if value == "" { - configErrs.Add(fmt.Sprintf("missing config key %q", key)) - return - } - url, err := url.Parse(value) - if err != nil { - configErrs.Add(fmt.Sprintf("config key %q contains invalid URL (%s)", key, err.Error())) - return - } - switch url.Scheme { - case "http": - case "https": - default: - configErrs.Add(fmt.Sprintf("config key %q URL should be http:// or https://", key)) - return - } -} - // checkLogging verifies the parameters logging.* are valid. func (config *Dendrite) checkLogging(configErrs *ConfigErrors) { for _, logrusHook := range config.Logging { @@ -461,7 +420,7 @@ func (config *Dendrite) checkLogging(configErrs *ConfigErrors) { // check returns an error type containing all errors found within the config // file. -func (config *Dendrite) check(_ bool) error { // monolithic +func (config *Dendrite) check() error { // monolithic var configErrs ConfigErrors if config.Version != Version { @@ -528,58 +487,13 @@ func readKeyPEM(path string, data []byte, enforceKeyIDFormat bool) (gomatrixserv } } -// AppServiceURL returns a HTTP URL for where the appservice component is listening. -func (config *Dendrite) AppServiceURL() string { - // Hard code the appservice server to talk HTTP for now. - // If we support HTTPS we need to think of a practical way to do certificate validation. - // People setting up servers shouldn't need to get a certificate valid for the public - // internet for an internal API. - return string(config.AppServiceAPI.InternalAPI.Connect) -} - -// FederationAPIURL returns an HTTP URL for where the federation API is listening. -func (config *Dendrite) FederationAPIURL() string { - // Hard code the federationapi to talk HTTP for now. - // If we support HTTPS we need to think of a practical way to do certificate validation. - // People setting up servers shouldn't need to get a certificate valid for the public - // internet for an internal API. - return string(config.FederationAPI.InternalAPI.Connect) -} - -// RoomServerURL returns an HTTP URL for where the roomserver is listening. -func (config *Dendrite) RoomServerURL() string { - // Hard code the roomserver to talk HTTP for now. - // If we support HTTPS we need to think of a practical way to do certificate validation. - // People setting up servers shouldn't need to get a certificate valid for the public - // internet for an internal API. - return string(config.RoomServer.InternalAPI.Connect) -} - -// UserAPIURL returns an HTTP URL for where the userapi is listening. -func (config *Dendrite) UserAPIURL() string { - // Hard code the userapi to talk HTTP for now. - // If we support HTTPS we need to think of a practical way to do certificate validation. - // People setting up servers shouldn't need to get a certificate valid for the public - // internet for an internal API. - return string(config.UserAPI.InternalAPI.Connect) -} - -// KeyServerURL returns an HTTP URL for where the key server is listening. -func (config *Dendrite) KeyServerURL() string { - // Hard code the key server to talk HTTP for now. - // If we support HTTPS we need to think of a practical way to do certificate validation. - // People setting up servers shouldn't need to get a certificate valid for the public - // internet for an internal API. - return string(config.KeyServer.InternalAPI.Connect) -} - // SetupTracing configures the opentracing using the supplied configuration. -func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err error) { +func (config *Dendrite) SetupTracing() (closer io.Closer, err error) { if !config.Tracing.Enabled { return io.NopCloser(bytes.NewReader([]byte{})), nil } return config.Tracing.Jaeger.InitGlobalTracer( - serviceName, + "Dendrite", jaegerconfig.Logger(logrusLogger{logrus.StandardLogger()}), jaegerconfig.Metrics(jaegermetrics.NullFactory), ) diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go index bd21826fe..37e20a978 100644 --- a/setup/config/config_appservice.go +++ b/setup/config/config_appservice.go @@ -22,15 +22,13 @@ import ( "strings" log "github.com/sirupsen/logrus" - yaml "gopkg.in/yaml.v2" + "gopkg.in/yaml.v2" ) type AppServiceAPI struct { Matrix *Global `yaml:"-"` Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit - InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` - // DisableTLSValidation disables the validation of X.509 TLS certs // on appservice endpoints. This is not recommended in production! DisableTLSValidation bool `yaml:"disable_tls_validation"` @@ -39,18 +37,9 @@ type AppServiceAPI struct { } func (c *AppServiceAPI) Defaults(opts DefaultOpts) { - if !opts.Monolithic { - c.InternalAPI.Listen = "http://localhost:7777" - c.InternalAPI.Connect = "http://localhost:7777" - } } -func (c *AppServiceAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { - if isMonolith { // polylith required configs below - return - } - checkURL(configErrs, "app_service_api.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "app_service_api.internal_api.connect", string(c.InternalAPI.Connect)) +func (c *AppServiceAPI) Verify(configErrs *ConfigErrors) { } // ApplicationServiceNamespace is the namespace that a specific application diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go index 1deba6bb5..b6c74a75f 100644 --- a/setup/config/config_clientapi.go +++ b/setup/config/config_clientapi.go @@ -9,9 +9,6 @@ type ClientAPI struct { Matrix *Global `yaml:"-"` Derived *Derived `yaml:"-"` // TODO: Nuke Derived from orbit - InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` - ExternalAPI ExternalAPIOptions `yaml:"external_api,omitempty"` - // If set disables new users from registering (except via shared // secrets) RegistrationDisabled bool `yaml:"registration_disabled"` @@ -58,11 +55,6 @@ type ClientAPI struct { } func (c *ClientAPI) Defaults(opts DefaultOpts) { - if !opts.Monolithic { - c.InternalAPI.Listen = "http://localhost:7771" - c.InternalAPI.Connect = "http://localhost:7771" - c.ExternalAPI.Listen = "http://[::]:8071" - } c.RegistrationSharedSecret = "" c.RecaptchaPublicKey = "" c.RecaptchaPrivateKey = "" @@ -74,7 +66,7 @@ func (c *ClientAPI) Defaults(opts DefaultOpts) { c.RateLimiting.Defaults() } -func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { +func (c *ClientAPI) Verify(configErrs *ConfigErrors) { c.TURN.Verify(configErrs) c.RateLimiting.Verify(configErrs) if c.RecaptchaEnabled { @@ -108,12 +100,6 @@ func (c *ClientAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { ) } } - if isMonolith { // polylith required configs below - return - } - checkURL(configErrs, "client_api.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "client_api.internal_api.connect", string(c.InternalAPI.Connect)) - checkURL(configErrs, "client_api.external_api.listen", string(c.ExternalAPI.Listen)) } type TURN struct { diff --git a/setup/config/config_federationapi.go b/setup/config/config_federationapi.go index cd7d90562..8c1540b57 100644 --- a/setup/config/config_federationapi.go +++ b/setup/config/config_federationapi.go @@ -1,13 +1,12 @@ package config -import "github.com/matrix-org/gomatrixserverlib" +import ( + "github.com/matrix-org/gomatrixserverlib" +) type FederationAPI struct { Matrix *Global `yaml:"-"` - InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` - ExternalAPI ExternalAPIOptions `yaml:"external_api,omitempty"` - // The database stores information used by the federation destination queues to // send transactions to remote servers. Database DatabaseOptions `yaml:"database,omitempty"` @@ -42,12 +41,6 @@ type FederationAPI struct { } func (c *FederationAPI) Defaults(opts DefaultOpts) { - if !opts.Monolithic { - c.InternalAPI.Listen = "http://localhost:7772" - c.InternalAPI.Connect = "http://localhost:7772" - c.ExternalAPI.Listen = "http://[::]:8072" - c.Database.Defaults(10) - } c.FederationMaxRetries = 16 c.P2PFederationRetriesUntilAssumedOffline = 1 c.DisableTLSValidation = false @@ -68,22 +61,16 @@ func (c *FederationAPI) Defaults(opts DefaultOpts) { }, }, } - if !opts.Monolithic { + if !opts.SingleDatabase { c.Database.ConnectionString = "file:federationapi.db" } } } -func (c *FederationAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { - if isMonolith { // polylith required configs below - return - } +func (c *FederationAPI) Verify(configErrs *ConfigErrors) { if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "federation_api.database.connection_string", string(c.Database.ConnectionString)) } - checkURL(configErrs, "federation_api.external_api.listen", string(c.ExternalAPI.Listen)) - checkURL(configErrs, "federation_api.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "federation_api.internal_api.connect", string(c.InternalAPI.Connect)) } // The config for setting a proxy to use for server->server requests diff --git a/setup/config/config_global.go b/setup/config/config_global.go index 804eb1a2d..ed980afaa 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -38,7 +38,6 @@ type Global struct { // component does not specify any database options of its own, then this pool of // connections will be used instead. This way we don't have to manage connection // counts on a per-component basis, but can instead do it for the entire monolith. - // In a polylith deployment, this will be ignored. DatabaseOptions DatabaseOptions `yaml:"database,omitempty"` // The server name to delegate server-server communications to, with optional port @@ -93,7 +92,7 @@ func (c *Global) Defaults(opts DefaultOpts) { } } c.KeyValidityPeriod = time.Hour * 24 * 7 - if opts.Monolithic { + if opts.SingleDatabase { c.DatabaseOptions.Defaults(90) } c.JetStream.Defaults(opts) @@ -105,7 +104,7 @@ func (c *Global) Defaults(opts DefaultOpts) { c.Cache.Defaults() } -func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) { +func (c *Global) Verify(configErrs *ConfigErrors) { checkNotEmpty(configErrs, "global.server_name", string(c.ServerName)) checkNotEmpty(configErrs, "global.private_key", string(c.PrivateKeyPath)) @@ -113,13 +112,13 @@ func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) { v.Verify(configErrs) } - c.JetStream.Verify(configErrs, isMonolith) - c.Metrics.Verify(configErrs, isMonolith) - c.Sentry.Verify(configErrs, isMonolith) - c.DNSCache.Verify(configErrs, isMonolith) - c.ServerNotices.Verify(configErrs, isMonolith) - c.ReportStats.Verify(configErrs, isMonolith) - c.Cache.Verify(configErrs, isMonolith) + c.JetStream.Verify(configErrs) + c.Metrics.Verify(configErrs) + c.Sentry.Verify(configErrs) + c.DNSCache.Verify(configErrs) + c.ServerNotices.Verify(configErrs) + c.ReportStats.Verify(configErrs) + c.Cache.Verify(configErrs) } func (c *Global) IsLocalServerName(serverName gomatrixserverlib.ServerName) bool { @@ -267,7 +266,7 @@ func (c *Metrics) Defaults(opts DefaultOpts) { } } -func (c *Metrics) Verify(configErrs *ConfigErrors, isMonolith bool) { +func (c *Metrics) Verify(configErrs *ConfigErrors) { } // ServerNotices defines the configuration used for sending server notices @@ -293,7 +292,7 @@ func (c *ServerNotices) Defaults(opts DefaultOpts) { } } -func (c *ServerNotices) Verify(errors *ConfigErrors, isMonolith bool) {} +func (c *ServerNotices) Verify(errors *ConfigErrors) {} type Cache struct { EstimatedMaxSize DataUnit `yaml:"max_size_estimated"` @@ -305,7 +304,7 @@ func (c *Cache) Defaults() { c.MaxAge = time.Hour } -func (c *Cache) Verify(errors *ConfigErrors, isMonolith bool) { +func (c *Cache) Verify(errors *ConfigErrors) { checkPositive(errors, "max_size_estimated", int64(c.EstimatedMaxSize)) } @@ -323,7 +322,7 @@ func (c *ReportStats) Defaults() { c.Endpoint = "https://matrix.org/report-usage-stats/push" } -func (c *ReportStats) Verify(configErrs *ConfigErrors, isMonolith bool) { +func (c *ReportStats) Verify(configErrs *ConfigErrors) { if c.Enabled { checkNotEmpty(configErrs, "global.report_stats.endpoint", c.Endpoint) } @@ -344,7 +343,7 @@ func (c *Sentry) Defaults() { c.Enabled = false } -func (c *Sentry) Verify(configErrs *ConfigErrors, isMonolith bool) { +func (c *Sentry) Verify(configErrs *ConfigErrors) { } type DatabaseOptions struct { @@ -364,8 +363,7 @@ func (c *DatabaseOptions) Defaults(conns int) { c.ConnMaxLifetimeSeconds = -1 } -func (c *DatabaseOptions) Verify(configErrs *ConfigErrors, isMonolith bool) { -} +func (c *DatabaseOptions) Verify(configErrs *ConfigErrors) {} // MaxIdleConns returns maximum idle connections to the DB func (c DatabaseOptions) MaxIdleConns() int { @@ -397,7 +395,7 @@ func (c *DNSCacheOptions) Defaults() { c.CacheLifetime = time.Minute * 5 } -func (c *DNSCacheOptions) Verify(configErrs *ConfigErrors, isMonolith bool) { +func (c *DNSCacheOptions) Verify(configErrs *ConfigErrors) { checkPositive(configErrs, "cache_size", int64(c.CacheSize)) checkPositive(configErrs, "cache_lifetime", int64(c.CacheLifetime)) } diff --git a/setup/config/config_jetstream.go b/setup/config/config_jetstream.go index ef8bf014b..b8abed25c 100644 --- a/setup/config/config_jetstream.go +++ b/setup/config/config_jetstream.go @@ -41,11 +41,4 @@ func (c *JetStream) Defaults(opts DefaultOpts) { } } -func (c *JetStream) Verify(configErrs *ConfigErrors, isMonolith bool) { - if isMonolith { // polylith required configs below - return - } - // If we are running in a polylith deployment then we need at least - // one NATS JetStream server to talk to. - checkNotZero(configErrs, "global.jetstream.addresses", int64(len(c.Addresses))) -} +func (c *JetStream) Verify(configErrs *ConfigErrors) {} diff --git a/setup/config/config_keyserver.go b/setup/config/config_keyserver.go index dca9ca9f5..64710d957 100644 --- a/setup/config/config_keyserver.go +++ b/setup/config/config_keyserver.go @@ -3,31 +3,19 @@ package config type KeyServer struct { Matrix *Global `yaml:"-"` - InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` - Database DatabaseOptions `yaml:"database,omitempty"` } func (c *KeyServer) Defaults(opts DefaultOpts) { - if !opts.Monolithic { - c.InternalAPI.Listen = "http://localhost:7779" - c.InternalAPI.Connect = "http://localhost:7779" - c.Database.Defaults(10) - } if opts.Generate { - if !opts.Monolithic { + if !opts.SingleDatabase { c.Database.ConnectionString = "file:keyserver.db" } } } -func (c *KeyServer) Verify(configErrs *ConfigErrors, isMonolith bool) { - if isMonolith { // polylith required configs below - return - } +func (c *KeyServer) Verify(configErrs *ConfigErrors) { if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "key_server.database.connection_string", string(c.Database.ConnectionString)) } - checkURL(configErrs, "key_server.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "key_server.internal_api.connect", string(c.InternalAPI.Connect)) } diff --git a/setup/config/config_mediaapi.go b/setup/config/config_mediaapi.go index 53a8219eb..030bc3754 100644 --- a/setup/config/config_mediaapi.go +++ b/setup/config/config_mediaapi.go @@ -7,9 +7,6 @@ import ( type MediaAPI struct { Matrix *Global `yaml:"-"` - InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` - ExternalAPI ExternalAPIOptions `yaml:"external_api,omitempty"` - // The MediaAPI database stores information about files uploaded and downloaded // by local users. It is only accessed by the MediaAPI. Database DatabaseOptions `yaml:"database,omitempty"` @@ -39,12 +36,6 @@ type MediaAPI struct { var DefaultMaxFileSizeBytes = FileSizeBytes(10485760) func (c *MediaAPI) Defaults(opts DefaultOpts) { - if !opts.Monolithic { - c.InternalAPI.Listen = "http://localhost:7774" - c.InternalAPI.Connect = "http://localhost:7774" - c.ExternalAPI.Listen = "http://[::]:8074" - c.Database.Defaults(5) - } c.MaxFileSizeBytes = DefaultMaxFileSizeBytes c.MaxThumbnailGenerators = 10 if opts.Generate { @@ -65,14 +56,14 @@ func (c *MediaAPI) Defaults(opts DefaultOpts) { ResizeMethod: "scale", }, } - if !opts.Monolithic { + if !opts.SingleDatabase { c.Database.ConnectionString = "file:mediaapi.db" } c.BasePath = "./media_store" } } -func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { +func (c *MediaAPI) Verify(configErrs *ConfigErrors) { checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath)) checkPositive(configErrs, "media_api.max_file_size_bytes", int64(c.MaxFileSizeBytes)) checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators)) @@ -81,13 +72,8 @@ func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].width", i), int64(size.Width)) checkPositive(configErrs, fmt.Sprintf("media_api.thumbnail_sizes[%d].height", i), int64(size.Height)) } - if isMonolith { // polylith required configs below - return - } + if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString)) } - checkURL(configErrs, "media_api.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "media_api.internal_api.connect", string(c.InternalAPI.Connect)) - checkURL(configErrs, "media_api.external_api.listen", string(c.ExternalAPI.Listen)) } diff --git a/setup/config/config_mscs.go b/setup/config/config_mscs.go index 6d5ff39a5..21d4b4da0 100644 --- a/setup/config/config_mscs.go +++ b/setup/config/config_mscs.go @@ -14,11 +14,8 @@ type MSCs struct { } func (c *MSCs) Defaults(opts DefaultOpts) { - if !opts.Monolithic { - c.Database.Defaults(5) - } if opts.Generate { - if !opts.Monolithic { + if !opts.SingleDatabase { c.Database.ConnectionString = "file:mscs.db" } } @@ -34,10 +31,7 @@ func (c *MSCs) Enabled(msc string) bool { return false } -func (c *MSCs) Verify(configErrs *ConfigErrors, isMonolith bool) { - if isMonolith { // polylith required configs below - return - } +func (c *MSCs) Verify(configErrs *ConfigErrors) { if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "mscs.database.connection_string", string(c.Database.ConnectionString)) } diff --git a/setup/config/config_relayapi.go b/setup/config/config_relayapi.go index 5a6b093d4..ba7b78082 100644 --- a/setup/config/config_relayapi.go +++ b/setup/config/config_relayapi.go @@ -17,36 +17,21 @@ package config type RelayAPI struct { Matrix *Global `yaml:"-"` - InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` - ExternalAPI ExternalAPIOptions `yaml:"external_api,omitempty"` - // The database stores information used by the relay queue to // forward transactions to remote servers. Database DatabaseOptions `yaml:"database,omitempty"` } func (c *RelayAPI) Defaults(opts DefaultOpts) { - if !opts.Monolithic { - c.InternalAPI.Listen = "http://localhost:7775" - c.InternalAPI.Connect = "http://localhost:7775" - c.ExternalAPI.Listen = "http://[::]:8075" - c.Database.Defaults(10) - } if opts.Generate { - if !opts.Monolithic { + if !opts.SingleDatabase { c.Database.ConnectionString = "file:relayapi.db" } } } -func (c *RelayAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { - if isMonolith { // polylith required configs below - return - } +func (c *RelayAPI) Verify(configErrs *ConfigErrors) { if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "relay_api.database.connection_string", string(c.Database.ConnectionString)) } - checkURL(configErrs, "relay_api.external_api.listen", string(c.ExternalAPI.Listen)) - checkURL(configErrs, "relay_api.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "relay_api.internal_api.connect", string(c.InternalAPI.Connect)) } diff --git a/setup/config/config_roomserver.go b/setup/config/config_roomserver.go index 5e3b7f2ec..319c2419c 100644 --- a/setup/config/config_roomserver.go +++ b/setup/config/config_roomserver.go @@ -3,31 +3,19 @@ package config type RoomServer struct { Matrix *Global `yaml:"-"` - InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` - Database DatabaseOptions `yaml:"database,omitempty"` } func (c *RoomServer) Defaults(opts DefaultOpts) { - if !opts.Monolithic { - c.InternalAPI.Listen = "http://localhost:7770" - c.InternalAPI.Connect = "http://localhost:7770" - c.Database.Defaults(20) - } if opts.Generate { - if !opts.Monolithic { + if !opts.SingleDatabase { c.Database.ConnectionString = "file:roomserver.db" } } } -func (c *RoomServer) Verify(configErrs *ConfigErrors, isMonolith bool) { - if isMonolith { // polylith required configs below - return - } +func (c *RoomServer) Verify(configErrs *ConfigErrors) { if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "room_server.database.connection_string", string(c.Database.ConnectionString)) } - checkURL(configErrs, "room_server.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "room_server.internal_ap.connect", string(c.InternalAPI.Connect)) } diff --git a/setup/config/config_syncapi.go b/setup/config/config_syncapi.go index a87da3732..756f4cfb3 100644 --- a/setup/config/config_syncapi.go +++ b/setup/config/config_syncapi.go @@ -3,9 +3,6 @@ package config type SyncAPI struct { Matrix *Global `yaml:"-"` - InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` - ExternalAPI ExternalAPIOptions `yaml:"external_api,omitempty"` - Database DatabaseOptions `yaml:"database,omitempty"` RealIPHeader string `yaml:"real_ip_header"` @@ -14,31 +11,19 @@ type SyncAPI struct { } func (c *SyncAPI) Defaults(opts DefaultOpts) { - if !opts.Monolithic { - c.InternalAPI.Listen = "http://localhost:7773" - c.InternalAPI.Connect = "http://localhost:7773" - c.ExternalAPI.Listen = "http://localhost:8073" - c.Database.Defaults(20) - } c.Fulltext.Defaults(opts) if opts.Generate { - if !opts.Monolithic { + if !opts.SingleDatabase { c.Database.ConnectionString = "file:syncapi.db" } } } -func (c *SyncAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { - c.Fulltext.Verify(configErrs, isMonolith) - if isMonolith { // polylith required configs below - return - } +func (c *SyncAPI) Verify(configErrs *ConfigErrors) { + c.Fulltext.Verify(configErrs) if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "sync_api.database", string(c.Database.ConnectionString)) } - checkURL(configErrs, "sync_api.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "sync_api.internal_api.connect", string(c.InternalAPI.Connect)) - checkURL(configErrs, "sync_api.external_api.listen", string(c.ExternalAPI.Listen)) } type Fulltext struct { @@ -54,7 +39,7 @@ func (f *Fulltext) Defaults(opts DefaultOpts) { f.Language = "en" } -func (f *Fulltext) Verify(configErrs *ConfigErrors, isMonolith bool) { +func (f *Fulltext) Verify(configErrs *ConfigErrors) { if !f.Enabled { return } diff --git a/setup/config/config_test.go b/setup/config/config_test.go index ffbf4c3c5..79407f30d 100644 --- a/setup/config/config_test.go +++ b/setup/config/config_test.go @@ -30,14 +30,13 @@ func TestLoadConfigRelative(t *testing.T) { "/my/config/dir/matrix_key.pem": testKey, "/my/config/dir/tls_cert.pem": testCert, }.readFile, - false, ) if err != nil { t.Error("failed to load config:", err) } configErrors := &ConfigErrors{} - cfg.Verify(configErrors, false) + cfg.Verify(configErrors) if len(*configErrors) > 0 { for _, err := range *configErrors { logrus.Errorf("Configuration error: %s", err) @@ -81,9 +80,6 @@ global: jetstream: addresses: ["test"] app_service_api: - internal_api: - listen: http://localhost:7777 - connect: http://localhost:7777 database: connection_string: file:appservice.db max_open_conns: 100 @@ -91,11 +87,6 @@ app_service_api: conn_max_lifetime: -1 config_files: [] client_api: - internal_api: - listen: http://localhost:7771 - connect: http://localhost:7771 - external_api: - listen: http://[::]:8071 registration_disabled: true registration_shared_secret: "" enable_registration_captcha: false @@ -109,38 +100,16 @@ client_api: turn_shared_secret: "" turn_username: "" turn_password: "" -current_state_server: - internal_api: - listen: http://localhost:7782 - connect: http://localhost:7782 - database: - connection_string: file:currentstate.db - max_open_conns: 100 - max_idle_conns: 2 - conn_max_lifetime: -1 federation_api: - internal_api: - listen: http://localhost:7772 - connect: http://localhost:7772 - external_api: - listen: http://[::]:8072 database: connection_string: file:federationapi.db key_server: - internal_api: - listen: http://localhost:7779 - connect: http://localhost:7779 database: connection_string: file:keyserver.db max_open_conns: 100 max_idle_conns: 2 conn_max_lifetime: -1 media_api: - internal_api: - listen: http://localhost:7774 - connect: http://localhost:7774 - external_api: - listen: http://[::]:8074 database: connection_string: file:mediaapi.db max_open_conns: 100 @@ -161,18 +130,12 @@ media_api: height: 480 method: scale room_server: - internal_api: - listen: http://localhost:7770 - connect: http://localhost:7770 database: connection_string: file:roomserver.db max_open_conns: 100 max_idle_conns: 2 conn_max_lifetime: -1 server_key_api: - internal_api: - listen: http://localhost:7780 - connect: http://localhost:7780 database: connection_string: file:serverkeyapi.db max_open_conns: 100 @@ -186,18 +149,12 @@ server_key_api: - key_id: ed25519:a_RXGa public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ sync_api: - internal_api: - listen: http://localhost:7773 - connect: http://localhost:7773 database: connection_string: file:syncapi.db max_open_conns: 100 max_idle_conns: 2 conn_max_lifetime: -1 user_api: - internal_api: - listen: http://localhost:7781 - connect: http://localhost:7781 account_database: connection_string: file:userapi_accounts.db max_open_conns: 100 @@ -209,11 +166,6 @@ user_api: max_idle_conns: 2 conn_max_lifetime: -1 relay_api: - internal_api: - listen: http://localhost:7775 - connect: http://localhost:7775 - external_api: - listen: http://[::]:8075 database: connection_string: file:relayapi.db mscs: diff --git a/setup/config/config_userapi.go b/setup/config/config_userapi.go index f8ad41d93..e64a3910c 100644 --- a/setup/config/config_userapi.go +++ b/setup/config/config_userapi.go @@ -5,8 +5,6 @@ import "golang.org/x/crypto/bcrypt" type UserAPI struct { Matrix *Global `yaml:"-"` - InternalAPI InternalAPIOptions `yaml:"internal_api,omitempty"` - // The cost when hashing passwords. BCryptCost int `yaml:"bcrypt_cost"` @@ -28,28 +26,18 @@ type UserAPI struct { const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes func (c *UserAPI) Defaults(opts DefaultOpts) { - if !opts.Monolithic { - c.InternalAPI.Listen = "http://localhost:7781" - c.InternalAPI.Connect = "http://localhost:7781" - c.AccountDatabase.Defaults(10) - } c.BCryptCost = bcrypt.DefaultCost c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS if opts.Generate { - if !opts.Monolithic { + if !opts.SingleDatabase { c.AccountDatabase.ConnectionString = "file:userapi_accounts.db" } } } -func (c *UserAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { +func (c *UserAPI) Verify(configErrs *ConfigErrors) { checkPositive(configErrs, "user_api.openid_token_lifetime_ms", c.OpenIDTokenLifetimeMS) - if isMonolith { // polylith required configs below - return - } if c.Matrix.DatabaseOptions.ConnectionString == "" { checkNotEmpty(configErrs, "user_api.account_database.connection_string", string(c.AccountDatabase.ConnectionString)) } - checkURL(configErrs, "user_api.internal_api.listen", string(c.InternalAPI.Listen)) - checkURL(configErrs, "user_api.internal_api.connect", string(c.InternalAPI.Connect)) } diff --git a/setup/flags.go b/setup/flags.go index a9dac61a1..869caa280 100644 --- a/setup/flags.go +++ b/setup/flags.go @@ -43,7 +43,7 @@ func ParseFlags(monolith bool) *config.Dendrite { logrus.Fatal("--config must be supplied") } - cfg, err := config.Load(*configPath, monolith) + cfg, err := config.Load(*configPath) if err != nil { logrus.Fatalf("Invalid config file: %s", err) diff --git a/setup/monolith.go b/setup/monolith.go index 5bbe4019e..d8c652234 100644 --- a/setup/monolith.go +++ b/setup/monolith.go @@ -21,7 +21,6 @@ import ( "github.com/matrix-org/dendrite/federationapi" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/transactions" - keyAPI "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/mediaapi" "github.com/matrix-org/dendrite/relayapi" relayAPI "github.com/matrix-org/dendrite/relayapi/api" @@ -45,7 +44,6 @@ type Monolith struct { FederationAPI federationAPI.FederationInternalAPI RoomserverAPI roomserverAPI.RoomserverInternalAPI UserAPI userapi.UserInternalAPI - KeyAPI keyAPI.KeyInternalAPI RelayAPI relayAPI.RelayInternalAPI // Optional @@ -61,19 +59,14 @@ func (m *Monolith) AddAllPublicRoutes(base *base.BaseDendrite) { } clientapi.AddPublicRoutes( base, m.FedClient, m.RoomserverAPI, m.AppserviceAPI, transactions.New(), - m.FederationAPI, m.UserAPI, userDirectoryProvider, m.KeyAPI, + m.FederationAPI, m.UserAPI, userDirectoryProvider, m.ExtPublicRoomsProvider, ) federationapi.AddPublicRoutes( - base, m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationAPI, - m.KeyAPI, nil, - ) - mediaapi.AddPublicRoutes( - base, m.UserAPI, m.Client, - ) - syncapi.AddPublicRoutes( - base, m.UserAPI, m.RoomserverAPI, m.KeyAPI, + base, m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationAPI, nil, ) + mediaapi.AddPublicRoutes(base, m.UserAPI, m.Client) + syncapi.AddPublicRoutes(base, m.UserAPI, m.RoomserverAPI) if m.RelayAPI != nil { relayapi.AddPublicRoutes(base, m.KeyRing, m.RelayAPI) diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go index 0388fcc53..f12fbbfcb 100644 --- a/setup/mscs/msc2836/msc2836_test.go +++ b/setup/mscs/msc2836/msc2836_test.go @@ -499,7 +499,7 @@ func assertUnsignedChildren(t *testing.T, ev gomatrixserverlib.ClientEvent, relT } type testUserAPI struct { - userapi.UserInternalAPITrace + userapi.UserInternalAPI accessTokens map[string]userapi.Device } @@ -516,7 +516,7 @@ func (u *testUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAc type testRoomserverAPI struct { // use a trace API as it implements method stubs so we don't need to have them here. // We'll override the functions we care about. - roomserver.RoomserverInternalAPITrace + roomserver.RoomserverInternalAPI userToJoinedRooms map[string][]string events map[string]*gomatrixserverlib.HeaderedEvent } @@ -548,8 +548,8 @@ func injectEvents(t *testing.T, userAPI userapi.UserInternalAPI, rsAPI roomserve t.Helper() cfg := &config.Dendrite{} cfg.Defaults(config.DefaultOpts{ - Generate: true, - Monolithic: true, + Generate: true, + SingleDatabase: true, }) cfg.Global.ServerName = "localhost" cfg.MSCs.Database.ConnectionString = "file:msc2836_test.db" diff --git a/syncapi/consumers/keychange.go b/syncapi/consumers/keychange.go index 92f081500..5faaefb8e 100644 --- a/syncapi/consumers/keychange.go +++ b/syncapi/consumers/keychange.go @@ -19,7 +19,6 @@ import ( "encoding/json" "github.com/getsentry/sentry-go" - "github.com/matrix-org/dendrite/keyserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" @@ -28,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/streams" "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/userapi/api" "github.com/nats-io/nats.go" "github.com/sirupsen/logrus" ) diff --git a/syncapi/consumers/sendtodevice.go b/syncapi/consumers/sendtodevice.go index 356e83263..32208c585 100644 --- a/syncapi/consumers/sendtodevice.go +++ b/syncapi/consumers/sendtodevice.go @@ -25,7 +25,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/tidwall/gjson" - keyapi "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" @@ -33,6 +32,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/streams" "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/userapi/api" ) // OutputSendToDeviceEventConsumer consumes events that originated in the EDU server. @@ -42,7 +42,7 @@ type OutputSendToDeviceEventConsumer struct { durable string topic string db storage.Database - keyAPI keyapi.SyncKeyAPI + userAPI api.SyncKeyAPI isLocalServerName func(gomatrixserverlib.ServerName) bool stream streams.StreamProvider notifier *notifier.Notifier @@ -55,7 +55,7 @@ func NewOutputSendToDeviceEventConsumer( cfg *config.SyncAPI, js nats.JetStreamContext, store storage.Database, - keyAPI keyapi.SyncKeyAPI, + userAPI api.SyncKeyAPI, notifier *notifier.Notifier, stream streams.StreamProvider, ) *OutputSendToDeviceEventConsumer { @@ -65,7 +65,7 @@ func NewOutputSendToDeviceEventConsumer( topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), durable: cfg.Matrix.JetStream.Durable("SyncAPISendToDeviceConsumer"), db: store, - keyAPI: keyAPI, + userAPI: userAPI, isLocalServerName: cfg.Matrix.IsLocalServerName, notifier: notifier, stream: stream, @@ -116,7 +116,7 @@ func (s *OutputSendToDeviceEventConsumer) onMessage(ctx context.Context, msgs [] _, senderDomain, _ := gomatrixserverlib.SplitID('@', output.Sender) if requestingDeviceID != "" && !s.isLocalServerName(senderDomain) { // Mark the requesting device as stale, if we don't know about it. - if err = s.keyAPI.PerformMarkAsStaleIfNeeded(ctx, &keyapi.PerformMarkAsStaleRequest{ + if err = s.userAPI.PerformMarkAsStaleIfNeeded(ctx, &api.PerformMarkAsStaleRequest{ UserID: output.Sender, Domain: senderDomain, DeviceID: requestingDeviceID, }, &struct{}{}); err != nil { logger.WithError(err).Errorf("failed to mark as stale if needed") diff --git a/syncapi/internal/history_visibility.go b/syncapi/internal/history_visibility.go index 71d7ddd15..ee695f0f5 100644 --- a/syncapi/internal/history_visibility.go +++ b/syncapi/internal/history_visibility.go @@ -33,8 +33,7 @@ func init() { } // calculateHistoryVisibilityDuration stores the time it takes to -// calculate the history visibility. In polylith mode the roundtrip -// to the roomserver is included in this time. +// calculate the history visibility. var calculateHistoryVisibilityDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "dendrite", @@ -121,10 +120,7 @@ func ApplyHistoryVisibilityFilter( // Get the mapping from eventID -> eventVisibility eventsFiltered := make([]*gomatrixserverlib.HeaderedEvent, 0, len(events)) - visibilities, err := visibilityForEvents(ctx, rsAPI, events, userID, events[0].RoomID()) - if err != nil { - return eventsFiltered, err - } + visibilities := visibilityForEvents(ctx, rsAPI, events, userID, events[0].RoomID()) for _, ev := range events { evVis := visibilities[ev.EventID()] evVis.membershipCurrent = membershipCurrent @@ -175,7 +171,7 @@ func visibilityForEvents( rsAPI api.SyncRoomserverAPI, events []*gomatrixserverlib.HeaderedEvent, userID, roomID string, -) (map[string]eventVisibility, error) { +) map[string]eventVisibility { eventIDs := make([]string, len(events)) for i := range events { eventIDs[i] = events[i].EventID() @@ -185,6 +181,7 @@ func visibilityForEvents( // get the membership events for all eventIDs membershipResp := &api.QueryMembershipAtEventResponse{} + err := rsAPI.QueryMembershipAtEvent(ctx, &api.QueryMembershipAtEventRequest{ RoomID: roomID, EventIDs: eventIDs, @@ -201,19 +198,20 @@ func visibilityForEvents( membershipAtEvent: gomatrixserverlib.Leave, // default to leave, to not expose events by accident visibility: event.Visibility, } - membershipEvs, ok := membershipResp.Memberships[eventID] - if !ok { + ev, ok := membershipResp.Membership[eventID] + if !ok || ev == nil { result[eventID] = vis continue } - for _, ev := range membershipEvs { - membership, err := ev.Membership() - if err != nil { - return result, err - } - vis.membershipAtEvent = membership + + membership, err := ev.Membership() + if err != nil { + result[eventID] = vis + continue } + vis.membershipAtEvent = membership + result[eventID] = vis } - return result, nil + return result } diff --git a/syncapi/internal/keychange.go b/syncapi/internal/keychange.go index 4867f7d9e..e7f677c85 100644 --- a/syncapi/internal/keychange.go +++ b/syncapi/internal/keychange.go @@ -18,22 +18,22 @@ import ( "context" "strings" + keytypes "github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/sirupsen/logrus" "github.com/tidwall/gjson" - keyapi "github.com/matrix-org/dendrite/keyserver/api" - keytypes "github.com/matrix-org/dendrite/keyserver/types" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/userapi/api" ) // DeviceOTKCounts adds one-time key counts to the /sync response -func DeviceOTKCounts(ctx context.Context, keyAPI keyapi.SyncKeyAPI, userID, deviceID string, res *types.Response) error { - var queryRes keyapi.QueryOneTimeKeysResponse - _ = keyAPI.QueryOneTimeKeys(ctx, &keyapi.QueryOneTimeKeysRequest{ +func DeviceOTKCounts(ctx context.Context, keyAPI api.SyncKeyAPI, userID, deviceID string, res *types.Response) error { + var queryRes api.QueryOneTimeKeysResponse + _ = keyAPI.QueryOneTimeKeys(ctx, &api.QueryOneTimeKeysRequest{ UserID: userID, DeviceID: deviceID, }, &queryRes) @@ -48,7 +48,7 @@ func DeviceOTKCounts(ctx context.Context, keyAPI keyapi.SyncKeyAPI, userID, devi // was filled in, else false if there are no new device list changes because there is nothing to catch up on. The response MUST // be already filled in with join/leave information. func DeviceListCatchup( - ctx context.Context, db storage.SharedUsers, keyAPI keyapi.SyncKeyAPI, rsAPI roomserverAPI.SyncRoomserverAPI, + ctx context.Context, db storage.SharedUsers, userAPI api.SyncKeyAPI, rsAPI roomserverAPI.SyncRoomserverAPI, userID string, res *types.Response, from, to types.StreamPosition, ) (newPos types.StreamPosition, hasNew bool, err error) { @@ -74,8 +74,8 @@ func DeviceListCatchup( if from > 0 { offset = int64(from) } - var queryRes keyapi.QueryKeyChangesResponse - _ = keyAPI.QueryKeyChanges(ctx, &keyapi.QueryKeyChangesRequest{ + var queryRes api.QueryKeyChangesResponse + _ = userAPI.QueryKeyChanges(ctx, &api.QueryKeyChangesRequest{ Offset: offset, ToOffset: toOffset, }, &queryRes) diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go index 53f3e5a40..4bb851668 100644 --- a/syncapi/internal/keychange_test.go +++ b/syncapi/internal/keychange_test.go @@ -9,7 +9,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" - keyapi "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -22,49 +21,49 @@ var ( type mockKeyAPI struct{} -func (k *mockKeyAPI) PerformMarkAsStaleIfNeeded(ctx context.Context, req *keyapi.PerformMarkAsStaleRequest, res *struct{}) error { +func (k *mockKeyAPI) PerformMarkAsStaleIfNeeded(ctx context.Context, req *userapi.PerformMarkAsStaleRequest, res *struct{}) error { return nil } -func (k *mockKeyAPI) PerformUploadKeys(ctx context.Context, req *keyapi.PerformUploadKeysRequest, res *keyapi.PerformUploadKeysResponse) error { +func (k *mockKeyAPI) PerformUploadKeys(ctx context.Context, req *userapi.PerformUploadKeysRequest, res *userapi.PerformUploadKeysResponse) error { return nil } func (k *mockKeyAPI) SetUserAPI(i userapi.UserInternalAPI) {} // PerformClaimKeys claims one-time keys for use in pre-key messages -func (k *mockKeyAPI) PerformClaimKeys(ctx context.Context, req *keyapi.PerformClaimKeysRequest, res *keyapi.PerformClaimKeysResponse) error { +func (k *mockKeyAPI) PerformClaimKeys(ctx context.Context, req *userapi.PerformClaimKeysRequest, res *userapi.PerformClaimKeysResponse) error { return nil } -func (k *mockKeyAPI) PerformDeleteKeys(ctx context.Context, req *keyapi.PerformDeleteKeysRequest, res *keyapi.PerformDeleteKeysResponse) error { +func (k *mockKeyAPI) PerformDeleteKeys(ctx context.Context, req *userapi.PerformDeleteKeysRequest, res *userapi.PerformDeleteKeysResponse) error { return nil } -func (k *mockKeyAPI) PerformUploadDeviceKeys(ctx context.Context, req *keyapi.PerformUploadDeviceKeysRequest, res *keyapi.PerformUploadDeviceKeysResponse) error { +func (k *mockKeyAPI) PerformUploadDeviceKeys(ctx context.Context, req *userapi.PerformUploadDeviceKeysRequest, res *userapi.PerformUploadDeviceKeysResponse) error { return nil } -func (k *mockKeyAPI) PerformUploadDeviceSignatures(ctx context.Context, req *keyapi.PerformUploadDeviceSignaturesRequest, res *keyapi.PerformUploadDeviceSignaturesResponse) error { +func (k *mockKeyAPI) PerformUploadDeviceSignatures(ctx context.Context, req *userapi.PerformUploadDeviceSignaturesRequest, res *userapi.PerformUploadDeviceSignaturesResponse) error { return nil } -func (k *mockKeyAPI) QueryKeys(ctx context.Context, req *keyapi.QueryKeysRequest, res *keyapi.QueryKeysResponse) error { +func (k *mockKeyAPI) QueryKeys(ctx context.Context, req *userapi.QueryKeysRequest, res *userapi.QueryKeysResponse) error { return nil } -func (k *mockKeyAPI) QueryKeyChanges(ctx context.Context, req *keyapi.QueryKeyChangesRequest, res *keyapi.QueryKeyChangesResponse) error { +func (k *mockKeyAPI) QueryKeyChanges(ctx context.Context, req *userapi.QueryKeyChangesRequest, res *userapi.QueryKeyChangesResponse) error { return nil } -func (k *mockKeyAPI) QueryOneTimeKeys(ctx context.Context, req *keyapi.QueryOneTimeKeysRequest, res *keyapi.QueryOneTimeKeysResponse) error { +func (k *mockKeyAPI) QueryOneTimeKeys(ctx context.Context, req *userapi.QueryOneTimeKeysRequest, res *userapi.QueryOneTimeKeysResponse) error { return nil } -func (k *mockKeyAPI) QueryDeviceMessages(ctx context.Context, req *keyapi.QueryDeviceMessagesRequest, res *keyapi.QueryDeviceMessagesResponse) error { +func (k *mockKeyAPI) QueryDeviceMessages(ctx context.Context, req *userapi.QueryDeviceMessagesRequest, res *userapi.QueryDeviceMessagesResponse) error { return nil } -func (k *mockKeyAPI) QuerySignatures(ctx context.Context, req *keyapi.QuerySignaturesRequest, res *keyapi.QuerySignaturesResponse) error { +func (k *mockKeyAPI) QuerySignatures(ctx context.Context, req *userapi.QuerySignaturesRequest, res *userapi.QuerySignaturesResponse) error { return nil } type mockRoomserverAPI struct { - api.RoomserverInternalAPITrace + api.RoomserverInternalAPI roomIDToJoinedMembers map[string][]string } diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go index 095a868c7..76f003671 100644 --- a/syncapi/routing/context.go +++ b/syncapi/routing/context.go @@ -67,6 +67,8 @@ func Context( errMsg = "unable to parse filter" case *strconv.NumError: errMsg = "unable to parse limit" + default: + errMsg = err.Error() } return util.JSONResponse{ Code: http.StatusBadRequest, @@ -167,7 +169,18 @@ func Context( eventsBeforeClient := gomatrixserverlib.HeaderedToClientEvents(eventsBeforeFiltered, gomatrixserverlib.FormatAll) eventsAfterClient := gomatrixserverlib.HeaderedToClientEvents(eventsAfterFiltered, gomatrixserverlib.FormatAll) - newState := applyLazyLoadMembers(device, filter, eventsAfterClient, eventsBeforeClient, state, lazyLoadCache) + + newState := state + if filter.LazyLoadMembers { + allEvents := append(eventsBeforeFiltered, eventsAfterFiltered...) + allEvents = append(allEvents, &requestedEvent) + evs := gomatrixserverlib.HeaderedToClientEvents(allEvents, gomatrixserverlib.FormatAll) + newState, err = applyLazyLoadMembers(ctx, device, snapshot, roomID, evs, lazyLoadCache) + if err != nil { + logrus.WithError(err).Error("unable to load membership events") + return jsonerror.InternalServerError() + } + } ev := gomatrixserverlib.HeaderedToClientEvent(&requestedEvent, gomatrixserverlib.FormatAll) response := ContextRespsonse{ @@ -244,41 +257,43 @@ func getStartEnd(ctx context.Context, snapshot storage.DatabaseTransaction, star } func applyLazyLoadMembers( + ctx context.Context, device *userapi.Device, - filter *gomatrixserverlib.RoomEventFilter, - eventsAfter, eventsBefore []gomatrixserverlib.ClientEvent, - state []*gomatrixserverlib.HeaderedEvent, + snapshot storage.DatabaseTransaction, + roomID string, + events []gomatrixserverlib.ClientEvent, lazyLoadCache caching.LazyLoadCache, -) []*gomatrixserverlib.HeaderedEvent { - if filter == nil || !filter.LazyLoadMembers { - return state - } - allEvents := append(eventsBefore, eventsAfter...) - x := make(map[string]struct{}) +) ([]*gomatrixserverlib.HeaderedEvent, error) { + eventSenders := make(map[string]struct{}) // get members who actually send an event - for _, e := range allEvents { + for _, e := range events { // Don't add membership events the client should already know about if _, cached := lazyLoadCache.IsLazyLoadedUserCached(device, e.RoomID, e.Sender); cached { continue } - x[e.Sender] = struct{}{} + eventSenders[e.Sender] = struct{}{} } - newState := []*gomatrixserverlib.HeaderedEvent{} - membershipEvents := []*gomatrixserverlib.HeaderedEvent{} - for _, event := range state { - if event.Type() != gomatrixserverlib.MRoomMember { - newState = append(newState, event) - } else { - // did the user send an event? - if _, ok := x[event.Sender()]; ok { - membershipEvents = append(membershipEvents, event) - lazyLoadCache.StoreLazyLoadedUser(device, event.RoomID(), event.Sender(), event.EventID()) - } - } + wantUsers := make([]string, 0, len(eventSenders)) + for userID := range eventSenders { + wantUsers = append(wantUsers, userID) } - // Add the membershipEvents to the end of the list, to make Sytest happy - return append(newState, membershipEvents...) + + // Query missing membership events + filter := gomatrixserverlib.DefaultStateFilter() + filter.Senders = &wantUsers + filter.Types = &[]string{gomatrixserverlib.MRoomMember} + memberships, err := snapshot.GetStateEventsForRoom(ctx, roomID, &filter) + if err != nil { + return nil, err + } + + // cache the membership events + for _, membership := range memberships { + lazyLoadCache.StoreLazyLoadedUser(device, roomID, *membership.StateKey(), membership.EventID()) + } + + return memberships, nil } func parseRoomEventFilter(req *http.Request) (*gomatrixserverlib.RoomEventFilter, error) { diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 4a01ec357..02d8fcc7e 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -64,6 +64,7 @@ type messagesResp struct { // OnIncomingMessagesRequest implements the /messages endpoint from the // client-server API. // See: https://matrix.org/docs/spec/client_server/latest.html#get-matrix-client-r0-rooms-roomid-messages +// nolint:gocyclo func OnIncomingMessagesRequest( req *http.Request, db storage.Database, roomID string, device *userapi.Device, rsAPI api.SyncRoomserverAPI, @@ -246,7 +247,14 @@ func OnIncomingMessagesRequest( Start: start.String(), End: end.String(), } - res.applyLazyLoadMembers(req.Context(), snapshot, roomID, device, filter.LazyLoadMembers, lazyLoadCache) + if filter.LazyLoadMembers { + membershipEvents, err := applyLazyLoadMembers(req.Context(), device, snapshot, roomID, clientEvents, lazyLoadCache) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("failed to apply lazy loading") + return jsonerror.InternalServerError() + } + res.State = append(res.State, gomatrixserverlib.HeaderedToClientEvents(membershipEvents, gomatrixserverlib.FormatAll)...) + } // If we didn't return any events, set the end to an empty string, so it will be omitted // in the response JSON. @@ -265,40 +273,6 @@ func OnIncomingMessagesRequest( } } -// applyLazyLoadMembers loads membership events for users returned in Chunk, if the filter has -// LazyLoadMembers enabled. -func (m *messagesResp) applyLazyLoadMembers( - ctx context.Context, - db storage.DatabaseTransaction, - roomID string, - device *userapi.Device, - lazyLoad bool, - lazyLoadCache caching.LazyLoadCache, -) { - if !lazyLoad { - return - } - membershipToUser := make(map[string]*gomatrixserverlib.HeaderedEvent) - for _, evt := range m.Chunk { - // Don't add membership events the client should already know about - if _, cached := lazyLoadCache.IsLazyLoadedUserCached(device, roomID, evt.Sender); cached { - continue - } - membership, err := db.GetStateEvent(ctx, roomID, gomatrixserverlib.MRoomMember, evt.Sender) - if err != nil { - util.GetLogger(ctx).WithError(err).Error("failed to get membership event for user") - continue - } - if membership != nil { - membershipToUser[evt.Sender] = membership - lazyLoadCache.StoreLazyLoadedUser(device, roomID, evt.Sender, membership.EventID()) - } - } - for _, evt := range membershipToUser { - m.State = append(m.State, gomatrixserverlib.HeaderedToClientEvent(evt, gomatrixserverlib.FormatAll)) - } -} - func getMembershipForUser(ctx context.Context, roomID, userID string, rsAPI api.SyncRoomserverAPI) (resp api.QueryMembershipForUserResponse, err error) { req := api.QueryMembershipForUserRequest{ RoomID: roomID, diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index a7a127e3a..04c2020a0 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -46,7 +46,7 @@ type DatabaseTransaction interface { RoomIDsWithMembership(ctx context.Context, userID string, membership string) ([]string, error) MembershipCount(ctx context.Context, roomID, membership string, pos types.StreamPosition) (int, error) GetRoomSummary(ctx context.Context, roomID, userID string) (summary *types.Summary, err error) - RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) + RecentEvents(ctx context.Context, roomIDs []string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) (map[string]types.RecentEvents, error) GetBackwardTopologyPos(ctx context.Context, events []*gomatrixserverlib.HeaderedEvent) (types.TopologyToken, error) PositionInTopology(ctx context.Context, eventID string) (pos types.StreamPosition, spos types.StreamPosition, err error) InviteEventsInRange(ctx context.Context, targetUserID string, r types.Range) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, types.StreamPosition, error) diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 3caafa14b..0d607b7c0 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -275,6 +275,15 @@ func (s *currentRoomStateStatements) SelectCurrentState( ) ([]*gomatrixserverlib.HeaderedEvent, error) { stmt := sqlutil.TxStmt(txn, s.selectCurrentStateStmt) senders, notSenders := getSendersStateFilterFilter(stateFilter) + // We're going to query members later, so remove them from this request + if stateFilter.LazyLoadMembers && !stateFilter.IncludeRedundantMembers { + notTypes := &[]string{gomatrixserverlib.MRoomMember} + if stateFilter.NotTypes != nil { + *stateFilter.NotTypes = append(*stateFilter.NotTypes, gomatrixserverlib.MRoomMember) + } else { + stateFilter.NotTypes = notTypes + } + } rows, err := stmt.QueryContext(ctx, roomID, pq.StringArray(senders), pq.StringArray(notSenders), diff --git a/cmd/dendrite-polylith-multi/personalities/mediaapi.go b/syncapi/storage/postgres/deltas/20230201152200_rename_index.go similarity index 52% rename from cmd/dendrite-polylith-multi/personalities/mediaapi.go rename to syncapi/storage/postgres/deltas/20230201152200_rename_index.go index 69d5fd5a8..5a0ec5050 100644 --- a/cmd/dendrite-polylith-multi/personalities/mediaapi.go +++ b/syncapi/storage/postgres/deltas/20230201152200_rename_index.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. +// Copyright 2023 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,25 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -package personalities +package deltas import ( - "github.com/matrix-org/dendrite/mediaapi" - basepkg "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" + "context" + "database/sql" + "fmt" ) -func MediaAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { - userAPI := base.UserAPIClient() - client := base.CreateClient() - - mediaapi.AddPublicRoutes( - base, userAPI, client, - ) - - base.SetupAndServeHTTP( - base.Cfg.MediaAPI.InternalAPI.Listen, - base.Cfg.MediaAPI.ExternalAPI.Listen, - nil, nil, - ) +func UpRenameOutputRoomEventsIndex(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, `ALTER TABLE syncapi_output_room_events RENAME CONSTRAINT syncapi_event_id_idx TO syncapi_output_room_event_id_idx;`) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil } diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 0075fc8d3..59fb99aa3 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -19,18 +19,17 @@ import ( "context" "database/sql" "encoding/json" + "fmt" "sort" + "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/postgres/deltas" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" - - "github.com/lib/pq" "github.com/matrix-org/gomatrixserverlib" - - "github.com/matrix-org/dendrite/internal/sqlutil" ) const outputRoomEventsSchema = ` @@ -44,7 +43,7 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( -- This isn't a problem for us since we just want to order by this field. id BIGINT PRIMARY KEY DEFAULT nextval('syncapi_stream_id'), -- The event ID for the event - event_id TEXT NOT NULL CONSTRAINT syncapi_event_id_idx UNIQUE, + event_id TEXT NOT NULL CONSTRAINT syncapi_output_room_event_id_idx UNIQUE, -- The 'room_id' key for the event. room_id TEXT NOT NULL, -- The headered JSON for the event, containing potentially additional metadata such as @@ -79,13 +78,16 @@ CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_out CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync); CREATE INDEX IF NOT EXISTS syncapi_output_room_events_add_state_ids_idx ON syncapi_output_room_events ((add_state_ids IS NOT NULL)); CREATE INDEX IF NOT EXISTS syncapi_output_room_events_remove_state_ids_idx ON syncapi_output_room_events ((remove_state_ids IS NOT NULL)); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_recent_events_idx ON syncapi_output_room_events (room_id, exclude_from_sync, id, sender, type); + + ` const insertEventSQL = "" + "INSERT INTO syncapi_output_room_events (" + "room_id, event_id, headered_event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync, history_visibility" + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) " + - "ON CONFLICT ON CONSTRAINT syncapi_event_id_idx DO UPDATE SET exclude_from_sync = (excluded.exclude_from_sync AND $11) " + + "ON CONFLICT ON CONSTRAINT syncapi_output_room_event_id_idx DO UPDATE SET exclude_from_sync = (excluded.exclude_from_sync AND $11) " + "RETURNING id" const selectEventsSQL = "" + @@ -109,14 +111,29 @@ const selectRecentEventsSQL = "" + " AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" + " ORDER BY id DESC LIMIT $8" -const selectRecentEventsForSyncSQL = "" + - "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events" + - " WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" + - " AND ( $4::text[] IS NULL OR sender = ANY($4) )" + - " AND ( $5::text[] IS NULL OR NOT(sender = ANY($5)) )" + - " AND ( $6::text[] IS NULL OR type LIKE ANY($6) )" + - " AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" + - " ORDER BY id DESC LIMIT $8" +// selectRecentEventsForSyncSQL contains an optimization to get the recent events for a list of rooms, using a LATERAL JOIN +// The sub select inside LATERAL () is executed for all room_ids it gets as a parameter $1 +const selectRecentEventsForSyncSQL = ` +WITH room_ids AS ( + SELECT unnest($1::text[]) AS room_id +) +SELECT x.* +FROM room_ids, + LATERAL ( + SELECT room_id, event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility + FROM syncapi_output_room_events recent_events + WHERE + recent_events.room_id = room_ids.room_id + AND recent_events.exclude_from_sync = FALSE + AND id > $2 AND id <= $3 + AND ( $4::text[] IS NULL OR sender = ANY($4) ) + AND ( $5::text[] IS NULL OR NOT(sender = ANY($5)) ) + AND ( $6::text[] IS NULL OR type LIKE ANY($6) ) + AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) ) + ORDER BY recent_events.id DESC + LIMIT $8 + ) AS x +` const selectEarlyEventsSQL = "" + "SELECT event_id, id, headered_event_json, session_id, exclude_from_sync, transaction_id, history_visibility FROM syncapi_output_room_events" + @@ -207,12 +224,30 @@ func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) { return nil, err } + migrationName := "syncapi: rename dupe index (output_room_events)" + + var cName string + err = db.QueryRowContext(context.Background(), "select constraint_name from information_schema.table_constraints where table_name = 'syncapi_output_room_events' AND constraint_name = 'syncapi_event_id_idx'").Scan(&cName) + switch err { + case sql.ErrNoRows: // migration was already executed, as the index was renamed + if err = sqlutil.InsertMigration(context.Background(), db, migrationName); err != nil { + return nil, fmt.Errorf("unable to manually insert migration '%s': %w", migrationName, err) + } + case nil: + default: + return nil, err + } + m := sqlutil.NewMigrator(db) m.AddMigrations( sqlutil.Migration{ Version: "syncapi: add history visibility column (output_room_events)", Up: deltas.UpAddHistoryVisibilityColumnOutputRoomEvents, }, + sqlutil.Migration{ + Version: migrationName, + Up: deltas.UpRenameOutputRoomEventsIndex, + }, ) err = m.Up(context.Background()) if err != nil { @@ -398,9 +433,9 @@ func (s *outputRoomEventsStatements) InsertEvent( // from sync. func (s *outputRoomEventsStatements) SelectRecentEvents( ctx context.Context, txn *sql.Tx, - roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, + roomIDs []string, ra types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool, -) ([]types.StreamEvent, bool, error) { +) (map[string]types.RecentEvents, error) { var stmt *sql.Stmt if onlySyncEvents { stmt = sqlutil.TxStmt(txn, s.selectRecentEventsForSyncStmt) @@ -408,8 +443,9 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( stmt = sqlutil.TxStmt(txn, s.selectRecentEventsStmt) } senders, notSenders := getSendersRoomEventFilter(eventFilter) + rows, err := stmt.QueryContext( - ctx, roomID, r.Low(), r.High(), + ctx, pq.StringArray(roomIDs), ra.Low(), ra.High(), pq.StringArray(senders), pq.StringArray(notSenders), pq.StringArray(filterConvertTypeWildcardToSQL(eventFilter.Types)), @@ -417,34 +453,80 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( eventFilter.Limit+1, ) if err != nil { - return nil, false, err + return nil, err } defer internal.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed") - events, err := rowsToStreamEvents(rows) - if err != nil { - return nil, false, err - } - if chronologicalOrder { - // The events need to be returned from oldest to latest, which isn't - // necessary the way the SQL query returns them, so a sort is necessary to - // ensure the events are in the right order in the slice. - sort.SliceStable(events, func(i int, j int) bool { - return events[i].StreamPosition < events[j].StreamPosition - }) - } - // we queried for 1 more than the limit, so if we returned one more mark limited=true - limited := false - if len(events) > eventFilter.Limit { - limited = true - // re-slice the extra (oldest) event out: in chronological order this is the first entry, else the last. - if chronologicalOrder { - events = events[1:] - } else { - events = events[:len(events)-1] + + result := make(map[string]types.RecentEvents) + + for rows.Next() { + var ( + roomID string + eventID string + streamPos types.StreamPosition + eventBytes []byte + excludeFromSync bool + sessionID *int64 + txnID *string + transactionID *api.TransactionID + historyVisibility gomatrixserverlib.HistoryVisibility + ) + if err := rows.Scan(&roomID, &eventID, &streamPos, &eventBytes, &sessionID, &excludeFromSync, &txnID, &historyVisibility); err != nil { + return nil, err } + // TODO: Handle redacted events + var ev gomatrixserverlib.HeaderedEvent + if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil { + return nil, err + } + + if sessionID != nil && txnID != nil { + transactionID = &api.TransactionID{ + SessionID: *sessionID, + TransactionID: *txnID, + } + } + + r := result[roomID] + + ev.Visibility = historyVisibility + r.Events = append(r.Events, types.StreamEvent{ + HeaderedEvent: &ev, + StreamPosition: streamPos, + TransactionID: transactionID, + ExcludeFromSync: excludeFromSync, + }) + + result[roomID] = r } - return events, limited, nil + if chronologicalOrder { + for roomID, evs := range result { + // The events need to be returned from oldest to latest, which isn't + // necessary the way the SQL query returns them, so a sort is necessary to + // ensure the events are in the right order in the slice. + sort.SliceStable(evs.Events, func(i int, j int) bool { + return evs.Events[i].StreamPosition < evs.Events[j].StreamPosition + }) + + if len(evs.Events) > eventFilter.Limit { + evs.Limited = true + evs.Events = evs.Events[1:] + } + + result[roomID] = evs + } + } else { + for roomID, evs := range result { + if len(evs.Events) > eventFilter.Limit { + evs.Limited = true + evs.Events = evs.Events[:len(evs.Events)-1] + } + + result[roomID] = evs + } + } + return result, rows.Err() } // selectEarlyEvents returns the earliest events in the given room, starting diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index 8385b95a5..931bc9e23 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -151,8 +151,8 @@ func (d *DatabaseTransaction) GetRoomSummary(ctx context.Context, roomID, userID return summary, nil } -func (d *DatabaseTransaction) RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) { - return d.OutputEvents.SelectRecentEvents(ctx, d.txn, roomID, r, eventFilter, chronologicalOrder, onlySyncEvents) +func (d *DatabaseTransaction) RecentEvents(ctx context.Context, roomIDs []string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) (map[string]types.RecentEvents, error) { + return d.OutputEvents.SelectRecentEvents(ctx, d.txn, roomIDs, r, eventFilter, chronologicalOrder, onlySyncEvents) } func (d *DatabaseTransaction) PositionInTopology(ctx context.Context, eventID string) (pos types.StreamPosition, spos types.StreamPosition, err error) { @@ -370,19 +370,25 @@ func (d *DatabaseTransaction) GetStateDeltas( } // get all the state events ever (i.e. for all available rooms) between these two positions - stateNeededFiltered, eventMapFiltered, err := d.OutputEvents.SelectStateInRange(ctx, d.txn, r, stateFilter, allRoomIDs) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil, nil + stateFiltered := state + // avoid hitting the database if the result would be the same as above + if !isStatefilterEmpty(stateFilter) { + var stateNeededFiltered map[string]map[string]bool + var eventMapFiltered map[string]types.StreamEvent + stateNeededFiltered, eventMapFiltered, err = d.OutputEvents.SelectStateInRange(ctx, d.txn, r, stateFilter, allRoomIDs) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } + return nil, nil, err } - return nil, nil, err - } - stateFiltered, err := d.fetchStateEvents(ctx, d.txn, stateNeededFiltered, eventMapFiltered) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil, nil + stateFiltered, err = d.fetchStateEvents(ctx, d.txn, stateNeededFiltered, eventMapFiltered) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } + return nil, nil, err } - return nil, nil, err } // find out which rooms this user is peeking, if any. @@ -701,6 +707,28 @@ func (d *DatabaseTransaction) MaxStreamPositionForRelations(ctx context.Context) return types.StreamPosition(id), err } +func isStatefilterEmpty(filter *gomatrixserverlib.StateFilter) bool { + if filter == nil { + return true + } + switch { + case filter.NotTypes != nil && len(*filter.NotTypes) > 0: + return false + case filter.Types != nil && len(*filter.Types) > 0: + return false + case filter.Senders != nil && len(*filter.Senders) > 0: + return false + case filter.NotSenders != nil && len(*filter.NotSenders) > 0: + return false + case filter.NotRooms != nil && len(*filter.NotRooms) > 0: + return false + case filter.ContainsURL != nil: + return false + default: + return true + } +} + func (d *DatabaseTransaction) RelationsFor(ctx context.Context, roomID, eventID, relType, eventType string, from, to types.StreamPosition, backwards bool, limit int) ( events []types.StreamEvent, prevBatch, nextBatch string, err error, ) { diff --git a/syncapi/storage/shared/storage_sync_test.go b/syncapi/storage/shared/storage_sync_test.go new file mode 100644 index 000000000..c56720db7 --- /dev/null +++ b/syncapi/storage/shared/storage_sync_test.go @@ -0,0 +1,72 @@ +package shared + +import ( + "testing" + + "github.com/matrix-org/gomatrixserverlib" +) + +func Test_isStatefilterEmpty(t *testing.T) { + filterSet := []string{"a"} + boolValue := false + + tests := []struct { + name string + filter *gomatrixserverlib.StateFilter + want bool + }{ + { + name: "nil filter is empty", + filter: nil, + want: true, + }, + { + name: "Empty filter is empty", + filter: &gomatrixserverlib.StateFilter{}, + want: true, + }, + { + name: "NotTypes is set", + filter: &gomatrixserverlib.StateFilter{ + NotTypes: &filterSet, + }, + }, + { + name: "Types is set", + filter: &gomatrixserverlib.StateFilter{ + Types: &filterSet, + }, + }, + { + name: "Senders is set", + filter: &gomatrixserverlib.StateFilter{ + Senders: &filterSet, + }, + }, + { + name: "NotSenders is set", + filter: &gomatrixserverlib.StateFilter{ + NotSenders: &filterSet, + }, + }, + { + name: "NotRooms is set", + filter: &gomatrixserverlib.StateFilter{ + NotRooms: &filterSet, + }, + }, + { + name: "ContainsURL is set", + filter: &gomatrixserverlib.StateFilter{ + ContainsURL: &boolValue, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isStatefilterEmpty(tt.filter); got != tt.want { + t.Errorf("isStatefilterEmpty() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index d2301e538..41b8b24a3 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -106,9 +106,8 @@ func (s *accountDataStatements) SelectAccountDataInRange( filter.Types, filter.NotTypes, []string{}, nil, filter.Limit, FilterOrderAsc) if err != nil { - return nil, 0, err + return } - rows, err := stmt.QueryContext(ctx, params...) if err != nil { return diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 6bc1b267a..35b746c5c 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -267,6 +267,15 @@ func (s *currentRoomStateStatements) SelectCurrentState( stateFilter *gomatrixserverlib.StateFilter, excludeEventIDs []string, ) ([]*gomatrixserverlib.HeaderedEvent, error) { + // We're going to query members later, so remove them from this request + if stateFilter.LazyLoadMembers && !stateFilter.IncludeRedundantMembers { + notTypes := &[]string{gomatrixserverlib.MRoomMember} + if stateFilter.NotTypes != nil { + *stateFilter.NotTypes = append(*stateFilter.NotTypes, gomatrixserverlib.MRoomMember) + } else { + stateFilter.NotTypes = notTypes + } + } stmt, params, err := prepareWithFilters( s.db, txn, selectCurrentStateSQL, []interface{}{ diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index db708c083..23bc68a41 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -368,9 +368,9 @@ func (s *outputRoomEventsStatements) InsertEvent( func (s *outputRoomEventsStatements) SelectRecentEvents( ctx context.Context, txn *sql.Tx, - roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, + roomIDs []string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool, -) ([]types.StreamEvent, bool, error) { +) (map[string]types.RecentEvents, error) { var query string if onlySyncEvents { query = selectRecentEventsForSyncSQL @@ -378,49 +378,55 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( query = selectRecentEventsSQL } - stmt, params, err := prepareWithFilters( - s.db, txn, query, - []interface{}{ - roomID, r.Low(), r.High(), - }, - eventFilter.Senders, eventFilter.NotSenders, - eventFilter.Types, eventFilter.NotTypes, - nil, eventFilter.ContainsURL, eventFilter.Limit+1, FilterOrderDesc, - ) - if err != nil { - return nil, false, fmt.Errorf("s.prepareWithFilters: %w", err) - } - defer internal.CloseAndLogIfError(ctx, stmt, "selectRecentEvents: stmt.close() failed") - - rows, err := stmt.QueryContext(ctx, params...) - if err != nil { - return nil, false, err - } - defer internal.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed") - events, err := rowsToStreamEvents(rows) - if err != nil { - return nil, false, err - } - if chronologicalOrder { - // The events need to be returned from oldest to latest, which isn't - // necessary the way the SQL query returns them, so a sort is necessary to - // ensure the events are in the right order in the slice. - sort.SliceStable(events, func(i int, j int) bool { - return events[i].StreamPosition < events[j].StreamPosition - }) - } - // we queried for 1 more than the limit, so if we returned one more mark limited=true - limited := false - if len(events) > eventFilter.Limit { - limited = true - // re-slice the extra (oldest) event out: in chronological order this is the first entry, else the last. - if chronologicalOrder { - events = events[1:] - } else { - events = events[:len(events)-1] + result := make(map[string]types.RecentEvents, len(roomIDs)) + for _, roomID := range roomIDs { + stmt, params, err := prepareWithFilters( + s.db, txn, query, + []interface{}{ + roomID, r.Low(), r.High(), + }, + eventFilter.Senders, eventFilter.NotSenders, + eventFilter.Types, eventFilter.NotTypes, + nil, eventFilter.ContainsURL, eventFilter.Limit+1, FilterOrderDesc, + ) + if err != nil { + return nil, fmt.Errorf("s.prepareWithFilters: %w", err) } + defer internal.CloseAndLogIfError(ctx, stmt, "selectRecentEvents: stmt.close() failed") + + rows, err := stmt.QueryContext(ctx, params...) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed") + events, err := rowsToStreamEvents(rows) + if err != nil { + return nil, err + } + if chronologicalOrder { + // The events need to be returned from oldest to latest, which isn't + // necessary the way the SQL query returns them, so a sort is necessary to + // ensure the events are in the right order in the slice. + sort.SliceStable(events, func(i int, j int) bool { + return events[i].StreamPosition < events[j].StreamPosition + }) + } + res := types.RecentEvents{} + // we queried for 1 more than the limit, so if we returned one more mark limited=true + if len(events) > eventFilter.Limit { + res.Limited = true + // re-slice the extra (oldest) event out: in chronological order this is the first entry, else the last. + if chronologicalOrder { + events = events[1:] + } else { + events = events[:len(events)-1] + } + } + res.Events = events + result[roomID] = res } - return events, limited, nil + + return result, nil } func (s *outputRoomEventsStatements) SelectEarlyEvents( diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index e65367d8b..05d498bc2 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -156,12 +156,12 @@ func TestRecentEventsPDU(t *testing.T) { tc := testCases[i] t.Run(tc.Name, func(st *testing.T) { var filter gomatrixserverlib.RoomEventFilter - var gotEvents []types.StreamEvent + var gotEvents map[string]types.RecentEvents var limited bool filter.Limit = tc.Limit WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) { var err error - gotEvents, limited, err = snapshot.RecentEvents(ctx, r.ID, types.Range{ + gotEvents, err = snapshot.RecentEvents(ctx, []string{r.ID}, types.Range{ From: tc.From, To: tc.To, }, &filter, !tc.ReverseOrder, true) @@ -169,15 +169,18 @@ func TestRecentEventsPDU(t *testing.T) { st.Fatalf("failed to do sync: %s", err) } }) + streamEvents := gotEvents[r.ID] + limited = streamEvents.Limited if limited != tc.WantLimited { st.Errorf("got limited=%v want %v", limited, tc.WantLimited) } - if len(gotEvents) != len(tc.WantEvents) { + if len(streamEvents.Events) != len(tc.WantEvents) { st.Errorf("got %d events, want %d", len(gotEvents), len(tc.WantEvents)) } - for j := range gotEvents { - if !reflect.DeepEqual(gotEvents[j].JSON(), tc.WantEvents[j].JSON()) { - st.Errorf("event %d got %s want %s", j, string(gotEvents[j].JSON()), string(tc.WantEvents[j].JSON())) + + for j := range streamEvents.Events { + if !reflect.DeepEqual(streamEvents.Events[j].JSON(), tc.WantEvents[j].JSON()) { + st.Errorf("event %d got %s want %s", j, string(streamEvents.Events[j].JSON()), string(tc.WantEvents[j].JSON())) } } }) @@ -923,3 +926,61 @@ func TestRoomSummary(t *testing.T) { } }) } + +func TestRecentEvents(t *testing.T) { + alice := test.NewUser(t) + room1 := test.NewRoom(t, alice) + room2 := test.NewRoom(t, alice) + roomIDs := []string{room1.ID, room2.ID} + rooms := map[string]*test.Room{ + room1.ID: room1, + room2.ID: room2, + } + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + filter := gomatrixserverlib.DefaultRoomEventFilter() + db, close, closeBase := MustCreateDatabase(t, dbType) + t.Cleanup(func() { + close() + closeBase() + }) + + MustWriteEvents(t, db, room1.Events()) + MustWriteEvents(t, db, room2.Events()) + + transaction, err := db.NewDatabaseTransaction(ctx) + assert.NoError(t, err) + defer transaction.Rollback() + + // get all recent events from 0 to 100 (we only created 5 events, so we should get 5 back) + roomEvs, err := transaction.RecentEvents(ctx, roomIDs, types.Range{From: 0, To: 100}, &filter, true, true) + assert.NoError(t, err) + assert.Equal(t, len(roomEvs), 2, "unexpected recent events response") + for _, recentEvents := range roomEvs { + assert.Equal(t, 5, len(recentEvents.Events), "unexpected recent events for room") + } + + // update the filter to only return one event + filter.Limit = 1 + roomEvs, err = transaction.RecentEvents(ctx, roomIDs, types.Range{From: 0, To: 100}, &filter, true, true) + assert.NoError(t, err) + assert.Equal(t, len(roomEvs), 2, "unexpected recent events response") + for roomID, recentEvents := range roomEvs { + origEvents := rooms[roomID].Events() + assert.Equal(t, true, recentEvents.Limited, "expected events to be limited") + assert.Equal(t, 1, len(recentEvents.Events), "unexpected recent events for room") + assert.Equal(t, origEvents[len(origEvents)-1].EventID(), recentEvents.Events[0].EventID()) + } + + // not chronologically ordered still returns the events in order (given ORDER BY id DESC) + roomEvs, err = transaction.RecentEvents(ctx, roomIDs, types.Range{From: 0, To: 100}, &filter, false, true) + assert.NoError(t, err) + assert.Equal(t, len(roomEvs), 2, "unexpected recent events response") + for roomID, recentEvents := range roomEvs { + origEvents := rooms[roomID].Events() + assert.Equal(t, true, recentEvents.Limited, "expected events to be limited") + assert.Equal(t, 1, len(recentEvents.Events), "unexpected recent events for room") + assert.Equal(t, origEvents[len(origEvents)-1].EventID(), recentEvents.Events[0].EventID()) + } + }) +} diff --git a/syncapi/storage/tables/current_room_state_test.go b/syncapi/storage/tables/current_room_state_test.go index 23287c500..c7af4f977 100644 --- a/syncapi/storage/tables/current_room_state_test.go +++ b/syncapi/storage/tables/current_room_state_test.go @@ -13,6 +13,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/gomatrixserverlib" ) func newCurrentRoomStateTable(t *testing.T, dbType test.DBType) (tables.CurrentRoomState, *sql.DB, func()) { @@ -79,6 +80,9 @@ func TestCurrentRoomStateTable(t *testing.T) { return fmt.Errorf("SelectEventsWithEventIDs\nexpected id %q not returned", id) } } + + testCurrentState(t, ctx, txn, tab, room) + return nil }) if err != nil { @@ -86,3 +90,39 @@ func TestCurrentRoomStateTable(t *testing.T) { } }) } + +func testCurrentState(t *testing.T, ctx context.Context, txn *sql.Tx, tab tables.CurrentRoomState, room *test.Room) { + t.Run("test currentState", func(t *testing.T) { + // returns the complete state of the room with a default filter + filter := gomatrixserverlib.DefaultStateFilter() + evs, err := tab.SelectCurrentState(ctx, txn, room.ID, &filter, nil) + if err != nil { + t.Fatal(err) + } + expectCount := 5 + if gotCount := len(evs); gotCount != expectCount { + t.Fatalf("expected %d state events, got %d", expectCount, gotCount) + } + // When lazy loading, we expect no membership event, so only 4 events + filter.LazyLoadMembers = true + expectCount = 4 + evs, err = tab.SelectCurrentState(ctx, txn, room.ID, &filter, nil) + if err != nil { + t.Fatal(err) + } + if gotCount := len(evs); gotCount != expectCount { + t.Fatalf("expected %d state events, got %d", expectCount, gotCount) + } + // same as above, but with existing NotTypes defined + notTypes := []string{gomatrixserverlib.MRoomMember} + filter.NotTypes = ¬Types + evs, err = tab.SelectCurrentState(ctx, txn, room.ID, &filter, nil) + if err != nil { + t.Fatal(err) + } + if gotCount := len(evs); gotCount != expectCount { + t.Fatalf("expected %d state events, got %d", expectCount, gotCount) + } + }) + +} diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 145e197cc..727d6bf2c 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -66,7 +66,7 @@ type Events interface { // SelectRecentEvents returns events between the two stream positions: exclusive of low and inclusive of high. // If onlySyncEvents has a value of true, only returns the events that aren't marked as to exclude from sync. // Returns up to `limit` events. Returns `limited=true` if there are more events in this range but we hit the `limit`. - SelectRecentEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) + SelectRecentEvents(ctx context.Context, txn *sql.Tx, roomIDs []string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) (map[string]types.RecentEvents, error) // SelectEarlyEvents returns the earliest events in the given room. SelectEarlyEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter) ([]types.StreamEvent, error) SelectEvents(ctx context.Context, txn *sql.Tx, eventIDs []string, filter *gomatrixserverlib.RoomEventFilter, preserveOrder bool) ([]types.StreamEvent, error) diff --git a/syncapi/streams/stream_devicelist.go b/syncapi/streams/stream_devicelist.go index 7996c2038..e8189c352 100644 --- a/syncapi/streams/stream_devicelist.go +++ b/syncapi/streams/stream_devicelist.go @@ -3,17 +3,17 @@ package streams import ( "context" - keyapi "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/internal" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" ) type DeviceListStreamProvider struct { DefaultStreamProvider - rsAPI api.SyncRoomserverAPI - keyAPI keyapi.SyncKeyAPI + rsAPI api.SyncRoomserverAPI + userAPI userapi.SyncKeyAPI } func (p *DeviceListStreamProvider) CompleteSync( @@ -31,12 +31,12 @@ func (p *DeviceListStreamProvider) IncrementalSync( from, to types.StreamPosition, ) types.StreamPosition { var err error - to, _, err = internal.DeviceListCatchup(context.Background(), snapshot, p.keyAPI, p.rsAPI, req.Device.UserID, req.Response, from, to) + to, _, err = internal.DeviceListCatchup(context.Background(), snapshot, p.userAPI, p.rsAPI, req.Device.UserID, req.Response, from, to) if err != nil { req.Log.WithError(err).Error("internal.DeviceListCatchup failed") return from } - err = internal.DeviceOTKCounts(req.Context, p.keyAPI, req.Device.UserID, req.Device.ID, req.Response) + err = internal.DeviceOTKCounts(req.Context, p.userAPI, req.Device.UserID, req.Device.ID, req.Response) if err != nil { req.Log.WithError(err).Error("internal.DeviceListCatchup failed") return from diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 44013e37c..6af25c028 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -82,19 +82,24 @@ func (p *PDUStreamProvider) CompleteSync( req.Log.WithError(err).Error("unable to update event filter with ignored users") } - // Invalidate the lazyLoadCache, otherwise we end up with missing displaynames/avatars - // TODO: This might be inefficient, when joined to many and/or large rooms. + recentEvents, err := snapshot.RecentEvents(ctx, joinedRoomIDs, r, &eventFilter, true, true) + if err != nil { + return from + } + // Build up a /sync response. Add joined rooms. for _, roomID := range joinedRoomIDs { + events := recentEvents[roomID] + // Invalidate the lazyLoadCache, otherwise we end up with missing displaynames/avatars + // TODO: This might be inefficient, when joined to many and/or large rooms. joinedUsers := p.notifier.JoinedUsers(roomID) for _, sharedUser := range joinedUsers { p.lazyLoadCache.InvalidateLazyLoadedUser(req.Device, roomID, sharedUser) } - } - // Build up a /sync response. Add joined rooms. - for _, roomID := range joinedRoomIDs { + // get the join response for each room jr, jerr := p.getJoinResponseForCompleteSync( - ctx, snapshot, roomID, r, &stateFilter, &eventFilter, req.WantFullState, req.Device, false, + ctx, snapshot, roomID, &stateFilter, req.WantFullState, req.Device, false, + events.Events, events.Limited, ) if jerr != nil { req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed") @@ -113,11 +118,25 @@ func (p *PDUStreamProvider) CompleteSync( req.Log.WithError(err).Error("p.DB.PeeksInRange failed") return from } - for _, peek := range peeks { - if !peek.Deleted { + if len(peeks) > 0 { + peekRooms := make([]string, 0, len(peeks)) + for _, peek := range peeks { + if !peek.Deleted { + peekRooms = append(peekRooms, peek.RoomID) + } + } + + recentEvents, err = snapshot.RecentEvents(ctx, peekRooms, r, &eventFilter, true, true) + if err != nil { + return from + } + + for _, roomID := range peekRooms { var jr *types.JoinResponse + events := recentEvents[roomID] jr, err = p.getJoinResponseForCompleteSync( - ctx, snapshot, peek.RoomID, r, &stateFilter, &eventFilter, req.WantFullState, req.Device, true, + ctx, snapshot, roomID, &stateFilter, req.WantFullState, req.Device, true, + events.Events, events.Limited, ) if err != nil { req.Log.WithError(err).Error("p.getJoinResponseForCompleteSync failed") @@ -126,7 +145,7 @@ func (p *PDUStreamProvider) CompleteSync( } continue } - req.Response.Rooms.Peek[peek.RoomID] = jr + req.Response.Rooms.Peek[roomID] = jr } } @@ -227,7 +246,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( stateFilter *gomatrixserverlib.StateFilter, req *types.SyncRequest, ) (types.StreamPosition, error) { - + var err error originalLimit := eventFilter.Limit // If we're going backwards, grep at least X events, this is mostly to satisfy Sytest if r.Backwards && originalLimit < recentEventBackwardsLimit { @@ -238,8 +257,8 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( } } - recentStreamEvents, limited, err := snapshot.RecentEvents( - ctx, delta.RoomID, r, + dbEvents, err := snapshot.RecentEvents( + ctx, []string{delta.RoomID}, r, eventFilter, true, true, ) if err != nil { @@ -248,6 +267,10 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( } return r.From, fmt.Errorf("p.DB.RecentEvents: %w", err) } + + recentStreamEvents := dbEvents[delta.RoomID].Events + limited := dbEvents[delta.RoomID].Limited + recentEvents := gomatrixserverlib.HeaderedReverseTopologicalOrdering( snapshot.StreamEventsToEvents(device, recentStreamEvents), gomatrixserverlib.TopologicalOrderByPrevEvents, @@ -420,7 +443,7 @@ func applyHistoryVisibilityFilter( "room_id": roomID, "before": len(recentEvents), "after": len(events), - }).Trace("Applied history visibility (sync)") + }).Debugf("Applied history visibility (sync)") return events, nil } @@ -428,25 +451,16 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( ctx context.Context, snapshot storage.DatabaseTransaction, roomID string, - r types.Range, stateFilter *gomatrixserverlib.StateFilter, - eventFilter *gomatrixserverlib.RoomEventFilter, wantFullState bool, device *userapi.Device, isPeek bool, + recentStreamEvents []types.StreamEvent, + limited bool, ) (jr *types.JoinResponse, err error) { jr = types.NewJoinResponse() // TODO: When filters are added, we may need to call this multiple times to get enough events. // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 - recentStreamEvents, limited, err := snapshot.RecentEvents( - ctx, roomID, r, eventFilter, true, true, - ) - if err != nil { - if err == sql.ErrNoRows { - return jr, nil - } - return - } // Work our way through the timeline events and pick out the event IDs // of any state events that appear in the timeline. We'll specifically diff --git a/syncapi/streams/streams.go b/syncapi/streams/streams.go index dc8547621..a35491acf 100644 --- a/syncapi/streams/streams.go +++ b/syncapi/streams/streams.go @@ -5,7 +5,6 @@ import ( "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" - keyapi "github.com/matrix-org/dendrite/keyserver/api" rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/notifier" "github.com/matrix-org/dendrite/syncapi/storage" @@ -27,7 +26,7 @@ type Streams struct { func NewSyncStreamProviders( d storage.Database, userAPI userapi.SyncUserAPI, - rsAPI rsapi.SyncRoomserverAPI, keyAPI keyapi.SyncKeyAPI, + rsAPI rsapi.SyncRoomserverAPI, eduCache *caching.EDUCache, lazyLoadCache caching.LazyLoadCache, notifier *notifier.Notifier, ) *Streams { streams := &Streams{ @@ -60,7 +59,7 @@ func NewSyncStreamProviders( DeviceListStreamProvider: &DeviceListStreamProvider{ DefaultStreamProvider: DefaultStreamProvider{DB: d}, rsAPI: rsAPI, - keyAPI: keyAPI, + userAPI: userAPI, }, PresenceStreamProvider: &PresenceStreamProvider{ DefaultStreamProvider: DefaultStreamProvider{DB: d}, diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index b086567b8..68f913871 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -32,7 +32,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/sqlutil" - keyapi "github.com/matrix-org/dendrite/keyserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/syncapi/internal" @@ -48,7 +47,6 @@ type RequestPool struct { db storage.Database cfg *config.SyncAPI userAPI userapi.SyncUserAPI - keyAPI keyapi.SyncKeyAPI rsAPI roomserverAPI.SyncRoomserverAPI lastseen *sync.Map presence *sync.Map @@ -69,7 +67,7 @@ type PresenceConsumer interface { // NewRequestPool makes a new RequestPool func NewRequestPool( db storage.Database, cfg *config.SyncAPI, - userAPI userapi.SyncUserAPI, keyAPI keyapi.SyncKeyAPI, + userAPI userapi.SyncUserAPI, rsAPI roomserverAPI.SyncRoomserverAPI, streams *streams.Streams, notifier *notifier.Notifier, producer PresencePublisher, consumer PresenceConsumer, enableMetrics bool, @@ -83,7 +81,6 @@ func NewRequestPool( db: db, cfg: cfg, userAPI: userAPI, - keyAPI: keyAPI, rsAPI: rsAPI, lastseen: &sync.Map{}, presence: &sync.Map{}, @@ -280,7 +277,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. // https://github.com/matrix-org/synapse/blob/29f06704b8871a44926f7c99e73cf4a978fb8e81/synapse/rest/client/sync.py#L276-L281 // Only try to get OTKs if the context isn't already done. if syncReq.Context.Err() == nil { - err = internal.DeviceOTKCounts(syncReq.Context, rp.keyAPI, syncReq.Device.UserID, syncReq.Device.ID, syncReq.Response) + err = internal.DeviceOTKCounts(syncReq.Context, rp.userAPI, syncReq.Device.UserID, syncReq.Device.ID, syncReq.Response) if err != nil && err != context.Canceled { syncReq.Log.WithError(err).Warn("failed to get OTK counts") } @@ -551,7 +548,7 @@ func (rp *RequestPool) OnIncomingKeyChangeRequest(req *http.Request, device *use defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) rp.streams.PDUStreamProvider.IncrementalSync(req.Context(), snapshot, syncReq, fromToken.PDUPosition, toToken.PDUPosition) _, _, err = internal.DeviceListCatchup( - req.Context(), snapshot, rp.keyAPI, rp.rsAPI, syncReq.Device.UserID, + req.Context(), snapshot, rp.userAPI, rp.rsAPI, syncReq.Device.UserID, syncReq.Response, fromToken.DeviceListPosition, toToken.DeviceListPosition, ) if err != nil { diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index be19310f2..153f7af5d 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -21,7 +21,6 @@ import ( "github.com/matrix-org/dendrite/internal/caching" - keyapi "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/jetstream" @@ -42,7 +41,6 @@ func AddPublicRoutes( base *base.BaseDendrite, userAPI userapi.SyncUserAPI, rsAPI api.SyncRoomserverAPI, - keyAPI keyapi.SyncKeyAPI, ) { cfg := &base.Cfg.SyncAPI @@ -55,7 +53,7 @@ func AddPublicRoutes( eduCache := caching.NewTypingCache() notifier := notifier.NewNotifier() - streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache, base.Caches, notifier) + streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, eduCache, base.Caches, notifier) notifier.SetCurrentPosition(streams.Latest(context.Background())) if err = notifier.Load(context.Background(), syncDB); err != nil { logrus.WithError(err).Panicf("failed to load notifier ") @@ -71,7 +69,7 @@ func AddPublicRoutes( userAPI, ) - requestPool := sync.NewRequestPool(syncDB, cfg, userAPI, keyAPI, rsAPI, streams, notifier, federationPresenceProducer, presenceConsumer, base.EnableMetrics) + requestPool := sync.NewRequestPool(syncDB, cfg, userAPI, rsAPI, streams, notifier, federationPresenceProducer, presenceConsumer, base.EnableMetrics) if err = presenceConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start presence consumer") @@ -117,7 +115,7 @@ func AddPublicRoutes( } sendToDeviceConsumer := consumers.NewOutputSendToDeviceEventConsumer( - base.ProcessContext, cfg, js, syncDB, keyAPI, notifier, streams.SendToDeviceStreamProvider, + base.ProcessContext, cfg, js, syncDB, userAPI, notifier, streams.SendToDeviceStreamProvider, ) if err = sendToDeviceConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start send-to-device consumer") diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go index 666a872f8..e748660f7 100644 --- a/syncapi/syncapi_test.go +++ b/syncapi/syncapi_test.go @@ -10,12 +10,12 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/syncapi/routing" "github.com/matrix-org/gomatrixserverlib" "github.com/nats-io/nats.go" "github.com/tidwall/gjson" "github.com/matrix-org/dendrite/clientapi/producers" - keyapi "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" rsapi "github.com/matrix-org/dendrite/roomserver/api" @@ -82,21 +82,18 @@ func (s *syncUserAPI) QueryAccessToken(ctx context.Context, req *userapi.QueryAc return nil } +func (s *syncUserAPI) QueryKeyChanges(ctx context.Context, req *userapi.QueryKeyChangesRequest, res *userapi.QueryKeyChangesResponse) error { + return nil +} + +func (s *syncUserAPI) QueryOneTimeKeys(ctx context.Context, req *userapi.QueryOneTimeKeysRequest, res *userapi.QueryOneTimeKeysResponse) error { + return nil +} + func (s *syncUserAPI) PerformLastSeenUpdate(ctx context.Context, req *userapi.PerformLastSeenUpdateRequest, res *userapi.PerformLastSeenUpdateResponse) error { return nil } -type syncKeyAPI struct { - keyapi.SyncKeyAPI -} - -func (s *syncKeyAPI) QueryKeyChanges(ctx context.Context, req *keyapi.QueryKeyChangesRequest, res *keyapi.QueryKeyChangesResponse) error { - return nil -} -func (s *syncKeyAPI) QueryOneTimeKeys(ctx context.Context, req *keyapi.QueryOneTimeKeysRequest, res *keyapi.QueryOneTimeKeysResponse) error { - return nil -} - func TestSyncAPIAccessTokens(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { testSyncAccessTokens(t, dbType) @@ -120,7 +117,7 @@ func testSyncAccessTokens(t *testing.T, dbType test.DBType) { jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) msgs := toNATSMsgs(t, base, room.Events()...) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, &syncKeyAPI{}) + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}) testrig.MustPublishMsgs(t, jsctx, msgs...) testCases := []struct { @@ -219,7 +216,7 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) { // m.room.history_visibility msgs := toNATSMsgs(t, base, room.Events()...) sinceTokens := make([]string, len(msgs)) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, &syncKeyAPI{}) + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}) for i, msg := range msgs { testrig.MustPublishMsgs(t, jsctx, msg) time.Sleep(100 * time.Millisecond) @@ -303,7 +300,7 @@ func testSyncAPIUpdatePresenceImmediately(t *testing.T, dbType test.DBType) { jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}, &syncKeyAPI{}) + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}) w := httptest.NewRecorder() base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ "access_token": alice.AccessToken, @@ -421,7 +418,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) { rsAPI := roomserver.NewInternalAPI(base) rsAPI.SetFederationAPI(nil, nil) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{aliceDev, bobDev}}, rsAPI, &syncKeyAPI{}) + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{aliceDev, bobDev}}, rsAPI) for _, tc := range testCases { testname := fmt.Sprintf("%s - %s", tc.historyVisibility, userType) @@ -448,6 +445,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) { base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/messages", room.ID), test.WithQueryParams(map[string]string{ "access_token": bobDev.AccessToken, "dir": "b", + "filter": `{"lazy_load_members":true}`, // check that lazy loading doesn't break history visibility }))) if w.Code != 200 { t.Logf("%s", w.Body.String()) @@ -720,7 +718,7 @@ func TestGetMembership(t *testing.T) { rsAPI := roomserver.NewInternalAPI(base) rsAPI.SetFederationAPI(nil, nil) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{aliceDev, bobDev}}, rsAPI, &syncKeyAPI{}) + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{aliceDev, bobDev}}, rsAPI) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -787,7 +785,7 @@ func testSendToDevice(t *testing.T, dbType test.DBType) { jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) - AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}, &syncKeyAPI{}) + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}) producer := producers.SyncAPIProducer{ TopicSendToDeviceEvent: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), @@ -905,6 +903,177 @@ func testSendToDevice(t *testing.T, dbType test.DBType) { } } +func TestContext(t *testing.T) { + test.WithAllDatabases(t, testContext) +} + +func testContext(t *testing.T, dbType test.DBType) { + + tests := []struct { + name string + roomID string + eventID string + params map[string]string + wantError bool + wantStateLength int + wantBeforeLength int + wantAfterLength int + }{ + { + name: "invalid filter", + params: map[string]string{ + "filter": "{", + }, + wantError: true, + }, + { + name: "invalid limit", + params: map[string]string{ + "limit": "abc", + }, + wantError: true, + }, + { + name: "high limit", + params: map[string]string{ + "limit": "100000", + }, + }, + { + name: "fine limit", + params: map[string]string{ + "limit": "10", + }, + }, + { + name: "last event without lazy loading", + wantStateLength: 5, + }, + { + name: "last event with lazy loading", + params: map[string]string{ + "filter": `{"lazy_load_members":true}`, + }, + wantStateLength: 1, + }, + { + name: "invalid room", + roomID: "!doesnotexist", + wantError: true, + }, + { + name: "invalid eventID", + eventID: "$doesnotexist", + wantError: true, + }, + { + name: "state is limited", + params: map[string]string{ + "limit": "1", + }, + wantStateLength: 1, + }, + { + name: "events are not limited", + wantBeforeLength: 7, + }, + { + name: "all events are limited", + params: map[string]string{ + "limit": "1", + }, + wantStateLength: 1, + wantBeforeLength: 1, + wantAfterLength: 1, + }, + } + + user := test.NewUser(t) + alice := userapi.Device{ + ID: "ALICEID", + UserID: user.ID, + AccessToken: "ALICE_BEARER_TOKEN", + DisplayName: "Alice", + AccountType: userapi.AccountTypeUser, + } + + base, baseClose := testrig.CreateBaseDendrite(t, dbType) + defer baseClose() + + // Use an actual roomserver for this + rsAPI := roomserver.NewInternalAPI(base) + rsAPI.SetFederationAPI(nil, nil) + + AddPublicRoutes(base, &syncUserAPI{accounts: []userapi.Device{alice}}, rsAPI) + + room := test.NewRoom(t, user) + + room.CreateAndInsert(t, user, "m.room.message", map[string]interface{}{"body": "hello world 1!"}) + room.CreateAndInsert(t, user, "m.room.message", map[string]interface{}{"body": "hello world 2!"}) + thirdMsg := room.CreateAndInsert(t, user, "m.room.message", map[string]interface{}{"body": "hello world3!"}) + room.CreateAndInsert(t, user, "m.room.message", map[string]interface{}{"body": "hello world4!"}) + + if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { + t.Fatalf("failed to send events: %v", err) + } + + jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) + + syncUntil(t, base, alice.AccessToken, false, func(syncBody string) bool { + // wait for the last sent eventID to come down sync + path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(event_id=="%s")`, room.ID, thirdMsg.EventID()) + return gjson.Get(syncBody, path).Exists() + }) + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + params := map[string]string{ + "access_token": alice.AccessToken, + } + w := httptest.NewRecorder() + // test overrides + roomID := room.ID + if tc.roomID != "" { + roomID = tc.roomID + } + eventID := thirdMsg.EventID() + if tc.eventID != "" { + eventID = tc.eventID + } + requestPath := fmt.Sprintf("/_matrix/client/v3/rooms/%s/context/%s", roomID, eventID) + if tc.params != nil { + for k, v := range tc.params { + params[k] = v + } + } + base.PublicClientAPIMux.ServeHTTP(w, test.NewRequest(t, "GET", requestPath, test.WithQueryParams(params))) + + if tc.wantError && w.Code == 200 { + t.Fatalf("Expected an error, but got none") + } + t.Log(w.Body.String()) + resp := routing.ContextRespsonse{} + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if tc.wantStateLength > 0 && tc.wantStateLength != len(resp.State) { + t.Fatalf("expected %d state events, got %d", tc.wantStateLength, len(resp.State)) + } + if tc.wantBeforeLength > 0 && tc.wantBeforeLength != len(resp.EventsBefore) { + t.Fatalf("expected %d before events, got %d", tc.wantBeforeLength, len(resp.EventsBefore)) + } + if tc.wantAfterLength > 0 && tc.wantAfterLength != len(resp.EventsAfter) { + t.Fatalf("expected %d after events, got %d", tc.wantAfterLength, len(resp.EventsAfter)) + } + + if !tc.wantError && resp.Event.EventID != eventID { + t.Fatalf("unexpected eventID %s, expected %s", resp.Event.EventID, eventID) + } + }) + } +} + func syncUntil(t *testing.T, base *base.BaseDendrite, accessToken string, skip bool, diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 9fbadc06c..6495dd535 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -63,6 +63,11 @@ type StreamEvent struct { ExcludeFromSync bool } +type RecentEvents struct { + Limited bool + Events []StreamEvent +} + // Range represents a range between two stream positions. type Range struct { // From is the position the client has already received. diff --git a/test/testrig/base.go b/test/testrig/base.go index dfc0d8aaf..bb8fded21 100644 --- a/test/testrig/base.go +++ b/test/testrig/base.go @@ -28,24 +28,24 @@ import ( func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, func()) { var cfg config.Dendrite cfg.Defaults(config.DefaultOpts{ - Generate: false, - Monolithic: true, + Generate: false, + SingleDatabase: true, }) cfg.Global.JetStream.InMemory = true cfg.FederationAPI.KeyPerspectives = nil switch dbType { case test.DBTypePostgres: cfg.Global.Defaults(config.DefaultOpts{ // autogen a signing key - Generate: true, - Monolithic: true, + Generate: true, + SingleDatabase: true, }) cfg.MediaAPI.Defaults(config.DefaultOpts{ // autogen a media path - Generate: true, - Monolithic: true, + Generate: true, + SingleDatabase: true, }) cfg.SyncAPI.Fulltext.Defaults(config.DefaultOpts{ // use in memory fts - Generate: true, - Monolithic: true, + Generate: true, + SingleDatabase: true, }) cfg.Global.ServerName = "test" // use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use @@ -58,7 +58,7 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f MaxIdleConnections: 2, ConnMaxLifetimeSeconds: 60, } - base := base.NewBaseDendrite(&cfg, "Test", base.DisableMetrics) + base := base.NewBaseDendrite(&cfg, base.DisableMetrics) return base, func() { base.ShutdownDendrite() base.WaitForShutdown() @@ -66,8 +66,8 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f } case test.DBTypeSQLite: cfg.Defaults(config.DefaultOpts{ - Generate: true, - Monolithic: true, + Generate: true, + SingleDatabase: false, }) cfg.Global.ServerName = "test" @@ -86,7 +86,7 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "userapi.db")) cfg.RelayAPI.Database.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "relayapi.db")) - base := base.NewBaseDendrite(&cfg, "Test", base.DisableMetrics) + base := base.NewBaseDendrite(&cfg, base.DisableMetrics) return base, func() { base.ShutdownDendrite() base.WaitForShutdown() @@ -102,14 +102,14 @@ func Base(cfg *config.Dendrite) (*base.BaseDendrite, nats.JetStreamContext, *nat if cfg == nil { cfg = &config.Dendrite{} cfg.Defaults(config.DefaultOpts{ - Generate: true, - Monolithic: true, + Generate: true, + SingleDatabase: false, }) } cfg.Global.JetStream.InMemory = true cfg.SyncAPI.Fulltext.InMemory = true cfg.FederationAPI.KeyPerspectives = nil - base := base.NewBaseDendrite(cfg, "Tests", base.DisableMetrics) + base := base.NewBaseDendrite(cfg, base.DisableMetrics) js, jc := base.NATS.Prepare(base.ProcessContext, &cfg.Global.JetStream) return base, js, jc } diff --git a/userapi/api/api.go b/userapi/api/api.go index 4ea2e91c3..fa297f773 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -15,9 +15,13 @@ package api import ( + "bytes" "context" "encoding/json" + "strings" + "time" + "github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -26,15 +30,12 @@ import ( // UserInternalAPI is the internal API for information about users and devices. type UserInternalAPI interface { - AppserviceUserAPI SyncUserAPI ClientUserAPI - MediaUserAPI FederationUserAPI - RoomserverUserAPI - KeyserverUserAPI QuerySearchProfilesAPI // used by p2p demos + QueryAccountByLocalpart(ctx context.Context, req *QueryAccountByLocalpartRequest, res *QueryAccountByLocalpartResponse) (err error) } // api functions required by the appservice api @@ -43,11 +44,6 @@ type AppserviceUserAPI interface { PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error } -type KeyserverUserAPI interface { - QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error - QueryDeviceInfos(ctx context.Context, req *QueryDeviceInfosRequest, res *QueryDeviceInfosResponse) error -} - type RoomserverUserAPI interface { QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error QueryAccountByLocalpart(ctx context.Context, req *QueryAccountByLocalpartRequest, res *QueryAccountByLocalpartResponse) (err error) @@ -60,13 +56,20 @@ type MediaUserAPI interface { // api functions required by the federation api type FederationUserAPI interface { + UploadDeviceKeysAPI QueryOpenIDToken(ctx context.Context, req *QueryOpenIDTokenRequest, res *QueryOpenIDTokenResponse) error QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error + QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error + QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse) error + QuerySignatures(ctx context.Context, req *QuerySignaturesRequest, res *QuerySignaturesResponse) error + QueryDeviceMessages(ctx context.Context, req *QueryDeviceMessagesRequest, res *QueryDeviceMessagesResponse) error + PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse) error } // api functions required by the sync api type SyncUserAPI interface { QueryAcccessTokenAPI + SyncKeyAPI QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error PerformLastSeenUpdate(ctx context.Context, req *PerformLastSeenUpdateRequest, res *PerformLastSeenUpdateResponse) error PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error @@ -79,6 +82,7 @@ type ClientUserAPI interface { QueryAcccessTokenAPI LoginTokenInternalAPI UserLoginAPI + ClientKeyAPI QueryNumericLocalpart(ctx context.Context, req *QueryNumericLocalpartRequest, res *QueryNumericLocalpartResponse) error QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error @@ -681,3 +685,310 @@ type QueryAccountByLocalpartRequest struct { type QueryAccountByLocalpartResponse struct { Account *Account } + +// API functions required by the clientapi +type ClientKeyAPI interface { + UploadDeviceKeysAPI + QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse) error + PerformUploadKeys(ctx context.Context, req *PerformUploadKeysRequest, res *PerformUploadKeysResponse) error + + PerformUploadDeviceSignatures(ctx context.Context, req *PerformUploadDeviceSignaturesRequest, res *PerformUploadDeviceSignaturesResponse) error + // PerformClaimKeys claims one-time keys for use in pre-key messages + PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse) error + PerformMarkAsStaleIfNeeded(ctx context.Context, req *PerformMarkAsStaleRequest, res *struct{}) error +} + +type UploadDeviceKeysAPI interface { + PerformUploadDeviceKeys(ctx context.Context, req *PerformUploadDeviceKeysRequest, res *PerformUploadDeviceKeysResponse) error +} + +// API functions required by the syncapi +type SyncKeyAPI interface { + QueryKeyChanges(ctx context.Context, req *QueryKeyChangesRequest, res *QueryKeyChangesResponse) error + QueryOneTimeKeys(ctx context.Context, req *QueryOneTimeKeysRequest, res *QueryOneTimeKeysResponse) error + PerformMarkAsStaleIfNeeded(ctx context.Context, req *PerformMarkAsStaleRequest, res *struct{}) error +} + +type FederationKeyAPI interface { + UploadDeviceKeysAPI + QueryKeys(ctx context.Context, req *QueryKeysRequest, res *QueryKeysResponse) error + QuerySignatures(ctx context.Context, req *QuerySignaturesRequest, res *QuerySignaturesResponse) error + QueryDeviceMessages(ctx context.Context, req *QueryDeviceMessagesRequest, res *QueryDeviceMessagesResponse) error + PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse) error +} + +// KeyError is returned if there was a problem performing/querying the server +type KeyError struct { + Err string `json:"error"` + IsInvalidSignature bool `json:"is_invalid_signature,omitempty"` // M_INVALID_SIGNATURE + IsMissingParam bool `json:"is_missing_param,omitempty"` // M_MISSING_PARAM + IsInvalidParam bool `json:"is_invalid_param,omitempty"` // M_INVALID_PARAM +} + +func (k *KeyError) Error() string { + return k.Err +} + +type DeviceMessageType int + +const ( + TypeDeviceKeyUpdate DeviceMessageType = iota + TypeCrossSigningUpdate +) + +// DeviceMessage represents the message produced into Kafka by the key server. +type DeviceMessage struct { + Type DeviceMessageType `json:"Type,omitempty"` + *DeviceKeys `json:"DeviceKeys,omitempty"` + *OutputCrossSigningKeyUpdate `json:"CrossSigningKeyUpdate,omitempty"` + // A monotonically increasing number which represents device changes for this user. + StreamID int64 + DeviceChangeID int64 +} + +// OutputCrossSigningKeyUpdate is an entry in the signing key update output kafka log +type OutputCrossSigningKeyUpdate struct { + CrossSigningKeyUpdate `json:"signing_keys"` +} + +type CrossSigningKeyUpdate struct { + MasterKey *gomatrixserverlib.CrossSigningKey `json:"master_key,omitempty"` + SelfSigningKey *gomatrixserverlib.CrossSigningKey `json:"self_signing_key,omitempty"` + UserID string `json:"user_id"` +} + +// DeviceKeysEqual returns true if the device keys updates contain the +// same display name and key JSON. This will return false if either of +// the updates is not a device keys update, or if the user ID/device ID +// differ between the two. +func (m1 *DeviceMessage) DeviceKeysEqual(m2 *DeviceMessage) bool { + if m1.DeviceKeys == nil || m2.DeviceKeys == nil { + return false + } + if m1.UserID != m2.UserID || m1.DeviceID != m2.DeviceID { + return false + } + if m1.DisplayName != m2.DisplayName { + return false // different display names + } + if len(m1.KeyJSON) == 0 || len(m2.KeyJSON) == 0 { + return false // either is empty + } + return bytes.Equal(m1.KeyJSON, m2.KeyJSON) +} + +// DeviceKeys represents a set of device keys for a single device +// https://matrix.org/docs/spec/client_server/r0.6.1#post-matrix-client-r0-keys-upload +type DeviceKeys struct { + // The user who owns this device + UserID string + // The device ID of this device + DeviceID string + // The device display name + DisplayName string + // The raw device key JSON + KeyJSON []byte +} + +// WithStreamID returns a copy of this device message with the given stream ID +func (k *DeviceKeys) WithStreamID(streamID int64) DeviceMessage { + return DeviceMessage{ + DeviceKeys: k, + StreamID: streamID, + } +} + +// OneTimeKeys represents a set of one-time keys for a single device +// https://matrix.org/docs/spec/client_server/r0.6.1#post-matrix-client-r0-keys-upload +type OneTimeKeys struct { + // The user who owns this device + UserID string + // The device ID of this device + DeviceID string + // A map of algorithm:key_id => key JSON + KeyJSON map[string]json.RawMessage +} + +// Split a key in KeyJSON into algorithm and key ID +func (k *OneTimeKeys) Split(keyIDWithAlgo string) (algo string, keyID string) { + segments := strings.Split(keyIDWithAlgo, ":") + return segments[0], segments[1] +} + +// OneTimeKeysCount represents the counts of one-time keys for a single device +type OneTimeKeysCount struct { + // The user who owns this device + UserID string + // The device ID of this device + DeviceID string + // algorithm to count e.g: + // { + // "curve25519": 10, + // "signed_curve25519": 20 + // } + KeyCount map[string]int +} + +// PerformUploadKeysRequest is the request to PerformUploadKeys +type PerformUploadKeysRequest struct { + UserID string // Required - User performing the request + DeviceID string // Optional - Device performing the request, for fetching OTK count + DeviceKeys []DeviceKeys + OneTimeKeys []OneTimeKeys + // OnlyDisplayNameUpdates should be `true` if ALL the DeviceKeys are present to update + // the display name for their respective device, and NOT to modify the keys. The key + // itself doesn't change but it's easier to pretend upload new keys and reuse the same code paths. + // Without this flag, requests to modify device display names would delete device keys. + OnlyDisplayNameUpdates bool +} + +// PerformUploadKeysResponse is the response to PerformUploadKeys +type PerformUploadKeysResponse struct { + // A fatal error when processing e.g database failures + Error *KeyError + // A map of user_id -> device_id -> Error for tracking failures. + KeyErrors map[string]map[string]*KeyError + OneTimeKeyCounts []OneTimeKeysCount +} + +// PerformDeleteKeysRequest asks the keyserver to forget about certain +// keys, and signatures related to those keys. +type PerformDeleteKeysRequest struct { + UserID string + KeyIDs []gomatrixserverlib.KeyID +} + +// PerformDeleteKeysResponse is the response to PerformDeleteKeysRequest. +type PerformDeleteKeysResponse struct { + Error *KeyError +} + +// KeyError sets a key error field on KeyErrors +func (r *PerformUploadKeysResponse) KeyError(userID, deviceID string, err *KeyError) { + if r.KeyErrors[userID] == nil { + r.KeyErrors[userID] = make(map[string]*KeyError) + } + r.KeyErrors[userID][deviceID] = err +} + +type PerformClaimKeysRequest struct { + // Map of user_id to device_id to algorithm name + OneTimeKeys map[string]map[string]string + Timeout time.Duration +} + +type PerformClaimKeysResponse struct { + // Map of user_id to device_id to algorithm:key_id to key JSON + OneTimeKeys map[string]map[string]map[string]json.RawMessage + // Map of remote server domain to error JSON + Failures map[string]interface{} + // Set if there was a fatal error processing this action + Error *KeyError +} + +type PerformUploadDeviceKeysRequest struct { + gomatrixserverlib.CrossSigningKeys + // The user that uploaded the key, should be populated by the clientapi. + UserID string +} + +type PerformUploadDeviceKeysResponse struct { + Error *KeyError +} + +type PerformUploadDeviceSignaturesRequest struct { + Signatures map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice + // The user that uploaded the sig, should be populated by the clientapi. + UserID string +} + +type PerformUploadDeviceSignaturesResponse struct { + Error *KeyError +} + +type QueryKeysRequest struct { + // The user ID asking for the keys, e.g. if from a client API request. + // Will not be populated if the key request came from federation. + UserID string + // Maps user IDs to a list of devices + UserToDevices map[string][]string + Timeout time.Duration +} + +type QueryKeysResponse struct { + // Map of remote server domain to error JSON + Failures map[string]interface{} + // Map of user_id to device_id to device_key + DeviceKeys map[string]map[string]json.RawMessage + // Maps of user_id to cross signing key + MasterKeys map[string]gomatrixserverlib.CrossSigningKey + SelfSigningKeys map[string]gomatrixserverlib.CrossSigningKey + UserSigningKeys map[string]gomatrixserverlib.CrossSigningKey + // Set if there was a fatal error processing this query + Error *KeyError +} + +type QueryKeyChangesRequest struct { + // The offset of the last received key event, or sarama.OffsetOldest if this is from the beginning + Offset int64 + // The inclusive offset where to track key changes up to. Messages with this offset are included in the response. + // Use types.OffsetNewest if the offset is unknown (then check the response Offset to avoid racing). + ToOffset int64 +} + +type QueryKeyChangesResponse struct { + // The set of users who have had their keys change. + UserIDs []string + // The latest offset represented in this response. + Offset int64 + // Set if there was a problem handling the request. + Error *KeyError +} + +type QueryOneTimeKeysRequest struct { + // The local user to query OTK counts for + UserID string + // The device to query OTK counts for + DeviceID string +} + +type QueryOneTimeKeysResponse struct { + // OTK key counts, in the extended /sync form described by https://matrix.org/docs/spec/client_server/r0.6.1#id84 + Count OneTimeKeysCount + Error *KeyError +} + +type QueryDeviceMessagesRequest struct { + UserID string +} + +type QueryDeviceMessagesResponse struct { + // The latest stream ID + StreamID int64 + Devices []DeviceMessage + Error *KeyError +} + +type QuerySignaturesRequest struct { + // A map of target user ID -> target key/device IDs to retrieve signatures for + TargetIDs map[string][]gomatrixserverlib.KeyID `json:"target_ids"` +} + +type QuerySignaturesResponse struct { + // A map of target user ID -> target key/device ID -> origin user ID -> origin key/device ID -> signatures + Signatures map[string]map[gomatrixserverlib.KeyID]types.CrossSigningSigMap + // A map of target user ID -> cross-signing master key + MasterKeys map[string]gomatrixserverlib.CrossSigningKey + // A map of target user ID -> cross-signing self-signing key + SelfSigningKeys map[string]gomatrixserverlib.CrossSigningKey + // A map of target user ID -> cross-signing user-signing key + UserSigningKeys map[string]gomatrixserverlib.CrossSigningKey + // The request error, if any + Error *KeyError +} + +type PerformMarkAsStaleRequest struct { + UserID string + Domain gomatrixserverlib.ServerName + DeviceID string +} diff --git a/userapi/api/api_trace.go b/userapi/api/api_trace.go deleted file mode 100644 index d10b5767b..000000000 --- a/userapi/api/api_trace.go +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2021 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/matrix-org/util" -) - -// UserInternalAPITrace wraps a RoomserverInternalAPI and logs the -// complete request/response/error -type UserInternalAPITrace struct { - Impl UserInternalAPI -} - -func (t *UserInternalAPITrace) InputAccountData(ctx context.Context, req *InputAccountDataRequest, res *InputAccountDataResponse) error { - err := t.Impl.InputAccountData(ctx, req, res) - util.GetLogger(ctx).Infof("InputAccountData req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) PerformAccountCreation(ctx context.Context, req *PerformAccountCreationRequest, res *PerformAccountCreationResponse) error { - err := t.Impl.PerformAccountCreation(ctx, req, res) - util.GetLogger(ctx).Infof("PerformAccountCreation req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) PerformPasswordUpdate(ctx context.Context, req *PerformPasswordUpdateRequest, res *PerformPasswordUpdateResponse) error { - err := t.Impl.PerformPasswordUpdate(ctx, req, res) - util.GetLogger(ctx).Infof("PerformPasswordUpdate req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *UserInternalAPITrace) PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error { - err := t.Impl.PerformDeviceCreation(ctx, req, res) - util.GetLogger(ctx).Infof("PerformDeviceCreation req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) PerformDeviceDeletion(ctx context.Context, req *PerformDeviceDeletionRequest, res *PerformDeviceDeletionResponse) error { - err := t.Impl.PerformDeviceDeletion(ctx, req, res) - util.GetLogger(ctx).Infof("PerformDeviceDeletion req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) PerformLastSeenUpdate(ctx context.Context, req *PerformLastSeenUpdateRequest, res *PerformLastSeenUpdateResponse) error { - err := t.Impl.PerformLastSeenUpdate(ctx, req, res) - util.GetLogger(ctx).Infof("PerformLastSeenUpdate req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) PerformDeviceUpdate(ctx context.Context, req *PerformDeviceUpdateRequest, res *PerformDeviceUpdateResponse) error { - err := t.Impl.PerformDeviceUpdate(ctx, req, res) - util.GetLogger(ctx).Infof("PerformDeviceUpdate req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) PerformAccountDeactivation(ctx context.Context, req *PerformAccountDeactivationRequest, res *PerformAccountDeactivationResponse) error { - err := t.Impl.PerformAccountDeactivation(ctx, req, res) - util.GetLogger(ctx).Infof("PerformAccountDeactivation req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) PerformOpenIDTokenCreation(ctx context.Context, req *PerformOpenIDTokenCreationRequest, res *PerformOpenIDTokenCreationResponse) error { - err := t.Impl.PerformOpenIDTokenCreation(ctx, req, res) - util.GetLogger(ctx).Infof("PerformOpenIDTokenCreation req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) error { - err := t.Impl.PerformKeyBackup(ctx, req, res) - util.GetLogger(ctx).Infof("PerformKeyBackup req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) PerformPusherSet(ctx context.Context, req *PerformPusherSetRequest, res *struct{}) error { - err := t.Impl.PerformPusherSet(ctx, req, res) - util.GetLogger(ctx).Infof("PerformPusherSet req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) PerformPusherDeletion(ctx context.Context, req *PerformPusherDeletionRequest, res *struct{}) error { - err := t.Impl.PerformPusherDeletion(ctx, req, res) - util.GetLogger(ctx).Infof("PerformPusherDeletion req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) PerformPushRulesPut(ctx context.Context, req *PerformPushRulesPutRequest, res *struct{}) error { - err := t.Impl.PerformPushRulesPut(ctx, req, res) - util.GetLogger(ctx).Infof("PerformPushRulesPut req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) QueryKeyBackup(ctx context.Context, req *QueryKeyBackupRequest, res *QueryKeyBackupResponse) error { - err := t.Impl.QueryKeyBackup(ctx, req, res) - util.GetLogger(ctx).Infof("QueryKeyBackup req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error { - err := t.Impl.QueryProfile(ctx, req, res) - util.GetLogger(ctx).Infof("QueryProfile req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error { - err := t.Impl.QueryAccessToken(ctx, req, res) - util.GetLogger(ctx).Infof("QueryAccessToken req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error { - err := t.Impl.QueryDevices(ctx, req, res) - util.GetLogger(ctx).Infof("QueryDevices req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error { - err := t.Impl.QueryAccountData(ctx, req, res) - util.GetLogger(ctx).Infof("QueryAccountData req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) QueryDeviceInfos(ctx context.Context, req *QueryDeviceInfosRequest, res *QueryDeviceInfosResponse) error { - err := t.Impl.QueryDeviceInfos(ctx, req, res) - util.GetLogger(ctx).Infof("QueryDeviceInfos req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) QuerySearchProfiles(ctx context.Context, req *QuerySearchProfilesRequest, res *QuerySearchProfilesResponse) error { - err := t.Impl.QuerySearchProfiles(ctx, req, res) - util.GetLogger(ctx).Infof("QuerySearchProfiles req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) QueryOpenIDToken(ctx context.Context, req *QueryOpenIDTokenRequest, res *QueryOpenIDTokenResponse) error { - err := t.Impl.QueryOpenIDToken(ctx, req, res) - util.GetLogger(ctx).Infof("QueryOpenIDToken req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) QueryPushers(ctx context.Context, req *QueryPushersRequest, res *QueryPushersResponse) error { - err := t.Impl.QueryPushers(ctx, req, res) - util.GetLogger(ctx).Infof("QueryPushers req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) QueryPushRules(ctx context.Context, req *QueryPushRulesRequest, res *QueryPushRulesResponse) error { - err := t.Impl.QueryPushRules(ctx, req, res) - util.GetLogger(ctx).Infof("QueryPushRules req=%+v res=%+v", js(req), js(res)) - return err -} -func (t *UserInternalAPITrace) QueryNotifications(ctx context.Context, req *QueryNotificationsRequest, res *QueryNotificationsResponse) error { - err := t.Impl.QueryNotifications(ctx, req, res) - util.GetLogger(ctx).Infof("QueryNotifications req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *UserInternalAPITrace) SetAvatarURL(ctx context.Context, req *PerformSetAvatarURLRequest, res *PerformSetAvatarURLResponse) error { - err := t.Impl.SetAvatarURL(ctx, req, res) - util.GetLogger(ctx).Infof("SetAvatarURL req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *UserInternalAPITrace) QueryNumericLocalpart(ctx context.Context, req *QueryNumericLocalpartRequest, res *QueryNumericLocalpartResponse) error { - err := t.Impl.QueryNumericLocalpart(ctx, req, res) - util.GetLogger(ctx).Infof("QueryNumericLocalpart req= res=%+v", js(res)) - return err -} - -func (t *UserInternalAPITrace) QueryAccountAvailability(ctx context.Context, req *QueryAccountAvailabilityRequest, res *QueryAccountAvailabilityResponse) error { - err := t.Impl.QueryAccountAvailability(ctx, req, res) - util.GetLogger(ctx).Infof("QueryAccountAvailability req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *UserInternalAPITrace) SetDisplayName(ctx context.Context, req *PerformUpdateDisplayNameRequest, res *PerformUpdateDisplayNameResponse) error { - err := t.Impl.SetDisplayName(ctx, req, res) - util.GetLogger(ctx).Infof("SetDisplayName req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *UserInternalAPITrace) QueryAccountByPassword(ctx context.Context, req *QueryAccountByPasswordRequest, res *QueryAccountByPasswordResponse) error { - err := t.Impl.QueryAccountByPassword(ctx, req, res) - util.GetLogger(ctx).Infof("QueryAccountByPassword req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *UserInternalAPITrace) QueryLocalpartForThreePID(ctx context.Context, req *QueryLocalpartForThreePIDRequest, res *QueryLocalpartForThreePIDResponse) error { - err := t.Impl.QueryLocalpartForThreePID(ctx, req, res) - util.GetLogger(ctx).Infof("QueryLocalpartForThreePID req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *UserInternalAPITrace) QueryThreePIDsForLocalpart(ctx context.Context, req *QueryThreePIDsForLocalpartRequest, res *QueryThreePIDsForLocalpartResponse) error { - err := t.Impl.QueryThreePIDsForLocalpart(ctx, req, res) - util.GetLogger(ctx).Infof("QueryThreePIDsForLocalpart req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *UserInternalAPITrace) PerformForgetThreePID(ctx context.Context, req *PerformForgetThreePIDRequest, res *struct{}) error { - err := t.Impl.PerformForgetThreePID(ctx, req, res) - util.GetLogger(ctx).Infof("PerformForgetThreePID req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *UserInternalAPITrace) PerformSaveThreePIDAssociation(ctx context.Context, req *PerformSaveThreePIDAssociationRequest, res *struct{}) error { - err := t.Impl.PerformSaveThreePIDAssociation(ctx, req, res) - util.GetLogger(ctx).Infof("PerformSaveThreePIDAssociation req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *UserInternalAPITrace) QueryAccountByLocalpart(ctx context.Context, req *QueryAccountByLocalpartRequest, res *QueryAccountByLocalpartResponse) error { - err := t.Impl.QueryAccountByLocalpart(ctx, req, res) - util.GetLogger(ctx).Infof("QueryAccountByLocalpart req=%+v res=%+v", js(req), js(res)) - return err -} - -func js(thing interface{}) string { - b, err := json.Marshal(thing) - if err != nil { - return fmt.Sprintf("Marshal error:%s", err) - } - return string(b) -} diff --git a/userapi/api/api_trace_logintoken.go b/userapi/api/api_trace_logintoken.go deleted file mode 100644 index e60dae594..000000000 --- a/userapi/api/api_trace_logintoken.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2021 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "context" - - "github.com/matrix-org/util" -) - -func (t *UserInternalAPITrace) PerformLoginTokenCreation(ctx context.Context, req *PerformLoginTokenCreationRequest, res *PerformLoginTokenCreationResponse) error { - err := t.Impl.PerformLoginTokenCreation(ctx, req, res) - util.GetLogger(ctx).Infof("PerformLoginTokenCreation req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *UserInternalAPITrace) PerformLoginTokenDeletion(ctx context.Context, req *PerformLoginTokenDeletionRequest, res *PerformLoginTokenDeletionResponse) error { - err := t.Impl.PerformLoginTokenDeletion(ctx, req, res) - util.GetLogger(ctx).Infof("PerformLoginTokenDeletion req=%+v res=%+v", js(req), js(res)) - return err -} - -func (t *UserInternalAPITrace) QueryLoginToken(ctx context.Context, req *QueryLoginTokenRequest, res *QueryLoginTokenResponse) error { - err := t.Impl.QueryLoginToken(ctx, req, res) - util.GetLogger(ctx).Infof("QueryLoginToken req=%+v res=%+v", js(req), js(res)) - return err -} diff --git a/userapi/consumers/clientapi.go b/userapi/consumers/clientapi.go index 42ae72e77..51bd2753a 100644 --- a/userapi/consumers/clientapi.go +++ b/userapi/consumers/clientapi.go @@ -37,7 +37,7 @@ type OutputReceiptEventConsumer struct { jetstream nats.JetStreamContext durable string topic string - db storage.Database + db storage.UserDatabase serverName gomatrixserverlib.ServerName syncProducer *producers.SyncAPI pgClient pushgateway.Client @@ -49,7 +49,7 @@ func NewOutputReceiptEventConsumer( process *process.ProcessContext, cfg *config.UserAPI, js nats.JetStreamContext, - store storage.Database, + store storage.UserDatabase, syncProducer *producers.SyncAPI, pgClient pushgateway.Client, ) *OutputReceiptEventConsumer { diff --git a/keyserver/consumers/devicelistupdate.go b/userapi/consumers/devicelistupdate.go similarity index 97% rename from keyserver/consumers/devicelistupdate.go rename to userapi/consumers/devicelistupdate.go index cd911f8c6..a65889fcc 100644 --- a/keyserver/consumers/devicelistupdate.go +++ b/userapi/consumers/devicelistupdate.go @@ -18,11 +18,11 @@ import ( "context" "encoding/json" + "github.com/matrix-org/dendrite/userapi/internal" "github.com/matrix-org/gomatrixserverlib" "github.com/nats-io/nats.go" "github.com/sirupsen/logrus" - "github.com/matrix-org/dendrite/keyserver/internal" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" @@ -41,7 +41,7 @@ type DeviceListUpdateConsumer struct { // NewDeviceListUpdateConsumer creates a new DeviceListConsumer. Call Start() to begin consuming from key servers. func NewDeviceListUpdateConsumer( process *process.ProcessContext, - cfg *config.KeyServer, + cfg *config.UserAPI, js nats.JetStreamContext, updater *internal.DeviceListUpdater, ) *DeviceListUpdateConsumer { diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go index 3ce5af621..47d330959 100644 --- a/userapi/consumers/roomserver.go +++ b/userapi/consumers/roomserver.go @@ -38,7 +38,7 @@ type OutputRoomEventConsumer struct { rsAPI rsapi.UserRoomserverAPI jetstream nats.JetStreamContext durable string - db storage.Database + db storage.UserDatabase topic string pgClient pushgateway.Client syncProducer *producers.SyncAPI @@ -53,7 +53,7 @@ func NewOutputRoomEventConsumer( process *process.ProcessContext, cfg *config.UserAPI, js nats.JetStreamContext, - store storage.Database, + store storage.UserDatabase, pgClient pushgateway.Client, rsAPI rsapi.UserRoomserverAPI, syncProducer *producers.SyncAPI, diff --git a/userapi/consumers/roomserver_test.go b/userapi/consumers/roomserver_test.go index 39f4aab4a..bc5ae652d 100644 --- a/userapi/consumers/roomserver_test.go +++ b/userapi/consumers/roomserver_test.go @@ -18,11 +18,11 @@ import ( userAPITypes "github.com/matrix-org/dendrite/userapi/types" ) -func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { +func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.UserDatabase, func()) { base, baseclose := testrig.CreateBaseDendrite(t, dbType) t.Helper() connStr, close := test.PrepareDBConnectionString(t, dbType) - db, err := storage.NewUserAPIDatabase(base, &config.DatabaseOptions{ + db, err := storage.NewUserDatabase(base, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, "", 4, 0, 0, "") if err != nil { diff --git a/keyserver/consumers/signingkeyupdate.go b/userapi/consumers/signingkeyupdate.go similarity index 87% rename from keyserver/consumers/signingkeyupdate.go rename to userapi/consumers/signingkeyupdate.go index bcceaad15..f4ff017db 100644 --- a/keyserver/consumers/signingkeyupdate.go +++ b/userapi/consumers/signingkeyupdate.go @@ -22,11 +22,10 @@ import ( "github.com/nats-io/nats.go" "github.com/sirupsen/logrus" - keyapi "github.com/matrix-org/dendrite/keyserver/api" - "github.com/matrix-org/dendrite/keyserver/internal" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/dendrite/userapi/api" ) // SigningKeyUpdateConsumer consumes signing key updates that came in over federation. @@ -35,24 +34,24 @@ type SigningKeyUpdateConsumer struct { jetstream nats.JetStreamContext durable string topic string - keyAPI *internal.KeyInternalAPI - cfg *config.KeyServer + userAPI api.UploadDeviceKeysAPI + cfg *config.UserAPI isLocalServerName func(gomatrixserverlib.ServerName) bool } // NewSigningKeyUpdateConsumer creates a new SigningKeyUpdateConsumer. Call Start() to begin consuming from key servers. func NewSigningKeyUpdateConsumer( process *process.ProcessContext, - cfg *config.KeyServer, + cfg *config.UserAPI, js nats.JetStreamContext, - keyAPI *internal.KeyInternalAPI, + userAPI api.UploadDeviceKeysAPI, ) *SigningKeyUpdateConsumer { return &SigningKeyUpdateConsumer{ ctx: process.Context(), jetstream: js, durable: cfg.Matrix.JetStream.Prefixed("KeyServerSigningKeyConsumer"), topic: cfg.Matrix.JetStream.Prefixed(jetstream.InputSigningKeyUpdate), - keyAPI: keyAPI, + userAPI: userAPI, cfg: cfg, isLocalServerName: cfg.Matrix.IsLocalServerName, } @@ -70,7 +69,7 @@ func (t *SigningKeyUpdateConsumer) Start() error { // signing key update events topic from the key server. func (t *SigningKeyUpdateConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called - var updatePayload keyapi.CrossSigningKeyUpdate + var updatePayload api.CrossSigningKeyUpdate if err := json.Unmarshal(msg.Data, &updatePayload); err != nil { logrus.WithError(err).Errorf("Failed to read from signing key update input topic") return true @@ -94,12 +93,12 @@ func (t *SigningKeyUpdateConsumer) onMessage(ctx context.Context, msgs []*nats.M if updatePayload.SelfSigningKey != nil { keys.SelfSigningKey = *updatePayload.SelfSigningKey } - uploadReq := &keyapi.PerformUploadDeviceKeysRequest{ + uploadReq := &api.PerformUploadDeviceKeysRequest{ CrossSigningKeys: keys, UserID: updatePayload.UserID, } - uploadRes := &keyapi.PerformUploadDeviceKeysResponse{} - if err := t.keyAPI.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes); err != nil { + uploadRes := &api.PerformUploadDeviceKeysResponse{} + if err := t.userAPI.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes); err != nil { logrus.WithError(err).Error("failed to upload device keys") return false } diff --git a/keyserver/internal/cross_signing.go b/userapi/internal/cross_signing.go similarity index 91% rename from keyserver/internal/cross_signing.go rename to userapi/internal/cross_signing.go index 99859dff6..8b9704d1b 100644 --- a/keyserver/internal/cross_signing.go +++ b/userapi/internal/cross_signing.go @@ -22,8 +22,8 @@ import ( "fmt" "strings" - "github.com/matrix-org/dendrite/keyserver/api" - "github.com/matrix-org/dendrite/keyserver/types" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" "golang.org/x/crypto/curve25519" @@ -103,7 +103,7 @@ func sanityCheckKey(key gomatrixserverlib.CrossSigningKey, userID string, purpos } // nolint:gocyclo -func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) error { +func (a *UserInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) error { // Find the keys to store. byPurpose := map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey{} toStore := types.CrossSigningKeyMap{} @@ -169,7 +169,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P // something if any of the specified keys in the request are different // to what we've got in the database, to avoid generating key change // notifications unnecessarily. - existingKeys, err := a.DB.CrossSigningKeysDataForUser(ctx, req.UserID) + existingKeys, err := a.KeyDatabase.CrossSigningKeysDataForUser(ctx, req.UserID) if err != nil { res.Error = &api.KeyError{ Err: "Retrieving cross-signing keys from database failed: " + err.Error(), @@ -216,7 +216,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P } // Store the keys. - if err := a.DB.StoreCrossSigningKeysForUser(ctx, req.UserID, toStore); err != nil { + if err := a.KeyDatabase.StoreCrossSigningKeysForUser(ctx, req.UserID, toStore); err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("a.DB.StoreCrossSigningKeysForUser: %s", err), } @@ -234,7 +234,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P continue } for sigKeyID, sigBytes := range forSigUserID { - if err := a.DB.StoreCrossSigningSigsForTarget(ctx, sigUserID, sigKeyID, req.UserID, targetKeyID, sigBytes); err != nil { + if err := a.KeyDatabase.StoreCrossSigningSigsForTarget(ctx, sigUserID, sigKeyID, req.UserID, targetKeyID, sigBytes); err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("a.DB.StoreCrossSigningSigsForTarget: %s", err), } @@ -257,7 +257,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P if update.MasterKey == nil && update.SelfSigningKey == nil { return nil } - if err := a.Producer.ProduceSigningKeyUpdate(update); err != nil { + if err := a.KeyChangeProducer.ProduceSigningKeyUpdate(update); err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("a.Producer.ProduceSigningKeyUpdate: %s", err), } @@ -266,7 +266,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.P return nil } -func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req *api.PerformUploadDeviceSignaturesRequest, res *api.PerformUploadDeviceSignaturesResponse) error { +func (a *UserInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req *api.PerformUploadDeviceSignaturesRequest, res *api.PerformUploadDeviceSignaturesResponse) error { // Before we do anything, we need the master and self-signing keys for this user. // Then we can verify the signatures make sense. queryReq := &api.QueryKeysRequest{ @@ -342,7 +342,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req MasterKey: &masterKey, SelfSigningKey: &selfSigningKey, } - if err := a.Producer.ProduceSigningKeyUpdate(update); err != nil { + if err := a.KeyChangeProducer.ProduceSigningKeyUpdate(update); err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("a.Producer.ProduceSigningKeyUpdate: %s", err), } @@ -352,7 +352,7 @@ func (a *KeyInternalAPI) PerformUploadDeviceSignatures(ctx context.Context, req return nil } -func (a *KeyInternalAPI) processSelfSignatures( +func (a *UserInternalAPI) processSelfSignatures( ctx context.Context, signatures map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice, ) error { @@ -373,7 +373,7 @@ func (a *KeyInternalAPI) processSelfSignatures( } for originUserID, forOriginUserID := range sig.Signatures { for originKeyID, originSig := range forOriginUserID { - if err := a.DB.StoreCrossSigningSigsForTarget( + if err := a.KeyDatabase.StoreCrossSigningSigsForTarget( ctx, originUserID, originKeyID, targetUserID, targetKeyID, originSig, ); err != nil { return fmt.Errorf("a.DB.StoreCrossSigningKeysForTarget: %w", err) @@ -384,7 +384,7 @@ func (a *KeyInternalAPI) processSelfSignatures( case *gomatrixserverlib.DeviceKeys: for originUserID, forOriginUserID := range sig.Signatures { for originKeyID, originSig := range forOriginUserID { - if err := a.DB.StoreCrossSigningSigsForTarget( + if err := a.KeyDatabase.StoreCrossSigningSigsForTarget( ctx, originUserID, originKeyID, targetUserID, targetKeyID, originSig, ); err != nil { return fmt.Errorf("a.DB.StoreCrossSigningKeysForTarget: %w", err) @@ -401,7 +401,7 @@ func (a *KeyInternalAPI) processSelfSignatures( return nil } -func (a *KeyInternalAPI) processOtherSignatures( +func (a *UserInternalAPI) processOtherSignatures( ctx context.Context, userID string, queryRes *api.QueryKeysResponse, signatures map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.CrossSigningForKeyOrDevice, ) error { @@ -442,7 +442,7 @@ func (a *KeyInternalAPI) processOtherSignatures( } for originKeyID, originSig := range userSigs { - if err := a.DB.StoreCrossSigningSigsForTarget( + if err := a.KeyDatabase.StoreCrossSigningSigsForTarget( ctx, userID, originKeyID, targetUserID, targetKeyID, originSig, ); err != nil { return fmt.Errorf("a.DB.StoreCrossSigningKeysForTarget: %w", err) @@ -461,11 +461,11 @@ func (a *KeyInternalAPI) processOtherSignatures( return nil } -func (a *KeyInternalAPI) crossSigningKeysFromDatabase( +func (a *UserInternalAPI) crossSigningKeysFromDatabase( ctx context.Context, req *api.QueryKeysRequest, res *api.QueryKeysResponse, ) { for targetUserID := range req.UserToDevices { - keys, err := a.DB.CrossSigningKeysForUser(ctx, targetUserID) + keys, err := a.KeyDatabase.CrossSigningKeysForUser(ctx, targetUserID) if err != nil { logrus.WithError(err).Errorf("Failed to get cross-signing keys for user %q", targetUserID) continue @@ -478,7 +478,7 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase( break } - sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, keyID) + sigMap, err := a.KeyDatabase.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, keyID) if err != nil && err != sql.ErrNoRows { logrus.WithError(err).Errorf("Failed to get cross-signing signatures for user %q key %q", targetUserID, keyID) continue @@ -522,9 +522,9 @@ func (a *KeyInternalAPI) crossSigningKeysFromDatabase( } } -func (a *KeyInternalAPI) QuerySignatures(ctx context.Context, req *api.QuerySignaturesRequest, res *api.QuerySignaturesResponse) error { +func (a *UserInternalAPI) QuerySignatures(ctx context.Context, req *api.QuerySignaturesRequest, res *api.QuerySignaturesResponse) error { for targetUserID, forTargetUser := range req.TargetIDs { - keyMap, err := a.DB.CrossSigningKeysForUser(ctx, targetUserID) + keyMap, err := a.KeyDatabase.CrossSigningKeysForUser(ctx, targetUserID) if err != nil && err != sql.ErrNoRows { res.Error = &api.KeyError{ Err: fmt.Sprintf("a.DB.CrossSigningKeysForUser: %s", err), @@ -556,7 +556,7 @@ func (a *KeyInternalAPI) QuerySignatures(ctx context.Context, req *api.QuerySign for _, targetKeyID := range forTargetUser { // Get own signatures only. - sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, targetUserID, targetUserID, targetKeyID) + sigMap, err := a.KeyDatabase.CrossSigningSigsForTarget(ctx, targetUserID, targetUserID, targetKeyID) if err != nil && err != sql.ErrNoRows { res.Error = &api.KeyError{ Err: fmt.Sprintf("a.DB.CrossSigningSigsForTarget: %s", err), diff --git a/keyserver/internal/device_list_update.go b/userapi/internal/device_list_update.go similarity index 98% rename from keyserver/internal/device_list_update.go rename to userapi/internal/device_list_update.go index c7bf8da53..3b4dcf98e 100644 --- a/keyserver/internal/device_list_update.go +++ b/userapi/internal/device_list_update.go @@ -33,8 +33,8 @@ import ( "github.com/sirupsen/logrus" fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" - "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/dendrite/userapi/api" ) var ( @@ -477,10 +477,6 @@ func (u *DeviceListUpdater) processServerUser(ctx context.Context, serverName go return e.RetryAfter, err } else if e.Blacklisted { return time.Hour * 8, err - } else if e.Code >= 300 { - // We didn't get a real FederationClientError (e.g. in polylith mode, where gomatrix.HTTPError - // are "converted" to FederationClientError), but we probably shouldn't hit them every $waitTime seconds. - return hourWaitTime, err } case net.Error: // Use the default waitTime, if it's a timeout. diff --git a/keyserver/internal/device_list_update_default.go b/userapi/internal/device_list_update_default.go similarity index 100% rename from keyserver/internal/device_list_update_default.go rename to userapi/internal/device_list_update_default.go diff --git a/keyserver/internal/device_list_update_sytest.go b/userapi/internal/device_list_update_sytest.go similarity index 100% rename from keyserver/internal/device_list_update_sytest.go rename to userapi/internal/device_list_update_sytest.go diff --git a/keyserver/internal/device_list_update_test.go b/userapi/internal/device_list_update_test.go similarity index 98% rename from keyserver/internal/device_list_update_test.go rename to userapi/internal/device_list_update_test.go index 60a2c2f30..868fc9be8 100644 --- a/keyserver/internal/device_list_update_test.go +++ b/userapi/internal/device_list_update_test.go @@ -29,13 +29,13 @@ import ( "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/keyserver/api" - "github.com/matrix-org/dendrite/keyserver/storage" roomserver "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage" ) var ( @@ -360,12 +360,12 @@ func TestDebounce(t *testing.T) { } } -func mustCreateKeyserverDB(t *testing.T, dbType test.DBType) (storage.Database, func()) { +func mustCreateKeyserverDB(t *testing.T, dbType test.DBType) (storage.KeyDatabase, func()) { t.Helper() base, _, _ := testrig.Base(nil) connStr, clearDB := test.PrepareDBConnectionString(t, dbType) - db, err := storage.NewDatabase(base, &config.DatabaseOptions{ConnectionString: config.DataSource(connStr)}) + db, err := storage.NewKeyDatabase(base, &config.DatabaseOptions{ConnectionString: config.DataSource(connStr)}) if err != nil { t.Fatal(err) } diff --git a/keyserver/internal/internal.go b/userapi/internal/key_api.go similarity index 83% rename from keyserver/internal/internal.go rename to userapi/internal/key_api.go index 9a08a0bb7..be816fe5d 100644 --- a/keyserver/internal/internal.go +++ b/userapi/internal/key_api.go @@ -29,29 +29,11 @@ import ( "github.com/tidwall/gjson" "github.com/tidwall/sjson" - fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" - "github.com/matrix-org/dendrite/keyserver/api" - "github.com/matrix-org/dendrite/keyserver/producers" - "github.com/matrix-org/dendrite/keyserver/storage" - "github.com/matrix-org/dendrite/setup/config" - userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/api" ) -type KeyInternalAPI struct { - DB storage.Database - Cfg *config.KeyServer - FedClient fedsenderapi.KeyserverFederationAPI - UserAPI userapi.KeyserverUserAPI - Producer *producers.KeyChange - Updater *DeviceListUpdater -} - -func (a *KeyInternalAPI) SetUserAPI(i userapi.KeyserverUserAPI) { - a.UserAPI = i -} - -func (a *KeyInternalAPI) QueryKeyChanges(ctx context.Context, req *api.QueryKeyChangesRequest, res *api.QueryKeyChangesResponse) error { - userIDs, latest, err := a.DB.KeyChanges(ctx, req.Offset, req.ToOffset) +func (a *UserInternalAPI) QueryKeyChanges(ctx context.Context, req *api.QueryKeyChangesRequest, res *api.QueryKeyChangesResponse) error { + userIDs, latest, err := a.KeyDatabase.KeyChanges(ctx, req.Offset, req.ToOffset) if err != nil { res.Error = &api.KeyError{ Err: err.Error(), @@ -63,7 +45,7 @@ func (a *KeyInternalAPI) QueryKeyChanges(ctx context.Context, req *api.QueryKeyC return nil } -func (a *KeyInternalAPI) PerformUploadKeys(ctx context.Context, req *api.PerformUploadKeysRequest, res *api.PerformUploadKeysResponse) error { +func (a *UserInternalAPI) PerformUploadKeys(ctx context.Context, req *api.PerformUploadKeysRequest, res *api.PerformUploadKeysResponse) error { res.KeyErrors = make(map[string]map[string]*api.KeyError) if len(req.DeviceKeys) > 0 { a.uploadLocalDeviceKeys(ctx, req, res) @@ -71,7 +53,7 @@ func (a *KeyInternalAPI) PerformUploadKeys(ctx context.Context, req *api.Perform if len(req.OneTimeKeys) > 0 { a.uploadOneTimeKeys(ctx, req, res) } - otks, err := a.DB.OneTimeKeysCount(ctx, req.UserID, req.DeviceID) + otks, err := a.KeyDatabase.OneTimeKeysCount(ctx, req.UserID, req.DeviceID) if err != nil { return err } @@ -79,7 +61,7 @@ func (a *KeyInternalAPI) PerformUploadKeys(ctx context.Context, req *api.Perform return nil } -func (a *KeyInternalAPI) PerformClaimKeys(ctx context.Context, req *api.PerformClaimKeysRequest, res *api.PerformClaimKeysResponse) error { +func (a *UserInternalAPI) PerformClaimKeys(ctx context.Context, req *api.PerformClaimKeysRequest, res *api.PerformClaimKeysResponse) error { res.OneTimeKeys = make(map[string]map[string]map[string]json.RawMessage) res.Failures = make(map[string]interface{}) // wrap request map in a top-level by-domain map @@ -97,11 +79,11 @@ func (a *KeyInternalAPI) PerformClaimKeys(ctx context.Context, req *api.PerformC domainToDeviceKeys[string(serverName)] = nested } for domain, local := range domainToDeviceKeys { - if !a.Cfg.Matrix.IsLocalServerName(gomatrixserverlib.ServerName(domain)) { + if !a.Config.Matrix.IsLocalServerName(gomatrixserverlib.ServerName(domain)) { continue } // claim local keys - keys, err := a.DB.ClaimKeys(ctx, local) + keys, err := a.KeyDatabase.ClaimKeys(ctx, local) if err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("failed to ClaimKeys locally: %s", err), @@ -129,7 +111,7 @@ func (a *KeyInternalAPI) PerformClaimKeys(ctx context.Context, req *api.PerformC return nil } -func (a *KeyInternalAPI) claimRemoteKeys( +func (a *UserInternalAPI) claimRemoteKeys( ctx context.Context, timeout time.Duration, res *api.PerformClaimKeysResponse, domainToDeviceKeys map[string]map[string]map[string]string, ) { var wg sync.WaitGroup // Wait for fan-out goroutines to finish @@ -146,7 +128,7 @@ func (a *KeyInternalAPI) claimRemoteKeys( defer cancel() defer wg.Done() - claimKeyRes, err := a.FedClient.ClaimKeys(fedCtx, a.Cfg.Matrix.ServerName, gomatrixserverlib.ServerName(domain), keysToClaim) + claimKeyRes, err := a.FedClient.ClaimKeys(fedCtx, a.Config.Matrix.ServerName, gomatrixserverlib.ServerName(domain), keysToClaim) mu.Lock() defer mu.Unlock() @@ -177,8 +159,8 @@ func (a *KeyInternalAPI) claimRemoteKeys( }).Infof("Claimed remote keys from %d server(s)", len(domainToDeviceKeys)) } -func (a *KeyInternalAPI) PerformDeleteKeys(ctx context.Context, req *api.PerformDeleteKeysRequest, res *api.PerformDeleteKeysResponse) error { - if err := a.DB.DeleteDeviceKeys(ctx, req.UserID, req.KeyIDs); err != nil { +func (a *UserInternalAPI) PerformDeleteKeys(ctx context.Context, req *api.PerformDeleteKeysRequest, res *api.PerformDeleteKeysResponse) error { + if err := a.KeyDatabase.DeleteDeviceKeys(ctx, req.UserID, req.KeyIDs); err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("Failed to delete device keys: %s", err), } @@ -186,8 +168,8 @@ func (a *KeyInternalAPI) PerformDeleteKeys(ctx context.Context, req *api.Perform return nil } -func (a *KeyInternalAPI) QueryOneTimeKeys(ctx context.Context, req *api.QueryOneTimeKeysRequest, res *api.QueryOneTimeKeysResponse) error { - count, err := a.DB.OneTimeKeysCount(ctx, req.UserID, req.DeviceID) +func (a *UserInternalAPI) QueryOneTimeKeys(ctx context.Context, req *api.QueryOneTimeKeysRequest, res *api.QueryOneTimeKeysResponse) error { + count, err := a.KeyDatabase.OneTimeKeysCount(ctx, req.UserID, req.DeviceID) if err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("Failed to query OTK counts: %s", err), @@ -198,8 +180,8 @@ func (a *KeyInternalAPI) QueryOneTimeKeys(ctx context.Context, req *api.QueryOne return nil } -func (a *KeyInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.QueryDeviceMessagesRequest, res *api.QueryDeviceMessagesResponse) error { - msgs, err := a.DB.DeviceKeysForUser(ctx, req.UserID, nil, false) +func (a *UserInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.QueryDeviceMessagesRequest, res *api.QueryDeviceMessagesResponse) error { + msgs, err := a.KeyDatabase.DeviceKeysForUser(ctx, req.UserID, nil, false) if err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("failed to query DB for device keys: %s", err), @@ -225,8 +207,8 @@ func (a *KeyInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.Query // PerformMarkAsStaleIfNeeded marks the users device list as stale, if the given deviceID is not present // in our database. -func (a *KeyInternalAPI) PerformMarkAsStaleIfNeeded(ctx context.Context, req *api.PerformMarkAsStaleRequest, res *struct{}) error { - knownDevices, err := a.DB.DeviceKeysForUser(ctx, req.UserID, []string{}, true) +func (a *UserInternalAPI) PerformMarkAsStaleIfNeeded(ctx context.Context, req *api.PerformMarkAsStaleRequest, res *struct{}) error { + knownDevices, err := a.KeyDatabase.DeviceKeysForUser(ctx, req.UserID, []string{}, true) if err != nil { return err } @@ -244,7 +226,7 @@ func (a *KeyInternalAPI) PerformMarkAsStaleIfNeeded(ctx context.Context, req *ap } // nolint:gocyclo -func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysRequest, res *api.QueryKeysResponse) error { +func (a *UserInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysRequest, res *api.QueryKeysResponse) error { var respMu sync.Mutex res.DeviceKeys = make(map[string]map[string]json.RawMessage) res.MasterKeys = make(map[string]gomatrixserverlib.CrossSigningKey) @@ -262,8 +244,8 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques } domain := string(serverName) // query local devices - if a.Cfg.Matrix.IsLocalServerName(serverName) { - deviceKeys, err := a.DB.DeviceKeysForUser(ctx, userID, deviceIDs, false) + if a.Config.Matrix.IsLocalServerName(serverName) { + deviceKeys, err := a.KeyDatabase.DeviceKeysForUser(ctx, userID, deviceIDs, false) if err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("failed to query local device keys: %s", err), @@ -276,8 +258,8 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques for _, dk := range deviceKeys { dids = append(dids, dk.DeviceID) } - var queryRes userapi.QueryDeviceInfosResponse - err = a.UserAPI.QueryDeviceInfos(ctx, &userapi.QueryDeviceInfosRequest{ + var queryRes api.QueryDeviceInfosResponse + err = a.QueryDeviceInfos(ctx, &api.QueryDeviceInfosRequest{ DeviceIDs: dids, }, &queryRes) if err != nil { @@ -341,14 +323,14 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques masterKey.Signatures = map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{} } for targetKeyID := range masterKey.Keys { - sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, targetKeyID) + sigMap, err := a.KeyDatabase.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, targetKeyID) if err != nil { // Stop executing the function if the context was canceled/the deadline was exceeded, // as we can't continue without a valid context. if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return nil } - logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed") + logrus.WithError(err).Errorf("a.KeyDatabase.CrossSigningSigsForTarget failed") continue } if len(sigMap) == 0 { @@ -367,14 +349,14 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques for targetUserID, forUserID := range res.DeviceKeys { for targetKeyID, key := range forUserID { - sigMap, err := a.DB.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, gomatrixserverlib.KeyID(targetKeyID)) + sigMap, err := a.KeyDatabase.CrossSigningSigsForTarget(ctx, req.UserID, targetUserID, gomatrixserverlib.KeyID(targetKeyID)) if err != nil { // Stop executing the function if the context was canceled/the deadline was exceeded, // as we can't continue without a valid context. if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return nil } - logrus.WithError(err).Errorf("a.DB.CrossSigningSigsForTarget failed") + logrus.WithError(err).Errorf("a.KeyDatabase.CrossSigningSigsForTarget failed") continue } if len(sigMap) == 0 { @@ -403,7 +385,7 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques return nil } -func (a *KeyInternalAPI) remoteKeysFromDatabase( +func (a *UserInternalAPI) remoteKeysFromDatabase( ctx context.Context, res *api.QueryKeysResponse, respMu *sync.Mutex, domainToDeviceKeys map[string]map[string][]string, ) map[string]map[string][]string { fetchRemote := make(map[string]map[string][]string) @@ -429,7 +411,7 @@ func (a *KeyInternalAPI) remoteKeysFromDatabase( return fetchRemote } -func (a *KeyInternalAPI) queryRemoteKeys( +func (a *UserInternalAPI) queryRemoteKeys( ctx context.Context, timeout time.Duration, res *api.QueryKeysResponse, domainToDeviceKeys map[string]map[string][]string, domainToCrossSigningKeys map[string]map[string]struct{}, ) { @@ -441,13 +423,13 @@ func (a *KeyInternalAPI) queryRemoteKeys( domains := map[string]struct{}{} for domain := range domainToDeviceKeys { - if a.Cfg.Matrix.IsLocalServerName(gomatrixserverlib.ServerName(domain)) { + if a.Config.Matrix.IsLocalServerName(gomatrixserverlib.ServerName(domain)) { continue } domains[domain] = struct{}{} } for domain := range domainToCrossSigningKeys { - if a.Cfg.Matrix.IsLocalServerName(gomatrixserverlib.ServerName(domain)) { + if a.Config.Matrix.IsLocalServerName(gomatrixserverlib.ServerName(domain)) { continue } domains[domain] = struct{}{} @@ -499,7 +481,7 @@ func (a *KeyInternalAPI) queryRemoteKeys( } } -func (a *KeyInternalAPI) queryRemoteKeysOnServer( +func (a *UserInternalAPI) queryRemoteKeysOnServer( ctx context.Context, serverName string, devKeys map[string][]string, crossSigningKeys map[string]struct{}, wg *sync.WaitGroup, respMu *sync.Mutex, timeout time.Duration, resultCh chan<- *gomatrixserverlib.RespQueryKeys, res *api.QueryKeysResponse, @@ -559,7 +541,7 @@ func (a *KeyInternalAPI) queryRemoteKeysOnServer( if len(devKeys) == 0 { return } - queryKeysResp, err := a.FedClient.QueryKeys(fedCtx, a.Cfg.Matrix.ServerName, gomatrixserverlib.ServerName(serverName), devKeys) + queryKeysResp, err := a.FedClient.QueryKeys(fedCtx, a.Config.Matrix.ServerName, gomatrixserverlib.ServerName(serverName), devKeys) if err == nil { resultCh <- &queryKeysResp return @@ -586,10 +568,10 @@ func (a *KeyInternalAPI) queryRemoteKeysOnServer( respMu.Unlock() } -func (a *KeyInternalAPI) populateResponseWithDeviceKeysFromDatabase( +func (a *UserInternalAPI) populateResponseWithDeviceKeysFromDatabase( ctx context.Context, res *api.QueryKeysResponse, respMu *sync.Mutex, userID string, deviceIDs []string, ) error { - keys, err := a.DB.DeviceKeysForUser(ctx, userID, deviceIDs, false) + keys, err := a.KeyDatabase.DeviceKeysForUser(ctx, userID, deviceIDs, false) // if we can't query the db or there are fewer keys than requested, fetch from remote. if err != nil { return fmt.Errorf("DeviceKeysForUser %s %v failed: %w", userID, deviceIDs, err) @@ -621,11 +603,11 @@ func (a *KeyInternalAPI) populateResponseWithDeviceKeysFromDatabase( return nil } -func (a *KeyInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.PerformUploadKeysRequest, res *api.PerformUploadKeysResponse) { +func (a *UserInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.PerformUploadKeysRequest, res *api.PerformUploadKeysResponse) { // get a list of devices from the user API that actually exist, as // we won't store keys for devices that don't exist - uapidevices := &userapi.QueryDevicesResponse{} - if err := a.UserAPI.QueryDevices(ctx, &userapi.QueryDevicesRequest{UserID: req.UserID}, uapidevices); err != nil { + uapidevices := &api.QueryDevicesResponse{} + if err := a.QueryDevices(ctx, &api.QueryDevicesRequest{UserID: req.UserID}, uapidevices); err != nil { res.Error = &api.KeyError{ Err: err.Error(), } @@ -643,7 +625,7 @@ func (a *KeyInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.Per } // Get all of the user existing device keys so we can check for changes. - existingKeys, err := a.DB.DeviceKeysForUser(ctx, req.UserID, nil, true) + existingKeys, err := a.KeyDatabase.DeviceKeysForUser(ctx, req.UserID, nil, true) if err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("failed to query existing device keys: %s", err.Error()), @@ -662,7 +644,7 @@ func (a *KeyInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.Per } if len(toClean) > 0 { - if err = a.DB.DeleteDeviceKeys(ctx, req.UserID, toClean); err != nil { + if err = a.KeyDatabase.DeleteDeviceKeys(ctx, req.UserID, toClean); err != nil { logrus.WithField("user_id", req.UserID).WithError(err).Errorf("Failed to clean up %d stale keyserver device key entries", len(toClean)) } else { logrus.WithField("user_id", req.UserID).Debugf("Cleaned up %d stale keyserver device key entries", len(toClean)) @@ -693,7 +675,7 @@ func (a *KeyInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.Per if err != nil { continue // ignore invalid users } - if !a.Cfg.Matrix.IsLocalServerName(serverName) { + if !a.Config.Matrix.IsLocalServerName(serverName) { continue // ignore remote users } if len(key.KeyJSON) == 0 { @@ -722,30 +704,30 @@ func (a *KeyInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.Per } // store the device keys and emit changes - err = a.DB.StoreLocalDeviceKeys(ctx, keysToStore) + err = a.KeyDatabase.StoreLocalDeviceKeys(ctx, keysToStore) if err != nil { res.Error = &api.KeyError{ Err: fmt.Sprintf("failed to store device keys: %s", err.Error()), } return } - err = emitDeviceKeyChanges(a.Producer, existingKeys, keysToStore, req.OnlyDisplayNameUpdates) + err = emitDeviceKeyChanges(a.KeyChangeProducer, existingKeys, keysToStore, req.OnlyDisplayNameUpdates) if err != nil { util.GetLogger(ctx).Errorf("Failed to emitDeviceKeyChanges: %s", err) } } -func (a *KeyInternalAPI) uploadOneTimeKeys(ctx context.Context, req *api.PerformUploadKeysRequest, res *api.PerformUploadKeysResponse) { +func (a *UserInternalAPI) uploadOneTimeKeys(ctx context.Context, req *api.PerformUploadKeysRequest, res *api.PerformUploadKeysResponse) { if req.UserID == "" { res.Error = &api.KeyError{ Err: "user ID missing", } } if req.DeviceID != "" && len(req.OneTimeKeys) == 0 { - counts, err := a.DB.OneTimeKeysCount(ctx, req.UserID, req.DeviceID) + counts, err := a.KeyDatabase.OneTimeKeysCount(ctx, req.UserID, req.DeviceID) if err != nil { res.Error = &api.KeyError{ - Err: fmt.Sprintf("a.DB.OneTimeKeysCount: %s", err), + Err: fmt.Sprintf("a.KeyDatabase.OneTimeKeysCount: %s", err), } } if counts != nil { @@ -761,7 +743,7 @@ func (a *KeyInternalAPI) uploadOneTimeKeys(ctx context.Context, req *api.Perform keyIDsWithAlgorithms[i] = keyIDWithAlgo i++ } - existingKeys, err := a.DB.ExistingOneTimeKeys(ctx, req.UserID, req.DeviceID, keyIDsWithAlgorithms) + existingKeys, err := a.KeyDatabase.ExistingOneTimeKeys(ctx, req.UserID, req.DeviceID, keyIDsWithAlgorithms) if err != nil { res.KeyError(req.UserID, req.DeviceID, &api.KeyError{ Err: "failed to query existing one-time keys: " + err.Error(), @@ -778,7 +760,7 @@ func (a *KeyInternalAPI) uploadOneTimeKeys(ctx context.Context, req *api.Perform } } // store one-time keys - counts, err := a.DB.StoreOneTimeKeys(ctx, key) + counts, err := a.KeyDatabase.StoreOneTimeKeys(ctx, key) if err != nil { res.KeyError(req.UserID, req.DeviceID, &api.KeyError{ Err: fmt.Sprintf("%s device %s : failed to store one-time keys: %s", req.UserID, req.DeviceID, err.Error()), diff --git a/keyserver/internal/internal_test.go b/userapi/internal/key_api_test.go similarity index 89% rename from keyserver/internal/internal_test.go rename to userapi/internal/key_api_test.go index 8a2c9c5d9..fc7e7e0df 100644 --- a/keyserver/internal/internal_test.go +++ b/userapi/internal/key_api_test.go @@ -5,23 +5,28 @@ import ( "reflect" "testing" - "github.com/matrix-org/dendrite/keyserver/api" - "github.com/matrix-org/dendrite/keyserver/internal" - "github.com/matrix-org/dendrite/keyserver/storage" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/test/testrig" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/internal" + "github.com/matrix-org/dendrite/userapi/storage" ) -func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { +func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.KeyDatabase, func()) { t.Helper() connStr, close := test.PrepareDBConnectionString(t, dbType) - db, err := storage.NewDatabase(nil, &config.DatabaseOptions{ + base, _, _ := testrig.Base(nil) + db, err := storage.NewKeyDatabase(base, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }) if err != nil { t.Fatalf("failed to create new user db: %v", err) } - return db, close + return db, func() { + base.Close() + close() + } } func Test_QueryDeviceMessages(t *testing.T) { @@ -140,8 +145,8 @@ func Test_QueryDeviceMessages(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - a := &internal.KeyInternalAPI{ - DB: db, + a := &internal.UserInternalAPI{ + KeyDatabase: db, } if err := a.QueryDeviceMessages(ctx, tt.args.req, tt.args.res); (err != nil) != tt.wantErr { t.Errorf("QueryDeviceMessages() error = %v, wantErr %v", err, tt.wantErr) diff --git a/userapi/internal/api.go b/userapi/internal/user_api.go similarity index 95% rename from userapi/internal/api.go rename to userapi/internal/user_api.go index 0bb480da6..8977697b5 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/user_api.go @@ -23,6 +23,7 @@ import ( "strconv" "time" + fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/sirupsen/logrus" @@ -32,7 +33,6 @@ import ( "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/pushgateway" "github.com/matrix-org/dendrite/internal/sqlutil" - keyapi "github.com/matrix-org/dendrite/keyserver/api" rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" synctypes "github.com/matrix-org/dendrite/syncapi/types" @@ -44,17 +44,19 @@ import ( ) type UserInternalAPI struct { - DB storage.Database - SyncProducer *producers.SyncAPI - Config *config.UserAPI + DB storage.UserDatabase + KeyDatabase storage.KeyDatabase + SyncProducer *producers.SyncAPI + KeyChangeProducer *producers.KeyChange + Config *config.UserAPI DisableTLSValidation bool // AppServices is the list of all registered AS AppServices []config.ApplicationService - KeyAPI keyapi.UserKeyAPI RSAPI rsapi.UserRoomserverAPI PgClient pushgateway.Client - Cfg *config.UserAPI + FedClient fedsenderapi.KeyserverFederationAPI + Updater *DeviceListUpdater } func (a *UserInternalAPI) InputAccountData(ctx context.Context, req *api.InputAccountDataRequest, res *api.InputAccountDataResponse) error { @@ -221,7 +223,7 @@ func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.P return fmt.Errorf("a.DB.SetDisplayName: %w", err) } - postRegisterJoinRooms(a.Cfg, acc, a.RSAPI) + postRegisterJoinRooms(a.Config, acc, a.RSAPI) res.AccountCreated = true res.Account = acc @@ -252,6 +254,17 @@ func (a *UserInternalAPI) PerformDeviceCreation(ctx context.Context, req *api.Pe if !a.Config.Matrix.IsLocalServerName(serverName) { return fmt.Errorf("server name %s is not local", serverName) } + // If a device ID was specified, check if it already exists and + // avoid sending an empty device list update which would remove + // existing device keys. + isExisting := false + if req.DeviceID != nil && *req.DeviceID != "" { + existingDev, err := a.DB.GetDeviceByID(ctx, req.Localpart, req.ServerName, *req.DeviceID) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + isExisting = existingDev.ID == *req.DeviceID + } util.GetLogger(ctx).WithFields(logrus.Fields{ "localpart": req.Localpart, "device_id": req.DeviceID, @@ -263,7 +276,7 @@ func (a *UserInternalAPI) PerformDeviceCreation(ctx context.Context, req *api.Pe } res.DeviceCreated = true res.Device = dev - if req.NoDeviceListUpdate { + if req.NoDeviceListUpdate || isExisting { return nil } // create empty device keys and upload them to trigger device list changes @@ -293,14 +306,14 @@ func (a *UserInternalAPI) PerformDeviceDeletion(ctx context.Context, req *api.Pe return err } // Ask the keyserver to delete device keys and signatures for those devices - deleteReq := &keyapi.PerformDeleteKeysRequest{ + deleteReq := &api.PerformDeleteKeysRequest{ UserID: req.UserID, } for _, keyID := range req.DeviceIDs { deleteReq.KeyIDs = append(deleteReq.KeyIDs, gomatrixserverlib.KeyID(keyID)) } - deleteRes := &keyapi.PerformDeleteKeysResponse{} - if err := a.KeyAPI.PerformDeleteKeys(ctx, deleteReq, deleteRes); err != nil { + deleteRes := &api.PerformDeleteKeysResponse{} + if err := a.PerformDeleteKeys(ctx, deleteReq, deleteRes); err != nil { return err } if err := deleteRes.Error; err != nil { @@ -311,17 +324,17 @@ func (a *UserInternalAPI) PerformDeviceDeletion(ctx context.Context, req *api.Pe } func (a *UserInternalAPI) deviceListUpdate(userID string, deviceIDs []string) error { - deviceKeys := make([]keyapi.DeviceKeys, len(deviceIDs)) + deviceKeys := make([]api.DeviceKeys, len(deviceIDs)) for i, did := range deviceIDs { - deviceKeys[i] = keyapi.DeviceKeys{ + deviceKeys[i] = api.DeviceKeys{ UserID: userID, DeviceID: did, KeyJSON: nil, } } - var uploadRes keyapi.PerformUploadKeysResponse - if err := a.KeyAPI.PerformUploadKeys(context.Background(), &keyapi.PerformUploadKeysRequest{ + var uploadRes api.PerformUploadKeysResponse + if err := a.PerformUploadKeys(context.Background(), &api.PerformUploadKeysRequest{ UserID: userID, DeviceKeys: deviceKeys, }, &uploadRes); err != nil { @@ -385,10 +398,10 @@ func (a *UserInternalAPI) PerformDeviceUpdate(ctx context.Context, req *api.Perf } if req.DisplayName != nil && dev.DisplayName != *req.DisplayName { // display name has changed: update the device key - var uploadRes keyapi.PerformUploadKeysResponse - if err := a.KeyAPI.PerformUploadKeys(context.Background(), &keyapi.PerformUploadKeysRequest{ + var uploadRes api.PerformUploadKeysResponse + if err := a.PerformUploadKeys(context.Background(), &api.PerformUploadKeysRequest{ UserID: req.RequestingUserID, - DeviceKeys: []keyapi.DeviceKeys{ + DeviceKeys: []api.DeviceKeys{ { DeviceID: dev.ID, DisplayName: *req.DisplayName, diff --git a/userapi/inthttp/client.go b/userapi/inthttp/client.go deleted file mode 100644 index 51b0fe3ef..000000000 --- a/userapi/inthttp/client.go +++ /dev/null @@ -1,454 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package inthttp - -import ( - "context" - "errors" - "net/http" - - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/userapi/api" -) - -// HTTP paths for the internal HTTP APIs -const ( - InputAccountDataPath = "/userapi/inputAccountData" - - PerformDeviceCreationPath = "/userapi/performDeviceCreation" - PerformAccountCreationPath = "/userapi/performAccountCreation" - PerformPasswordUpdatePath = "/userapi/performPasswordUpdate" - PerformDeviceDeletionPath = "/userapi/performDeviceDeletion" - PerformLastSeenUpdatePath = "/userapi/performLastSeenUpdate" - PerformDeviceUpdatePath = "/userapi/performDeviceUpdate" - PerformAccountDeactivationPath = "/userapi/performAccountDeactivation" - PerformOpenIDTokenCreationPath = "/userapi/performOpenIDTokenCreation" - PerformKeyBackupPath = "/userapi/performKeyBackup" - PerformPusherSetPath = "/pushserver/performPusherSet" - PerformPusherDeletionPath = "/pushserver/performPusherDeletion" - PerformPushRulesPutPath = "/pushserver/performPushRulesPut" - PerformSetAvatarURLPath = "/userapi/performSetAvatarURL" - PerformSetDisplayNamePath = "/userapi/performSetDisplayName" - PerformForgetThreePIDPath = "/userapi/performForgetThreePID" - PerformSaveThreePIDAssociationPath = "/userapi/performSaveThreePIDAssociation" - - QueryKeyBackupPath = "/userapi/queryKeyBackup" - QueryProfilePath = "/userapi/queryProfile" - QueryAccessTokenPath = "/userapi/queryAccessToken" - QueryDevicesPath = "/userapi/queryDevices" - QueryAccountDataPath = "/userapi/queryAccountData" - QueryDeviceInfosPath = "/userapi/queryDeviceInfos" - QuerySearchProfilesPath = "/userapi/querySearchProfiles" - QueryOpenIDTokenPath = "/userapi/queryOpenIDToken" - QueryPushersPath = "/pushserver/queryPushers" - QueryPushRulesPath = "/pushserver/queryPushRules" - QueryNotificationsPath = "/pushserver/queryNotifications" - QueryNumericLocalpartPath = "/userapi/queryNumericLocalpart" - QueryAccountAvailabilityPath = "/userapi/queryAccountAvailability" - QueryAccountByPasswordPath = "/userapi/queryAccountByPassword" - QueryLocalpartForThreePIDPath = "/userapi/queryLocalpartForThreePID" - QueryThreePIDsForLocalpartPath = "/userapi/queryThreePIDsForLocalpart" - QueryAccountByLocalpartPath = "/userapi/queryAccountType" -) - -// NewUserAPIClient creates a UserInternalAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewUserAPIClient( - apiURL string, - httpClient *http.Client, -) (api.UserInternalAPI, error) { - if httpClient == nil { - return nil, errors.New("NewUserAPIClient: httpClient is ") - } - return &httpUserInternalAPI{ - apiURL: apiURL, - httpClient: httpClient, - }, nil -} - -type httpUserInternalAPI struct { - apiURL string - httpClient *http.Client -} - -func (h *httpUserInternalAPI) InputAccountData(ctx context.Context, req *api.InputAccountDataRequest, res *api.InputAccountDataResponse) error { - return httputil.CallInternalRPCAPI( - "InputAccountData", h.apiURL+InputAccountDataPath, - h.httpClient, ctx, req, res, - ) -} - -func (h *httpUserInternalAPI) PerformAccountCreation( - ctx context.Context, - request *api.PerformAccountCreationRequest, - response *api.PerformAccountCreationResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformAccountCreation", h.apiURL+PerformAccountCreationPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) PerformPasswordUpdate( - ctx context.Context, - request *api.PerformPasswordUpdateRequest, - response *api.PerformPasswordUpdateResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformPasswordUpdate", h.apiURL+PerformPasswordUpdatePath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) PerformDeviceCreation( - ctx context.Context, - request *api.PerformDeviceCreationRequest, - response *api.PerformDeviceCreationResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformDeviceCreation", h.apiURL+PerformDeviceCreationPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) PerformDeviceDeletion( - ctx context.Context, - request *api.PerformDeviceDeletionRequest, - response *api.PerformDeviceDeletionResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformDeviceDeletion", h.apiURL+PerformDeviceDeletionPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) PerformLastSeenUpdate( - ctx context.Context, - request *api.PerformLastSeenUpdateRequest, - response *api.PerformLastSeenUpdateResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformLastSeen", h.apiURL+PerformLastSeenUpdatePath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) PerformDeviceUpdate( - ctx context.Context, - request *api.PerformDeviceUpdateRequest, - response *api.PerformDeviceUpdateResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformDeviceUpdate", h.apiURL+PerformDeviceUpdatePath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) PerformAccountDeactivation( - ctx context.Context, - request *api.PerformAccountDeactivationRequest, - response *api.PerformAccountDeactivationResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformAccountDeactivation", h.apiURL+PerformAccountDeactivationPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) PerformOpenIDTokenCreation( - ctx context.Context, - request *api.PerformOpenIDTokenCreationRequest, - response *api.PerformOpenIDTokenCreationResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformOpenIDTokenCreation", h.apiURL+PerformOpenIDTokenCreationPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryProfile( - ctx context.Context, - request *api.QueryProfileRequest, - response *api.QueryProfileResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryProfile", h.apiURL+QueryProfilePath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryDeviceInfos( - ctx context.Context, - request *api.QueryDeviceInfosRequest, - response *api.QueryDeviceInfosResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryDeviceInfos", h.apiURL+QueryDeviceInfosPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryAccessToken( - ctx context.Context, - request *api.QueryAccessTokenRequest, - response *api.QueryAccessTokenResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryAccessToken", h.apiURL+QueryAccessTokenPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryDevices( - ctx context.Context, - request *api.QueryDevicesRequest, - response *api.QueryDevicesResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryDevices", h.apiURL+QueryDevicesPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryAccountData( - ctx context.Context, - request *api.QueryAccountDataRequest, - response *api.QueryAccountDataResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryAccountData", h.apiURL+QueryAccountDataPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QuerySearchProfiles( - ctx context.Context, - request *api.QuerySearchProfilesRequest, - response *api.QuerySearchProfilesResponse, -) error { - return httputil.CallInternalRPCAPI( - "QuerySearchProfiles", h.apiURL+QuerySearchProfilesPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryOpenIDToken( - ctx context.Context, - request *api.QueryOpenIDTokenRequest, - response *api.QueryOpenIDTokenResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryOpenIDToken", h.apiURL+QueryOpenIDTokenPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) PerformKeyBackup( - ctx context.Context, - request *api.PerformKeyBackupRequest, - response *api.PerformKeyBackupResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformKeyBackup", h.apiURL+PerformKeyBackupPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryKeyBackup( - ctx context.Context, - request *api.QueryKeyBackupRequest, - response *api.QueryKeyBackupResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryKeyBackup", h.apiURL+QueryKeyBackupPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryNotifications( - ctx context.Context, - request *api.QueryNotificationsRequest, - response *api.QueryNotificationsResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryNotifications", h.apiURL+QueryNotificationsPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) PerformPusherSet( - ctx context.Context, - request *api.PerformPusherSetRequest, - response *struct{}, -) error { - return httputil.CallInternalRPCAPI( - "PerformPusherSet", h.apiURL+PerformPusherSetPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) PerformPusherDeletion( - ctx context.Context, - request *api.PerformPusherDeletionRequest, - response *struct{}, -) error { - return httputil.CallInternalRPCAPI( - "PerformPusherDeletion", h.apiURL+PerformPusherDeletionPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryPushers( - ctx context.Context, - request *api.QueryPushersRequest, - response *api.QueryPushersResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryPushers", h.apiURL+QueryPushersPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) PerformPushRulesPut( - ctx context.Context, - request *api.PerformPushRulesPutRequest, - response *struct{}, -) error { - return httputil.CallInternalRPCAPI( - "PerformPushRulesPut", h.apiURL+PerformPushRulesPutPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryPushRules( - ctx context.Context, - request *api.QueryPushRulesRequest, - response *api.QueryPushRulesResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryPushRules", h.apiURL+QueryPushRulesPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) SetAvatarURL( - ctx context.Context, - request *api.PerformSetAvatarURLRequest, - response *api.PerformSetAvatarURLResponse, -) error { - return httputil.CallInternalRPCAPI( - "SetAvatarURL", h.apiURL+PerformSetAvatarURLPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryNumericLocalpart( - ctx context.Context, - request *api.QueryNumericLocalpartRequest, - response *api.QueryNumericLocalpartResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryNumericLocalpart", h.apiURL+QueryNumericLocalpartPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryAccountAvailability( - ctx context.Context, - request *api.QueryAccountAvailabilityRequest, - response *api.QueryAccountAvailabilityResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryAccountAvailability", h.apiURL+QueryAccountAvailabilityPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryAccountByPassword( - ctx context.Context, - request *api.QueryAccountByPasswordRequest, - response *api.QueryAccountByPasswordResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryAccountByPassword", h.apiURL+QueryAccountByPasswordPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) SetDisplayName( - ctx context.Context, - request *api.PerformUpdateDisplayNameRequest, - response *api.PerformUpdateDisplayNameResponse, -) error { - return httputil.CallInternalRPCAPI( - "SetDisplayName", h.apiURL+PerformSetDisplayNamePath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryLocalpartForThreePID( - ctx context.Context, - request *api.QueryLocalpartForThreePIDRequest, - response *api.QueryLocalpartForThreePIDResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryLocalpartForThreePID", h.apiURL+QueryLocalpartForThreePIDPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryThreePIDsForLocalpart( - ctx context.Context, - request *api.QueryThreePIDsForLocalpartRequest, - response *api.QueryThreePIDsForLocalpartResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryThreePIDsForLocalpart", h.apiURL+QueryThreePIDsForLocalpartPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) PerformForgetThreePID( - ctx context.Context, - request *api.PerformForgetThreePIDRequest, - response *struct{}, -) error { - return httputil.CallInternalRPCAPI( - "PerformForgetThreePID", h.apiURL+PerformForgetThreePIDPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) PerformSaveThreePIDAssociation( - ctx context.Context, - request *api.PerformSaveThreePIDAssociationRequest, - response *struct{}, -) error { - return httputil.CallInternalRPCAPI( - "PerformSaveThreePIDAssociation", h.apiURL+PerformSaveThreePIDAssociationPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryAccountByLocalpart( - ctx context.Context, - req *api.QueryAccountByLocalpartRequest, - res *api.QueryAccountByLocalpartResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryAccountByLocalpart", h.apiURL+QueryAccountByLocalpartPath, - h.httpClient, ctx, req, res, - ) -} diff --git a/userapi/inthttp/client_logintoken.go b/userapi/inthttp/client_logintoken.go deleted file mode 100644 index 211b1b7a1..000000000 --- a/userapi/inthttp/client_logintoken.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2021 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package inthttp - -import ( - "context" - - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/userapi/api" -) - -const ( - PerformLoginTokenCreationPath = "/userapi/performLoginTokenCreation" - PerformLoginTokenDeletionPath = "/userapi/performLoginTokenDeletion" - QueryLoginTokenPath = "/userapi/queryLoginToken" -) - -func (h *httpUserInternalAPI) PerformLoginTokenCreation( - ctx context.Context, - request *api.PerformLoginTokenCreationRequest, - response *api.PerformLoginTokenCreationResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformLoginTokenCreation", h.apiURL+PerformLoginTokenCreationPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) PerformLoginTokenDeletion( - ctx context.Context, - request *api.PerformLoginTokenDeletionRequest, - response *api.PerformLoginTokenDeletionResponse, -) error { - return httputil.CallInternalRPCAPI( - "PerformLoginTokenDeletion", h.apiURL+PerformLoginTokenDeletionPath, - h.httpClient, ctx, request, response, - ) -} - -func (h *httpUserInternalAPI) QueryLoginToken( - ctx context.Context, - request *api.QueryLoginTokenRequest, - response *api.QueryLoginTokenResponse, -) error { - return httputil.CallInternalRPCAPI( - "QueryLoginToken", h.apiURL+QueryLoginTokenPath, - h.httpClient, ctx, request, response, - ) -} diff --git a/userapi/inthttp/server.go b/userapi/inthttp/server.go deleted file mode 100644 index b40b507c2..000000000 --- a/userapi/inthttp/server.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package inthttp - -import ( - "github.com/gorilla/mux" - - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/userapi/api" -) - -// nolint: gocyclo -func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI, enableMetrics bool) { - addRoutesLoginToken(internalAPIMux, s, enableMetrics) - - internalAPIMux.Handle( - PerformAccountCreationPath, - httputil.MakeInternalRPCAPI("UserAPIPerformAccountCreation", enableMetrics, s.PerformAccountCreation), - ) - - internalAPIMux.Handle( - PerformPasswordUpdatePath, - httputil.MakeInternalRPCAPI("UserAPIPerformPasswordUpdate", enableMetrics, s.PerformPasswordUpdate), - ) - - internalAPIMux.Handle( - PerformDeviceCreationPath, - httputil.MakeInternalRPCAPI("UserAPIPerformDeviceCreation", enableMetrics, s.PerformDeviceCreation), - ) - - internalAPIMux.Handle( - PerformLastSeenUpdatePath, - httputil.MakeInternalRPCAPI("UserAPIPerformLastSeenUpdate", enableMetrics, s.PerformLastSeenUpdate), - ) - - internalAPIMux.Handle( - PerformDeviceUpdatePath, - httputil.MakeInternalRPCAPI("UserAPIPerformDeviceUpdate", enableMetrics, s.PerformDeviceUpdate), - ) - - internalAPIMux.Handle( - PerformDeviceDeletionPath, - httputil.MakeInternalRPCAPI("UserAPIPerformDeviceDeletion", enableMetrics, s.PerformDeviceDeletion), - ) - - internalAPIMux.Handle( - PerformAccountDeactivationPath, - httputil.MakeInternalRPCAPI("UserAPIPerformAccountDeactivation", enableMetrics, s.PerformAccountDeactivation), - ) - - internalAPIMux.Handle( - PerformOpenIDTokenCreationPath, - httputil.MakeInternalRPCAPI("UserAPIPerformOpenIDTokenCreation", enableMetrics, s.PerformOpenIDTokenCreation), - ) - - internalAPIMux.Handle( - QueryProfilePath, - httputil.MakeInternalRPCAPI("UserAPIQueryProfile", enableMetrics, s.QueryProfile), - ) - - internalAPIMux.Handle( - QueryAccessTokenPath, - httputil.MakeInternalRPCAPI("UserAPIQueryAccessToken", enableMetrics, s.QueryAccessToken), - ) - - internalAPIMux.Handle( - QueryDevicesPath, - httputil.MakeInternalRPCAPI("UserAPIQueryDevices", enableMetrics, s.QueryDevices), - ) - - internalAPIMux.Handle( - QueryAccountDataPath, - httputil.MakeInternalRPCAPI("UserAPIQueryAccountData", enableMetrics, s.QueryAccountData), - ) - - internalAPIMux.Handle( - QueryDeviceInfosPath, - httputil.MakeInternalRPCAPI("UserAPIQueryDeviceInfos", enableMetrics, s.QueryDeviceInfos), - ) - - internalAPIMux.Handle( - QuerySearchProfilesPath, - httputil.MakeInternalRPCAPI("UserAPIQuerySearchProfiles", enableMetrics, s.QuerySearchProfiles), - ) - - internalAPIMux.Handle( - QueryOpenIDTokenPath, - httputil.MakeInternalRPCAPI("UserAPIQueryOpenIDToken", enableMetrics, s.QueryOpenIDToken), - ) - - internalAPIMux.Handle( - InputAccountDataPath, - httputil.MakeInternalRPCAPI("UserAPIInputAccountData", enableMetrics, s.InputAccountData), - ) - - internalAPIMux.Handle( - QueryKeyBackupPath, - httputil.MakeInternalRPCAPI("UserAPIQueryKeyBackup", enableMetrics, s.QueryKeyBackup), - ) - - internalAPIMux.Handle( - PerformKeyBackupPath, - httputil.MakeInternalRPCAPI("UserAPIPerformKeyBackup", enableMetrics, s.PerformKeyBackup), - ) - - internalAPIMux.Handle( - QueryNotificationsPath, - httputil.MakeInternalRPCAPI("UserAPIQueryNotifications", enableMetrics, s.QueryNotifications), - ) - - internalAPIMux.Handle( - PerformPusherSetPath, - httputil.MakeInternalRPCAPI("UserAPIPerformPusherSet", enableMetrics, s.PerformPusherSet), - ) - - internalAPIMux.Handle( - PerformPusherDeletionPath, - httputil.MakeInternalRPCAPI("UserAPIPerformPusherDeletion", enableMetrics, s.PerformPusherDeletion), - ) - - internalAPIMux.Handle( - QueryPushersPath, - httputil.MakeInternalRPCAPI("UserAPIQueryPushers", enableMetrics, s.QueryPushers), - ) - - internalAPIMux.Handle( - PerformPushRulesPutPath, - httputil.MakeInternalRPCAPI("UserAPIPerformPushRulesPut", enableMetrics, s.PerformPushRulesPut), - ) - - internalAPIMux.Handle( - QueryPushRulesPath, - httputil.MakeInternalRPCAPI("UserAPIQueryPushRules", enableMetrics, s.QueryPushRules), - ) - - internalAPIMux.Handle( - PerformSetAvatarURLPath, - httputil.MakeInternalRPCAPI("UserAPIPerformSetAvatarURL", enableMetrics, s.SetAvatarURL), - ) - - internalAPIMux.Handle( - QueryNumericLocalpartPath, - httputil.MakeInternalRPCAPI("UserAPIQueryNumericLocalpart", enableMetrics, s.QueryNumericLocalpart), - ) - - internalAPIMux.Handle( - QueryAccountAvailabilityPath, - httputil.MakeInternalRPCAPI("UserAPIQueryAccountAvailability", enableMetrics, s.QueryAccountAvailability), - ) - - internalAPIMux.Handle( - QueryAccountByPasswordPath, - httputil.MakeInternalRPCAPI("UserAPIQueryAccountByPassword", enableMetrics, s.QueryAccountByPassword), - ) - - internalAPIMux.Handle( - PerformSetDisplayNamePath, - httputil.MakeInternalRPCAPI("UserAPISetDisplayName", enableMetrics, s.SetDisplayName), - ) - - internalAPIMux.Handle( - QueryLocalpartForThreePIDPath, - httputil.MakeInternalRPCAPI("UserAPIQueryLocalpartForThreePID", enableMetrics, s.QueryLocalpartForThreePID), - ) - - internalAPIMux.Handle( - QueryThreePIDsForLocalpartPath, - httputil.MakeInternalRPCAPI("UserAPIQueryThreePIDsForLocalpart", enableMetrics, s.QueryThreePIDsForLocalpart), - ) - - internalAPIMux.Handle( - PerformForgetThreePIDPath, - httputil.MakeInternalRPCAPI("UserAPIPerformForgetThreePID", enableMetrics, s.PerformForgetThreePID), - ) - - internalAPIMux.Handle( - PerformSaveThreePIDAssociationPath, - httputil.MakeInternalRPCAPI("UserAPIPerformSaveThreePIDAssociation", enableMetrics, s.PerformSaveThreePIDAssociation), - ) - - internalAPIMux.Handle( - QueryAccountByLocalpartPath, - httputil.MakeInternalRPCAPI("AccountByLocalpart", enableMetrics, s.QueryAccountByLocalpart), - ) -} diff --git a/userapi/inthttp/server_logintoken.go b/userapi/inthttp/server_logintoken.go deleted file mode 100644 index dc116428b..000000000 --- a/userapi/inthttp/server_logintoken.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2021 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package inthttp - -import ( - "github.com/gorilla/mux" - - "github.com/matrix-org/dendrite/internal/httputil" - "github.com/matrix-org/dendrite/userapi/api" -) - -// addRoutesLoginToken adds routes for all login token API calls. -func addRoutesLoginToken(internalAPIMux *mux.Router, s api.UserInternalAPI, enableMetrics bool) { - internalAPIMux.Handle( - PerformLoginTokenCreationPath, - httputil.MakeInternalRPCAPI("UserAPIPerformLoginTokenCreation", enableMetrics, s.PerformLoginTokenCreation), - ) - - internalAPIMux.Handle( - PerformLoginTokenDeletionPath, - httputil.MakeInternalRPCAPI("UserAPIPerformLoginTokenDeletion", enableMetrics, s.PerformLoginTokenDeletion), - ) - - internalAPIMux.Handle( - QueryLoginTokenPath, - httputil.MakeInternalRPCAPI("UserAPIQueryLoginToken", enableMetrics, s.QueryLoginToken), - ) -} diff --git a/keyserver/producers/keychange.go b/userapi/producers/keychange.go similarity index 94% rename from keyserver/producers/keychange.go rename to userapi/producers/keychange.go index f86c34177..da6cea31a 100644 --- a/keyserver/producers/keychange.go +++ b/userapi/producers/keychange.go @@ -18,9 +18,9 @@ import ( "context" "encoding/json" - "github.com/matrix-org/dendrite/keyserver/api" - "github.com/matrix-org/dendrite/keyserver/storage" "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage" "github.com/nats-io/nats.go" "github.com/sirupsen/logrus" ) @@ -28,8 +28,8 @@ import ( // KeyChange produces key change events for the sync API and federation sender to consume type KeyChange struct { Topic string - JetStream nats.JetStreamContext - DB storage.Database + JetStream JetStreamPublisher + DB storage.KeyChangeDatabase } // ProduceKeyChanges creates new change events for each key diff --git a/userapi/producers/syncapi.go b/userapi/producers/syncapi.go index 51eaa9856..165de8994 100644 --- a/userapi/producers/syncapi.go +++ b/userapi/producers/syncapi.go @@ -19,13 +19,13 @@ type JetStreamPublisher interface { // SyncAPI produces messages for the Sync API server to consume. type SyncAPI struct { - db storage.Database + db storage.Notification producer JetStreamPublisher clientDataTopic string notificationDataTopic string } -func NewSyncAPI(db storage.Database, js JetStreamPublisher, clientDataTopic string, notificationDataTopic string) *SyncAPI { +func NewSyncAPI(db storage.UserDatabase, js JetStreamPublisher, clientDataTopic string, notificationDataTopic string) *SyncAPI { return &SyncAPI{ db: db, producer: js, diff --git a/userapi/storage/interface.go b/userapi/storage/interface.go index c22b7658f..278378861 100644 --- a/userapi/storage/interface.go +++ b/userapi/storage/interface.go @@ -90,7 +90,7 @@ type KeyBackup interface { type LoginToken interface { // CreateLoginToken generates a token, stores and returns it. The lifetime is - // determined by the loginTokenLifetime given to the Database constructor. + // determined by the loginTokenLifetime given to the UserDatabase constructor. CreateLoginToken(ctx context.Context, data *api.LoginTokenData) (*api.LoginTokenMetadata, error) // RemoveLoginToken removes the named token (and may clean up other expired tokens). @@ -130,7 +130,7 @@ type Notification interface { DeleteOldNotifications(ctx context.Context) error } -type Database interface { +type UserDatabase interface { Account AccountData Device @@ -144,6 +144,78 @@ type Database interface { ThreePID } +type KeyChangeDatabase interface { + // StoreKeyChange stores key change metadata and returns the device change ID which represents the position in the /sync stream for this device change. + // `userID` is the the user who has changed their keys in some way. + StoreKeyChange(ctx context.Context, userID string) (int64, error) +} + +type KeyDatabase interface { + KeyChangeDatabase + // ExistingOneTimeKeys returns a map of keyIDWithAlgorithm to key JSON for the given parameters. If no keys exist with this combination + // of user/device/key/algorithm 4-uple then it is omitted from the map. Returns an error when failing to communicate with the database. + ExistingOneTimeKeys(ctx context.Context, userID, deviceID string, keyIDsWithAlgorithms []string) (map[string]json.RawMessage, error) + + // StoreOneTimeKeys persists the given one-time keys. + StoreOneTimeKeys(ctx context.Context, keys api.OneTimeKeys) (*api.OneTimeKeysCount, error) + + // OneTimeKeysCount returns a count of all OTKs for this device. + OneTimeKeysCount(ctx context.Context, userID, deviceID string) (*api.OneTimeKeysCount, error) + + // DeviceKeysJSON populates the KeyJSON for the given keys. If any proided `keys` have a `KeyJSON` or `StreamID` already then it will be replaced. + DeviceKeysJSON(ctx context.Context, keys []api.DeviceMessage) error + + // StoreLocalDeviceKeys persists the given keys. Keys with the same user ID and device ID will be replaced. An empty KeyJSON removes the key + // for this (user, device). + // The `StreamID` for each message is set on successful insertion. In the event the key already exists, the existing StreamID is set. + // Returns an error if there was a problem storing the keys. + StoreLocalDeviceKeys(ctx context.Context, keys []api.DeviceMessage) error + + // StoreRemoteDeviceKeys persists the given keys. Keys with the same user ID and device ID will be replaced. An empty KeyJSON removes the key + // for this (user, device). Does not modify the stream ID for keys. User IDs in `clearUserIDs` will have all their device keys deleted prior + // to insertion - use this when you have a complete snapshot of a user's keys in order to track device deletions correctly. + StoreRemoteDeviceKeys(ctx context.Context, keys []api.DeviceMessage, clearUserIDs []string) error + + // PrevIDsExists returns true if all prev IDs exist for this user. + PrevIDsExists(ctx context.Context, userID string, prevIDs []int64) (bool, error) + + // DeviceKeysForUser returns the device keys for the device IDs given. If the length of deviceIDs is 0, all devices are selected. + // If there are some missing keys, they are omitted from the returned slice. There is no ordering on the returned slice. + DeviceKeysForUser(ctx context.Context, userID string, deviceIDs []string, includeEmpty bool) ([]api.DeviceMessage, error) + + // DeleteDeviceKeys removes the device keys for a given user/device, and any accompanying + // cross-signing signatures relating to that device. + DeleteDeviceKeys(ctx context.Context, userID string, deviceIDs []gomatrixserverlib.KeyID) error + + // ClaimKeys based on the 3-uple of user_id, device_id and algorithm name. Returns the keys claimed. Returns no error if a key + // cannot be claimed or if none exist for this (user, device, algorithm), instead it is omitted from the returned slice. + ClaimKeys(ctx context.Context, userToDeviceToAlgorithm map[string]map[string]string) ([]api.OneTimeKeys, error) + + // KeyChanges returns a list of user IDs who have modified their keys from the offset given (exclusive) to the offset given (inclusive). + // A to offset of types.OffsetNewest means no upper limit. + // Returns the offset of the latest key change. + KeyChanges(ctx context.Context, fromOffset, toOffset int64) (userIDs []string, latestOffset int64, err error) + + // StaleDeviceLists returns a list of user IDs ending with the domains provided who have stale device lists. + // If no domains are given, all user IDs with stale device lists are returned. + StaleDeviceLists(ctx context.Context, domains []gomatrixserverlib.ServerName) ([]string, error) + + // MarkDeviceListStale sets the stale bit for this user to isStale. + MarkDeviceListStale(ctx context.Context, userID string, isStale bool) error + + CrossSigningKeysForUser(ctx context.Context, userID string) (map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey, error) + CrossSigningKeysDataForUser(ctx context.Context, userID string) (types.CrossSigningKeyMap, error) + CrossSigningSigsForTarget(ctx context.Context, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error) + + StoreCrossSigningKeysForUser(ctx context.Context, userID string, keyMap types.CrossSigningKeyMap) error + StoreCrossSigningSigsForTarget(ctx context.Context, originUserID string, originKeyID gomatrixserverlib.KeyID, targetUserID string, targetKeyID gomatrixserverlib.KeyID, signature gomatrixserverlib.Base64Bytes) error + + DeleteStaleDeviceLists( + ctx context.Context, + userIDs []string, + ) error +} + type Statistics interface { UserStatistics(ctx context.Context) (*types.UserStatistics, *types.DatabaseEngine, error) DailyRoomsMessages(ctx context.Context, serverName gomatrixserverlib.ServerName) (stats types.MessageStats, activeRooms, activeE2EERooms int64, err error) diff --git a/userapi/storage/postgres/account_data_table.go b/userapi/storage/postgres/account_data_table.go index 2a4777d74..057160374 100644 --- a/userapi/storage/postgres/account_data_table.go +++ b/userapi/storage/postgres/account_data_table.go @@ -78,7 +78,13 @@ func (s *accountDataStatements) InsertAccountData( roomID, dataType string, content json.RawMessage, ) (err error) { stmt := sqlutil.TxStmt(txn, s.insertAccountDataStmt) - _, err = stmt.ExecContext(ctx, localpart, serverName, roomID, dataType, content) + // Empty/nil json.RawMessage is not interpreted as "nil", so use *json.RawMessage + // when passing the data to trigger "NOT NULL" constraint + var data *json.RawMessage + if len(content) > 0 { + data = &content + } + _, err = stmt.ExecContext(ctx, localpart, serverName, roomID, dataType, data) return } diff --git a/keyserver/storage/postgres/cross_signing_keys_table.go b/userapi/storage/postgres/cross_signing_keys_table.go similarity index 96% rename from keyserver/storage/postgres/cross_signing_keys_table.go rename to userapi/storage/postgres/cross_signing_keys_table.go index 1022157e8..c0ecbd303 100644 --- a/keyserver/storage/postgres/cross_signing_keys_table.go +++ b/userapi/storage/postgres/cross_signing_keys_table.go @@ -21,8 +21,8 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/keyserver/storage/tables" - "github.com/matrix-org/dendrite/keyserver/types" + "github.com/matrix-org/dendrite/userapi/storage/tables" + "github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/keyserver/storage/postgres/cross_signing_sigs_table.go b/userapi/storage/postgres/cross_signing_sigs_table.go similarity index 96% rename from keyserver/storage/postgres/cross_signing_sigs_table.go rename to userapi/storage/postgres/cross_signing_sigs_table.go index 4536b7d80..b0117145c 100644 --- a/keyserver/storage/postgres/cross_signing_sigs_table.go +++ b/userapi/storage/postgres/cross_signing_sigs_table.go @@ -21,9 +21,9 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/keyserver/storage/postgres/deltas" - "github.com/matrix-org/dendrite/keyserver/storage/tables" - "github.com/matrix-org/dendrite/keyserver/types" + "github.com/matrix-org/dendrite/userapi/storage/postgres/deltas" + "github.com/matrix-org/dendrite/userapi/storage/tables" + "github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/keyserver/storage/postgres/deltas/2022012016470000_key_changes.go b/userapi/storage/postgres/deltas/2022012016470000_key_changes.go similarity index 100% rename from keyserver/storage/postgres/deltas/2022012016470000_key_changes.go rename to userapi/storage/postgres/deltas/2022012016470000_key_changes.go diff --git a/keyserver/storage/postgres/deltas/2022042612000000_xsigning_idx.go b/userapi/storage/postgres/deltas/2022042612000000_xsigning_idx.go similarity index 100% rename from keyserver/storage/postgres/deltas/2022042612000000_xsigning_idx.go rename to userapi/storage/postgres/deltas/2022042612000000_xsigning_idx.go diff --git a/keyserver/storage/postgres/device_keys_table.go b/userapi/storage/postgres/device_keys_table.go similarity index 87% rename from keyserver/storage/postgres/device_keys_table.go rename to userapi/storage/postgres/device_keys_table.go index 2aa11c520..a9203857f 100644 --- a/keyserver/storage/postgres/device_keys_table.go +++ b/userapi/storage/postgres/device_keys_table.go @@ -23,8 +23,8 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/keyserver/api" - "github.com/matrix-org/dendrite/keyserver/storage/tables" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/tables" ) var deviceKeysSchema = ` @@ -92,31 +92,16 @@ func NewPostgresDeviceKeysTable(db *sql.DB) (tables.DeviceKeys, error) { if err != nil { return nil, err } - if s.upsertDeviceKeysStmt, err = db.Prepare(upsertDeviceKeysSQL); err != nil { - return nil, err - } - if s.selectDeviceKeysStmt, err = db.Prepare(selectDeviceKeysSQL); err != nil { - return nil, err - } - if s.selectBatchDeviceKeysStmt, err = db.Prepare(selectBatchDeviceKeysSQL); err != nil { - return nil, err - } - if s.selectBatchDeviceKeysWithEmptiesStmt, err = db.Prepare(selectBatchDeviceKeysWithEmptiesSQL); err != nil { - return nil, err - } - if s.selectMaxStreamForUserStmt, err = db.Prepare(selectMaxStreamForUserSQL); err != nil { - return nil, err - } - if s.countStreamIDsForUserStmt, err = db.Prepare(countStreamIDsForUserSQL); err != nil { - return nil, err - } - if s.deleteDeviceKeysStmt, err = db.Prepare(deleteDeviceKeysSQL); err != nil { - return nil, err - } - if s.deleteAllDeviceKeysStmt, err = db.Prepare(deleteAllDeviceKeysSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.upsertDeviceKeysStmt, upsertDeviceKeysSQL}, + {&s.selectDeviceKeysStmt, selectDeviceKeysSQL}, + {&s.selectBatchDeviceKeysStmt, selectBatchDeviceKeysSQL}, + {&s.selectBatchDeviceKeysWithEmptiesStmt, selectBatchDeviceKeysWithEmptiesSQL}, + {&s.selectMaxStreamForUserStmt, selectMaxStreamForUserSQL}, + {&s.countStreamIDsForUserStmt, countStreamIDsForUserSQL}, + {&s.deleteDeviceKeysStmt, deleteDeviceKeysSQL}, + {&s.deleteAllDeviceKeysStmt, deleteAllDeviceKeysSQL}, + }.Prepare(db) } func (s *deviceKeysStatements) SelectDeviceKeysJSON(ctx context.Context, keys []api.DeviceMessage) error { diff --git a/userapi/storage/postgres/devices_table.go b/userapi/storage/postgres/devices_table.go index 2dd216189..88f8839c5 100644 --- a/userapi/storage/postgres/devices_table.go +++ b/userapi/storage/postgres/devices_table.go @@ -81,7 +81,7 @@ const selectDeviceByIDSQL = "" + "SELECT display_name, last_seen_ts, ip FROM userapi_devices WHERE localpart = $1 AND server_name = $2 AND device_id = $3" const selectDevicesByLocalpartSQL = "" + - "SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM userapi_devices WHERE localpart = $1 AND server_name = $2 AND device_id != $3 ORDER BY last_seen_ts DESC" + "SELECT device_id, display_name, last_seen_ts, ip, user_agent, session_id FROM userapi_devices WHERE localpart = $1 AND server_name = $2 AND device_id != $3 ORDER BY last_seen_ts DESC" const updateDeviceNameSQL = "" + "UPDATE userapi_devices SET display_name = $1 WHERE localpart = $2 AND server_name = $3 AND device_id = $4" @@ -96,7 +96,7 @@ const deleteDevicesSQL = "" + "DELETE FROM userapi_devices WHERE localpart = $1 AND server_name = $2 AND device_id = ANY($3)" const selectDevicesByIDSQL = "" + - "SELECT device_id, localpart, server_name, display_name, last_seen_ts FROM userapi_devices WHERE device_id = ANY($1) ORDER BY last_seen_ts DESC" + "SELECT device_id, localpart, server_name, display_name, last_seen_ts, session_id FROM userapi_devices WHERE device_id = ANY($1) ORDER BY last_seen_ts DESC" const updateDeviceLastSeen = "" + "UPDATE userapi_devices SET last_seen_ts = $1, ip = $2, user_agent = $3 WHERE localpart = $4 AND server_name = $5 AND device_id = $6" @@ -160,7 +160,7 @@ func (s *devicesStatements) InsertDevice( if err := stmt.QueryRowContext(ctx, id, localpart, serverName, accessToken, createdTimeMS, displayName, createdTimeMS, ipAddr, userAgent).Scan(&sessionID); err != nil { return nil, fmt.Errorf("insertDeviceStmt: %w", err) } - return &api.Device{ + dev := &api.Device{ ID: id, UserID: userutil.MakeUserID(localpart, serverName), AccessToken: accessToken, @@ -168,7 +168,19 @@ func (s *devicesStatements) InsertDevice( LastSeenTS: createdTimeMS, LastSeenIP: ipAddr, UserAgent: userAgent, - }, nil + } + if displayName != nil { + dev.DisplayName = *displayName + } + return dev, nil +} + +func (s *devicesStatements) InsertDeviceWithSessionID(ctx context.Context, txn *sql.Tx, id, + localpart string, serverName gomatrixserverlib.ServerName, + accessToken string, displayName *string, ipAddr, userAgent string, + sessionID int64, +) (*api.Device, error) { + return s.InsertDevice(ctx, txn, id, localpart, serverName, accessToken, displayName, ipAddr, userAgent) } // deleteDevice removes a single device by id and user localpart. @@ -271,7 +283,7 @@ func (s *devicesStatements) SelectDevicesByID(ctx context.Context, deviceIDs []s var lastseents sql.NullInt64 var displayName sql.NullString for rows.Next() { - if err := rows.Scan(&dev.ID, &localpart, &serverName, &displayName, &lastseents); err != nil { + if err := rows.Scan(&dev.ID, &localpart, &serverName, &displayName, &lastseents, &dev.SessionID); err != nil { return nil, err } if displayName.Valid { @@ -303,7 +315,7 @@ func (s *devicesStatements) SelectDevicesByLocalpart( var lastseents sql.NullInt64 var id, displayname, ip, useragent sql.NullString for rows.Next() { - err = rows.Scan(&id, &displayname, &lastseents, &ip, &useragent) + err = rows.Scan(&id, &displayname, &lastseents, &ip, &useragent, &dev.SessionID) if err != nil { return devices, err } diff --git a/userapi/storage/postgres/key_backup_table.go b/userapi/storage/postgres/key_backup_table.go index 7b58f7bae..91a34c357 100644 --- a/userapi/storage/postgres/key_backup_table.go +++ b/userapi/storage/postgres/key_backup_table.go @@ -52,7 +52,7 @@ const updateBackupKeySQL = "" + const countKeysSQL = "" + "SELECT COUNT(*) FROM userapi_key_backups WHERE user_id = $1 AND version = $2" -const selectKeysSQL = "" + +const selectBackupKeysSQL = "" + "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM userapi_key_backups " + "WHERE user_id = $1 AND version = $2" @@ -83,7 +83,7 @@ func NewPostgresKeyBackupTable(db *sql.DB) (tables.KeyBackupTable, error) { {&s.insertBackupKeyStmt, insertBackupKeySQL}, {&s.updateBackupKeyStmt, updateBackupKeySQL}, {&s.countKeysStmt, countKeysSQL}, - {&s.selectKeysStmt, selectKeysSQL}, + {&s.selectKeysStmt, selectBackupKeysSQL}, {&s.selectKeysByRoomIDStmt, selectKeysByRoomIDSQL}, {&s.selectKeysByRoomIDAndSessionIDStmt, selectKeysByRoomIDAndSessionIDSQL}, }.Prepare(db) diff --git a/keyserver/storage/postgres/key_changes_table.go b/userapi/storage/postgres/key_changes_table.go similarity index 90% rename from keyserver/storage/postgres/key_changes_table.go rename to userapi/storage/postgres/key_changes_table.go index c0e3429c7..a00494140 100644 --- a/keyserver/storage/postgres/key_changes_table.go +++ b/userapi/storage/postgres/key_changes_table.go @@ -22,8 +22,8 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/keyserver/storage/postgres/deltas" - "github.com/matrix-org/dendrite/keyserver/storage/tables" + "github.com/matrix-org/dendrite/userapi/storage/postgres/deltas" + "github.com/matrix-org/dendrite/userapi/storage/tables" ) var keyChangesSchema = ` @@ -66,7 +66,10 @@ func NewPostgresKeyChangesTable(db *sql.DB) (tables.KeyChanges, error) { if err = executeMigration(context.Background(), db); err != nil { return nil, err } - return s, nil + return s, sqlutil.StatementList{ + {&s.upsertKeyChangeStmt, upsertKeyChangeSQL}, + {&s.selectKeyChangesStmt, selectKeyChangesSQL}, + }.Prepare(db) } func executeMigration(ctx context.Context, db *sql.DB) error { @@ -95,16 +98,6 @@ func executeMigration(ctx context.Context, db *sql.DB) error { return m.Up(ctx) } -func (s *keyChangesStatements) Prepare() (err error) { - if s.upsertKeyChangeStmt, err = s.db.Prepare(upsertKeyChangeSQL); err != nil { - return err - } - if s.selectKeyChangesStmt, err = s.db.Prepare(selectKeyChangesSQL); err != nil { - return err - } - return nil -} - func (s *keyChangesStatements) InsertKeyChange(ctx context.Context, userID string) (changeID int64, err error) { err = s.upsertKeyChangeStmt.QueryRowContext(ctx, userID).Scan(&changeID) return diff --git a/keyserver/storage/postgres/one_time_keys_table.go b/userapi/storage/postgres/one_time_keys_table.go similarity index 89% rename from keyserver/storage/postgres/one_time_keys_table.go rename to userapi/storage/postgres/one_time_keys_table.go index 2117efcae..972a59147 100644 --- a/keyserver/storage/postgres/one_time_keys_table.go +++ b/userapi/storage/postgres/one_time_keys_table.go @@ -23,8 +23,8 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/keyserver/api" - "github.com/matrix-org/dendrite/keyserver/storage/tables" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/tables" ) var oneTimeKeysSchema = ` @@ -49,7 +49,7 @@ const upsertKeysSQL = "" + " ON CONFLICT ON CONSTRAINT keyserver_one_time_keys_unique" + " DO UPDATE SET key_json = $6" -const selectKeysSQL = "" + +const selectOneTimeKeysSQL = "" + "SELECT concat(algorithm, ':', key_id) as algorithmwithid, key_json FROM keyserver_one_time_keys WHERE user_id=$1 AND device_id=$2 AND concat(algorithm, ':', key_id) = ANY($3);" const selectKeysCountSQL = "" + @@ -84,25 +84,14 @@ func NewPostgresOneTimeKeysTable(db *sql.DB) (tables.OneTimeKeys, error) { if err != nil { return nil, err } - if s.upsertKeysStmt, err = db.Prepare(upsertKeysSQL); err != nil { - return nil, err - } - if s.selectKeysStmt, err = db.Prepare(selectKeysSQL); err != nil { - return nil, err - } - if s.selectKeysCountStmt, err = db.Prepare(selectKeysCountSQL); err != nil { - return nil, err - } - if s.selectKeyByAlgorithmStmt, err = db.Prepare(selectKeyByAlgorithmSQL); err != nil { - return nil, err - } - if s.deleteOneTimeKeyStmt, err = db.Prepare(deleteOneTimeKeySQL); err != nil { - return nil, err - } - if s.deleteOneTimeKeysStmt, err = db.Prepare(deleteOneTimeKeysSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.upsertKeysStmt, upsertKeysSQL}, + {&s.selectKeysStmt, selectOneTimeKeysSQL}, + {&s.selectKeysCountStmt, selectKeysCountSQL}, + {&s.selectKeyByAlgorithmStmt, selectKeyByAlgorithmSQL}, + {&s.deleteOneTimeKeyStmt, deleteOneTimeKeySQL}, + {&s.deleteOneTimeKeysStmt, deleteOneTimeKeysSQL}, + }.Prepare(db) } func (s *oneTimeKeysStatements) SelectOneTimeKeys(ctx context.Context, userID, deviceID string, keyIDsWithAlgorithms []string) (map[string]json.RawMessage, error) { diff --git a/keyserver/storage/postgres/stale_device_lists.go b/userapi/storage/postgres/stale_device_lists.go similarity index 98% rename from keyserver/storage/postgres/stale_device_lists.go rename to userapi/storage/postgres/stale_device_lists.go index 248ddfb45..c823b58c6 100644 --- a/keyserver/storage/postgres/stale_device_lists.go +++ b/userapi/storage/postgres/stale_device_lists.go @@ -24,7 +24,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/keyserver/storage/tables" + "github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/userapi/storage/postgres/storage.go b/userapi/storage/postgres/storage.go index 92dc48081..673d123ba 100644 --- a/userapi/storage/postgres/storage.go +++ b/userapi/storage/postgres/storage.go @@ -136,3 +136,44 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, OpenIDTokenLifetimeMS: openIDTokenLifetimeMS, }, nil } + +func NewKeyDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*shared.KeyDatabase, error) { + db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()) + if err != nil { + return nil, err + } + otk, err := NewPostgresOneTimeKeysTable(db) + if err != nil { + return nil, err + } + dk, err := NewPostgresDeviceKeysTable(db) + if err != nil { + return nil, err + } + kc, err := NewPostgresKeyChangesTable(db) + if err != nil { + return nil, err + } + sdl, err := NewPostgresStaleDeviceListsTable(db) + if err != nil { + return nil, err + } + csk, err := NewPostgresCrossSigningKeysTable(db) + if err != nil { + return nil, err + } + css, err := NewPostgresCrossSigningSigsTable(db) + if err != nil { + return nil, err + } + + return &shared.KeyDatabase{ + OneTimeKeysTable: otk, + DeviceKeysTable: dk, + KeyChangesTable: kc, + StaleDeviceListsTable: sdl, + CrossSigningKeysTable: csk, + CrossSigningSigsTable: css, + Writer: writer, + }, nil +} diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index f549dcef9..d3272a032 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -59,6 +59,17 @@ type Database struct { OpenIDTokenLifetimeMS int64 } +type KeyDatabase struct { + OneTimeKeysTable tables.OneTimeKeys + DeviceKeysTable tables.DeviceKeys + KeyChangesTable tables.KeyChanges + StaleDeviceListsTable tables.StaleDeviceLists + CrossSigningKeysTable tables.CrossSigningKeys + CrossSigningSigsTable tables.CrossSigningSigs + DB *sql.DB + Writer sqlutil.Writer +} + const ( // The length of generated device IDs deviceIDByteLength = 6 @@ -588,16 +599,42 @@ func (d *Database) CreateDevice( deviceID *string, accessToken string, displayName *string, ipAddr, userAgent string, ) (dev *api.Device, returnErr error) { if deviceID != nil { - returnErr = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - var err error - // Revoke existing tokens for this device - if err = d.Devices.DeleteDevice(ctx, txn, *deviceID, localpart, serverName); err != nil { - return err - } + _, ok := d.Writer.(*sqlutil.ExclusiveWriter) + if ok { // we're using most likely using SQLite, so do things a little different + returnErr = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + var err error - dev, err = d.Devices.InsertDevice(ctx, txn, *deviceID, localpart, serverName, accessToken, displayName, ipAddr, userAgent) - return err - }) + devices, err := d.Devices.SelectDevicesByLocalpart(ctx, txn, localpart, serverName, "") + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + // No devices yet, only create a new one + if len(devices) == 0 { + dev, err = d.Devices.InsertDevice(ctx, txn, *deviceID, localpart, serverName, accessToken, displayName, ipAddr, userAgent) + return err + } + sessionID := devices[0].SessionID + 1 + + // Revoke existing tokens for this device + if err = d.Devices.DeleteDevice(ctx, txn, *deviceID, localpart, serverName); err != nil { + return err + } + // Create a new device with the session ID incremented + dev, err = d.Devices.InsertDeviceWithSessionID(ctx, txn, *deviceID, localpart, serverName, accessToken, displayName, ipAddr, userAgent, sessionID) + return err + }) + } else { + returnErr = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + var err error + // Revoke existing tokens for this device + if err = d.Devices.DeleteDevice(ctx, txn, *deviceID, localpart, serverName); err != nil { + return err + } + + dev, err = d.Devices.InsertDevice(ctx, txn, *deviceID, localpart, serverName, accessToken, displayName, ipAddr, userAgent) + return err + }) + } } else { // We generate device IDs in a loop in case its already taken. // We cap this at going round 5 times to ensure we don't spin forever @@ -618,7 +655,7 @@ func (d *Database) CreateDevice( } } } - return + return dev, returnErr } // generateDeviceID creates a new device id. Returns an error if failed to generate @@ -849,3 +886,227 @@ func (d *Database) DailyRoomsMessages( ) (stats types.MessageStats, activeRooms, activeE2EERooms int64, err error) { return d.Stats.DailyRoomsMessages(ctx, nil, serverName) } + +// + +func (d *KeyDatabase) ExistingOneTimeKeys(ctx context.Context, userID, deviceID string, keyIDsWithAlgorithms []string) (map[string]json.RawMessage, error) { + return d.OneTimeKeysTable.SelectOneTimeKeys(ctx, userID, deviceID, keyIDsWithAlgorithms) +} + +func (d *KeyDatabase) StoreOneTimeKeys(ctx context.Context, keys api.OneTimeKeys) (counts *api.OneTimeKeysCount, err error) { + _ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + counts, err = d.OneTimeKeysTable.InsertOneTimeKeys(ctx, txn, keys) + return err + }) + return +} + +func (d *KeyDatabase) OneTimeKeysCount(ctx context.Context, userID, deviceID string) (*api.OneTimeKeysCount, error) { + return d.OneTimeKeysTable.CountOneTimeKeys(ctx, userID, deviceID) +} + +func (d *KeyDatabase) DeviceKeysJSON(ctx context.Context, keys []api.DeviceMessage) error { + return d.DeviceKeysTable.SelectDeviceKeysJSON(ctx, keys) +} + +func (d *KeyDatabase) PrevIDsExists(ctx context.Context, userID string, prevIDs []int64) (bool, error) { + count, err := d.DeviceKeysTable.CountStreamIDsForUser(ctx, userID, prevIDs) + if err != nil { + return false, err + } + return count == len(prevIDs), nil +} + +func (d *KeyDatabase) StoreRemoteDeviceKeys(ctx context.Context, keys []api.DeviceMessage, clearUserIDs []string) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + for _, userID := range clearUserIDs { + err := d.DeviceKeysTable.DeleteAllDeviceKeys(ctx, txn, userID) + if err != nil { + return err + } + } + return d.DeviceKeysTable.InsertDeviceKeys(ctx, txn, keys) + }) +} + +func (d *KeyDatabase) StoreLocalDeviceKeys(ctx context.Context, keys []api.DeviceMessage) error { + // work out the latest stream IDs for each user + userIDToStreamID := make(map[string]int64) + for _, k := range keys { + userIDToStreamID[k.UserID] = 0 + } + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + for userID := range userIDToStreamID { + streamID, err := d.DeviceKeysTable.SelectMaxStreamIDForUser(ctx, txn, userID) + if err != nil { + return err + } + userIDToStreamID[userID] = streamID + } + // set the stream IDs for each key + for i := range keys { + k := keys[i] + userIDToStreamID[k.UserID]++ // start stream from 1 + k.StreamID = userIDToStreamID[k.UserID] + keys[i] = k + } + return d.DeviceKeysTable.InsertDeviceKeys(ctx, txn, keys) + }) +} + +func (d *KeyDatabase) DeviceKeysForUser(ctx context.Context, userID string, deviceIDs []string, includeEmpty bool) ([]api.DeviceMessage, error) { + return d.DeviceKeysTable.SelectBatchDeviceKeys(ctx, userID, deviceIDs, includeEmpty) +} + +func (d *KeyDatabase) ClaimKeys(ctx context.Context, userToDeviceToAlgorithm map[string]map[string]string) ([]api.OneTimeKeys, error) { + var result []api.OneTimeKeys + err := d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + for userID, deviceToAlgo := range userToDeviceToAlgorithm { + for deviceID, algo := range deviceToAlgo { + keyJSON, err := d.OneTimeKeysTable.SelectAndDeleteOneTimeKey(ctx, txn, userID, deviceID, algo) + if err != nil { + return err + } + if keyJSON != nil { + result = append(result, api.OneTimeKeys{ + UserID: userID, + DeviceID: deviceID, + KeyJSON: keyJSON, + }) + } + } + } + return nil + }) + return result, err +} + +func (d *KeyDatabase) StoreKeyChange(ctx context.Context, userID string) (id int64, err error) { + err = d.Writer.Do(nil, nil, func(_ *sql.Tx) error { + id, err = d.KeyChangesTable.InsertKeyChange(ctx, userID) + return err + }) + return +} + +func (d *KeyDatabase) KeyChanges(ctx context.Context, fromOffset, toOffset int64) (userIDs []string, latestOffset int64, err error) { + return d.KeyChangesTable.SelectKeyChanges(ctx, fromOffset, toOffset) +} + +// StaleDeviceLists returns a list of user IDs ending with the domains provided who have stale device lists. +// If no domains are given, all user IDs with stale device lists are returned. +func (d *KeyDatabase) StaleDeviceLists(ctx context.Context, domains []gomatrixserverlib.ServerName) ([]string, error) { + return d.StaleDeviceListsTable.SelectUserIDsWithStaleDeviceLists(ctx, domains) +} + +// MarkDeviceListStale sets the stale bit for this user to isStale. +func (d *KeyDatabase) MarkDeviceListStale(ctx context.Context, userID string, isStale bool) error { + return d.Writer.Do(nil, nil, func(_ *sql.Tx) error { + return d.StaleDeviceListsTable.InsertStaleDeviceList(ctx, userID, isStale) + }) +} + +// DeleteDeviceKeys removes the device keys for a given user/device, and any accompanying +// cross-signing signatures relating to that device. +func (d *KeyDatabase) DeleteDeviceKeys(ctx context.Context, userID string, deviceIDs []gomatrixserverlib.KeyID) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + for _, deviceID := range deviceIDs { + if err := d.CrossSigningSigsTable.DeleteCrossSigningSigsForTarget(ctx, txn, userID, deviceID); err != nil && err != sql.ErrNoRows { + return fmt.Errorf("d.CrossSigningSigsTable.DeleteCrossSigningSigsForTarget: %w", err) + } + if err := d.DeviceKeysTable.DeleteDeviceKeys(ctx, txn, userID, string(deviceID)); err != nil && err != sql.ErrNoRows { + return fmt.Errorf("d.DeviceKeysTable.DeleteDeviceKeys: %w", err) + } + if err := d.OneTimeKeysTable.DeleteOneTimeKeys(ctx, txn, userID, string(deviceID)); err != nil && err != sql.ErrNoRows { + return fmt.Errorf("d.OneTimeKeysTable.DeleteOneTimeKeys: %w", err) + } + } + return nil + }) +} + +// CrossSigningKeysForUser returns the latest known cross-signing keys for a user, if any. +func (d *KeyDatabase) CrossSigningKeysForUser(ctx context.Context, userID string) (map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey, error) { + keyMap, err := d.CrossSigningKeysTable.SelectCrossSigningKeysForUser(ctx, nil, userID) + if err != nil { + return nil, fmt.Errorf("d.CrossSigningKeysTable.SelectCrossSigningKeysForUser: %w", err) + } + results := map[gomatrixserverlib.CrossSigningKeyPurpose]gomatrixserverlib.CrossSigningKey{} + for purpose, key := range keyMap { + keyID := gomatrixserverlib.KeyID("ed25519:" + key.Encode()) + result := gomatrixserverlib.CrossSigningKey{ + UserID: userID, + Usage: []gomatrixserverlib.CrossSigningKeyPurpose{purpose}, + Keys: map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{ + keyID: key, + }, + } + sigMap, err := d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, userID, userID, keyID) + if err != nil { + continue + } + for sigUserID, forSigUserID := range sigMap { + if userID != sigUserID { + continue + } + if result.Signatures == nil { + result.Signatures = map[string]map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{} + } + if _, ok := result.Signatures[sigUserID]; !ok { + result.Signatures[sigUserID] = map[gomatrixserverlib.KeyID]gomatrixserverlib.Base64Bytes{} + } + for sigKeyID, sigBytes := range forSigUserID { + result.Signatures[sigUserID][sigKeyID] = sigBytes + } + } + results[purpose] = result + } + return results, nil +} + +// CrossSigningKeysForUser returns the latest known cross-signing keys for a user, if any. +func (d *KeyDatabase) CrossSigningKeysDataForUser(ctx context.Context, userID string) (types.CrossSigningKeyMap, error) { + return d.CrossSigningKeysTable.SelectCrossSigningKeysForUser(ctx, nil, userID) +} + +// CrossSigningSigsForTarget returns the signatures for a given user's key ID, if any. +func (d *KeyDatabase) CrossSigningSigsForTarget(ctx context.Context, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (types.CrossSigningSigMap, error) { + return d.CrossSigningSigsTable.SelectCrossSigningSigsForTarget(ctx, nil, originUserID, targetUserID, targetKeyID) +} + +// StoreCrossSigningKeysForUser stores the latest known cross-signing keys for a user. +func (d *KeyDatabase) StoreCrossSigningKeysForUser(ctx context.Context, userID string, keyMap types.CrossSigningKeyMap) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + for keyType, keyData := range keyMap { + if err := d.CrossSigningKeysTable.UpsertCrossSigningKeysForUser(ctx, txn, userID, keyType, keyData); err != nil { + return fmt.Errorf("d.CrossSigningKeysTable.InsertCrossSigningKeysForUser: %w", err) + } + } + return nil + }) +} + +// StoreCrossSigningSigsForTarget stores a signature for a target user ID and key/dvice. +func (d *KeyDatabase) StoreCrossSigningSigsForTarget( + ctx context.Context, + originUserID string, originKeyID gomatrixserverlib.KeyID, + targetUserID string, targetKeyID gomatrixserverlib.KeyID, + signature gomatrixserverlib.Base64Bytes, +) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + if err := d.CrossSigningSigsTable.UpsertCrossSigningSigsForTarget(ctx, nil, originUserID, originKeyID, targetUserID, targetKeyID, signature); err != nil { + return fmt.Errorf("d.CrossSigningSigsTable.InsertCrossSigningSigsForTarget: %w", err) + } + return nil + }) +} + +// DeleteStaleDeviceLists deletes stale device list entries for users we don't share a room with anymore. +func (d *KeyDatabase) DeleteStaleDeviceLists( + ctx context.Context, + userIDs []string, +) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + return d.StaleDeviceListsTable.DeleteStaleDeviceLists(ctx, txn, userIDs) + }) +} diff --git a/keyserver/storage/sqlite3/cross_signing_keys_table.go b/userapi/storage/sqlite3/cross_signing_keys_table.go similarity index 96% rename from keyserver/storage/sqlite3/cross_signing_keys_table.go rename to userapi/storage/sqlite3/cross_signing_keys_table.go index e103d9883..10721fcc8 100644 --- a/keyserver/storage/sqlite3/cross_signing_keys_table.go +++ b/userapi/storage/sqlite3/cross_signing_keys_table.go @@ -21,8 +21,8 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/keyserver/storage/tables" - "github.com/matrix-org/dendrite/keyserver/types" + "github.com/matrix-org/dendrite/userapi/storage/tables" + "github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/keyserver/storage/sqlite3/cross_signing_sigs_table.go b/userapi/storage/sqlite3/cross_signing_sigs_table.go similarity index 96% rename from keyserver/storage/sqlite3/cross_signing_sigs_table.go rename to userapi/storage/sqlite3/cross_signing_sigs_table.go index 7a153e8fb..2be00c9c1 100644 --- a/keyserver/storage/sqlite3/cross_signing_sigs_table.go +++ b/userapi/storage/sqlite3/cross_signing_sigs_table.go @@ -21,9 +21,9 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/keyserver/storage/sqlite3/deltas" - "github.com/matrix-org/dendrite/keyserver/storage/tables" - "github.com/matrix-org/dendrite/keyserver/types" + "github.com/matrix-org/dendrite/userapi/storage/sqlite3/deltas" + "github.com/matrix-org/dendrite/userapi/storage/tables" + "github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/keyserver/storage/sqlite3/deltas/2022012016470000_key_changes.go b/userapi/storage/sqlite3/deltas/2022012016470000_key_changes.go similarity index 100% rename from keyserver/storage/sqlite3/deltas/2022012016470000_key_changes.go rename to userapi/storage/sqlite3/deltas/2022012016470000_key_changes.go diff --git a/keyserver/storage/sqlite3/deltas/2022042612000000_xsigning_idx.go b/userapi/storage/sqlite3/deltas/2022042612000000_xsigning_idx.go similarity index 100% rename from keyserver/storage/sqlite3/deltas/2022042612000000_xsigning_idx.go rename to userapi/storage/sqlite3/deltas/2022042612000000_xsigning_idx.go diff --git a/keyserver/storage/sqlite3/device_keys_table.go b/userapi/storage/sqlite3/device_keys_table.go similarity index 88% rename from keyserver/storage/sqlite3/device_keys_table.go rename to userapi/storage/sqlite3/device_keys_table.go index 73768da5b..15e69cc4c 100644 --- a/keyserver/storage/sqlite3/device_keys_table.go +++ b/userapi/storage/sqlite3/device_keys_table.go @@ -22,8 +22,8 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/keyserver/api" - "github.com/matrix-org/dendrite/keyserver/storage/tables" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/tables" ) var deviceKeysSchema = ` @@ -86,28 +86,16 @@ func NewSqliteDeviceKeysTable(db *sql.DB) (tables.DeviceKeys, error) { if err != nil { return nil, err } - if s.upsertDeviceKeysStmt, err = db.Prepare(upsertDeviceKeysSQL); err != nil { - return nil, err - } - if s.selectDeviceKeysStmt, err = db.Prepare(selectDeviceKeysSQL); err != nil { - return nil, err - } - if s.selectBatchDeviceKeysStmt, err = db.Prepare(selectBatchDeviceKeysSQL); err != nil { - return nil, err - } - if s.selectBatchDeviceKeysWithEmptiesStmt, err = db.Prepare(selectBatchDeviceKeysWithEmptiesSQL); err != nil { - return nil, err - } - if s.selectMaxStreamForUserStmt, err = db.Prepare(selectMaxStreamForUserSQL); err != nil { - return nil, err - } - if s.deleteDeviceKeysStmt, err = db.Prepare(deleteDeviceKeysSQL); err != nil { - return nil, err - } - if s.deleteAllDeviceKeysStmt, err = db.Prepare(deleteAllDeviceKeysSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.upsertDeviceKeysStmt, upsertDeviceKeysSQL}, + {&s.selectDeviceKeysStmt, selectDeviceKeysSQL}, + {&s.selectBatchDeviceKeysStmt, selectBatchDeviceKeysSQL}, + {&s.selectBatchDeviceKeysWithEmptiesStmt, selectBatchDeviceKeysWithEmptiesSQL}, + {&s.selectMaxStreamForUserStmt, selectMaxStreamForUserSQL}, + // {&s.countStreamIDsForUserStmt, countStreamIDsForUserSQL}, // prepared at runtime + {&s.deleteDeviceKeysStmt, deleteDeviceKeysSQL}, + {&s.deleteAllDeviceKeysStmt, deleteAllDeviceKeysSQL}, + }.Prepare(db) } func (s *deviceKeysStatements) DeleteDeviceKeys(ctx context.Context, txn *sql.Tx, userID, deviceID string) error { diff --git a/userapi/storage/sqlite3/devices_table.go b/userapi/storage/sqlite3/devices_table.go index c5db34bd7..65e17527d 100644 --- a/userapi/storage/sqlite3/devices_table.go +++ b/userapi/storage/sqlite3/devices_table.go @@ -65,7 +65,7 @@ const selectDeviceByIDSQL = "" + "SELECT display_name, last_seen_ts, ip FROM userapi_devices WHERE localpart = $1 AND server_name = $2 AND device_id = $3" const selectDevicesByLocalpartSQL = "" + - "SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM userapi_devices WHERE localpart = $1 AND server_name = $2 AND device_id != $3 ORDER BY last_seen_ts DESC" + "SELECT device_id, display_name, last_seen_ts, ip, user_agent, session_id FROM userapi_devices WHERE localpart = $1 AND server_name = $2 AND device_id != $3 ORDER BY last_seen_ts DESC" const updateDeviceNameSQL = "" + "UPDATE userapi_devices SET display_name = $1 WHERE localpart = $2 AND server_name = $3 AND device_id = $4" @@ -80,7 +80,7 @@ const deleteDevicesSQL = "" + "DELETE FROM userapi_devices WHERE localpart = $1 AND server_name = $2 AND device_id IN ($3)" const selectDevicesByIDSQL = "" + - "SELECT device_id, localpart, server_name, display_name, last_seen_ts FROM userapi_devices WHERE device_id IN ($1) ORDER BY last_seen_ts DESC" + "SELECT device_id, localpart, server_name, display_name, last_seen_ts, session_id FROM userapi_devices WHERE device_id IN ($1) ORDER BY last_seen_ts DESC" const updateDeviceLastSeen = "" + "UPDATE userapi_devices SET last_seen_ts = $1, ip = $2, user_agent = $3 WHERE localpart = $4 AND server_name = $5 AND device_id = $6" @@ -151,7 +151,7 @@ func (s *devicesStatements) InsertDevice( if _, err := insertStmt.ExecContext(ctx, id, localpart, serverName, accessToken, createdTimeMS, displayName, sessionID, createdTimeMS, ipAddr, userAgent); err != nil { return nil, err } - return &api.Device{ + dev := &api.Device{ ID: id, UserID: userutil.MakeUserID(localpart, serverName), AccessToken: accessToken, @@ -159,7 +159,36 @@ func (s *devicesStatements) InsertDevice( LastSeenTS: createdTimeMS, LastSeenIP: ipAddr, UserAgent: userAgent, - }, nil + } + if displayName != nil { + dev.DisplayName = *displayName + } + return dev, nil +} + +func (s *devicesStatements) InsertDeviceWithSessionID(ctx context.Context, txn *sql.Tx, id, + localpart string, serverName gomatrixserverlib.ServerName, + accessToken string, displayName *string, ipAddr, userAgent string, + sessionID int64, +) (*api.Device, error) { + createdTimeMS := time.Now().UnixNano() / 1000000 + insertStmt := sqlutil.TxStmt(txn, s.insertDeviceStmt) + if _, err := insertStmt.ExecContext(ctx, id, localpart, serverName, accessToken, createdTimeMS, displayName, sessionID, createdTimeMS, ipAddr, userAgent); err != nil { + return nil, err + } + dev := &api.Device{ + ID: id, + UserID: userutil.MakeUserID(localpart, serverName), + AccessToken: accessToken, + SessionID: sessionID, + LastSeenTS: createdTimeMS, + LastSeenIP: ipAddr, + UserAgent: userAgent, + } + if displayName != nil { + dev.DisplayName = *displayName + } + return dev, nil } func (s *devicesStatements) DeleteDevice( @@ -181,6 +210,7 @@ func (s *devicesStatements) DeleteDevices( if err != nil { return err } + defer internal.CloseAndLogIfError(ctx, prep, "DeleteDevices.StmtClose() failed") stmt := sqlutil.TxStmt(txn, prep) params := make([]interface{}, len(devices)+2) params[0] = localpart @@ -271,7 +301,7 @@ func (s *devicesStatements) SelectDevicesByLocalpart( var lastseents sql.NullInt64 var id, displayname, ip, useragent sql.NullString for rows.Next() { - err = rows.Scan(&id, &displayname, &lastseents, &ip, &useragent) + err = rows.Scan(&id, &displayname, &lastseents, &ip, &useragent, &dev.SessionID) if err != nil { return devices, err } @@ -317,7 +347,7 @@ func (s *devicesStatements) SelectDevicesByID(ctx context.Context, deviceIDs []s var displayName sql.NullString var lastseents sql.NullInt64 for rows.Next() { - if err := rows.Scan(&dev.ID, &localpart, &serverName, &displayName, &lastseents); err != nil { + if err := rows.Scan(&dev.ID, &localpart, &serverName, &displayName, &lastseents, &dev.SessionID); err != nil { return nil, err } if displayName.Valid { diff --git a/userapi/storage/sqlite3/key_backup_table.go b/userapi/storage/sqlite3/key_backup_table.go index 7883ffb19..ed2746310 100644 --- a/userapi/storage/sqlite3/key_backup_table.go +++ b/userapi/storage/sqlite3/key_backup_table.go @@ -52,7 +52,7 @@ const updateBackupKeySQL = "" + const countKeysSQL = "" + "SELECT COUNT(*) FROM userapi_key_backups WHERE user_id = $1 AND version = $2" -const selectKeysSQL = "" + +const selectBackupKeysSQL = "" + "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM userapi_key_backups " + "WHERE user_id = $1 AND version = $2" @@ -83,7 +83,7 @@ func NewSQLiteKeyBackupTable(db *sql.DB) (tables.KeyBackupTable, error) { {&s.insertBackupKeyStmt, insertBackupKeySQL}, {&s.updateBackupKeyStmt, updateBackupKeySQL}, {&s.countKeysStmt, countKeysSQL}, - {&s.selectKeysStmt, selectKeysSQL}, + {&s.selectKeysStmt, selectBackupKeysSQL}, {&s.selectKeysByRoomIDStmt, selectKeysByRoomIDSQL}, {&s.selectKeysByRoomIDAndSessionIDStmt, selectKeysByRoomIDAndSessionIDSQL}, }.Prepare(db) diff --git a/keyserver/storage/sqlite3/key_changes_table.go b/userapi/storage/sqlite3/key_changes_table.go similarity index 90% rename from keyserver/storage/sqlite3/key_changes_table.go rename to userapi/storage/sqlite3/key_changes_table.go index 0c844d67a..923bb57eb 100644 --- a/keyserver/storage/sqlite3/key_changes_table.go +++ b/userapi/storage/sqlite3/key_changes_table.go @@ -22,8 +22,8 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/keyserver/storage/sqlite3/deltas" - "github.com/matrix-org/dendrite/keyserver/storage/tables" + "github.com/matrix-org/dendrite/userapi/storage/sqlite3/deltas" + "github.com/matrix-org/dendrite/userapi/storage/tables" ) var keyChangesSchema = ` @@ -65,7 +65,10 @@ func NewSqliteKeyChangesTable(db *sql.DB) (tables.KeyChanges, error) { return nil, err } - return s, nil + return s, sqlutil.StatementList{ + {&s.upsertKeyChangeStmt, upsertKeyChangeSQL}, + {&s.selectKeyChangesStmt, selectKeyChangesSQL}, + }.Prepare(db) } func executeMigration(ctx context.Context, db *sql.DB) error { @@ -93,16 +96,6 @@ func executeMigration(ctx context.Context, db *sql.DB) error { return m.Up(ctx) } -func (s *keyChangesStatements) Prepare() (err error) { - if s.upsertKeyChangeStmt, err = s.db.Prepare(upsertKeyChangeSQL); err != nil { - return err - } - if s.selectKeyChangesStmt, err = s.db.Prepare(selectKeyChangesSQL); err != nil { - return err - } - return nil -} - func (s *keyChangesStatements) InsertKeyChange(ctx context.Context, userID string) (changeID int64, err error) { err = s.upsertKeyChangeStmt.QueryRowContext(ctx, userID).Scan(&changeID) return diff --git a/keyserver/storage/sqlite3/one_time_keys_table.go b/userapi/storage/sqlite3/one_time_keys_table.go similarity index 89% rename from keyserver/storage/sqlite3/one_time_keys_table.go rename to userapi/storage/sqlite3/one_time_keys_table.go index 7a923d0e5..a992d399c 100644 --- a/keyserver/storage/sqlite3/one_time_keys_table.go +++ b/userapi/storage/sqlite3/one_time_keys_table.go @@ -22,8 +22,8 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/keyserver/api" - "github.com/matrix-org/dendrite/keyserver/storage/tables" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/tables" ) var oneTimeKeysSchema = ` @@ -48,7 +48,7 @@ const upsertKeysSQL = "" + " ON CONFLICT (user_id, device_id, key_id, algorithm)" + " DO UPDATE SET key_json = $6" -const selectKeysSQL = "" + +const selectOneTimeKeysSQL = "" + "SELECT key_id, algorithm, key_json FROM keyserver_one_time_keys WHERE user_id=$1 AND device_id=$2" const selectKeysCountSQL = "" + @@ -83,25 +83,14 @@ func NewSqliteOneTimeKeysTable(db *sql.DB) (tables.OneTimeKeys, error) { if err != nil { return nil, err } - if s.upsertKeysStmt, err = db.Prepare(upsertKeysSQL); err != nil { - return nil, err - } - if s.selectKeysStmt, err = db.Prepare(selectKeysSQL); err != nil { - return nil, err - } - if s.selectKeysCountStmt, err = db.Prepare(selectKeysCountSQL); err != nil { - return nil, err - } - if s.selectKeyByAlgorithmStmt, err = db.Prepare(selectKeyByAlgorithmSQL); err != nil { - return nil, err - } - if s.deleteOneTimeKeyStmt, err = db.Prepare(deleteOneTimeKeySQL); err != nil { - return nil, err - } - if s.deleteOneTimeKeysStmt, err = db.Prepare(deleteOneTimeKeysSQL); err != nil { - return nil, err - } - return s, nil + return s, sqlutil.StatementList{ + {&s.upsertKeysStmt, upsertKeysSQL}, + {&s.selectKeysStmt, selectOneTimeKeysSQL}, + {&s.selectKeysCountStmt, selectKeysCountSQL}, + {&s.selectKeyByAlgorithmStmt, selectKeyByAlgorithmSQL}, + {&s.deleteOneTimeKeyStmt, deleteOneTimeKeySQL}, + {&s.deleteOneTimeKeysStmt, deleteOneTimeKeysSQL}, + }.Prepare(db) } func (s *oneTimeKeysStatements) SelectOneTimeKeys(ctx context.Context, userID, deviceID string, keyIDsWithAlgorithms []string) (map[string]json.RawMessage, error) { diff --git a/keyserver/storage/sqlite3/stale_device_lists.go b/userapi/storage/sqlite3/stale_device_lists.go similarity index 98% rename from keyserver/storage/sqlite3/stale_device_lists.go rename to userapi/storage/sqlite3/stale_device_lists.go index fd76a6e3b..f078fc99f 100644 --- a/keyserver/storage/sqlite3/stale_device_lists.go +++ b/userapi/storage/sqlite3/stale_device_lists.go @@ -23,7 +23,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/keyserver/storage/tables" + "github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/userapi/storage/sqlite3/stats_table.go b/userapi/storage/sqlite3/stats_table.go index a1365c944..72b3ba49d 100644 --- a/userapi/storage/sqlite3/stats_table.go +++ b/userapi/storage/sqlite3/stats_table.go @@ -256,6 +256,7 @@ func (s *statsStatements) allUsers(ctx context.Context, txn *sql.Tx) (result int if err != nil { return 0, err } + defer internal.CloseAndLogIfError(ctx, queryStmt, "allUsers.StmtClose() failed") stmt := sqlutil.TxStmt(txn, queryStmt) err = stmt.QueryRowContext(ctx, 1, 2, 3, 4, @@ -269,6 +270,7 @@ func (s *statsStatements) nonBridgedUsers(ctx context.Context, txn *sql.Tx) (res if err != nil { return 0, err } + defer internal.CloseAndLogIfError(ctx, queryStmt, "nonBridgedUsers.StmtClose() failed") stmt := sqlutil.TxStmt(txn, queryStmt) err = stmt.QueryRowContext(ctx, 1, 2, 3, @@ -286,6 +288,7 @@ func (s *statsStatements) registeredUserByType(ctx context.Context, txn *sql.Tx) if err != nil { return nil, err } + defer internal.CloseAndLogIfError(ctx, queryStmt, "registeredUserByType.StmtClose() failed") stmt := sqlutil.TxStmt(txn, queryStmt) registeredAfter := time.Now().AddDate(0, 0, -30) diff --git a/userapi/storage/sqlite3/storage.go b/userapi/storage/sqlite3/storage.go index 85a1f7063..0f3eeed1b 100644 --- a/userapi/storage/sqlite3/storage.go +++ b/userapi/storage/sqlite3/storage.go @@ -30,8 +30,8 @@ import ( "github.com/matrix-org/dendrite/userapi/storage/sqlite3/deltas" ) -// NewDatabase creates a new accounts and profiles database -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) { +// NewUserDatabase creates a new accounts and profiles database +func NewUserDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (*shared.Database, error) { db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()) if err != nil { return nil, err @@ -134,3 +134,44 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, OpenIDTokenLifetimeMS: openIDTokenLifetimeMS, }, nil } + +func NewKeyDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (*shared.KeyDatabase, error) { + db, writer, err := base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()) + if err != nil { + return nil, err + } + otk, err := NewSqliteOneTimeKeysTable(db) + if err != nil { + return nil, err + } + dk, err := NewSqliteDeviceKeysTable(db) + if err != nil { + return nil, err + } + kc, err := NewSqliteKeyChangesTable(db) + if err != nil { + return nil, err + } + sdl, err := NewSqliteStaleDeviceListsTable(db) + if err != nil { + return nil, err + } + csk, err := NewSqliteCrossSigningKeysTable(db) + if err != nil { + return nil, err + } + css, err := NewSqliteCrossSigningSigsTable(db) + if err != nil { + return nil, err + } + + return &shared.KeyDatabase{ + OneTimeKeysTable: otk, + DeviceKeysTable: dk, + KeyChangesTable: kc, + StaleDeviceListsTable: sdl, + CrossSigningKeysTable: csk, + CrossSigningSigsTable: css, + Writer: writer, + }, nil +} diff --git a/userapi/storage/storage.go b/userapi/storage/storage.go index 42221e752..0329fb46a 100644 --- a/userapi/storage/storage.go +++ b/userapi/storage/storage.go @@ -29,15 +29,36 @@ import ( "github.com/matrix-org/dendrite/userapi/storage/sqlite3" ) -// NewUserAPIDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) +// NewUserDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets postgres connection parameters -func NewUserAPIDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, serverName gomatrixserverlib.ServerName, bcryptCost int, openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string) (Database, error) { +func NewUserDatabase( + base *base.BaseDendrite, + dbProperties *config.DatabaseOptions, + serverName gomatrixserverlib.ServerName, + bcryptCost int, + openIDTokenLifetimeMS int64, + loginTokenLifetime time.Duration, + serverNoticesLocalpart string, +) (UserDatabase, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(base, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) + return sqlite3.NewUserDatabase(base, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) case dbProperties.ConnectionString.IsPostgres(): return postgres.NewDatabase(base, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) default: return nil, fmt.Errorf("unexpected database type") } } + +// NewKeyDatabase opens a new Postgres or Sqlite database (base on dataSourceName) scheme) +// and sets postgres connection parameters. +func NewKeyDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (KeyDatabase, error) { + switch { + case dbProperties.ConnectionString.IsSQLite(): + return sqlite3.NewKeyDatabase(base, dbProperties) + case dbProperties.ConnectionString.IsPostgres(): + return postgres.NewKeyDatabase(base, dbProperties) + default: + return nil, fmt.Errorf("unexpected database type") + } +} diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index 23aafff03..f52e7e17d 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -4,9 +4,12 @@ import ( "context" "encoding/json" "fmt" + "reflect" + "sync" "testing" "time" + "github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/stretchr/testify/assert" @@ -29,14 +32,14 @@ var ( ctx = context.Background() ) -func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { +func mustCreateUserDatabase(t *testing.T, dbType test.DBType) (storage.UserDatabase, func()) { base, baseclose := testrig.CreateBaseDendrite(t, dbType) connStr, close := test.PrepareDBConnectionString(t, dbType) - db, err := storage.NewUserAPIDatabase(base, &config.DatabaseOptions{ + db, err := storage.NewUserDatabase(base, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, "localhost", bcrypt.MinCost, openIDLifetimeMS, loginTokenLifetime, "_server") if err != nil { - t.Fatalf("NewUserAPIDatabase returned %s", err) + t.Fatalf("NewUserDatabase returned %s", err) } return db, func() { close() @@ -47,7 +50,7 @@ func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, fun // Tests storing and getting account data func Test_AccountData(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, close := mustCreateDatabase(t, dbType) + db, close := mustCreateUserDatabase(t, dbType) defer close() alice := test.NewUser(t) localpart, domain, err := gomatrixserverlib.SplitID('@', alice.ID) @@ -78,7 +81,7 @@ func Test_AccountData(t *testing.T) { // Tests the creation of accounts func Test_Accounts(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, close := mustCreateDatabase(t, dbType) + db, close := mustCreateUserDatabase(t, dbType) defer close() alice := test.NewUser(t) aliceLocalpart, aliceDomain, err := gomatrixserverlib.SplitID('@', alice.ID) @@ -158,7 +161,7 @@ func Test_Devices(t *testing.T) { accessToken := util.RandomString(16) test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, close := mustCreateDatabase(t, dbType) + db, close := mustCreateUserDatabase(t, dbType) defer close() deviceWithID, err := db.CreateDevice(ctx, localpart, domain, &deviceID, accessToken, nil, "", "") @@ -238,7 +241,7 @@ func Test_KeyBackup(t *testing.T) { room := test.NewRoom(t, alice) test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, close := mustCreateDatabase(t, dbType) + db, close := mustCreateUserDatabase(t, dbType) defer close() wantAuthData := json.RawMessage("my auth data") @@ -315,7 +318,7 @@ func Test_KeyBackup(t *testing.T) { func Test_LoginToken(t *testing.T) { alice := test.NewUser(t) test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, close := mustCreateDatabase(t, dbType) + db, close := mustCreateUserDatabase(t, dbType) defer close() // create a new token @@ -347,7 +350,7 @@ func Test_OpenID(t *testing.T) { token := util.RandomString(24) test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, close := mustCreateDatabase(t, dbType) + db, close := mustCreateUserDatabase(t, dbType) defer close() expiresAtMS := time.Now().UnixNano()/int64(time.Millisecond) + openIDLifetimeMS @@ -368,7 +371,7 @@ func Test_Profile(t *testing.T) { assert.NoError(t, err) test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, close := mustCreateDatabase(t, dbType) + db, close := mustCreateUserDatabase(t, dbType) defer close() // create account, which also creates a profile @@ -417,7 +420,7 @@ func Test_Pusher(t *testing.T) { assert.NoError(t, err) test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, close := mustCreateDatabase(t, dbType) + db, close := mustCreateUserDatabase(t, dbType) defer close() appID := util.RandomString(8) @@ -468,7 +471,7 @@ func Test_ThreePID(t *testing.T) { assert.NoError(t, err) test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, close := mustCreateDatabase(t, dbType) + db, close := mustCreateUserDatabase(t, dbType) defer close() threePID := util.RandomString(8) medium := util.RandomString(8) @@ -507,7 +510,7 @@ func Test_Notification(t *testing.T) { room := test.NewRoom(t, alice) room2 := test.NewRoom(t, alice) test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - db, close := mustCreateDatabase(t, dbType) + db, close := mustCreateUserDatabase(t, dbType) defer close() // generate some dummy notifications for i := 0; i < 10; i++ { @@ -571,3 +574,184 @@ func Test_Notification(t *testing.T) { assert.Equal(t, int64(0), total) }) } + +func mustCreateKeyDatabase(t *testing.T, dbType test.DBType) (storage.KeyDatabase, func()) { + base, close := testrig.CreateBaseDendrite(t, dbType) + db, err := storage.NewKeyDatabase(base, &base.Cfg.KeyServer.Database) + if err != nil { + t.Fatalf("failed to create new database: %v", err) + } + return db, close +} + +func MustNotError(t *testing.T, err error) { + t.Helper() + if err == nil { + return + } + t.Fatalf("operation failed: %s", err) +} + +func TestKeyChanges(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, clean := mustCreateKeyDatabase(t, dbType) + defer clean() + _, err := db.StoreKeyChange(ctx, "@alice:localhost") + MustNotError(t, err) + deviceChangeIDB, err := db.StoreKeyChange(ctx, "@bob:localhost") + MustNotError(t, err) + deviceChangeIDC, err := db.StoreKeyChange(ctx, "@charlie:localhost") + MustNotError(t, err) + userIDs, latest, err := db.KeyChanges(ctx, deviceChangeIDB, types.OffsetNewest) + if err != nil { + t.Fatalf("Failed to KeyChanges: %s", err) + } + if latest != deviceChangeIDC { + t.Fatalf("KeyChanges: got latest=%d want %d", latest, deviceChangeIDC) + } + if !reflect.DeepEqual(userIDs, []string{"@charlie:localhost"}) { + t.Fatalf("KeyChanges: wrong user_ids: %v", userIDs) + } + }) +} + +func TestKeyChangesNoDupes(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, clean := mustCreateKeyDatabase(t, dbType) + defer clean() + deviceChangeIDA, err := db.StoreKeyChange(ctx, "@alice:localhost") + MustNotError(t, err) + deviceChangeIDB, err := db.StoreKeyChange(ctx, "@alice:localhost") + MustNotError(t, err) + if deviceChangeIDA == deviceChangeIDB { + t.Fatalf("Expected change ID to be different even when inserting key change for the same user, got %d for both changes", deviceChangeIDA) + } + deviceChangeID, err := db.StoreKeyChange(ctx, "@alice:localhost") + MustNotError(t, err) + userIDs, latest, err := db.KeyChanges(ctx, 0, types.OffsetNewest) + if err != nil { + t.Fatalf("Failed to KeyChanges: %s", err) + } + if latest != deviceChangeID { + t.Fatalf("KeyChanges: got latest=%d want %d", latest, deviceChangeID) + } + if !reflect.DeepEqual(userIDs, []string{"@alice:localhost"}) { + t.Fatalf("KeyChanges: wrong user_ids: %v", userIDs) + } + }) +} + +func TestKeyChangesUpperLimit(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, clean := mustCreateKeyDatabase(t, dbType) + defer clean() + deviceChangeIDA, err := db.StoreKeyChange(ctx, "@alice:localhost") + MustNotError(t, err) + deviceChangeIDB, err := db.StoreKeyChange(ctx, "@bob:localhost") + MustNotError(t, err) + _, err = db.StoreKeyChange(ctx, "@charlie:localhost") + MustNotError(t, err) + userIDs, latest, err := db.KeyChanges(ctx, deviceChangeIDA, deviceChangeIDB) + if err != nil { + t.Fatalf("Failed to KeyChanges: %s", err) + } + if latest != deviceChangeIDB { + t.Fatalf("KeyChanges: got latest=%d want %d", latest, deviceChangeIDB) + } + if !reflect.DeepEqual(userIDs, []string{"@bob:localhost"}) { + t.Fatalf("KeyChanges: wrong user_ids: %v", userIDs) + } + }) +} + +var dbLock sync.Mutex +var deviceArray = []string{"AAA", "another_device"} + +// The purpose of this test is to make sure that the storage layer is generating sequential stream IDs per user, +// and that they are returned correctly when querying for device keys. +func TestDeviceKeysStreamIDGeneration(t *testing.T) { + var err error + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, clean := mustCreateKeyDatabase(t, dbType) + defer clean() + alice := "@alice:TestDeviceKeysStreamIDGeneration" + bob := "@bob:TestDeviceKeysStreamIDGeneration" + msgs := []api.DeviceMessage{ + { + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{ + DeviceID: "AAA", + UserID: alice, + KeyJSON: []byte(`{"key":"v1"}`), + }, + // StreamID: 1 + }, + { + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{ + DeviceID: "AAA", + UserID: bob, + KeyJSON: []byte(`{"key":"v1"}`), + }, + // StreamID: 1 as this is a different user + }, + { + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{ + DeviceID: "another_device", + UserID: alice, + KeyJSON: []byte(`{"key":"v1"}`), + }, + // StreamID: 2 as this is a 2nd device key + }, + } + MustNotError(t, db.StoreLocalDeviceKeys(ctx, msgs)) + if msgs[0].StreamID != 1 { + t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=1 but got %d", msgs[0].StreamID) + } + if msgs[1].StreamID != 1 { + t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=1 (different user) but got %d", msgs[1].StreamID) + } + if msgs[2].StreamID != 2 { + t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=2 (another device) but got %d", msgs[2].StreamID) + } + + // updating a device sets the next stream ID for that user + msgs = []api.DeviceMessage{ + { + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{ + DeviceID: "AAA", + UserID: alice, + KeyJSON: []byte(`{"key":"v2"}`), + }, + // StreamID: 3 + }, + } + MustNotError(t, db.StoreLocalDeviceKeys(ctx, msgs)) + if msgs[0].StreamID != 3 { + t.Fatalf("Expected StoreLocalDeviceKeys to set StreamID=3 (new key same device) but got %d", msgs[0].StreamID) + } + + dbLock.Lock() + defer dbLock.Unlock() + // Querying for device keys returns the latest stream IDs + msgs, err = db.DeviceKeysForUser(ctx, alice, deviceArray, false) + + if err != nil { + t.Fatalf("DeviceKeysForUser returned error: %s", err) + } + wantStreamIDs := map[string]int64{ + "AAA": 3, + "another_device": 2, + } + if len(msgs) != len(wantStreamIDs) { + t.Fatalf("DeviceKeysForUser: wrong number of devices, got %d want %d", len(msgs), len(wantStreamIDs)) + } + for _, m := range msgs { + if m.StreamID != wantStreamIDs[m.DeviceID] { + t.Errorf("DeviceKeysForUser: wrong returned stream ID for key, got %d want %d", m.StreamID, wantStreamIDs[m.DeviceID]) + } + } + }) +} diff --git a/userapi/storage/storage_wasm.go b/userapi/storage/storage_wasm.go index 5d5d292e6..163e3e173 100644 --- a/userapi/storage/storage_wasm.go +++ b/userapi/storage/storage_wasm.go @@ -32,10 +32,10 @@ func NewUserAPIDatabase( openIDTokenLifetimeMS int64, loginTokenLifetime time.Duration, serverNoticesLocalpart string, -) (Database, error) { +) (UserDatabase, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(base, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) + return sqlite3.NewUserDatabase(base, dbProperties, serverName, bcryptCost, openIDTokenLifetimeMS, loginTokenLifetime, serverNoticesLocalpart) case dbProperties.ConnectionString.IsPostgres(): return nil, fmt.Errorf("can't use Postgres implementation") default: diff --git a/userapi/storage/tables/interface.go b/userapi/storage/tables/interface.go index e14776cf3..693e73038 100644 --- a/userapi/storage/tables/interface.go +++ b/userapi/storage/tables/interface.go @@ -20,10 +20,10 @@ import ( "encoding/json" "time" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/types" ) @@ -44,6 +44,7 @@ type AccountsTable interface { type DevicesTable interface { InsertDevice(ctx context.Context, txn *sql.Tx, id, localpart string, serverName gomatrixserverlib.ServerName, accessToken string, displayName *string, ipAddr, userAgent string) (*api.Device, error) + InsertDeviceWithSessionID(ctx context.Context, txn *sql.Tx, id, localpart string, serverName gomatrixserverlib.ServerName, accessToken string, displayName *string, ipAddr, userAgent string, sessionID int64) (*api.Device, error) DeleteDevice(ctx context.Context, txn *sql.Tx, id, localpart string, serverName gomatrixserverlib.ServerName) error DeleteDevices(ctx context.Context, txn *sql.Tx, localpart string, serverName gomatrixserverlib.ServerName, devices []string) error DeleteDevicesByLocalpart(ctx context.Context, txn *sql.Tx, localpart string, serverName gomatrixserverlib.ServerName, exceptDeviceID string) error @@ -144,3 +145,47 @@ const ( // uint32. AllNotifications NotificationFilter = (1 << 31) - 1 ) + +type OneTimeKeys interface { + SelectOneTimeKeys(ctx context.Context, userID, deviceID string, keyIDsWithAlgorithms []string) (map[string]json.RawMessage, error) + CountOneTimeKeys(ctx context.Context, userID, deviceID string) (*api.OneTimeKeysCount, error) + InsertOneTimeKeys(ctx context.Context, txn *sql.Tx, keys api.OneTimeKeys) (*api.OneTimeKeysCount, error) + // SelectAndDeleteOneTimeKey selects a single one time key matching the user/device/algorithm specified and returns the algo:key_id => JSON. + // Returns an empty map if the key does not exist. + SelectAndDeleteOneTimeKey(ctx context.Context, txn *sql.Tx, userID, deviceID, algorithm string) (map[string]json.RawMessage, error) + DeleteOneTimeKeys(ctx context.Context, txn *sql.Tx, userID, deviceID string) error +} + +type DeviceKeys interface { + SelectDeviceKeysJSON(ctx context.Context, keys []api.DeviceMessage) error + InsertDeviceKeys(ctx context.Context, txn *sql.Tx, keys []api.DeviceMessage) error + SelectMaxStreamIDForUser(ctx context.Context, txn *sql.Tx, userID string) (streamID int64, err error) + CountStreamIDsForUser(ctx context.Context, userID string, streamIDs []int64) (int, error) + SelectBatchDeviceKeys(ctx context.Context, userID string, deviceIDs []string, includeEmpty bool) ([]api.DeviceMessage, error) + DeleteDeviceKeys(ctx context.Context, txn *sql.Tx, userID, deviceID string) error + DeleteAllDeviceKeys(ctx context.Context, txn *sql.Tx, userID string) error +} + +type KeyChanges interface { + InsertKeyChange(ctx context.Context, userID string) (int64, error) + // SelectKeyChanges returns the set (de-duplicated) of users who have changed their keys between the two offsets. + // Results are exclusive of fromOffset and inclusive of toOffset. A toOffset of types.OffsetNewest means no upper offset. + SelectKeyChanges(ctx context.Context, fromOffset, toOffset int64) (userIDs []string, latestOffset int64, err error) +} + +type StaleDeviceLists interface { + InsertStaleDeviceList(ctx context.Context, userID string, isStale bool) error + SelectUserIDsWithStaleDeviceLists(ctx context.Context, domains []gomatrixserverlib.ServerName) ([]string, error) + DeleteStaleDeviceLists(ctx context.Context, txn *sql.Tx, userIDs []string) error +} + +type CrossSigningKeys interface { + SelectCrossSigningKeysForUser(ctx context.Context, txn *sql.Tx, userID string) (r types.CrossSigningKeyMap, err error) + UpsertCrossSigningKeysForUser(ctx context.Context, txn *sql.Tx, userID string, keyType gomatrixserverlib.CrossSigningKeyPurpose, keyData gomatrixserverlib.Base64Bytes) error +} + +type CrossSigningSigs interface { + SelectCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID) (r types.CrossSigningSigMap, err error) + UpsertCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, originUserID string, originKeyID gomatrixserverlib.KeyID, targetUserID string, targetKeyID gomatrixserverlib.KeyID, signature gomatrixserverlib.Base64Bytes) error + DeleteCrossSigningSigsForTarget(ctx context.Context, txn *sql.Tx, targetUserID string, targetKeyID gomatrixserverlib.KeyID) error +} diff --git a/keyserver/storage/tables/stale_device_lists_test.go b/userapi/storage/tables/stale_device_lists_test.go similarity index 94% rename from keyserver/storage/tables/stale_device_lists_test.go rename to userapi/storage/tables/stale_device_lists_test.go index 76d3baddd..b9bdafdaa 100644 --- a/keyserver/storage/tables/stale_device_lists_test.go +++ b/userapi/storage/tables/stale_device_lists_test.go @@ -4,15 +4,15 @@ import ( "context" "testing" + "github.com/matrix-org/dendrite/userapi/storage/postgres" + "github.com/matrix-org/dendrite/userapi/storage/sqlite3" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/keyserver/storage/sqlite3" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/keyserver/storage/postgres" - "github.com/matrix-org/dendrite/keyserver/storage/tables" "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/userapi/storage/tables" ) func mustCreateTable(t *testing.T, dbType test.DBType) (tab tables.StaleDeviceLists, close func()) { diff --git a/keyserver/types/storage.go b/userapi/types/storage.go similarity index 100% rename from keyserver/types/storage.go rename to userapi/types/storage.go diff --git a/userapi/userapi.go b/userapi/userapi.go index 183ca3123..826bd7213 100644 --- a/userapi/userapi.go +++ b/userapi/userapi.go @@ -17,40 +17,34 @@ package userapi import ( "time" - "github.com/gorilla/mux" + fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" "github.com/sirupsen/logrus" - "github.com/matrix-org/dendrite/internal/pushgateway" - keyapi "github.com/matrix-org/dendrite/keyserver/api" rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/base" - "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/consumers" "github.com/matrix-org/dendrite/userapi/internal" - "github.com/matrix-org/dendrite/userapi/inthttp" "github.com/matrix-org/dendrite/userapi/producers" "github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/dendrite/userapi/util" ) -// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions -// on the given input API. -func AddInternalRoutes(router *mux.Router, intAPI api.UserInternalAPI, enableMetrics bool) { - inthttp.AddRoutes(router, intAPI, enableMetrics) -} - -// NewInternalAPI returns a concerete implementation of the internal API. Callers +// NewInternalAPI returns a concrete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *base.BaseDendrite, cfg *config.UserAPI, - appServices []config.ApplicationService, keyAPI keyapi.UserKeyAPI, - rsAPI rsapi.UserRoomserverAPI, pgClient pushgateway.Client, -) api.UserInternalAPI { + base *base.BaseDendrite, + rsAPI rsapi.UserRoomserverAPI, + fedClient fedsenderapi.KeyserverFederationAPI, +) *internal.UserInternalAPI { + cfg := &base.Cfg.UserAPI js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream) + appServices := base.Cfg.Derived.ApplicationServices - db, err := storage.NewUserAPIDatabase( + pgClient := base.PushGatewayHTTPClient() + + db, err := storage.NewUserDatabase( base, &cfg.AccountDatabase, cfg.Matrix.ServerName, @@ -63,6 +57,11 @@ func NewInternalAPI( logrus.WithError(err).Panicf("failed to connect to accounts db") } + keyDB, err := storage.NewKeyDatabase(base, &base.Cfg.KeyServer.Database) + if err != nil { + logrus.WithError(err).Panicf("failed to connect to key db") + } + syncProducer := producers.NewSyncAPI( db, js, // TODO: user API should handle syncs for account data. Right now, @@ -72,17 +71,50 @@ func NewInternalAPI( cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData), cfg.Matrix.JetStream.Prefixed(jetstream.OutputNotificationData), ) + keyChangeProducer := &producers.KeyChange{ + Topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputKeyChangeEvent), + JetStream: js, + DB: keyDB, + } userAPI := &internal.UserInternalAPI{ DB: db, + KeyDatabase: keyDB, SyncProducer: syncProducer, + KeyChangeProducer: keyChangeProducer, Config: cfg, AppServices: appServices, - KeyAPI: keyAPI, RSAPI: rsAPI, DisableTLSValidation: cfg.PushGatewayDisableTLSValidation, PgClient: pgClient, - Cfg: cfg, + FedClient: fedClient, + } + + updater := internal.NewDeviceListUpdater(base.ProcessContext, keyDB, userAPI, keyChangeProducer, fedClient, 8, rsAPI, cfg.Matrix.ServerName) // 8 workers TODO: configurable + userAPI.Updater = updater + // Remove users which we don't share a room with anymore + if err := updater.CleanUp(); err != nil { + logrus.WithError(err).Error("failed to cleanup stale device lists") + } + + go func() { + if err := updater.Start(); err != nil { + logrus.WithError(err).Panicf("failed to start device list updater") + } + }() + + dlConsumer := consumers.NewDeviceListUpdateConsumer( + base.ProcessContext, cfg, js, updater, + ) + if err := dlConsumer.Start(); err != nil { + logrus.WithError(err).Panic("failed to start device list consumer") + } + + sigConsumer := consumers.NewSigningKeyUpdateConsumer( + base.ProcessContext, cfg, js, userAPI, + ) + if err := sigConsumer.Start(); err != nil { + logrus.WithError(err).Panic("failed to start signing key consumer") } receiptConsumer := consumers.NewOutputReceiptEventConsumer( diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index dada56de4..01e491cb6 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -17,23 +17,22 @@ package userapi_test import ( "context" "fmt" - "net/http" "reflect" + "sync" "testing" "time" - "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/userapi/producers" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/nats-io/nats.go" "golang.org/x/crypto/bcrypt" - "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" - "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/internal" - "github.com/matrix-org/dendrite/userapi/inthttp" "github.com/matrix-org/dendrite/userapi/storage" ) @@ -43,32 +42,71 @@ const ( type apiTestOpts struct { loginTokenLifetime time.Duration + serverName string } -func MustMakeInternalAPI(t *testing.T, opts apiTestOpts, dbType test.DBType) (api.UserInternalAPI, storage.Database, func()) { +type dummyProducer struct { + callCount sync.Map + t *testing.T +} + +func (d *dummyProducer) PublishMsg(msg *nats.Msg, opts ...nats.PubOpt) (*nats.PubAck, error) { + count, loaded := d.callCount.LoadOrStore(msg.Subject, 1) + if loaded { + c, ok := count.(int) + if !ok { + d.t.Fatalf("unexpected type: %T with value %q", c, c) + } + d.callCount.Store(msg.Subject, c+1) + d.t.Logf("Incrementing call counter for %s", msg.Subject) + } + return &nats.PubAck{}, nil +} + +func MustMakeInternalAPI(t *testing.T, opts apiTestOpts, dbType test.DBType, publisher producers.JetStreamPublisher) (api.UserInternalAPI, storage.UserDatabase, func()) { if opts.loginTokenLifetime == 0 { opts.loginTokenLifetime = api.DefaultLoginTokenLifetime * time.Millisecond } base, baseclose := testrig.CreateBaseDendrite(t, dbType) connStr, close := test.PrepareDBConnectionString(t, dbType) - accountDB, err := storage.NewUserAPIDatabase(base, &config.DatabaseOptions{ + sName := serverName + if opts.serverName != "" { + sName = gomatrixserverlib.ServerName(opts.serverName) + } + accountDB, err := storage.NewUserDatabase(base, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), - }, serverName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "") + }, sName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "") if err != nil { t.Fatalf("failed to create account DB: %s", err) } + keyDB, err := storage.NewKeyDatabase(base, &config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }) + if err != nil { + t.Fatalf("failed to create key DB: %s", err) + } + cfg := &config.UserAPI{ Matrix: &config.Global{ SigningIdentity: gomatrixserverlib.SigningIdentity{ - ServerName: serverName, + ServerName: sName, }, }, } + if publisher == nil { + publisher = &dummyProducer{t: t} + } + + syncProducer := producers.NewSyncAPI(accountDB, publisher, "client_data", "notification_data") + keyChangeProducer := &producers.KeyChange{DB: keyDB, JetStream: publisher, Topic: "keychange"} return &internal.UserInternalAPI{ - DB: accountDB, - Config: cfg, + DB: accountDB, + KeyDatabase: keyDB, + Config: cfg, + SyncProducer: syncProducer, + KeyChangeProducer: keyChangeProducer, }, accountDB, func() { close() baseclose() @@ -129,7 +167,7 @@ func TestQueryProfile(t *testing.T) { } test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType) + userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType, nil) defer close() _, err := accountDB.CreateAccount(context.TODO(), "alice", serverName, "foobar", "", api.AccountTypeUser) if err != nil { @@ -142,20 +180,7 @@ func TestQueryProfile(t *testing.T) { t.Fatalf("failed to set display name: %s", err) } - t.Run("HTTP API", func(t *testing.T) { - router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter() - userapi.AddInternalRoutes(router, userAPI, false) - apiURL, cancel := test.ListenAndServe(t, router, false) - defer cancel() - httpAPI, err := inthttp.NewUserAPIClient(apiURL, &http.Client{}) - if err != nil { - t.Fatalf("failed to create HTTP client") - } - runCases(httpAPI, true) - }) - t.Run("Monolith", func(t *testing.T) { - runCases(userAPI, false) - }) + runCases(userAPI, false) }) } @@ -165,7 +190,7 @@ func TestQueryProfile(t *testing.T) { func TestPasswordlessLoginFails(t *testing.T) { ctx := context.Background() test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType) + userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType, nil) defer close() _, err := accountDB.CreateAccount(ctx, "auser", serverName, "", "", api.AccountTypeAppService) if err != nil { @@ -191,7 +216,7 @@ func TestLoginToken(t *testing.T) { t.Run("tokenLoginFlow", func(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType) + userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType, nil) defer close() _, err := accountDB.CreateAccount(ctx, "auser", serverName, "apassword", "", api.AccountTypeUser) if err != nil { @@ -241,7 +266,7 @@ func TestLoginToken(t *testing.T) { t.Run("expiredTokenIsNotReturned", func(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - userAPI, _, close := MustMakeInternalAPI(t, apiTestOpts{loginTokenLifetime: -1 * time.Second}, dbType) + userAPI, _, close := MustMakeInternalAPI(t, apiTestOpts{loginTokenLifetime: -1 * time.Second}, dbType, nil) defer close() creq := api.PerformLoginTokenCreationRequest{ @@ -266,7 +291,7 @@ func TestLoginToken(t *testing.T) { t.Run("deleteWorks", func(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - userAPI, _, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType) + userAPI, _, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType, nil) defer close() creq := api.PerformLoginTokenCreationRequest{ @@ -297,7 +322,7 @@ func TestLoginToken(t *testing.T) { t.Run("deleteUnknownIsNoOp", func(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - userAPI, _, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType) + userAPI, _, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType, nil) defer close() dreq := api.PerformLoginTokenDeletionRequest{Token: "non-existent token"} var dresp api.PerformLoginTokenDeletionResponse @@ -315,7 +340,7 @@ func TestQueryAccountByLocalpart(t *testing.T) { ctx := context.Background() test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - intAPI, db, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType) + intAPI, db, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType, nil) defer close() createdAcc, err := db.CreateAccount(ctx, localpart, userServername, "", "", alice.AccountType) @@ -347,24 +372,306 @@ func TestQueryAccountByLocalpart(t *testing.T) { } } - t.Run("Monolith", func(t *testing.T) { - testCases(t, intAPI) - // also test tracing - testCases(t, &api.UserInternalAPITrace{Impl: intAPI}) - }) + testCases(t, intAPI) + }) +} - t.Run("HTTP API", func(t *testing.T) { - router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter() - userapi.AddInternalRoutes(router, intAPI, false) - apiURL, cancel := test.ListenAndServe(t, router, false) - defer cancel() +func TestAccountData(t *testing.T) { + ctx := context.Background() + alice := test.NewUser(t) - userHTTPApi, err := inthttp.NewUserAPIClient(apiURL, &http.Client{Timeout: time.Second * 5}) - if err != nil { - t.Fatalf("failed to create HTTP client: %s", err) + testCases := []struct { + name string + inputData *api.InputAccountDataRequest + wantErr bool + }{ + { + name: "not a local user", + inputData: &api.InputAccountDataRequest{UserID: "@notlocal:example.com"}, + wantErr: true, + }, + { + name: "local user missing datatype", + inputData: &api.InputAccountDataRequest{UserID: alice.ID}, + wantErr: true, + }, + { + name: "missing json", + inputData: &api.InputAccountDataRequest{UserID: alice.ID, DataType: "m.push_rules", AccountData: nil}, + wantErr: true, + }, + { + name: "with json", + inputData: &api.InputAccountDataRequest{UserID: alice.ID, DataType: "m.push_rules", AccountData: []byte("{}")}, + }, + { + name: "room data", + inputData: &api.InputAccountDataRequest{UserID: alice.ID, DataType: "m.push_rules", AccountData: []byte("{}"), RoomID: "!dummy:test"}, + }, + { + name: "ignored users", + inputData: &api.InputAccountDataRequest{UserID: alice.ID, DataType: "m.ignored_user_list", AccountData: []byte("{}")}, + }, + { + name: "m.fully_read", + inputData: &api.InputAccountDataRequest{UserID: alice.ID, DataType: "m.fully_read", AccountData: []byte("{}")}, + }, + } + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + intAPI, _, close := MustMakeInternalAPI(t, apiTestOpts{serverName: "test"}, dbType, nil) + defer close() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + res := api.InputAccountDataResponse{} + err := intAPI.InputAccountData(ctx, tc.inputData, &res) + if tc.wantErr && err == nil { + t.Fatalf("expected an error, but got none") + } + if !tc.wantErr && err != nil { + t.Fatalf("expected no error, but got: %s", err) + } + + // query the data again and compare + queryRes := api.QueryAccountDataResponse{} + queryReq := api.QueryAccountDataRequest{ + UserID: tc.inputData.UserID, + DataType: tc.inputData.DataType, + RoomID: tc.inputData.RoomID, + } + err = intAPI.QueryAccountData(ctx, &queryReq, &queryRes) + if err != nil && !tc.wantErr { + t.Fatal(err) + } + // verify global data + if tc.inputData.RoomID == "" { + if !reflect.DeepEqual(tc.inputData.AccountData, queryRes.GlobalAccountData[tc.inputData.DataType]) { + t.Fatalf("expected accountdata to be %s, got %s", string(tc.inputData.AccountData), string(queryRes.GlobalAccountData[tc.inputData.DataType])) + } + } else { + // verify room data + if !reflect.DeepEqual(tc.inputData.AccountData, queryRes.RoomAccountData[tc.inputData.RoomID][tc.inputData.DataType]) { + t.Fatalf("expected accountdata to be %s, got %s", string(tc.inputData.AccountData), string(queryRes.RoomAccountData[tc.inputData.RoomID][tc.inputData.DataType])) + } + } + }) + } + }) +} + +func TestDevices(t *testing.T) { + ctx := context.Background() + + dupeAccessToken := util.RandomString(8) + + displayName := "testing" + + creationTests := []struct { + name string + inputData *api.PerformDeviceCreationRequest + wantErr bool + wantNewDevID bool + }{ + { + name: "not a local user", + inputData: &api.PerformDeviceCreationRequest{Localpart: "test1", ServerName: "notlocal"}, + wantErr: true, + }, + { + name: "implicit local user", + inputData: &api.PerformDeviceCreationRequest{Localpart: "test1", AccessToken: util.RandomString(8), NoDeviceListUpdate: true, DeviceDisplayName: &displayName}, + }, + { + name: "explicit local user", + inputData: &api.PerformDeviceCreationRequest{Localpart: "test2", ServerName: "test", AccessToken: util.RandomString(8), NoDeviceListUpdate: true}, + }, + { + name: "dupe token - ok", + inputData: &api.PerformDeviceCreationRequest{Localpart: "test3", ServerName: "test", AccessToken: dupeAccessToken, NoDeviceListUpdate: true}, + }, + { + name: "dupe token - not ok", + inputData: &api.PerformDeviceCreationRequest{Localpart: "test3", ServerName: "test", AccessToken: dupeAccessToken, NoDeviceListUpdate: true}, + wantErr: true, + }, + { + name: "test3 second device", // used to test deletion later + inputData: &api.PerformDeviceCreationRequest{Localpart: "test3", ServerName: "test", AccessToken: util.RandomString(8), NoDeviceListUpdate: true}, + }, + { + name: "test3 third device", // used to test deletion later + wantNewDevID: true, + inputData: &api.PerformDeviceCreationRequest{Localpart: "test3", ServerName: "test", AccessToken: util.RandomString(8), NoDeviceListUpdate: true}, + }, + } + + deletionTests := []struct { + name string + inputData *api.PerformDeviceDeletionRequest + wantErr bool + wantDevices int + }{ + { + name: "deletion - not a local user", + inputData: &api.PerformDeviceDeletionRequest{UserID: "@test:notlocalhost"}, + wantErr: true, + }, + { + name: "deleting not existing devices should not error", + inputData: &api.PerformDeviceDeletionRequest{UserID: "@test1:test", DeviceIDs: []string{"iDontExist"}}, + wantDevices: 1, + }, + { + name: "delete all devices", + inputData: &api.PerformDeviceDeletionRequest{UserID: "@test1:test"}, + wantDevices: 0, + }, + { + name: "delete all devices", + inputData: &api.PerformDeviceDeletionRequest{UserID: "@test3:test"}, + wantDevices: 0, + }, + } + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + intAPI, _, close := MustMakeInternalAPI(t, apiTestOpts{serverName: "test"}, dbType, nil) + defer close() + + for _, tc := range creationTests { + t.Run(tc.name, func(t *testing.T) { + res := api.PerformDeviceCreationResponse{} + deviceID := util.RandomString(8) + tc.inputData.DeviceID = &deviceID + if tc.wantNewDevID { + tc.inputData.DeviceID = nil + } + err := intAPI.PerformDeviceCreation(ctx, tc.inputData, &res) + if tc.wantErr && err == nil { + t.Fatalf("expected an error, but got none") + } + if !tc.wantErr && err != nil { + t.Fatalf("expected no error, but got: %s", err) + } + if !res.DeviceCreated { + return + } + + queryDevicesRes := api.QueryDevicesResponse{} + queryDevicesReq := api.QueryDevicesRequest{UserID: res.Device.UserID} + if err = intAPI.QueryDevices(ctx, &queryDevicesReq, &queryDevicesRes); err != nil { + t.Fatal(err) + } + // We only want to verify one device + if len(queryDevicesRes.Devices) > 1 { + return + } + res.Device.AccessToken = "" + + // At this point, there should only be one device + if !reflect.DeepEqual(*res.Device, queryDevicesRes.Devices[0]) { + t.Fatalf("expected device to be\n%#v, got \n%#v", *res.Device, queryDevicesRes.Devices[0]) + } + + newDisplayName := "new name" + if tc.inputData.DeviceDisplayName == nil { + updateRes := api.PerformDeviceUpdateResponse{} + updateReq := api.PerformDeviceUpdateRequest{ + RequestingUserID: fmt.Sprintf("@%s:%s", tc.inputData.Localpart, "test"), + DeviceID: deviceID, + DisplayName: &newDisplayName, + } + + if err = intAPI.PerformDeviceUpdate(ctx, &updateReq, &updateRes); err != nil { + t.Fatal(err) + } + } + + queryDeviceInfosRes := api.QueryDeviceInfosResponse{} + queryDeviceInfosReq := api.QueryDeviceInfosRequest{DeviceIDs: []string{*tc.inputData.DeviceID}} + if err = intAPI.QueryDeviceInfos(ctx, &queryDeviceInfosReq, &queryDeviceInfosRes); err != nil { + t.Fatal(err) + } + gotDisplayName := queryDeviceInfosRes.DeviceInfo[*tc.inputData.DeviceID].DisplayName + if tc.inputData.DeviceDisplayName != nil { + wantDisplayName := *tc.inputData.DeviceDisplayName + if wantDisplayName != gotDisplayName { + t.Fatalf("expected displayName to be %s, got %s", wantDisplayName, gotDisplayName) + } + } else { + wantDisplayName := newDisplayName + if wantDisplayName != gotDisplayName { + t.Fatalf("expected displayName to be %s, got %s", wantDisplayName, gotDisplayName) + } + } + }) + } + + for _, tc := range deletionTests { + t.Run(tc.name, func(t *testing.T) { + delRes := api.PerformDeviceDeletionResponse{} + err := intAPI.PerformDeviceDeletion(ctx, tc.inputData, &delRes) + if tc.wantErr && err == nil { + t.Fatalf("expected an error, but got none") + } + if !tc.wantErr && err != nil { + t.Fatalf("expected no error, but got: %s", err) + } + if tc.wantErr { + return + } + + queryDevicesRes := api.QueryDevicesResponse{} + queryDevicesReq := api.QueryDevicesRequest{UserID: tc.inputData.UserID} + if err = intAPI.QueryDevices(ctx, &queryDevicesReq, &queryDevicesRes); err != nil { + t.Fatal(err) + } + + if len(queryDevicesRes.Devices) != tc.wantDevices { + t.Fatalf("expected %d devices, got %d", tc.wantDevices, len(queryDevicesRes.Devices)) + } + + }) + } + }) +} + +// Tests that the session ID of a device is not reused when reusing the same device ID. +func TestDeviceIDReuse(t *testing.T) { + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + publisher := &dummyProducer{t: t} + intAPI, _, close := MustMakeInternalAPI(t, apiTestOpts{serverName: "test"}, dbType, publisher) + defer close() + + res := api.PerformDeviceCreationResponse{} + // create a first device + deviceID := util.RandomString(8) + req := api.PerformDeviceCreationRequest{Localpart: "alice", ServerName: "test", DeviceID: &deviceID, NoDeviceListUpdate: true} + err := intAPI.PerformDeviceCreation(ctx, &req, &res) + if err != nil { + t.Fatal(err) + } + + // Do the same request again, we expect a different sessionID + res2 := api.PerformDeviceCreationResponse{} + // Set NoDeviceListUpdate to false, to verify we don't send device list updates when + // reusing the same device ID + req.NoDeviceListUpdate = false + err = intAPI.PerformDeviceCreation(ctx, &req, &res2) + if err != nil { + t.Fatalf("expected no error, but got: %v", err) + } + + if res2.Device.SessionID == res.Device.SessionID { + t.Fatalf("expected a different session ID, but they are the same") + } + + publisher.callCount.Range(func(key, value any) bool { + if value != nil { + t.Fatalf("expected publisher to not get called, but got value %d for subject %s", value, key) } - testCases(t, userHTTPApi) - + return true }) }) } diff --git a/userapi/util/devices.go b/userapi/util/devices.go index c55fc7999..31617d8c1 100644 --- a/userapi/util/devices.go +++ b/userapi/util/devices.go @@ -19,7 +19,7 @@ type PusherDevice struct { } // GetPushDevices pushes to the configured devices of a local user. -func GetPushDevices(ctx context.Context, localpart string, serverName gomatrixserverlib.ServerName, tweaks map[string]interface{}, db storage.Database) ([]*PusherDevice, error) { +func GetPushDevices(ctx context.Context, localpart string, serverName gomatrixserverlib.ServerName, tweaks map[string]interface{}, db storage.UserDatabase) ([]*PusherDevice, error) { pushers, err := db.GetPushers(ctx, localpart, serverName) if err != nil { return nil, fmt.Errorf("db.GetPushers: %w", err) diff --git a/userapi/util/notify.go b/userapi/util/notify.go index fc0ab39bf..08d1371d6 100644 --- a/userapi/util/notify.go +++ b/userapi/util/notify.go @@ -13,11 +13,11 @@ import ( ) // NotifyUserCountsAsync sends notifications to a local user's -// notification destinations. Database lookups run synchronously, but +// notification destinations. UserDatabase lookups run synchronously, but // a single goroutine is started when talking to the Push // gateways. There is no way to know when the background goroutine has // finished. -func NotifyUserCountsAsync(ctx context.Context, pgClient pushgateway.Client, localpart string, serverName gomatrixserverlib.ServerName, db storage.Database) error { +func NotifyUserCountsAsync(ctx context.Context, pgClient pushgateway.Client, localpart string, serverName gomatrixserverlib.ServerName, db storage.UserDatabase) error { pusherDevices, err := GetPushDevices(ctx, localpart, serverName, nil, db) if err != nil { return err diff --git a/userapi/util/notify_test.go b/userapi/util/notify_test.go index f1d20259c..421852d3f 100644 --- a/userapi/util/notify_test.go +++ b/userapi/util/notify_test.go @@ -79,7 +79,7 @@ func TestNotifyUserCountsAsync(t *testing.T) { defer close() base, _, _ := testrig.Base(nil) defer base.Close() - db, err := storage.NewUserAPIDatabase(base, &config.DatabaseOptions{ + db, err := storage.NewUserDatabase(base, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, "test", bcrypt.MinCost, 0, 0, "") if err != nil { diff --git a/userapi/util/phonehomestats.go b/userapi/util/phonehomestats.go index 42c8f5d7c..21035e045 100644 --- a/userapi/util/phonehomestats.go +++ b/userapi/util/phonehomestats.go @@ -55,7 +55,7 @@ func StartPhoneHomeCollector(startTime time.Time, cfg *config.Dendrite, statsDB serverName: cfg.Global.ServerName, cfg: cfg, db: statsDB, - isMonolith: cfg.IsMonolith, + isMonolith: true, client: &http.Client{ Timeout: time.Second * 30, Transport: http.DefaultTransport, diff --git a/userapi/util/phonehomestats_test.go b/userapi/util/phonehomestats_test.go index 6e62210e8..5f626b5bc 100644 --- a/userapi/util/phonehomestats_test.go +++ b/userapi/util/phonehomestats_test.go @@ -21,7 +21,7 @@ func TestCollect(t *testing.T) { b, _, _ := testrig.Base(nil) connStr, closeDB := test.PrepareDBConnectionString(t, dbType) defer closeDB() - db, err := storage.NewUserAPIDatabase(b, &config.DatabaseOptions{ + db, err := storage.NewUserDatabase(b, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, "localhost", bcrypt.MinCost, 1000, 1000, "") if err != nil {