From a9d3bdc058fa27a68865677592904031058974b1 Mon Sep 17 00:00:00 2001 From: Tak Wai Wong <64229756+tak-hntlabs@users.noreply.github.com> Date: Wed, 2 Nov 2022 16:28:06 -0700 Subject: [PATCH] subtree pull from dendrite fork for fix #2383 (#802) Pull the fix for https://github.com/matrix-org/dendrite/issues/2838 into the dendrite subtree. Includes latest dendrite main. Signed-off-by: `Devon Hudson ` Signed-off-by: dependabot[bot] Signed-off-by: Brian Meek Signed-off-by: Austin Ellis Signed-off-by: `Rubin Poster ` Signed-off-by: `ash lea ` Co-authored-by: Neil Alexander Co-authored-by: devonh Co-authored-by: Till <2353100+S7evinK@users.noreply.github.com> Co-authored-by: Ashley Nelson Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Brian Meek Co-authored-by: Tak Wai Wong Co-authored-by: Jean Lucas Co-authored-by: Kabir Kwatra Co-authored-by: sergekh2 Co-authored-by: texuf Co-authored-by: kegsay Co-authored-by: Brian Meek Co-authored-by: John Terzis Co-authored-by: John Terzis Co-authored-by: Kerem Co-authored-by: Neboer <43609792+Neboer@users.noreply.github.com> Co-authored-by: X. Ding Co-authored-by: dxl Co-authored-by: ash lea Co-authored-by: 0x1a8510f2 Co-authored-by: Till Faelligen Co-authored-by: Till Faelligen --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/dendrite.yml | 30 ++- .github/workflows/docker.yml | 184 +++++++++++++---- CHANGES.md | 14 ++ Dockerfile | 122 +++++++++++ appservice/api/query.go | 75 ++++++- appservice/appservice.go | 7 +- appservice/inthttp/client.go | 24 +++ appservice/inthttp/server.go | 16 ++ appservice/query/query.go | 190 +++++++++++++++++- .../dendritejs-pinecone/jsServer.go | 0 {cmd => build}/dendritejs-pinecone/main.go | 0 .../dendritejs-pinecone/main_noop.go | 0 .../dendritejs-pinecone/main_test.go | 0 build/docker/Dockerfile.demo-pinecone | 25 --- build/docker/Dockerfile.polylith | 25 --- build/docker/README.md | 15 +- build/docker/images-build.sh | 6 +- clientapi/routing/routing.go | 48 ++++- clientapi/routing/thirdparty.go | 106 ++++++++++ cmd/dendrite-demo-pinecone/README.md | 26 +++ cmd/dendrite-demo-pinecone/main.go | 2 - cmd/dendrite-demo-yggdrasil/main.go | 2 - cmd/dendrite-monolith-server/main.go | 2 - cmd/dendrite-polylith-multi/main.go | 2 - go.mod | 47 +++-- go.sum | 101 ++++++---- internal/sqlutil/migrate_test.go | 6 +- internal/sqlutil/sqlite_cgo.go | 19 ++ internal/sqlutil/sqlite_native.go | 29 +++ internal/sqlutil/sqlutil.go | 5 +- internal/sqlutil/trace_driver.go | 3 +- internal/sqlutil/unique_constraint.go | 13 +- internal/sqlutil/unique_constraint_cgo.go | 36 ++++ internal/sqlutil/unique_constraint_wasm.go | 11 +- internal/version.go | 2 +- roomserver/internal/helpers/helpers.go | 11 +- roomserver/internal/input/input_missing.go | 48 +++-- roomserver/internal/query/query.go | 30 ++- roomserver/state/state.go | 108 ++++------ roomserver/storage/interface.go | 1 + roomserver/storage/postgres/events_table.go | 31 ++- roomserver/storage/shared/room_updater.go | 7 +- roomserver/storage/shared/storage.go | 23 +++ .../20221027084407_published_appservice.go | 4 +- roomserver/storage/sqlite3/events_table.go | 42 +++- roomserver/storage/tables/interface.go | 1 + syncapi/routing/context.go | 1 - syncapi/routing/memberships.go | 2 +- syncapi/routing/search.go | 2 +- .../postgres/current_room_state_table.go | 4 +- .../postgres/output_room_events_table.go | 9 +- .../sqlite3/current_room_state_table.go | 3 +- syncapi/storage/sqlite3/filtering.go | 6 +- .../sqlite3/output_room_events_table.go | 2 +- syncapi/streams/stream_invite.go | 52 ++--- syncapi/streams/stream_pdu.go | 22 +- syncapi/sync/request.go | 1 - test/db.go | 2 +- userapi/consumers/roomserver.go | 78 +++++++ userapi/consumers/roomserver_test.go | 123 ++++++++++++ userapi/storage/interface.go | 4 + userapi/storage/postgres/stats_table.go | 73 ++++++- userapi/storage/shared/storage.go | 15 +- userapi/storage/sqlite3/stats_table.go | 68 +++++++ userapi/storage/tables/interface.go | 4 + userapi/storage/tables/stats_table_test.go | 9 +- userapi/types/statistics.go | 7 + userapi/util/phonehomestats.go | 22 +- 69 files changed, 1639 insertions(+), 371 deletions(-) create mode 100644 Dockerfile rename {cmd => build}/dendritejs-pinecone/jsServer.go (100%) rename {cmd => build}/dendritejs-pinecone/main.go (100%) rename {cmd => build}/dendritejs-pinecone/main_noop.go (100%) rename {cmd => build}/dendritejs-pinecone/main_test.go (100%) delete mode 100644 build/docker/Dockerfile.demo-pinecone delete mode 100644 build/docker/Dockerfile.polylith create mode 100644 clientapi/routing/thirdparty.go create mode 100644 cmd/dendrite-demo-pinecone/README.md create mode 100644 internal/sqlutil/sqlite_cgo.go create mode 100644 internal/sqlutil/sqlite_native.go create mode 100644 internal/sqlutil/unique_constraint_cgo.go diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d6c54ead1..bbbdd82a4 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,7 +2,7 @@ -* [ ] I have added tests for PR _or_ I have justified why this PR doesn't need tests. +* [ ] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests * [ ] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed-off-by: `Your Name ` diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index fef351de2..d6e3d7ffd 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -269,11 +269,18 @@ jobs: fail-fast: false matrix: include: - - label: SQLite + - label: SQLite native - - label: SQLite, full HTTP APIs + - label: SQLite Cgo + cgo: 1 + + - label: SQLite native, full HTTP APIs api: full-http + - label: SQLite Cgo, full HTTP APIs + api: full-http + cgo: 1 + - label: PostgreSQL postgres: postgres @@ -288,8 +295,9 @@ jobs: POSTGRES: ${{ matrix.postgres && 1}} API: ${{ matrix.api && 1 }} SYTEST_BRANCH: ${{ github.head_ref }} + CGO_ENABLED: ${{ matrix.cgo && 1 }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run Sytest run: /bootstrap.sh dendrite working-directory: /src @@ -323,11 +331,18 @@ jobs: fail-fast: false matrix: include: - - label: SQLite + - label: SQLite native - - label: SQLite, full HTTP APIs + - label: SQLite Cgo + cgo: 1 + + - label: SQLite native, full HTTP APIs api: full-http + - label: SQLite Cgo, full HTTP APIs + api: full-http + cgo: 1 + - label: PostgreSQL postgres: Postgres @@ -349,8 +364,8 @@ jobs: sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest - - name: Run actions/checkout@v2 for dendrite - uses: actions/checkout@v2 + - name: Run actions/checkout@v3 for dendrite + uses: actions/checkout@v3 with: path: dendrite @@ -393,6 +408,7 @@ jobs: env: COMPLEMENT_BASE_IMAGE: complement-dendrite:latest API: ${{ matrix.api && 1 }} + CGO_ENABLED: ${{ matrix.cgo && 1 }} working-directory: complement integration-tests-done: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4fa932f92..0cc025862 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -26,21 +26,26 @@ jobs: packages: write steps: - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag + uses: actions/checkout@v3 + - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -49,27 +54,41 @@ jobs: - name: Build main monolith image if: github.ref_name == 'main' id: docker_build_monolith - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.monolith + build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} + target: monolith platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} + format: "sarif" + output: "trivy-results.sarif" + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: "trivy-results.sarif" + - name: Build release monolith image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_monolith_release - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.monolith + build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} + target: monolith platforms: ${{ env.PLATFORMS }} push: true build-args: | @@ -88,21 +107,26 @@ jobs: packages: write steps: - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag + uses: actions/checkout@v3 + - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -111,27 +135,41 @@ jobs: - name: Build main polylith image if: github.ref_name == 'main' id: docker_build_polylith - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.polylith + build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} + target: polylith platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} + format: "sarif" + output: "trivy-results.sarif" + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: "trivy-results.sarif" + - name: Build release polylith image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_polylith_release - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.polylith + build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} + target: polylith platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -148,54 +186,128 @@ jobs: packages: write steps: - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag + uses: actions/checkout@v3 + - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build main pinecone demo image + - name: Build main Pinecone demo image if: github.ref_name == 'main' id: docker_build_demo_pinecone - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.demo-pinecone + build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} + target: demo-pinecone platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }} - - name: Build release pinecone demo image + - name: Build release Pinecone demo image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_demo_pinecone_release - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.demo-pinecone + build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} + target: demo-pinecone platforms: ${{ env.PLATFORMS }} push: true tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:latest - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ env.RELEASE_VERSION }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:latest - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ env.RELEASE_VERSION }} + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} + + demo-yggdrasil: + name: Yggdrasil demo image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Get release tag & build flags + if: github.event_name == 'release' # Only for GitHub releases + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ env.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + - name: Login to GitHub Containers + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build main Yggdrasil demo image + if: github.ref_name == 'main' + id: docker_build_demo_yggdrasil + uses: docker/build-push-action@v3 + with: + cache-from: type=gha + cache-to: type=gha,mode=max + context: . + build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} + target: demo-yggdrasil + platforms: ${{ env.PLATFORMS }} + push: true + tags: | + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }} + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }} + + - name: Build release Yggdrasil demo image + if: github.event_name == 'release' # Only for GitHub releases + id: docker_build_demo_yggdrasil_release + uses: docker/build-push-action@v3 + with: + cache-from: type=gha + cache-to: type=gha,mode=max + context: . + build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} + target: demo-yggdrasil + platforms: ${{ env.PLATFORMS }} + push: true + tags: | + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} diff --git a/CHANGES.md b/CHANGES.md index ba14dd07a..55df36f96 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,19 @@ # Changelog +## Dendrite 0.10.6 (2022-11-01) + +### Features + +* History visibility checks have been optimised, which should speed up response times on a variety of endpoints (including `/sync`, `/messages`, `/context` and others) and reduce database load +* The built-in NATS Server has been updated to version 2.9.4 +* Some other minor dependencies have been updated + +### Fixes + +* A panic has been fixed in the sync API PDU stream which could cause requests to fail +* The `/members` response now contains the `room_id` field, which may fix some E2EE problems with clients using the JS SDK (contributed by [ashkitten](https://github.com/ashkitten)) +* The auth difference calculation in state resolution v2 has been tweaked for clarity (and moved into gomatrixserverlib with the rest of the state resolution code) + ## Dendrite 0.10.5 (2022-10-31) ### Features diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..007d985b6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,122 @@ +#syntax=docker/dockerfile:1.2 + +# +# base installs required dependencies and runs go mod download to cache dependencies +# +FROM --platform=${BUILDPLATFORM} docker.io/golang:1.19-alpine AS base +RUN apk --update --no-cache add bash build-base curl + +# +# build creates all needed binaries +# +FROM --platform=${BUILDPLATFORM} base AS build +WORKDIR /src +ARG TARGETOS +ARG TARGETARCH +ARG FLAGS +RUN --mount=target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + USERARCH=`go env GOARCH` \ + GOARCH="$TARGETARCH" \ + GOOS="linux" \ + CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0") \ + go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/... + +# +# The dendrite base image; mainly creates a user and switches to it +# +FROM alpine:latest AS dendrite-base +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" +LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/" +LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C." +RUN addgroup dendrite && adduser dendrite -G dendrite -u 1337 -D +USER dendrite +WORKDIR /home/dendrite + +# +# Builds the polylith image and only contains the polylith binary +# +FROM dendrite-base AS polylith +LABEL org.opencontainers.image.title="Dendrite (Polylith)" + +COPY --from=build /out/dendrite-polylith-multi /usr/bin/ + +ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] + +# +# Builds the monolith image and contains all required binaries +# +FROM dendrite-base AS monolith +LABEL org.opencontainers.image.title="Dendrite (Monolith)" + +COPY --from=build /out/create-account /usr/bin/create-account +COPY --from=build /out/generate-config /usr/bin/generate-config +COPY --from=build /out/generate-keys /usr/bin/generate-keys +COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server + +ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] +EXPOSE 8008 8448 + +# +# Builds the Pinecone P2P demo image and contains all required binaries +# +FROM dendrite-base AS demo-pinecone +LABEL org.opencontainers.image.title="Dendrite (Pinecone P2P Demo)" + +COPY --from=build /out/create-account /usr/bin/create-account +COPY --from=build /out/generate-config /usr/bin/generate-config +COPY --from=build /out/generate-keys /usr/bin/generate-keys +COPY --from=build /out/dendrite-demo-pinecone /usr/bin/dendrite-demo-pinecone + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"] +EXPOSE 8008 8448 + +# +# Builds the Yggdrasil P2P demo image and contains all required binaries +# +FROM dendrite-base AS demo-yggdrasil +LABEL org.opencontainers.image.title="Dendrite (Yggdrasil P2P Demo)" + +COPY --from=build /out/create-account /usr/bin/create-account +COPY --from=build /out/generate-config /usr/bin/generate-config +COPY --from=build /out/generate-keys /usr/bin/generate-keys +COPY --from=build /out/dendrite-demo-yggdrasil /usr/bin/dendrite-demo-yggdrasil + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-demo-yggdrasil"] +EXPOSE 8008 8448 + +# +# Builds the Complement image, used for integration tests +# +FROM base AS complement +LABEL org.opencontainers.image.title="Dendrite (Complement)" +RUN apk add --no-cache sqlite openssl ca-certificates + +COPY --from=build /out/generate-config /usr/bin/generate-config +COPY --from=build /out/generate-keys /usr/bin/generate-keys +COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server + +WORKDIR /dendrite +RUN /usr/bin/generate-keys --private-key matrix_key.pem && \ + mkdir /ca && \ + openssl genrsa -out /ca/ca.key 2048 && \ + openssl req -new -x509 -key /ca/ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out /ca/ca.crt + +ENV SERVER_NAME=localhost +ENV API=0 +EXPOSE 8008 8448 + +# At runtime, generate TLS cert based on the CA now mounted at /ca +# At runtime, replace the SERVER_NAME with what we are told +CMD /usr/bin/generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /ca/ca.crt --tls-authority-key /ca/ca.key && \ + /usr/bin/generate-config -server $SERVER_NAME --ci > dendrite.yaml && \ + cp /ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ + /usr/bin/dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} \ No newline at end of file diff --git a/appservice/api/query.go b/appservice/api/query.go index 4d1cf9474..eb567b2ee 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -19,11 +19,13 @@ package api import ( "context" + "encoding/json" "errors" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" ) // AppServiceInternalAPI is used to query user and room alias data from application @@ -41,6 +43,10 @@ type AppServiceInternalAPI interface { req *UserIDExistsRequest, resp *UserIDExistsResponse, ) error + + Locations(ctx context.Context, req *LocationRequest, resp *LocationResponse) error + User(ctx context.Context, request *UserRequest, response *UserResponse) error + Protocols(ctx context.Context, req *ProtocolRequest, resp *ProtocolResponse) error } // RoomAliasExistsRequest is a request to an application service @@ -77,6 +83,73 @@ type UserIDExistsResponse struct { UserIDExists bool `json:"exists"` } +const ( + ASProtocolPath = "/_matrix/app/unstable/thirdparty/protocol/" + ASUserPath = "/_matrix/app/unstable/thirdparty/user" + ASLocationPath = "/_matrix/app/unstable/thirdparty/location" +) + +type ProtocolRequest struct { + Protocol string `json:"protocol,omitempty"` +} + +type ProtocolResponse struct { + Protocols map[string]ASProtocolResponse `json:"protocols"` + Exists bool `json:"exists"` +} + +type ASProtocolResponse struct { + FieldTypes map[string]FieldType `json:"field_types,omitempty"` // NOTSPEC: field_types is required by the spec + Icon string `json:"icon"` + Instances []ProtocolInstance `json:"instances"` + LocationFields []string `json:"location_fields"` + UserFields []string `json:"user_fields"` +} + +type FieldType struct { + Placeholder string `json:"placeholder"` + Regexp string `json:"regexp"` +} + +type ProtocolInstance struct { + Description string `json:"desc"` + Icon string `json:"icon,omitempty"` + NetworkID string `json:"network_id,omitempty"` // NOTSPEC: network_id is required by the spec + Fields json.RawMessage `json:"fields,omitempty"` // NOTSPEC: fields is required by the spec +} + +type UserRequest struct { + Protocol string `json:"protocol"` + Params string `json:"params"` +} + +type UserResponse struct { + Users []ASUserResponse `json:"users,omitempty"` + Exists bool `json:"exists,omitempty"` +} + +type ASUserResponse struct { + Protocol string `json:"protocol"` + UserID string `json:"userid"` + Fields json.RawMessage `json:"fields"` +} + +type LocationRequest struct { + Protocol string `json:"protocol"` + Params string `json:"params"` +} + +type LocationResponse struct { + Locations []ASLocationResponse `json:"locations,omitempty"` + Exists bool `json:"exists,omitempty"` +} + +type ASLocationResponse struct { + Alias string `json:"alias"` + Protocol string `json:"protocol"` + Fields json.RawMessage `json:"fields"` +} + // RetrieveUserProfile is a wrapper that queries both the local database and // application services for a given user's profile // TODO: Remove this, it's called from federationapi and clientapi but is a pure function diff --git a/appservice/appservice.go b/appservice/appservice.go index 9000adb1d..0c778b6ca 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -18,6 +18,7 @@ import ( "context" "crypto/tls" "net/http" + "sync" "time" "github.com/gorilla/mux" @@ -58,8 +59,10 @@ func NewInternalAPI( // Create appserivce query API with an HTTP client that will be used for all // outbound and inbound requests (inbound only for the internal API) appserviceQueryAPI := &query.AppServiceQueryAPI{ - HTTPClient: client, - Cfg: &base.Cfg.AppServiceAPI, + HTTPClient: client, + Cfg: &base.Cfg.AppServiceAPI, + ProtocolCache: map[string]appserviceAPI.ASProtocolResponse{}, + CacheMu: sync.Mutex{}, } if len(base.Cfg.Derived.ApplicationServices) == 0 { diff --git a/appservice/inthttp/client.go b/appservice/inthttp/client.go index 3ae2c9278..f7f164877 100644 --- a/appservice/inthttp/client.go +++ b/appservice/inthttp/client.go @@ -13,6 +13,9 @@ import ( const ( AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists" AppServiceUserIDExistsPath = "/appservice/UserIDExists" + AppServiceLocationsPath = "/appservice/locations" + AppServiceUserPath = "/appservice/users" + AppServiceProtocolsPath = "/appservice/protocols" ) // httpAppServiceQueryAPI contains the URL to an appservice query API and a @@ -58,3 +61,24 @@ func (h *httpAppServiceQueryAPI) UserIDExists( h.httpClient, ctx, request, response, ) } + +func (h *httpAppServiceQueryAPI) Locations(ctx context.Context, request *api.LocationRequest, response *api.LocationResponse) error { + return httputil.CallInternalRPCAPI( + "ASLocation", h.appserviceURL+AppServiceLocationsPath, + h.httpClient, ctx, request, response, + ) +} + +func (h *httpAppServiceQueryAPI) User(ctx context.Context, request *api.UserRequest, response *api.UserResponse) error { + return httputil.CallInternalRPCAPI( + "ASUser", h.appserviceURL+AppServiceUserPath, + h.httpClient, ctx, request, response, + ) +} + +func (h *httpAppServiceQueryAPI) Protocols(ctx context.Context, request *api.ProtocolRequest, response *api.ProtocolResponse) error { + return httputil.CallInternalRPCAPI( + "ASProtocols", h.appserviceURL+AppServiceProtocolsPath, + h.httpClient, ctx, request, response, + ) +} diff --git a/appservice/inthttp/server.go b/appservice/inthttp/server.go index 01d9f9895..ccf5c83d8 100644 --- a/appservice/inthttp/server.go +++ b/appservice/inthttp/server.go @@ -2,6 +2,7 @@ package inthttp import ( "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/internal/httputil" ) @@ -17,4 +18,19 @@ func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) { AppServiceUserIDExistsPath, httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists), ) + + internalAPIMux.Handle( + AppServiceProtocolsPath, + httputil.MakeInternalRPCAPI("AppserviceProtocols", a.Protocols), + ) + + internalAPIMux.Handle( + AppServiceLocationsPath, + httputil.MakeInternalRPCAPI("AppserviceLocations", a.Locations), + ) + + internalAPIMux.Handle( + AppServiceUserPath, + httputil.MakeInternalRPCAPI("AppserviceUser", a.User), + ) } diff --git a/appservice/query/query.go b/appservice/query/query.go index 53b34cb18..2348eab4b 100644 --- a/appservice/query/query.go +++ b/appservice/query/query.go @@ -18,13 +18,18 @@ package query import ( "context" + "encoding/json" + "io" "net/http" "net/url" + "strings" + "sync" + + "github.com/opentracing/opentracing-go" + log "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/setup/config" - opentracing "github.com/opentracing/opentracing-go" - log "github.com/sirupsen/logrus" ) const roomAliasExistsPath = "/rooms/" @@ -32,8 +37,10 @@ const userIDExistsPath = "/users/" // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI type AppServiceQueryAPI struct { - HTTPClient *http.Client - Cfg *config.AppServiceAPI + HTTPClient *http.Client + Cfg *config.AppServiceAPI + ProtocolCache map[string]api.ASProtocolResponse + CacheMu sync.Mutex } // RoomAliasExists performs a request to '/room/{roomAlias}' on all known @@ -165,3 +172,178 @@ func (a *AppServiceQueryAPI) UserIDExists( response.UserIDExists = false return nil } + +type thirdpartyResponses interface { + api.ASProtocolResponse | []api.ASUserResponse | []api.ASLocationResponse +} + +func requestDo[T thirdpartyResponses](client *http.Client, url string, response *T) (err error) { + origURL := url + // try v1 and unstable appservice endpoints + for _, version := range []string{"v1", "unstable"} { + var resp *http.Response + var body []byte + asURL := strings.Replace(origURL, "unstable", version, 1) + resp, err = client.Get(asURL) + if err != nil { + continue + } + defer resp.Body.Close() // nolint: errcheck + body, err = io.ReadAll(resp.Body) + if err != nil { + continue + } + return json.Unmarshal(body, &response) + } + return err +} + +func (a *AppServiceQueryAPI) Locations( + ctx context.Context, + req *api.LocationRequest, + resp *api.LocationResponse, +) error { + params, err := url.ParseQuery(req.Params) + if err != nil { + return err + } + + for _, as := range a.Cfg.Derived.ApplicationServices { + var asLocations []api.ASLocationResponse + params.Set("access_token", as.HSToken) + + url := as.URL + api.ASLocationPath + if req.Protocol != "" { + url += "/" + req.Protocol + } + + if err := requestDo[[]api.ASLocationResponse](a.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil { + log.WithError(err).Error("unable to get 'locations' from application service") + continue + } + + resp.Locations = append(resp.Locations, asLocations...) + } + + if len(resp.Locations) == 0 { + resp.Exists = false + return nil + } + resp.Exists = true + return nil +} + +func (a *AppServiceQueryAPI) User( + ctx context.Context, + req *api.UserRequest, + resp *api.UserResponse, +) error { + params, err := url.ParseQuery(req.Params) + if err != nil { + return err + } + + for _, as := range a.Cfg.Derived.ApplicationServices { + var asUsers []api.ASUserResponse + params.Set("access_token", as.HSToken) + + url := as.URL + api.ASUserPath + if req.Protocol != "" { + url += "/" + req.Protocol + } + + if err := requestDo[[]api.ASUserResponse](a.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil { + log.WithError(err).Error("unable to get 'user' from application service") + continue + } + + resp.Users = append(resp.Users, asUsers...) + } + + if len(resp.Users) == 0 { + resp.Exists = false + return nil + } + resp.Exists = true + return nil +} + +func (a *AppServiceQueryAPI) Protocols( + ctx context.Context, + req *api.ProtocolRequest, + resp *api.ProtocolResponse, +) error { + + // get a single protocol response + if req.Protocol != "" { + + a.CacheMu.Lock() + defer a.CacheMu.Unlock() + if proto, ok := a.ProtocolCache[req.Protocol]; ok { + resp.Exists = true + resp.Protocols = map[string]api.ASProtocolResponse{ + req.Protocol: proto, + } + return nil + } + + response := api.ASProtocolResponse{} + for _, as := range a.Cfg.Derived.ApplicationServices { + var proto api.ASProtocolResponse + if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+req.Protocol, &proto); err != nil { + log.WithError(err).Error("unable to get 'protocol' from application service") + continue + } + + if len(response.Instances) != 0 { + response.Instances = append(response.Instances, proto.Instances...) + } else { + response = proto + } + } + + if len(response.Instances) == 0 { + resp.Exists = false + return nil + } + + resp.Exists = true + resp.Protocols = map[string]api.ASProtocolResponse{ + req.Protocol: response, + } + a.ProtocolCache[req.Protocol] = response + return nil + } + + response := make(map[string]api.ASProtocolResponse, len(a.Cfg.Derived.ApplicationServices)) + + for _, as := range a.Cfg.Derived.ApplicationServices { + for _, p := range as.Protocols { + var proto api.ASProtocolResponse + if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+p, &proto); err != nil { + log.WithError(err).Error("unable to get 'protocol' from application service") + continue + } + existing, ok := response[p] + if !ok { + response[p] = proto + continue + } + existing.Instances = append(existing.Instances, proto.Instances...) + response[p] = existing + } + } + + if len(response) == 0 { + resp.Exists = false + return nil + } + + a.CacheMu.Lock() + defer a.CacheMu.Unlock() + a.ProtocolCache = response + + resp.Exists = true + resp.Protocols = response + return nil +} diff --git a/cmd/dendritejs-pinecone/jsServer.go b/build/dendritejs-pinecone/jsServer.go similarity index 100% rename from cmd/dendritejs-pinecone/jsServer.go rename to build/dendritejs-pinecone/jsServer.go diff --git a/cmd/dendritejs-pinecone/main.go b/build/dendritejs-pinecone/main.go similarity index 100% rename from cmd/dendritejs-pinecone/main.go rename to build/dendritejs-pinecone/main.go diff --git a/cmd/dendritejs-pinecone/main_noop.go b/build/dendritejs-pinecone/main_noop.go similarity index 100% rename from cmd/dendritejs-pinecone/main_noop.go rename to build/dendritejs-pinecone/main_noop.go diff --git a/cmd/dendritejs-pinecone/main_test.go b/build/dendritejs-pinecone/main_test.go similarity index 100% rename from cmd/dendritejs-pinecone/main_test.go rename to build/dendritejs-pinecone/main_test.go diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone deleted file mode 100644 index 133c63c53..000000000 --- a/build/docker/Dockerfile.demo-pinecone +++ /dev/null @@ -1,25 +0,0 @@ -FROM docker.io/golang:1.19-alpine AS base - -RUN apk --update --no-cache add bash build-base - -WORKDIR /build - -COPY . /build - -RUN mkdir -p bin -RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-pinecone -RUN go build -trimpath -o bin/ ./cmd/create-account -RUN go build -trimpath -o bin/ ./cmd/generate-keys - -FROM alpine:latest -LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)" -LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" -LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" -LABEL org.opencontainers.image.licenses="Apache-2.0" - -COPY --from=base /build/bin/* /usr/bin/ - -VOLUME /etc/dendrite -WORKDIR /etc/dendrite - -ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"] diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith deleted file mode 100644 index 79f8a5f23..000000000 --- a/build/docker/Dockerfile.polylith +++ /dev/null @@ -1,25 +0,0 @@ -FROM docker.io/golang:1.19-alpine AS base - -RUN apk --update --no-cache add bash build-base - -WORKDIR /build - -COPY . /build - -RUN mkdir -p bin -RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi -RUN go build -trimpath -o bin/ ./cmd/create-account -RUN go build -trimpath -o bin/ ./cmd/generate-keys - -FROM alpine:latest -LABEL org.opencontainers.image.title="Dendrite (Polylith)" -LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" -LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" -LABEL org.opencontainers.image.licenses="Apache-2.0" - -COPY --from=base /build/bin/* /usr/bin/ - -VOLUME /etc/dendrite -WORKDIR /etc/dendrite - -ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] diff --git a/build/docker/README.md b/build/docker/README.md index 261519fde..6111b8305 100644 --- a/build/docker/README.md +++ b/build/docker/README.md @@ -9,11 +9,16 @@ They can be found on Docker Hub: ## Dockerfiles -The `Dockerfile` builds the base image which contains all of the Dendrite -components. The `Dockerfile.component` file takes the given component, as -specified with `--buildarg component=` from the base image and produce -smaller component-specific images, which are substantially smaller and do -not contain the Go toolchain etc. +The `Dockerfile` is a multistage file which can build all four Dendrite +images depending on the supplied `--target`. From the root of the Dendrite +repository, run: + +``` +docker build . --target monolith -t matrixdotorg/dendrite-monolith +docker build . --target polylith -t matrixdotorg/dendrite-monolith +docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone +docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil +``` ## Compose files diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index c2c140685..d97a701ed 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -6,5 +6,7 @@ TAG=${1:-latest} echo "Building tag '${TAG}'" -docker build -t matrixdotorg/dendrite-monolith:${TAG} -f build/docker/Dockerfile.monolith . -docker build -t matrixdotorg/dendrite-polylith:${TAG} -f build/docker/Dockerfile.polylith . \ No newline at end of file +docker build . --target monolith -t matrixdotorg/dendrite-monolith:${TAG} +docker build . --target polylith -t matrixdotorg/dendrite-monolith:${TAG} +docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone:${TAG} +docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil:${TAG} \ No newline at end of file diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index db294a07a..4746f73c8 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -937,12 +937,50 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) v3mux.Handle("/thirdparty/protocols", - httputil.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse { - // TODO: Return the third party protcols - return util.JSONResponse{ - Code: http.StatusOK, - JSON: struct{}{}, + httputil.MakeAuthAPI("thirdparty_protocols", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return Protocols(req, asAPI, device, "") + }), + ).Methods(http.MethodGet, http.MethodOptions) + + v3mux.Handle("/thirdparty/protocol/{protocolID}", + httputil.MakeAuthAPI("thirdparty_protocols", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) } + return Protocols(req, asAPI, device, vars["protocolID"]) + }), + ).Methods(http.MethodGet, http.MethodOptions) + + v3mux.Handle("/thirdparty/user/{protocolID}", + httputil.MakeAuthAPI("thirdparty_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return User(req, asAPI, device, vars["protocolID"], req.URL.Query()) + }), + ).Methods(http.MethodGet, http.MethodOptions) + + v3mux.Handle("/thirdparty/user", + httputil.MakeAuthAPI("thirdparty_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return User(req, asAPI, device, "", req.URL.Query()) + }), + ).Methods(http.MethodGet, http.MethodOptions) + + v3mux.Handle("/thirdparty/location/{protocolID}", + httputil.MakeAuthAPI("thirdparty_location", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return Location(req, asAPI, device, vars["protocolID"], req.URL.Query()) + }), + ).Methods(http.MethodGet, http.MethodOptions) + + v3mux.Handle("/thirdparty/location", + httputil.MakeAuthAPI("thirdparty_location", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return Location(req, asAPI, device, "", req.URL.Query()) }), ).Methods(http.MethodGet, http.MethodOptions) diff --git a/clientapi/routing/thirdparty.go b/clientapi/routing/thirdparty.go new file mode 100644 index 000000000..e757cd411 --- /dev/null +++ b/clientapi/routing/thirdparty.go @@ -0,0 +1,106 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + "net/url" + + "github.com/matrix-org/util" + + appserviceAPI "github.com/matrix-org/dendrite/appservice/api" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/userapi/api" +) + +// Protocols implements +// +// GET /_matrix/client/v3/thirdparty/protocols/{protocol} +// GET /_matrix/client/v3/thirdparty/protocols +func Protocols(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, device *api.Device, protocol string) util.JSONResponse { + resp := &appserviceAPI.ProtocolResponse{} + + if err := asAPI.Protocols(req.Context(), &appserviceAPI.ProtocolRequest{Protocol: protocol}, resp); err != nil { + return jsonerror.InternalServerError() + } + if !resp.Exists { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The protocol is unknown."), + } + } + if protocol != "" { + return util.JSONResponse{ + Code: http.StatusOK, + JSON: resp.Protocols[protocol], + } + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: resp.Protocols, + } +} + +// User implements +// +// GET /_matrix/client/v3/thirdparty/user +// GET /_matrix/client/v3/thirdparty/user/{protocol} +func User(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, device *api.Device, protocol string, params url.Values) util.JSONResponse { + resp := &appserviceAPI.UserResponse{} + + params.Del("access_token") + if err := asAPI.User(req.Context(), &appserviceAPI.UserRequest{ + Protocol: protocol, + Params: params.Encode(), + }, resp); err != nil { + return jsonerror.InternalServerError() + } + if !resp.Exists { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The Matrix User ID was not found"), + } + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: resp.Users, + } +} + +// Location implements +// +// GET /_matrix/client/v3/thirdparty/location +// GET /_matrix/client/v3/thirdparty/location/{protocol} +func Location(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, device *api.Device, protocol string, params url.Values) util.JSONResponse { + resp := &appserviceAPI.LocationResponse{} + + params.Del("access_token") + if err := asAPI.Locations(req.Context(), &appserviceAPI.LocationRequest{ + Protocol: protocol, + Params: params.Encode(), + }, resp); err != nil { + return jsonerror.InternalServerError() + } + if !resp.Exists { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("No portal rooms were found."), + } + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: resp.Locations, + } +} diff --git a/cmd/dendrite-demo-pinecone/README.md b/cmd/dendrite-demo-pinecone/README.md new file mode 100644 index 000000000..d6dd95905 --- /dev/null +++ b/cmd/dendrite-demo-pinecone/README.md @@ -0,0 +1,26 @@ +# Pinecone Demo + +This is the Dendrite Pinecone demo! It's easy to get started. + +To run the homeserver, start at the root of the Dendrite repository and run: + +``` +go run ./cmd/dendrite-demo-pinecone +``` + +To connect to the static Pinecone peer used by the mobile demos run: + +``` +go run ./cmd/dendrite-demo-pinecone -peer wss://pinecone.matrix.org/public +``` + +The following command line arguments are accepted: + +* `-peer tcp://a.b.c.d:e` to specify a static Pinecone peer to connect to - you will need to supply this if you do not have another Pinecone node on your network +* `-port 12345` to specify a port to listen on for client connections + +Then point your favourite Matrix client to the homeserver URL`http://localhost:8008` (or whichever `-port` you specified), create an account and log in. + +If your peering connection is operational then you should see a `Connected TCP:` line in the log output. If not then try a different peer. + +Once logged in, you should be able to open the room directory or join a room by its ID. diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index be34365b4..6c719a1ee 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -54,8 +54,6 @@ import ( pineconeSessions "github.com/matrix-org/pinecone/sessions" "github.com/sirupsen/logrus" - - _ "github.com/mattn/go-sqlite3" ) var ( diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 38c25cdec..1226496c3 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -48,8 +48,6 @@ import ( "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/userapi" "github.com/sirupsen/logrus" - - _ "github.com/mattn/go-sqlite3" ) var ( diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 845b9e465..ff980dc1c 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -30,8 +30,6 @@ import ( "github.com/matrix-org/dendrite/userapi" uapi "github.com/matrix-org/dendrite/userapi/api" "github.com/sirupsen/logrus" - - _ "github.com/mattn/go-sqlite3" ) var ( diff --git a/cmd/dendrite-polylith-multi/main.go b/cmd/dendrite-polylith-multi/main.go index e4845f649..c6a560b19 100644 --- a/cmd/dendrite-polylith-multi/main.go +++ b/cmd/dendrite-polylith-multi/main.go @@ -24,8 +24,6 @@ import ( "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" "github.com/sirupsen/logrus" - - _ "github.com/mattn/go-sqlite3" ) type entrypoint func(base *base.BaseDendrite, cfg *config.Dendrite) diff --git a/go.mod b/go.mod index e31af9bf1..c90e3b99d 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/blevesearch/bleve/v2 v2.3.4 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 github.com/codeclysm/extract v2.2.0+incompatible - github.com/dgraph-io/ristretto v0.1.1 + github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d github.com/docker/docker v20.10.19+incompatible github.com/docker/go-connections v0.4.0 github.com/ethereum/go-ethereum v1.10.25 @@ -25,12 +25,12 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20221031151122-0885c35ebe74 + github.com/matrix-org/gomatrixserverlib v0.0.0-20221101165746-0e4a8bb6db7e github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 - github.com/nats-io/nats-server/v2 v2.9.3 - github.com/nats-io/nats.go v1.18.0 + github.com/nats-io/nats-server/v2 v2.9.4 + github.com/nats-io/nats.go v1.19.0 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79 @@ -40,21 +40,22 @@ require ( github.com/prometheus/client_golang v1.13.0 github.com/sirupsen/logrus v1.9.0 github.com/spruceid/siwe-go v0.2.0 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.1 github.com/tidwall/gjson v1.14.3 github.com/tidwall/sjson v1.2.5 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible github.com/yggdrasil-network/yggdrasil-go v0.4.6 go.uber.org/atomic v1.10.0 - golang.org/x/crypto v0.0.0-20221012134737-56aed061732a - golang.org/x/image v0.0.0-20220902085622-e7cb96979f69 - golang.org/x/mobile v0.0.0-20221012134814-c746ac228303 - golang.org/x/net v0.0.0-20221014081412-f15817d10f9b - golang.org/x/term v0.0.0-20220919170432-7a66f970e087 + golang.org/x/crypto v0.1.0 + golang.org/x/image v0.1.0 + golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e + golang.org/x/net v0.1.0 + golang.org/x/term v0.1.0 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 gotest.tools/v3 v3.4.0 + modernc.org/sqlite v1.19.3 nhooyr.io/websocket v1.8.7 ) @@ -102,12 +103,14 @@ require ( github.com/h2non/filetype v1.1.3 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/juju/errors v1.0.0 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.15.11 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/lucas-clemente/quic-go v0.29.2 // indirect github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect + github.com/mattn/go-isatty v0.0.16 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/minio/highwayhash v1.0.2 // indirect github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -127,6 +130,7 @@ require ( github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/relvacode/iso8601 v1.1.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa // indirect github.com/rjeczalik/notify v0.9.1 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/tidwall/match v1.1.1 // indirect @@ -134,17 +138,26 @@ require ( github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect go.etcd.io/bbolt v1.3.6 // indirect - golang.org/x/exp v0.0.0-20221012211006-4de253d81b95 // indirect - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 // indirect - golang.org/x/text v0.3.8 // indirect - golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect - golang.org/x/tools v0.1.12 // indirect + golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 // indirect + golang.org/x/mod v0.6.0 // indirect + golang.org/x/sys v0.1.0 // indirect + golang.org/x/text v0.4.0 // indirect + golang.org/x/time v0.1.0 // indirect + golang.org/x/tools v0.2.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/macaroon.v2 v2.1.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + lukechampine.com/uint128 v1.2.0 // indirect + modernc.org/cc/v3 v3.40.0 // indirect + modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8 // indirect + modernc.org/libc v1.21.4 // indirect + modernc.org/mathutil v1.5.0 // indirect + modernc.org/memory v1.4.0 // indirect + modernc.org/opt v0.1.3 // indirect + modernc.org/strutil v1.1.3 // indirect + modernc.org/token v1.0.1 // indirect ) go 1.19 diff --git a/go.sum b/go.sum index 1d922a2fc..d8961843c 100644 --- a/go.sum +++ b/go.sum @@ -175,8 +175,8 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= -github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= -github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d h1:Wrc3UKTS+cffkOx0xRGFC+ZesNuTfn0ThvEC72N0krk= +github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d/go.mod h1:RAy2GVV4sTWVlNMavv3xhLsk18rxhfhDnombTe6EF5c= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= @@ -381,6 +381,8 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5lLdR0= github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= @@ -423,8 +425,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20221031151122-0885c35ebe74 h1:I4LUlFqxZ72m3s9wIvUIV2FpprsxW28dO/0lAgepCZY= -github.com/matrix-org/gomatrixserverlib v0.0.0-20221031151122-0885c35ebe74/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221101165746-0e4a8bb6db7e h1:6I34fdyiHMRCxL6GOb/G8ZyI1WWlb6ZxCF2hIGSMSCc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221101165746-0e4a8bb6db7e/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6 h1:nAT5w41Q9uWTSnpKW55/hBwP91j2IFYPDRs0jJ8TyFI= github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= @@ -432,13 +434,14 @@ github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/em github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattomatic/dijkstra v0.0.0-20130617153013-6f6d134eb237/go.mod h1:UOnLAUmVG5paym8pD3C4B9BQylUDC2vXFJJpT7JrlEA= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= -github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= @@ -465,10 +468,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI= github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= -github.com/nats-io/nats-server/v2 v2.9.3 h1:HrfzA7G9LNetKkm1z+jU/e9kuAe+E6uaBuuq9EB5sQQ= -github.com/nats-io/nats-server/v2 v2.9.3/go.mod h1:4sq8wvrpbvSzL1n3ZfEYnH4qeUuIl5W990j3kw13rRk= -github.com/nats-io/nats.go v1.18.0 h1:o480Ao6kuSSFyJO75rGTXCEPj7LGkY84C1Ye+Uhm4c0= -github.com/nats-io/nats.go v1.18.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= +github.com/nats-io/nats-server/v2 v2.9.4 h1:GvRgv1936J/zYUwMg/cqtYaJ6L+bgeIOIvPslbesdow= +github.com/nats-io/nats-server/v2 v2.9.4/go.mod h1:AB6hAnGZDlYfqb7CTAm66ZKMZy9DpfierY1/PbpvI2g= +github.com/nats-io/nats.go v1.19.0 h1:H6j8aBnTQFoVrTGB6Xjd903UMdE7jz6DS4YkmAqgZ9Q= +github.com/nats-io/nats.go v1.19.0/go.mod h1:tLqubohF7t4z3du1QDPYJIQQyhb4wl6DhjxEajSI7UA= github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= @@ -552,6 +555,9 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/relvacode/iso8601 v1.1.0 h1:2nV8sp0eOjpoKQ2vD3xSDygsjAx37NHG2UlZiCkDH4I= github.com/relvacode/iso8601 v1.1.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa h1:tEkEyxYeZ43TR55QU/hsIt9aRGBxbgGuz9CGykjvogY= +github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= @@ -608,8 +614,9 @@ github.com/spruceid/siwe-go v0.2.0/go.mod h1:rvV+8/z/ryBKqdw9RcexFgtcsrDlESOGR38 github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -617,8 +624,9 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -694,8 +702,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20221012134737-56aed061732a h1:NmSIgad6KjE6VvHciPZuNRTKxGhlPfD6OA87W/PLkqg= -golang.org/x/crypto v0.0.0-20221012134737-56aed061732a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -710,13 +718,13 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20221012211006-4de253d81b95 h1:sBdrWpxhGDdTAYNqbgBLAR+ULAPPhfgncLr1X0lyWtg= -golang.org/x/exp v0.0.0-20221012211006-4de253d81b95/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE= +golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20220902085622-e7cb96979f69 h1:Lj6HJGCSn5AjxRAH2+r35Mir4icalbqku+CLUtjnvXY= -golang.org/x/image v0.0.0-20220902085622-e7cb96979f69/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY= +golang.org/x/image v0.1.0 h1:r8Oj8ZA2Xy12/b5KZYj3tuv7NG/fBz3TwQVvpJ9l8Rk= +golang.org/x/image v0.1.0/go.mod h1:iyPr49SD/G/TBxYVB/9RRtGUT5eNbo2u4NamWeQcD5c= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -731,8 +739,8 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= -golang.org/x/mobile v0.0.0-20221012134814-c746ac228303 h1:K4fp1rDuJBz0FCPAWzIJwnzwNEM7S6yobdZzMrZ/Zws= -golang.org/x/mobile v0.0.0-20221012134814-c746ac228303/go.mod h1:M32cGdzp91A8Ex9qQtyZinr19EYxzkFqDjW2oyHzTDQ= +golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e h1:zSgtO19fpg781xknwqiQPmOHaASr6E7ZVlTseLd9Fx4= +golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e/go.mod h1:aAjjkJNdrh3PMckS4B10TGS2nag27cbKR1y2BpUxsiY= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -740,8 +748,9 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -787,8 +796,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU= -golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -811,7 +820,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220907140024-f12130a52804 h1:0SH2R3f1b1VmIMG7BXbEZCBUu2dKmHschSmjqGUrW8A= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -881,14 +889,14 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220730100132-1609e554cd39/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 h1:OK7RB6t2WQX54srQQYSXMW8dF5C6/8+oA/s5QBmmto4= -golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220919170432-7a66f970e087 h1:tPwmk4vmvVCMdr98VgL4JH+qZxPL8fqlUOHnyOM8N3w= -golang.org/x/term v0.0.0-20220919170432-7a66f970e087/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -896,14 +904,14 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -955,8 +963,9 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1100,6 +1109,30 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= +modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= +modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8 h1:0+dsXf0zeLx9ixj4nilg6jKe5Bg1ilzBwSFq4kJmIUc= +modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g= +modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= +modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= +modernc.org/libc v1.21.4 h1:CzTlumWeIbPV5/HVIMzYHNPCRP8uiU/CWiN2gtd/Qu8= +modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.4.0 h1:crykUfNSnMAXaOJnnxcSzbUGMqkLWjklJKkBK2nwZwk= +modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.19.3 h1:dIoagx6yIQT3V/zOSeAyZ8OqQyEr17YTgETOXTZNJMA= +modernc.org/sqlite v1.19.3/go.mod h1:xiyJD7FY8mTZXnQwE/gEL1STtFrrnDx03V8KhVQmcr8= +modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.15.0 h1:oY+JeD11qVVSgVvodMJsu7Edf8tr5E/7tuhF5cNYz34= +modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg= +modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/internal/sqlutil/migrate_test.go b/internal/sqlutil/migrate_test.go index 5116237a1..30aa6790c 100644 --- a/internal/sqlutil/migrate_test.go +++ b/internal/sqlutil/migrate_test.go @@ -7,8 +7,6 @@ import ( "reflect" "testing" - _ "github.com/mattn/go-sqlite3" - "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/test" ) @@ -88,7 +86,7 @@ func Test_migrations_Up(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - driverName := "sqlite3" + driverName := sqlutil.SQLITE_DRIVER_NAME if dbType == test.DBTypePostgres { driverName = "postgres" } @@ -117,7 +115,7 @@ func Test_insertMigration(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { conStr, close := test.PrepareDBConnectionString(t, dbType) defer close() - driverName := "sqlite3" + driverName := sqlutil.SQLITE_DRIVER_NAME if dbType == test.DBTypePostgres { driverName = "postgres" } diff --git a/internal/sqlutil/sqlite_cgo.go b/internal/sqlutil/sqlite_cgo.go new file mode 100644 index 000000000..efb743fc7 --- /dev/null +++ b/internal/sqlutil/sqlite_cgo.go @@ -0,0 +1,19 @@ +//go:build cgo +// +build cgo + +package sqlutil + +import ( + "github.com/mattn/go-sqlite3" + _ "github.com/mattn/go-sqlite3" +) + +const SQLITE_DRIVER_NAME = "sqlite3" + +func sqliteDSNExtension(dsn string) string { + return dsn +} + +func sqliteDriver() *sqlite3.SQLiteDriver { + return &sqlite3.SQLiteDriver{} +} diff --git a/internal/sqlutil/sqlite_native.go b/internal/sqlutil/sqlite_native.go new file mode 100644 index 000000000..ed500afc6 --- /dev/null +++ b/internal/sqlutil/sqlite_native.go @@ -0,0 +1,29 @@ +//go:build !cgo +// +build !cgo + +package sqlutil + +import ( + "modernc.org/sqlite" + "strings" +) + +const SQLITE_DRIVER_NAME = "sqlite" + +func sqliteDSNExtension(dsn string) string { + // add query parameters to the dsn + if strings.Contains(dsn, "?") { + dsn += "&" + } else { + dsn += "?" + } + + // wait some time before erroring if the db is locked + // https://gitlab.com/cznic/sqlite/-/issues/106#note_1058094993 + dsn += "_pragma=busy_timeout%3d10000" + return dsn +} + +func sqliteDriver() *sqlite.Driver { + return &sqlite.Driver{} +} diff --git a/internal/sqlutil/sqlutil.go b/internal/sqlutil/sqlutil.go index 789bceeac..39a067e52 100644 --- a/internal/sqlutil/sqlutil.go +++ b/internal/sqlutil/sqlutil.go @@ -20,11 +20,12 @@ func Open(dbProperties *config.DatabaseOptions, writer Writer) (*sql.DB, error) var driverName, dsn string switch { case dbProperties.ConnectionString.IsSQLite(): - driverName = "sqlite3" + driverName = SQLITE_DRIVER_NAME dsn, err = ParseFileURI(dbProperties.ConnectionString) if err != nil { return nil, fmt.Errorf("ParseFileURI: %w", err) } + dsn = sqliteDSNExtension(dsn) case dbProperties.ConnectionString.IsPostgres(): driverName = "postgres" dsn = string(dbProperties.ConnectionString) @@ -39,7 +40,7 @@ func Open(dbProperties *config.DatabaseOptions, writer Writer) (*sql.DB, error) if err != nil { return nil, err } - if driverName != "sqlite3" { + if driverName != SQLITE_DRIVER_NAME { logger := logrus.WithFields(logrus.Fields{ "max_open_conns": dbProperties.MaxOpenConns(), "max_idle_conns": dbProperties.MaxIdleConns(), diff --git a/internal/sqlutil/trace_driver.go b/internal/sqlutil/trace_driver.go index b7bb36764..a2e0d12e2 100644 --- a/internal/sqlutil/trace_driver.go +++ b/internal/sqlutil/trace_driver.go @@ -21,7 +21,6 @@ import ( "database/sql" "github.com/lib/pq" - sqlite "github.com/mattn/go-sqlite3" "github.com/ngrok/sqlmw" ) @@ -31,6 +30,6 @@ func registerDrivers() { } // install the wrapped drivers sql.Register("postgres-trace", sqlmw.Driver(&pq.Driver{}, new(traceInterceptor))) - sql.Register("sqlite3-trace", sqlmw.Driver(&sqlite.SQLiteDriver{}, new(traceInterceptor))) + sql.Register("sqlite3-trace", sqlmw.Driver(sqliteDriver(), new(traceInterceptor))) } diff --git a/internal/sqlutil/unique_constraint.go b/internal/sqlutil/unique_constraint.go index ed70f5ed2..767a586ec 100644 --- a/internal/sqlutil/unique_constraint.go +++ b/internal/sqlutil/unique_constraint.go @@ -12,14 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !wasm -// +build !wasm +//go:build !wasm && !cgo +// +build !wasm,!cgo package sqlutil import ( "github.com/lib/pq" - "github.com/mattn/go-sqlite3" + "modernc.org/sqlite" + lib "modernc.org/sqlite/lib" ) // IsUniqueConstraintViolationErr returns true if the error is an unique_violation error @@ -27,10 +28,8 @@ func IsUniqueConstraintViolationErr(err error) bool { switch e := err.(type) { case *pq.Error: return e.Code == "23505" - case *sqlite3.Error: - return e.Code == sqlite3.ErrConstraint - case sqlite3.Error: - return e.Code == sqlite3.ErrConstraint + case *sqlite.Error: + return e.Code() == lib.SQLITE_CONSTRAINT } return false } diff --git a/internal/sqlutil/unique_constraint_cgo.go b/internal/sqlutil/unique_constraint_cgo.go new file mode 100644 index 000000000..edeb7c450 --- /dev/null +++ b/internal/sqlutil/unique_constraint_cgo.go @@ -0,0 +1,36 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !wasm && cgo +// +build !wasm,cgo + +package sqlutil + +import ( + "github.com/lib/pq" + "github.com/mattn/go-sqlite3" +) + +// IsUniqueConstraintViolationErr returns true if the error is an unique_violation error +func IsUniqueConstraintViolationErr(err error) bool { + switch e := err.(type) { + case *pq.Error: + return e.Code == "23505" + case *sqlite3.Error: + return e.Code == sqlite3.ErrConstraint + case sqlite3.Error: + return e.Code == sqlite3.ErrConstraint + } + return false +} diff --git a/internal/sqlutil/unique_constraint_wasm.go b/internal/sqlutil/unique_constraint_wasm.go index 02ceb5851..ef393fa59 100644 --- a/internal/sqlutil/unique_constraint_wasm.go +++ b/internal/sqlutil/unique_constraint_wasm.go @@ -17,15 +17,16 @@ package sqlutil -import "github.com/mattn/go-sqlite3" +import ( + "modernc.org/sqlite" + lib "modernc.org/sqlite/lib" +) // IsUniqueConstraintViolationErr returns true if the error is an unique_violation error func IsUniqueConstraintViolationErr(err error) bool { switch e := err.(type) { - case *sqlite3.Error: - return e.Code == sqlite3.ErrConstraint - case sqlite3.Error: - return e.Code == sqlite3.ErrConstraint + case *sqlite.Error: + return e.Code() == lib.SQLITE_CONSTRAINT } return false } diff --git a/internal/version.go b/internal/version.go index 7254ab102..f762adf90 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 10 - VersionPatch = 5 + VersionPatch = 6 VersionTag = "" // example: "rc1" ) diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go index a6de8ac84..7efad7af6 100644 --- a/roomserver/internal/helpers/helpers.go +++ b/roomserver/internal/helpers/helpers.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "sort" "strings" "github.com/matrix-org/gomatrixserverlib" @@ -159,7 +160,7 @@ func GetMembershipsAtState( ctx context.Context, db storage.Database, stateEntries []types.StateEntry, joinedOnly bool, ) ([]types.Event, error) { - var eventNIDs []types.EventNID + var eventNIDs types.EventNIDs for _, entry := range stateEntries { // Filter the events to retrieve to only keep the membership events if entry.EventTypeNID == types.MRoomMemberNID { @@ -167,6 +168,14 @@ func GetMembershipsAtState( } } + // There are no events to get, don't bother asking the database + if len(eventNIDs) == 0 { + return []types.Event{}, nil + } + + sort.Sort(eventNIDs) + util.Unique(eventNIDs) + // Get all of the events in this state stateEvents, err := db.Events(ctx, eventNIDs) if err != nil { diff --git a/roomserver/internal/input/input_missing.go b/roomserver/internal/input/input_missing.go index d789c3a14..f788c5b3a 100644 --- a/roomserver/internal/input/input_missing.go +++ b/roomserver/internal/input/input_missing.go @@ -40,6 +40,7 @@ func (p *parsedRespState) Events() []*gomatrixserverlib.Event { } type missingStateReq struct { + log *logrus.Entry origin gomatrixserverlib.ServerName db storage.Database roomInfo *types.RoomInfo @@ -77,7 +78,7 @@ func (t *missingStateReq) processEventWithMissingState( // event ids and then use /event to fetch the individual events. // However not all version of synapse support /state_ids so you may // need to fallback to /state. - logger := util.GetLogger(ctx).WithFields(map[string]interface{}{ + t.log = util.GetLogger(ctx).WithFields(map[string]interface{}{ "txn_event": e.EventID(), "room_id": e.RoomID(), "txn_prev_events": e.PrevEventIDs(), @@ -96,7 +97,7 @@ func (t *missingStateReq) processEventWithMissingState( return nil, fmt.Errorf("expected to find missing events but didn't") } if isGapFilled { - logger.Infof("Gap filled by /get_missing_events, injecting %d new events", len(newEvents)) + t.log.Infof("Gap filled by /get_missing_events, injecting %d new events", len(newEvents)) // we can just inject all the newEvents as new as we may have only missed 1 or 2 events and have filled // in the gap in the DAG for _, newEvent := range newEvents { @@ -118,7 +119,7 @@ func (t *missingStateReq) processEventWithMissingState( // then there's nothing else to do, we have everything we need to deal // with the new event. if isGapFilled && prevStatesKnown { - logger.Infof("Gap filled and state found for all prev events") + t.log.Infof("Gap filled and state found for all prev events") return nil, nil } @@ -321,7 +322,7 @@ func (t *missingStateReq) lookupStateAfterEvent(ctx context.Context, roomVersion defer span.Finish() // try doing all this locally before we resort to querying federation - respState := t.lookupStateAfterEventLocally(ctx, roomID, eventID) + respState := t.lookupStateAfterEventLocally(ctx, eventID) if respState != nil { return respState, true, nil } @@ -373,7 +374,7 @@ func (t *missingStateReq) cacheAndReturn(ev *gomatrixserverlib.Event) *gomatrixs return ev } -func (t *missingStateReq) lookupStateAfterEventLocally(ctx context.Context, roomID, eventID string) *parsedRespState { +func (t *missingStateReq) lookupStateAfterEventLocally(ctx context.Context, eventID string) *parsedRespState { span, ctx := opentracing.StartSpanFromContext(ctx, "lookupStateAfterEventLocally") defer span.Finish() @@ -381,12 +382,12 @@ func (t *missingStateReq) lookupStateAfterEventLocally(ctx context.Context, room roomState := state.NewStateResolution(t.db, t.roomInfo) stateAtEvents, err := t.db.StateAtEventIDs(ctx, []string{eventID}) if err != nil { - util.GetLogger(ctx).WithField("room_id", roomID).WithError(err).Warnf("failed to get state after %s locally", eventID) + t.log.WithError(err).Warnf("failed to get state after %s locally", eventID) return nil } stateEntries, err := roomState.LoadCombinedStateAfterEvents(ctx, stateAtEvents) if err != nil { - util.GetLogger(ctx).WithField("room_id", roomID).WithError(err).Warnf("failed to load combined state after %s locally", eventID) + t.log.WithError(err).Warnf("failed to load combined state after %s locally", eventID) return nil } stateEventNIDs := make([]types.EventNID, 0, len(stateEntries)) @@ -395,7 +396,7 @@ func (t *missingStateReq) lookupStateAfterEventLocally(ctx context.Context, room } stateEvents, err := t.db.Events(ctx, stateEventNIDs) if err != nil { - util.GetLogger(ctx).WithField("room_id", roomID).WithError(err).Warnf("failed to load state events locally") + t.log.WithError(err).Warnf("failed to load state events locally") return nil } res.StateEvents = make([]*gomatrixserverlib.Event, 0, len(stateEvents)) @@ -429,7 +430,7 @@ func (t *missingStateReq) lookupStateAfterEventLocally(ctx context.Context, room for evID := range missingAuthEvents { missingEventList = append(missingEventList, evID) } - util.GetLogger(ctx).WithField("count", len(missingEventList)).Debugf("Fetching missing auth events") + t.log.WithField("count", len(missingEventList)).Debugf("Fetching missing auth events") events, err := t.db.EventsFromIDs(ctx, missingEventList) if err != nil { return nil @@ -485,7 +486,7 @@ retryAllowedState: default: return nil, fmt.Errorf("missing auth event %s and failed to look it up: %w", missing.AuthEventID, err2) } - util.GetLogger(ctx).Tracef("fetched event %s", missing.AuthEventID) + t.log.Tracef("fetched event %s", missing.AuthEventID) resolvedStateEvents = append(resolvedStateEvents, h) goto retryAllowedState default: @@ -504,7 +505,7 @@ func (t *missingStateReq) getMissingEvents(ctx context.Context, e *gomatrixserve span, ctx := opentracing.StartSpanFromContext(ctx, "getMissingEvents") defer span.Finish() - logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID()) + logger := t.log.WithField("event_id", e.EventID()).WithField("room_id", e.RoomID()) latest, _, _, err := t.db.LatestEventIDs(ctx, t.roomInfo.RoomNID) if err != nil { return nil, false, false, fmt.Errorf("t.DB.LatestEventIDs: %w", err) @@ -666,7 +667,7 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo span, ctx := opentracing.StartSpanFromContext(ctx, "lookupMissingStateViaStateIDs") defer span.Finish() - util.GetLogger(ctx).WithField("room_id", roomID).Infof("lookupMissingStateViaStateIDs %s", eventID) + t.log.Infof("lookupMissingStateViaStateIDs %s", eventID) // fetch the state event IDs at the time of the event var stateIDs gomatrixserverlib.RespStateIDs var err error @@ -719,16 +720,15 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo concurrentRequests := 8 missingCount := len(missing) - util.GetLogger(ctx).WithField("room_id", roomID).WithField("event_id", eventID).Debugf("lookupMissingStateViaStateIDs missing %d/%d events", missingCount, len(wantIDs)) + t.log.WithField("event_id", eventID).Debugf("lookupMissingStateViaStateIDs missing %d/%d events", missingCount, len(wantIDs)) // If over 50% of the auth/state events from /state_ids are missing // then we'll just call /state instead, otherwise we'll just end up // hammering the remote side with /event requests unnecessarily. if missingCount > concurrentRequests && missingCount > len(wantIDs)/2 { - util.GetLogger(ctx).WithFields(logrus.Fields{ + t.log.WithFields(logrus.Fields{ "missing": missingCount, "event_id": eventID, - "room_id": roomID, "total_state": len(stateIDs.StateEventIDs), "total_auth_events": len(stateIDs.AuthEventIDs), }).Debug("Fetching all state at event") @@ -736,10 +736,9 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo } if missingCount > 0 { - util.GetLogger(ctx).WithFields(logrus.Fields{ + t.log.WithFields(logrus.Fields{ "missing": missingCount, "event_id": eventID, - "room_id": roomID, "total_state": len(stateIDs.StateEventIDs), "total_auth_events": len(stateIDs.AuthEventIDs), "concurrent_requests": concurrentRequests, @@ -776,9 +775,8 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo case nil: break default: - util.GetLogger(ctx).WithFields(logrus.Fields{ - "event_id": missingEventID, - "room_id": roomID, + t.log.WithFields(logrus.Fields{ + "missing_event_id": missingEventID, }).WithError(herr).Warn("Failed to fetch missing event") return } @@ -847,7 +845,7 @@ func (t *missingStateReq) lookupEvent(ctx context.Context, roomVersion gomatrixs // fetch from the roomserver events, err := t.db.EventsFromIDs(ctx, []string{missingEventID}) if err != nil { - util.GetLogger(ctx).Warnf("Failed to query roomserver for missing event %s: %s - falling back to remote", missingEventID, err) + t.log.Warnf("Failed to query roomserver for missing event %s: %s - falling back to remote", missingEventID, err) } else if len(events) == 1 { return events[0].Event, nil } @@ -859,7 +857,7 @@ func (t *missingStateReq) lookupEvent(ctx context.Context, roomVersion gomatrixs defer cancel() txn, err := t.federation.GetEvent(reqctx, serverName, missingEventID) if err != nil || len(txn.PDUs) == 0 { - util.GetLogger(ctx).WithError(err).WithField("event_id", missingEventID).Warn("Failed to get missing /event for event ID") + t.log.WithError(err).WithField("missing_event_id", missingEventID).Warn("Failed to get missing /event for event ID") if errors.Is(err, context.DeadlineExceeded) { select { case <-reqctx.Done(): // this server took too long @@ -872,18 +870,18 @@ func (t *missingStateReq) lookupEvent(ctx context.Context, roomVersion gomatrixs } event, err = gomatrixserverlib.NewEventFromUntrustedJSON(txn.PDUs[0], roomVersion) if err != nil { - util.GetLogger(ctx).WithError(err).WithField("event_id", missingEventID).Warnf("Failed to parse event JSON of event returned from /event") + t.log.WithError(err).WithField("missing_event_id", missingEventID).Warnf("Failed to parse event JSON of event returned from /event") continue } found = true break } if !found { - util.GetLogger(ctx).WithField("event_id", missingEventID).Warnf("Failed to get missing /event for event ID from %d server(s)", len(t.servers)) + t.log.WithField("missing_event_id", missingEventID).Warnf("Failed to get missing /event for event ID from %d server(s)", len(t.servers)) return nil, fmt.Errorf("wasn't able to find event via %d server(s)", len(t.servers)) } if err := event.VerifyEventSignatures(ctx, t.keys); err != nil { - util.GetLogger(ctx).WithError(err).Warnf("Couldn't validate signature of event %q from /event", event.EventID()) + t.log.WithError(err).Warnf("Couldn't validate signature of event %q from /event", event.EventID()) return nil, verifySigError{event.EventID(), err} } return t.cacheAndReturn(event), nil diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 0db046a86..8850e5c46 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -239,16 +239,42 @@ func (r *Queryer) QueryMembershipAtEvent( return fmt.Errorf("unable to get state before event: %w", err) } + // If we only have one or less state entries, we can short circuit the below + // loop and avoid hitting the database + allStateEventNIDs := make(map[types.EventNID]types.StateEntry) + for _, eventID := range request.EventIDs { + stateEntry := stateEntries[eventID] + for _, s := range stateEntry { + allStateEventNIDs[s.EventNID] = s + } + } + + var canShortCircuit bool + if len(allStateEventNIDs) <= 1 { + canShortCircuit = true + } + + var memberships []types.Event for _, eventID := range request.EventIDs { stateEntry, ok := stateEntries[eventID] - if !ok { + if !ok || len(stateEntry) == 0 { response.Memberships[eventID] = []*gomatrixserverlib.HeaderedEvent{} continue } - memberships, err := helpers.GetMembershipsAtState(ctx, r.DB, stateEntry, false) + + // If we can short circuit, e.g. we only have 0 or 1 membership events, we only get the memberships + // once. If we have more than one membership event, we need to get the state for each state entry. + if canShortCircuit { + if len(memberships) == 0 { + memberships, err = helpers.GetMembershipsAtState(ctx, r.DB, stateEntry, false) + } + } else { + memberships, err = helpers.GetMembershipsAtState(ctx, r.DB, stateEntry, false) + } if err != nil { return fmt.Errorf("unable to get memberships at state: %w", err) } + res := make([]*gomatrixserverlib.HeaderedEvent, 0, len(memberships)) for i := range memberships { diff --git a/roomserver/state/state.go b/roomserver/state/state.go index cb96d83ec..1cfde5e4b 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -18,17 +18,17 @@ package state import ( "context" - "database/sql" "fmt" "sort" "sync" "time" - "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" + + "github.com/matrix-org/dendrite/roomserver/types" ) type StateResolutionStorage interface { @@ -37,6 +37,7 @@ type StateResolutionStorage interface { StateBlockNIDs(ctx context.Context, stateNIDs []types.StateSnapshotNID) ([]types.StateBlockNIDList, error) StateEntries(ctx context.Context, stateBlockNIDs []types.StateBlockNID) ([]types.StateEntryList, error) SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) + BulkSelectSnapshotsFromEventIDs(ctx context.Context, eventIDs []string) (map[types.StateSnapshotNID][]string, error) StateEntriesForTuples(ctx context.Context, stateBlockNIDs []types.StateBlockNID, stateKeyTuples []types.StateKeyTuple) ([]types.StateEntryList, error) StateAtEventIDs(ctx context.Context, eventIDs []string) ([]types.StateAtEvent, error) AddState(ctx context.Context, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, state []types.StateEntry) (types.StateSnapshotNID, error) @@ -130,21 +131,10 @@ func (v *StateResolution) LoadMembershipAtEvent( span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.LoadMembershipAtEvent") defer span.Finish() - // De-dupe snapshotNIDs - snapshotNIDMap := make(map[types.StateSnapshotNID][]string) // map from snapshot NID to eventIDs - for i := range eventIDs { - eventID := eventIDs[i] - snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID) - if err != nil && err != sql.ErrNoRows { - return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID failed for event %s : %w", eventID, err) - } - if snapshotNID == 0 { - // If we don't know a state snapshot for this event then we can't calculate - // memberships at the time of the event, so skip over it. This means that - // it isn't guaranteed that the response map will contain every single event. - continue - } - snapshotNIDMap[snapshotNID] = append(snapshotNIDMap[snapshotNID], eventID) + // Get a mapping from snapshotNID -> eventIDs + snapshotNIDMap, err := v.db.BulkSelectSnapshotsFromEventIDs(ctx, eventIDs) + if err != nil { + return nil, err } snapshotNIDs := make([]types.StateSnapshotNID, 0, len(snapshotNIDMap)) @@ -157,24 +147,45 @@ func (v *StateResolution) LoadMembershipAtEvent( return nil, err } + var wantStateBlocks []types.StateBlockNID + for _, x := range stateBlockNIDLists { + wantStateBlocks = append(wantStateBlocks, x.StateBlockNIDs...) + } + + stateEntryLists, err := v.db.StateEntriesForTuples(ctx, uniqueStateBlockNIDs(wantStateBlocks), []types.StateKeyTuple{ + { + EventTypeNID: types.MRoomMemberNID, + EventStateKeyNID: stateKeyNID, + }, + }) + if err != nil { + return nil, err + } + + stateBlockNIDsMap := stateBlockNIDListMap(stateBlockNIDLists) + stateEntriesMap := stateEntryListMap(stateEntryLists) + result := make(map[string][]types.StateEntry) for _, stateBlockNIDList := range stateBlockNIDLists { - // Query the membership event for the user at the given stateblocks - stateEntryLists, err := v.db.StateEntriesForTuples(ctx, stateBlockNIDList.StateBlockNIDs, []types.StateKeyTuple{ - { - EventTypeNID: types.MRoomMemberNID, - EventStateKeyNID: stateKeyNID, - }, - }) - if err != nil { - return nil, err + stateBlockNIDs, ok := stateBlockNIDsMap.lookup(stateBlockNIDList.StateSnapshotNID) + if !ok { + // This should only get hit if the database is corrupt. + // It should be impossible for an event to reference a NID that doesn't exist + return nil, fmt.Errorf("corrupt DB: Missing state snapshot numeric ID %d", stateBlockNIDList.StateSnapshotNID) } - evIDs := snapshotNIDMap[stateBlockNIDList.StateSnapshotNID] + for _, stateBlockNID := range stateBlockNIDs { + entries, ok := stateEntriesMap.lookup(stateBlockNID) + if !ok { + // This should only get hit if the database is corrupt. + // It should be impossible for an event to reference a NID that doesn't exist + return nil, fmt.Errorf("corrupt DB: Missing state block numeric ID %d", stateBlockNID) + } - for _, evID := range evIDs { - for _, x := range stateEntryLists { - result[evID] = append(result[evID], x.StateEntries...) + evIDs := snapshotNIDMap[stateBlockNIDList.StateSnapshotNID] + + for _, evID := range evIDs { + result[evID] = append(result[evID], entries...) } } } @@ -944,7 +955,6 @@ func (v *StateResolution) resolveConflictsV2( authSets := make(map[string][]*gomatrixserverlib.Event, len(conflicted)) authEvents := make([]*gomatrixserverlib.Event, 0, estimate*3) gotAuthEvents := make(map[string]struct{}, estimate*3) - authDifference := make([]*gomatrixserverlib.Event, 0, estimate) knownAuthEvents := make(map[string]types.Event, estimate*3) // For each conflicted event, let's try and get the needed auth events. @@ -992,41 +1002,6 @@ func (v *StateResolution) resolveConflictsV2( // longer need this after this point. gotAuthEvents = nil // nolint:ineffassign - // This function helps us to work out whether an event exists in one of the - // auth sets. - isInAuthList := func(k string, event *gomatrixserverlib.Event) bool { - for _, e := range authSets[k] { - if e.EventID() == event.EventID() { - return true - } - } - return false - } - - // This function works out if an event exists in all of the auth sets. - isInAllAuthLists := func(event *gomatrixserverlib.Event) bool { - for k := range authSets { - if !isInAuthList(k, event) { - return false - } - } - return true - } - - // Look through all of the auth events that we've been given and work out if - // there are any events which don't appear in all of the auth sets. If they - // don't then we add them to the auth difference. - func() { - span, _ := opentracing.StartSpanFromContext(ctx, "isInAllAuthLists") - defer span.Finish() - - for _, event := range authEvents { - if !isInAllAuthLists(event) { - authDifference = append(authDifference, event) - } - } - }() - // Resolve the conflicts. resolvedEvents := func() []*gomatrixserverlib.Event { span, _ := opentracing.StartSpanFromContext(ctx, "gomatrixserverlib.ResolveStateConflictsV2") @@ -1036,7 +1011,6 @@ func (v *StateResolution) resolveConflictsV2( conflictedEvents, nonConflictedEvents, authEvents, - authDifference, ) }() diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 094537948..c39a8cbba 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -72,6 +72,7 @@ type Database interface { Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error) // Look up snapshot NID for an event ID string SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) + BulkSelectSnapshotsFromEventIDs(ctx context.Context, eventIDs []string) (map[types.StateSnapshotNID][]string, error) // Stores a matrix room event in the database. Returns the room NID, the state snapshot and the redacted event ID if any, or an error. StoreEvent( ctx context.Context, event *gomatrixserverlib.Event, authEventNIDs []types.EventNID, diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index 1e7ca7669..9b5ed6eda 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -22,11 +22,12 @@ import ( "sort" "github.com/lib/pq" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" ) const eventsSchema = ` @@ -80,6 +81,9 @@ const insertEventSQL = "" + const selectEventSQL = "" + "SELECT event_nid, state_snapshot_nid FROM roomserver_events WHERE event_id = $1" +const bulkSelectSnapshotsForEventIDsSQL = "" + + "SELECT event_id, state_snapshot_nid FROM roomserver_events WHERE event_id = ANY($1)" + // Bulk lookup of events by string ID. // Sort by the numeric IDs for event type and state key. // This means we can use binary search to lookup entries by type and state key. @@ -150,6 +154,7 @@ const selectEventRejectedSQL = "" + type eventStatements struct { insertEventStmt *sql.Stmt selectEventStmt *sql.Stmt + bulkSelectSnapshotsForEventIDsStmt *sql.Stmt bulkSelectStateEventByIDStmt *sql.Stmt bulkSelectStateEventByIDExcludingRejectedStmt *sql.Stmt bulkSelectStateEventByNIDStmt *sql.Stmt @@ -179,6 +184,7 @@ func PrepareEventsTable(db *sql.DB) (tables.Events, error) { return s, sqlutil.StatementList{ {&s.insertEventStmt, insertEventSQL}, {&s.selectEventStmt, selectEventSQL}, + {&s.bulkSelectSnapshotsForEventIDsStmt, bulkSelectSnapshotsForEventIDsSQL}, {&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL}, {&s.bulkSelectStateEventByIDExcludingRejectedStmt, bulkSelectStateEventByIDExcludingRejectedSQL}, {&s.bulkSelectStateEventByNIDStmt, bulkSelectStateEventByNIDSQL}, @@ -230,6 +236,29 @@ func (s *eventStatements) SelectEvent( return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err } +func (s *eventStatements) BulkSelectSnapshotsFromEventIDs( + ctx context.Context, txn *sql.Tx, eventIDs []string, +) (map[types.StateSnapshotNID][]string, error) { + stmt := sqlutil.TxStmt(txn, s.bulkSelectSnapshotsForEventIDsStmt) + + rows, err := stmt.QueryContext(ctx, pq.Array(eventIDs)) + if err != nil { + return nil, err + } + + var eventID string + var stateNID types.StateSnapshotNID + result := make(map[types.StateSnapshotNID][]string) + for rows.Next() { + if err := rows.Scan(&eventID, &stateNID); err != nil { + return nil, err + } + result[stateNID] = append(result[stateNID], eventID) + } + + return result, rows.Err() +} + // bulkSelectStateEventByID lookups a list of state events by event ID. // If not excluding rejected events, and any of the requested events are missing from // the database it returns a types.MissingEventError. If excluding rejected events, diff --git a/roomserver/storage/shared/room_updater.go b/roomserver/storage/shared/room_updater.go index 42c0c8f2d..cc880a6c8 100644 --- a/roomserver/storage/shared/room_updater.go +++ b/roomserver/storage/shared/room_updater.go @@ -5,8 +5,9 @@ import ( "database/sql" "fmt" - "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" + + "github.com/matrix-org/dendrite/roomserver/types" ) type RoomUpdater struct { @@ -186,6 +187,10 @@ func (u *RoomUpdater) EventIDs( return u.d.EventsTable.BulkSelectEventID(ctx, u.txn, eventNIDs) } +func (u *RoomUpdater) BulkSelectSnapshotsFromEventIDs(ctx context.Context, eventIDs []string) (map[types.StateSnapshotNID][]string, error) { + return u.d.EventsTable.BulkSelectSnapshotsFromEventIDs(ctx, u.txn, eventIDs) +} + func (u *RoomUpdater) StateAtEventIDs( ctx context.Context, eventIDs []string, ) ([]types.StateAtEvent, error) { diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index ed86280bf..4455ec3bf 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -469,6 +469,23 @@ func (d *Database) events( eventNIDs = append(eventNIDs, nid) } } + // If we don't need to get any events from the database, short circuit now + if len(eventNIDs) == 0 { + results := make([]types.Event, 0, len(inputEventNIDs)) + for _, nid := range inputEventNIDs { + event, ok := events[nid] + if !ok || event == nil { + return nil, fmt.Errorf("event %d missing", nid) + } + results = append(results, types.Event{ + EventNID: nid, + Event: event, + }) + } + if !redactionsArePermanent { + d.applyRedactions(results) + } + } eventJSONs, err := d.EventJSONTable.BulkSelectEventJSON(ctx, txn, eventNIDs) if err != nil { return nil, err @@ -534,6 +551,12 @@ func (d *Database) events( return results, nil } +func (d *Database) BulkSelectSnapshotsFromEventIDs( + ctx context.Context, eventIDs []string, +) (map[types.StateSnapshotNID][]string, error) { + return d.EventsTable.BulkSelectSnapshotsFromEventIDs(ctx, nil, eventIDs) +} + func (d *Database) MembershipUpdater( ctx context.Context, roomID, targetUserID string, targetLocal bool, roomVersion gomatrixserverlib.RoomVersion, diff --git a/roomserver/storage/sqlite3/deltas/20221027084407_published_appservice.go b/roomserver/storage/sqlite3/deltas/20221027084407_published_appservice.go index cd923b1c1..410fb7cf6 100644 --- a/roomserver/storage/sqlite3/deltas/20221027084407_published_appservice.go +++ b/roomserver/storage/sqlite3/deltas/20221027084407_published_appservice.go @@ -24,8 +24,8 @@ func UpPulishedAppservice(ctx context.Context, tx *sql.Tx) error { _, err := tx.ExecContext(ctx, ` ALTER TABLE roomserver_published RENAME TO roomserver_published_tmp; CREATE TABLE IF NOT EXISTS roomserver_published ( room_id TEXT NOT NULL, - appservice_id TEXT NOT NULL, - network_id TEXT NOT NULL, + appservice_id TEXT NOT NULL DEFAULT '', + network_id TEXT NOT NULL DEFAULT '', published BOOLEAN NOT NULL DEFAULT false, CONSTRAINT unique_published_idx PRIMARY KEY (room_id, appservice_id, network_id) ); diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index 950d03b03..f39b9902d 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -23,11 +23,12 @@ import ( "sort" "strings" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" ) const eventsSchema = ` @@ -57,6 +58,9 @@ const insertEventSQL = ` const selectEventSQL = "" + "SELECT event_nid, state_snapshot_nid FROM roomserver_events WHERE event_id = $1" +const bulkSelectSnapshotsForEventIDsSQL = "" + + "SELECT event_id, state_snapshot_nid FROM roomserver_events WHERE event_id IN ($1)" + // Bulk lookup of events by string ID. // Sort by the numeric IDs for event type and state key. // This means we can use binary search to lookup entries by type and state key. @@ -124,6 +128,7 @@ type eventStatements struct { db *sql.DB insertEventStmt *sql.Stmt selectEventStmt *sql.Stmt + bulkSelectSnapshotsForEventIDsStmt *sql.Stmt bulkSelectStateEventByIDStmt *sql.Stmt bulkSelectStateEventByIDExcludingRejectedStmt *sql.Stmt bulkSelectStateAtEventByIDStmt *sql.Stmt @@ -153,6 +158,7 @@ func PrepareEventsTable(db *sql.DB) (tables.Events, error) { return s, sqlutil.StatementList{ {&s.insertEventStmt, insertEventSQL}, {&s.selectEventStmt, selectEventSQL}, + {&s.bulkSelectSnapshotsForEventIDsStmt, bulkSelectSnapshotsForEventIDsSQL}, {&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL}, {&s.bulkSelectStateEventByIDExcludingRejectedStmt, bulkSelectStateEventByIDExcludingRejectedSQL}, {&s.bulkSelectStateAtEventByIDStmt, bulkSelectStateAtEventByIDSQL}, @@ -203,6 +209,40 @@ func (s *eventStatements) SelectEvent( return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err } +func (s *eventStatements) BulkSelectSnapshotsFromEventIDs( + ctx context.Context, txn *sql.Tx, eventIDs []string, +) (map[types.StateSnapshotNID][]string, error) { + qry := strings.Replace(bulkSelectSnapshotsForEventIDsSQL, "($1)", sqlutil.QueryVariadic(len(eventIDs)), 1) + stmt, err := s.db.Prepare(qry) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, stmt, "BulkSelectSnapshotsFromEventIDs: stmt.close() failed") + + params := make([]interface{}, len(eventIDs)) + for i := range eventIDs { + params[i] = eventIDs[i] + } + + rows, err := stmt.QueryContext(ctx, params...) + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "BulkSelectSnapshotsFromEventIDs: rows.close() failed") + + var eventID string + var stateNID types.StateSnapshotNID + result := make(map[types.StateSnapshotNID][]string) + for rows.Next() { + if err := rows.Scan(&eventID, &stateNID); err != nil { + return nil, err + } + result[stateNID] = append(result[stateNID], eventID) + } + + return result, rows.Err() +} + // bulkSelectStateEventByID lookups a list of state events by event ID. // If not excluding rejected events, and any of the requested events are missing from // the database it returns a types.MissingEventError. If excluding rejected events, diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index 8d6ca324c..50d27c756 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -44,6 +44,7 @@ type Events interface { referenceSHA256 []byte, authEventNIDs []types.EventNID, depth int64, isRejected bool, ) (types.EventNID, types.StateSnapshotNID, error) SelectEvent(ctx context.Context, txn *sql.Tx, eventID string) (types.EventNID, types.StateSnapshotNID, error) + BulkSelectSnapshotsFromEventIDs(ctx context.Context, txn *sql.Tx, eventIDs []string) (map[types.StateSnapshotNID][]string, error) // bulkSelectStateEventByID lookups a list of state events by event ID. // If any of the requested events are missing from the database it returns a types.MissingEventError BulkSelectStateEventByID(ctx context.Context, txn *sql.Tx, eventIDs []string, excludeRejected bool) ([]types.StateEntry, error) diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go index 0ed164c7e..095a868c7 100644 --- a/syncapi/routing/context.go +++ b/syncapi/routing/context.go @@ -93,7 +93,6 @@ func Context( } stateFilter := gomatrixserverlib.StateFilter{ - Limit: 100, NotSenders: filter.NotSenders, NotTypes: filter.NotTypes, Senders: filter.Senders, diff --git a/syncapi/routing/memberships.go b/syncapi/routing/memberships.go index 05c7deef0..3fcc3235c 100644 --- a/syncapi/routing/memberships.go +++ b/syncapi/routing/memberships.go @@ -135,6 +135,6 @@ func GetMemberships( } return util.JSONResponse{ Code: http.StatusOK, - JSON: getMembershipResponse{gomatrixserverlib.HeaderedToClientEvents(result, gomatrixserverlib.FormatSync)}, + JSON: getMembershipResponse{gomatrixserverlib.HeaderedToClientEvents(result, gomatrixserverlib.FormatAll)}, } } diff --git a/syncapi/routing/search.go b/syncapi/routing/search.go index aef355def..081ec6cb1 100644 --- a/syncapi/routing/search.go +++ b/syncapi/routing/search.go @@ -294,7 +294,7 @@ type SearchRequest struct { BeforeLimit int `json:"before_limit,omitempty"` IncludeProfile bool `json:"include_profile,omitempty"` } `json:"event_context"` - Filter gomatrixserverlib.StateFilter `json:"filter"` + Filter gomatrixserverlib.RoomEventFilter `json:"filter"` Groupings struct { GroupBy []struct { Key string `json:"key"` diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 2ccf0be1a..48ed20021 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -91,8 +91,7 @@ const selectCurrentStateSQL = "" + " AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" + " AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" + " AND ( $6::bool IS NULL OR contains_url = $6 )" + - " AND (event_id = ANY($7)) IS NOT TRUE" + - " LIMIT $8" + " AND (event_id = ANY($7)) IS NOT TRUE" const selectJoinedUsersSQL = "" + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" @@ -290,7 +289,6 @@ func (s *currentRoomStateStatements) SelectCurrentState( pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.NotTypes)), stateFilter.ContainsURL, pq.StringArray(excludeEventIDs), - stateFilter.Limit, ) if err != nil { return nil, err diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 0ecbdf4d2..09340bbdf 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -144,17 +144,16 @@ const selectStateInRangeFilteredSQL = "" + " AND ( $6::text[] IS NULL OR type LIKE ANY($6) )" + " AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" + " AND ( $8::bool IS NULL OR contains_url = $8 )" + - " ORDER BY id ASC" + - " LIMIT $9" + " ORDER BY id ASC" +// In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). // In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). const selectStateInRangeSQL = "" + "SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids, history_visibility" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2) AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + " AND room_id = ANY($3)" + - " ORDER BY id ASC" + - " LIMIT $4" + " ORDER BY id ASC" const deleteEventsForRoomSQL = "" + "DELETE FROM syncapi_output_room_events WHERE room_id = $1" @@ -264,13 +263,11 @@ func (s *outputRoomEventsStatements) SelectStateInRange( pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.Types)), pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.NotTypes)), stateFilter.ContainsURL, - stateFilter.Limit, ) } else { stmt := sqlutil.TxStmt(txn, s.selectStateInRangeStmt) rows, err = stmt.QueryContext( ctx, r.Low(), r.High(), pq.StringArray(roomIDs), - r.High()-r.Low(), ) } diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index ff45e786e..7a381f68b 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -277,7 +277,8 @@ func (s *currentRoomStateStatements) SelectCurrentState( }, stateFilter.Senders, stateFilter.NotSenders, stateFilter.Types, stateFilter.NotTypes, - excludeEventIDs, stateFilter.ContainsURL, stateFilter.Limit, FilterOrderNone, + excludeEventIDs, stateFilter.ContainsURL, 0, + FilterOrderNone, ) if err != nil { return nil, fmt.Errorf("s.prepareWithFilters: %w", err) diff --git a/syncapi/storage/sqlite3/filtering.go b/syncapi/storage/sqlite3/filtering.go index 05edb7b8c..17a37a2df 100644 --- a/syncapi/storage/sqlite3/filtering.go +++ b/syncapi/storage/sqlite3/filtering.go @@ -84,8 +84,10 @@ func prepareWithFilters( case FilterOrderDesc: query += " ORDER BY id DESC" } - query += fmt.Sprintf(" LIMIT $%d", offset+1) - params = append(params, limit) + if limit > 0 { + query += fmt.Sprintf(" LIMIT $%d", offset+1) + params = append(params, limit) + } var stmt *sql.Stmt var err error diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 77c692ff0..1aa4bfff7 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -200,7 +200,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( s.db, txn, stmtSQL, inputParams, stateFilter.Senders, stateFilter.NotSenders, stateFilter.Types, stateFilter.NotTypes, - nil, stateFilter.ContainsURL, stateFilter.Limit, FilterOrderAsc, + nil, stateFilter.ContainsURL, 0, FilterOrderAsc, ) } else { stmt, params, err = prepareWithFilters( diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go index f1c5450fd..8c2b8fc89 100644 --- a/syncapi/streams/stream_invite.go +++ b/syncapi/streams/stream_invite.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/base64" + "math" "strconv" "time" @@ -73,43 +74,28 @@ func (p *InviteStreamProvider) IncrementalSync( return to } for roomID := range retiredInvites { - if req.Response.Rooms.Invite[roomID] != nil { - continue - } - if req.Response.Rooms.Join[roomID] != nil { + membership, _, err := snapshot.SelectMembershipForUser(ctx, roomID, req.Device.UserID, math.MaxInt64) + // Skip if the user is an existing member of the room. + // Otherwise, the NewLeaveResponse will eject the user from the room unintentionally + if membership == gomatrixserverlib.Join || + err != nil { continue } - joinedUsers, err := snapshot.AllJoinedUsersInRoom(ctx, []string{roomID}) - if err != nil { - continue - } - - if !contains(joinedUsers[roomID], req.Device.UserID) { - lr := types.NewLeaveResponse() - h := sha256.Sum256(append([]byte(roomID), []byte(strconv.FormatInt(int64(to), 10))...)) - lr.Timeline.Events = append(lr.Timeline.Events, gomatrixserverlib.ClientEvent{ - // fake event ID which muxes in the to position - EventID: "$" + base64.RawURLEncoding.EncodeToString(h[:]), - OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), - RoomID: roomID, - Sender: req.Device.UserID, - StateKey: &req.Device.UserID, - Type: "m.room.member", - Content: gomatrixserverlib.RawJSON(`{"membership":"leave"}`), - }) - req.Response.Rooms.Leave[roomID] = lr - } + lr := types.NewLeaveResponse() + h := sha256.Sum256(append([]byte(roomID), []byte(strconv.FormatInt(int64(to), 10))...)) + lr.Timeline.Events = append(lr.Timeline.Events, gomatrixserverlib.ClientEvent{ + // fake event ID which muxes in the to position + EventID: "$" + base64.RawURLEncoding.EncodeToString(h[:]), + OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), + RoomID: roomID, + Sender: req.Device.UserID, + StateKey: &req.Device.UserID, + Type: "m.room.member", + Content: gomatrixserverlib.RawJSON(`{"membership":"leave"}`), + }) + req.Response.Rooms.Leave[roomID] = lr } return maxID } - -func contains(values []string, findVal string) bool { - for _, v := range values { - if v == findVal { - return true - } - } - return false -} diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index b191f482b..e6aa19f54 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -301,7 +301,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( } // Applies the history visibility rules - events, err := applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, delta.RoomID, device.UserID, eventFilter.Limit, recentEvents) + events, err := applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, delta.RoomID, device.UserID, recentEvents) if err != nil { logrus.WithError(err).Error("unable to apply history visibility filter") } @@ -315,11 +315,6 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( // Now that we've filtered the timeline, work out which state events are still // left. Anything that appears in the filtered timeline will be removed from the // "state" section and kept in "timeline". - - // Don't remove the state events if this is a newly joined room. The member - // needs the full room state. For example, to figure out if the room is a space - // or not, it needs the m.room.create event which may be chronologically outside - // the filter limit. if !delta.NewlyJoined { delta.StateEvents = gomatrixserverlib.HeaderedReverseTopologicalOrdering( removeDuplicates(delta.StateEvents, recentEvents), @@ -328,10 +323,14 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( } if len(delta.StateEvents) > 0 { - updateLatestPosition(delta.StateEvents[len(delta.StateEvents)-1].EventID()) + if last := delta.StateEvents[len(delta.StateEvents)-1]; last != nil { + updateLatestPosition(last.EventID()) + } } if len(events) > 0 { - updateLatestPosition(events[len(events)-1].EventID()) + if last := events[len(events)-1]; last != nil { + updateLatestPosition(last.EventID()) + } } switch delta.Membership { @@ -381,12 +380,12 @@ func applyHistoryVisibilityFilter( snapshot storage.DatabaseTransaction, rsAPI roomserverAPI.SyncRoomserverAPI, roomID, userID string, - limit int, recentEvents []*gomatrixserverlib.HeaderedEvent, ) ([]*gomatrixserverlib.HeaderedEvent, error) { // We need to make sure we always include the latest states events, if they are in the timeline. // We grep at least limit * 2 events, to ensure we really get the needed events. - stateEvents, err := snapshot.CurrentState(ctx, roomID, &gomatrixserverlib.StateFilter{Limit: limit * 2}, nil) + filter := gomatrixserverlib.DefaultStateFilter() + stateEvents, err := snapshot.CurrentState(ctx, roomID, &filter, nil) if err != nil { // Not a fatal error, we can continue without the stateEvents, // they are only needed if there are state events in the timeline. @@ -524,7 +523,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( events := recentEvents // Only apply history visibility checks if the response is for joined rooms if !isPeek { - events, err = applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, roomID, device.UserID, eventFilter.Limit, recentEvents) + events, err = applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, roomID, device.UserID, recentEvents) if err != nil { logrus.WithError(err).Error("unable to apply history visibility filter") } @@ -604,7 +603,6 @@ func (p *PDUStreamProvider) lazyLoadMembers( } // Query missing membership events filter := gomatrixserverlib.DefaultStateFilter() - filter.Limit = stateFilter.Limit filter.Senders = &wantUsers filter.Types = &[]string{gomatrixserverlib.MRoomMember} memberships, err := snapshot.GetStateEventsForRoom(ctx, roomID, &filter) diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index 620dfdcdb..e5e5fdb5b 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -79,7 +79,6 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat // for the rest of the data to trickle down. filter.AccountData.Limit = math.MaxInt32 filter.Room.AccountData.Limit = math.MaxInt32 - filter.Room.State.Limit = math.MaxInt32 } logger := util.GetLogger(req.Context()).WithFields(logrus.Fields{ diff --git a/test/db.go b/test/db.go index c7cb919f6..17f637e18 100644 --- a/test/db.go +++ b/test/db.go @@ -176,7 +176,7 @@ func WithAllDatabases(t *testing.T, testFn func(t *testing.T, db DBType)) { for dbName, dbType := range dbs { dbt := dbType t.Run(dbName, func(tt *testing.T) { - tt.Parallel() + //tt.Parallel() testFn(tt, dbt) }) } diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go index a12876946..97c17e188 100644 --- a/userapi/consumers/roomserver.go +++ b/userapi/consumers/roomserver.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "strings" + "sync" "time" "github.com/matrix-org/gomatrixserverlib" @@ -23,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/userapi/producers" "github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/dendrite/userapi/storage/tables" + userAPITypes "github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/dendrite/userapi/util" ) @@ -36,6 +38,11 @@ type OutputRoomEventConsumer struct { topic string pgClient pushgateway.Client syncProducer *producers.SyncAPI + msgCounts map[gomatrixserverlib.ServerName]userAPITypes.MessageStats + roomCounts map[gomatrixserverlib.ServerName]map[string]bool // map from serverName to map from rommID to "isEncrypted" + lastUpdate time.Time + countsLock sync.Mutex + serverName gomatrixserverlib.ServerName } func NewOutputRoomEventConsumer( @@ -57,6 +64,11 @@ func NewOutputRoomEventConsumer( pgClient: pgClient, rsAPI: rsAPI, syncProducer: syncProducer, + msgCounts: map[gomatrixserverlib.ServerName]userAPITypes.MessageStats{}, + roomCounts: map[gomatrixserverlib.ServerName]map[string]bool{}, + lastUpdate: time.Now(), + countsLock: sync.Mutex{}, + serverName: cfg.Matrix.ServerName, } } @@ -88,6 +100,10 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms return true } + if s.cfg.Matrix.ReportStats.Enabled { + go s.storeMessageStats(ctx, event.Type(), event.Sender(), event.RoomID()) + } + log.WithFields(log.Fields{ "event_id": event.EventID(), "event_type": event.Type(), @@ -107,6 +123,68 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms return true } +func (s *OutputRoomEventConsumer) storeMessageStats(ctx context.Context, eventType, eventSender, roomID string) { + s.countsLock.Lock() + defer s.countsLock.Unlock() + + // reset the roomCounts on a day change + if s.lastUpdate.Day() != time.Now().Day() { + s.roomCounts[s.serverName] = make(map[string]bool) + s.lastUpdate = time.Now() + } + + _, sender, err := gomatrixserverlib.SplitID('@', eventSender) + if err != nil { + return + } + msgCount := s.msgCounts[s.serverName] + roomCount := s.roomCounts[s.serverName] + if roomCount == nil { + roomCount = make(map[string]bool) + } + switch eventType { + case "m.room.message": + roomCount[roomID] = false + msgCount.Messages++ + if sender == s.serverName { + msgCount.SentMessages++ + } + case "m.room.encrypted": + roomCount[roomID] = true + msgCount.MessagesE2EE++ + if sender == s.serverName { + msgCount.SentMessagesE2EE++ + } + default: + return + } + s.msgCounts[s.serverName] = msgCount + s.roomCounts[s.serverName] = roomCount + + for serverName, stats := range s.msgCounts { + var normalRooms, encryptedRooms int64 = 0, 0 + for _, isEncrypted := range s.roomCounts[s.serverName] { + if isEncrypted { + encryptedRooms++ + } else { + normalRooms++ + } + } + err := s.db.UpsertDailyRoomsMessages(ctx, serverName, stats, normalRooms, encryptedRooms) + if err != nil { + log.WithError(err).Errorf("failed to upsert daily messages") + } + // Clear stats if we successfully stored it + if err == nil { + stats.Messages = 0 + stats.SentMessages = 0 + stats.MessagesE2EE = 0 + stats.SentMessagesE2EE = 0 + s.msgCounts[serverName] = stats + } + } +} + func (s *OutputRoomEventConsumer) processMessage(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, streamPos uint64) error { members, roomSize, err := s.localRoomMembers(ctx, event.RoomID()) if err != nil { diff --git a/userapi/consumers/roomserver_test.go b/userapi/consumers/roomserver_test.go index e4587670f..265e3a3aa 100644 --- a/userapi/consumers/roomserver_test.go +++ b/userapi/consumers/roomserver_test.go @@ -2,7 +2,10 @@ package consumers import ( "context" + "reflect" + "sync" "testing" + "time" "github.com/matrix-org/gomatrixserverlib" "github.com/stretchr/testify/assert" @@ -12,6 +15,7 @@ import ( "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/userapi/storage" + userAPITypes "github.com/matrix-org/dendrite/userapi/types" ) func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { @@ -132,3 +136,122 @@ func Test_evaluatePushRules(t *testing.T) { } }) } + +func TestMessageStats(t *testing.T) { + type args struct { + eventType string + eventSender string + roomID string + } + tests := []struct { + name string + args args + ourServer gomatrixserverlib.ServerName + lastUpdate time.Time + initRoomCounts map[gomatrixserverlib.ServerName]map[string]bool + wantStats userAPITypes.MessageStats + }{ + { + name: "m.room.create does not count as a message", + ourServer: "localhost", + args: args{ + eventType: "m.room.create", + eventSender: "@alice:localhost", + }, + }, + { + name: "our server - message", + ourServer: "localhost", + args: args{ + eventType: "m.room.message", + eventSender: "@alice:localhost", + roomID: "normalRoom", + }, + wantStats: userAPITypes.MessageStats{Messages: 1, SentMessages: 1}, + }, + { + name: "our server - E2EE message", + ourServer: "localhost", + args: args{ + eventType: "m.room.encrypted", + eventSender: "@alice:localhost", + roomID: "encryptedRoom", + }, + wantStats: userAPITypes.MessageStats{Messages: 1, SentMessages: 1, MessagesE2EE: 1, SentMessagesE2EE: 1}, + }, + + { + name: "remote server - message", + ourServer: "localhost", + args: args{ + eventType: "m.room.message", + eventSender: "@alice:remote", + roomID: "normalRoom", + }, + wantStats: userAPITypes.MessageStats{Messages: 2, SentMessages: 1, MessagesE2EE: 1, SentMessagesE2EE: 1}, + }, + { + name: "remote server - E2EE message", + ourServer: "localhost", + args: args{ + eventType: "m.room.encrypted", + eventSender: "@alice:remote", + roomID: "encryptedRoom", + }, + wantStats: userAPITypes.MessageStats{Messages: 2, SentMessages: 1, MessagesE2EE: 2, SentMessagesE2EE: 1}, + }, + { + name: "day change creates a new room map", + ourServer: "localhost", + lastUpdate: time.Now().Add(-time.Hour * 24), + initRoomCounts: map[gomatrixserverlib.ServerName]map[string]bool{ + "localhost": {"encryptedRoom": true}, + }, + args: args{ + eventType: "m.room.encrypted", + eventSender: "@alice:remote", + roomID: "someOtherRoom", + }, + wantStats: userAPITypes.MessageStats{Messages: 2, SentMessages: 1, MessagesE2EE: 3, SentMessagesE2EE: 1}, + }, + } + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, close := mustCreateDatabase(t, dbType) + defer close() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.lastUpdate.IsZero() { + tt.lastUpdate = time.Now() + } + if tt.initRoomCounts == nil { + tt.initRoomCounts = map[gomatrixserverlib.ServerName]map[string]bool{} + } + s := &OutputRoomEventConsumer{ + db: db, + msgCounts: map[gomatrixserverlib.ServerName]userAPITypes.MessageStats{}, + roomCounts: tt.initRoomCounts, + countsLock: sync.Mutex{}, + lastUpdate: tt.lastUpdate, + serverName: tt.ourServer, + } + s.storeMessageStats(context.Background(), tt.args.eventType, tt.args.eventSender, tt.args.roomID) + t.Logf("%+v", s.roomCounts) + gotStats, activeRooms, activeE2EERooms, err := db.DailyRoomsMessages(context.Background(), tt.ourServer) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + if !reflect.DeepEqual(gotStats, tt.wantStats) { + t.Fatalf("expected %+v, got %+v", tt.wantStats, gotStats) + } + if tt.args.eventType == "m.room.encrypted" && activeE2EERooms != 1 { + t.Fatalf("expected room to be activeE2EE") + } + if tt.args.eventType == "m.room.message" && activeRooms != 1 { + t.Fatalf("expected room to be active") + } + }) + } + }) +} diff --git a/userapi/storage/interface.go b/userapi/storage/interface.go index fb12b53af..28ef26559 100644 --- a/userapi/storage/interface.go +++ b/userapi/storage/interface.go @@ -19,6 +19,8 @@ import ( "encoding/json" "errors" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/pushrules" "github.com/matrix-org/dendrite/userapi/api" @@ -144,6 +146,8 @@ type Database interface { type Statistics interface { UserStatistics(ctx context.Context) (*types.UserStatistics, *types.DatabaseEngine, error) + DailyRoomsMessages(ctx context.Context, serverName gomatrixserverlib.ServerName) (stats types.MessageStats, activeRooms, activeE2EERooms int64, err error) + UpsertDailyRoomsMessages(ctx context.Context, serverName gomatrixserverlib.ServerName, stats types.MessageStats, activeRooms, activeE2EERooms int64) error } // Err3PIDInUse is the error returned when trying to save an association involving diff --git a/userapi/storage/postgres/stats_table.go b/userapi/storage/postgres/stats_table.go index 20eb0bf46..f62467fa4 100644 --- a/userapi/storage/postgres/stats_table.go +++ b/userapi/storage/postgres/stats_table.go @@ -20,13 +20,14 @@ import ( "time" "github.com/lib/pq" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/dendrite/userapi/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/sirupsen/logrus" ) const userDailyVisitsSchema = ` @@ -43,6 +44,35 @@ CREATE INDEX IF NOT EXISTS userapi_daily_visits_timestamp_idx ON userapi_daily_v CREATE INDEX IF NOT EXISTS userapi_daily_visits_localpart_timestamp_idx ON userapi_daily_visits(localpart, timestamp); ` +const messagesDailySchema = ` +CREATE TABLE IF NOT EXISTS userapi_daily_stats ( + timestamp BIGINT NOT NULL, + server_name TEXT NOT NULL, + messages BIGINT NOT NULL, + sent_messages BIGINT NOT NULL, + e2ee_messages BIGINT NOT NULL, + sent_e2ee_messages BIGINT NOT NULL, + active_rooms BIGINT NOT NULL, + active_e2ee_rooms BIGINT NOT NULL, + CONSTRAINT daily_stats_unique UNIQUE (timestamp, server_name) +); +` + +const upsertDailyMessagesSQL = ` + INSERT INTO userapi_daily_stats AS u (timestamp, server_name, messages, sent_messages, e2ee_messages, sent_e2ee_messages, active_rooms, active_e2ee_rooms) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT ON CONSTRAINT daily_stats_unique + DO UPDATE SET + messages=u.messages+excluded.messages, sent_messages=u.sent_messages+excluded.sent_messages, + e2ee_messages=u.e2ee_messages+excluded.e2ee_messages, sent_e2ee_messages=u.sent_e2ee_messages+excluded.sent_e2ee_messages, + active_rooms=GREATEST($7, u.active_rooms), active_e2ee_rooms=GREATEST($8, u.active_e2ee_rooms) +` + +const selectDailyMessagesSQL = ` + SELECT messages, sent_messages, e2ee_messages, sent_e2ee_messages, active_rooms, active_e2ee_rooms + FROM userapi_daily_stats + WHERE server_name = $1 AND timestamp = $2; +` + const countUsersLastSeenAfterSQL = "" + "SELECT COUNT(*) FROM (" + " SELECT localpart FROM userapi_devices WHERE last_seen_ts > $1 " + @@ -170,6 +200,8 @@ type statsStatements struct { countUserByAccountTypeStmt *sql.Stmt countRegisteredUserByTypeStmt *sql.Stmt dbEngineVersionStmt *sql.Stmt + upsertMessagesStmt *sql.Stmt + selectDailyMessagesStmt *sql.Stmt } func NewPostgresStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (tables.StatsTable, error) { @@ -182,6 +214,10 @@ func NewPostgresStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) if err != nil { return nil, err } + _, err = db.Exec(messagesDailySchema) + if err != nil { + return nil, err + } go s.startTimers() return s, sqlutil.StatementList{ {&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL}, @@ -191,6 +227,8 @@ func NewPostgresStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) {&s.countUserByAccountTypeStmt, countUserByAccountTypeSQL}, {&s.countRegisteredUserByTypeStmt, countRegisteredUserByTypeStmt}, {&s.dbEngineVersionStmt, queryDBEngineVersion}, + {&s.upsertMessagesStmt, upsertDailyMessagesSQL}, + {&s.selectDailyMessagesStmt, selectDailyMessagesSQL}, }.Prepare(db) } @@ -435,3 +473,34 @@ func (s *statsStatements) UpdateUserDailyVisits( } return err } + +func (s *statsStatements) UpsertDailyStats( + ctx context.Context, txn *sql.Tx, + serverName gomatrixserverlib.ServerName, stats types.MessageStats, + activeRooms, activeE2EERooms int64, +) error { + stmt := sqlutil.TxStmt(txn, s.upsertMessagesStmt) + timestamp := time.Now().Truncate(time.Hour * 24) + _, err := stmt.ExecContext(ctx, + gomatrixserverlib.AsTimestamp(timestamp), + serverName, + stats.Messages, stats.SentMessages, stats.MessagesE2EE, stats.SentMessagesE2EE, + activeRooms, activeE2EERooms, + ) + return err +} + +func (s *statsStatements) DailyRoomsMessages( + ctx context.Context, txn *sql.Tx, + serverName gomatrixserverlib.ServerName, +) (msgStats types.MessageStats, activeRooms, activeE2EERooms int64, err error) { + stmt := sqlutil.TxStmt(txn, s.selectDailyMessagesStmt) + timestamp := time.Now().Truncate(time.Hour * 24) + + err = stmt.QueryRowContext(ctx, serverName, gomatrixserverlib.AsTimestamp(timestamp)). + Scan(&msgStats.Messages, &msgStats.SentMessages, &msgStats.MessagesE2EE, &msgStats.SentMessagesE2EE, &activeRooms, &activeE2EERooms) + if err != nil && err != sql.ErrNoRows { + return msgStats, 0, 0, err + } + return msgStats, activeRooms, activeE2EERooms, nil +} diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index f8b6ad311..f8b8d02c9 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -29,13 +29,12 @@ import ( "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" - "github.com/matrix-org/dendrite/userapi/types" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/pushrules" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/storage/tables" + "github.com/matrix-org/dendrite/userapi/types" ) // Database represents an account database @@ -808,3 +807,15 @@ func (d *Database) RemovePushers( func (d *Database) UserStatistics(ctx context.Context) (*types.UserStatistics, *types.DatabaseEngine, error) { return d.Stats.UserStatistics(ctx, nil) } + +func (d *Database) UpsertDailyRoomsMessages(ctx context.Context, serverName gomatrixserverlib.ServerName, stats types.MessageStats, activeRooms, activeE2EERooms int64) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + return d.Stats.UpsertDailyStats(ctx, txn, serverName, stats, activeRooms, activeE2EERooms) + }) +} + +func (d *Database) DailyRoomsMessages( + ctx context.Context, serverName gomatrixserverlib.ServerName, +) (stats types.MessageStats, activeRooms, activeE2EERooms int64, err error) { + return d.Stats.DailyRoomsMessages(ctx, nil, serverName) +} diff --git a/userapi/storage/sqlite3/stats_table.go b/userapi/storage/sqlite3/stats_table.go index 35e3c653e..a1365c944 100644 --- a/userapi/storage/sqlite3/stats_table.go +++ b/userapi/storage/sqlite3/stats_table.go @@ -44,6 +44,35 @@ CREATE INDEX IF NOT EXISTS userapi_daily_visits_timestamp_idx ON userapi_daily_v CREATE INDEX IF NOT EXISTS userapi_daily_visits_localpart_timestamp_idx ON userapi_daily_visits(localpart, timestamp); ` +const messagesDailySchema = ` +CREATE TABLE IF NOT EXISTS userapi_daily_stats ( + timestamp BIGINT NOT NULL, + server_name TEXT NOT NULL, + messages BIGINT NOT NULL, + sent_messages BIGINT NOT NULL, + e2ee_messages BIGINT NOT NULL, + sent_e2ee_messages BIGINT NOT NULL, + active_rooms BIGINT NOT NULL, + active_e2ee_rooms BIGINT NOT NULL, + CONSTRAINT daily_stats_unique UNIQUE (timestamp, server_name) +); +` + +const upsertDailyMessagesSQL = ` + INSERT INTO userapi_daily_stats (timestamp, server_name, messages, sent_messages, e2ee_messages, sent_e2ee_messages, active_rooms, active_e2ee_rooms) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (timestamp, server_name) + DO UPDATE SET + messages=messages+excluded.messages, sent_messages=sent_messages+excluded.sent_messages, + e2ee_messages=e2ee_messages+excluded.e2ee_messages, sent_e2ee_messages=sent_e2ee_messages+excluded.sent_e2ee_messages, + active_rooms=MAX($7, active_rooms), active_e2ee_rooms=MAX($8, active_e2ee_rooms) +` + +const selectDailyMessagesSQL = ` + SELECT messages, sent_messages, e2ee_messages, sent_e2ee_messages, active_rooms, active_e2ee_rooms + FROM userapi_daily_stats + WHERE server_name = $1 AND timestamp = $2; +` + const countUsersLastSeenAfterSQL = "" + "SELECT COUNT(*) FROM (" + " SELECT localpart FROM userapi_devices WHERE last_seen_ts > $1 " + @@ -176,6 +205,8 @@ type statsStatements struct { countUserByAccountTypeStmt *sql.Stmt countRegisteredUserByTypeStmt *sql.Stmt dbEngineVersionStmt *sql.Stmt + upsertMessagesStmt *sql.Stmt + selectDailyMessagesStmt *sql.Stmt } func NewSQLiteStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (tables.StatsTable, error) { @@ -189,6 +220,10 @@ func NewSQLiteStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (t if err != nil { return nil, err } + _, err = db.Exec(messagesDailySchema) + if err != nil { + return nil, err + } go s.startTimers() return s, sqlutil.StatementList{ {&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL}, @@ -198,6 +233,8 @@ func NewSQLiteStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (t {&s.countUserByAccountTypeStmt, countUserByAccountTypeSQL}, {&s.countRegisteredUserByTypeStmt, countRegisteredUserByTypeSQL}, {&s.dbEngineVersionStmt, queryDBEngineVersion}, + {&s.upsertMessagesStmt, upsertDailyMessagesSQL}, + {&s.selectDailyMessagesStmt, selectDailyMessagesSQL}, }.Prepare(db) } @@ -451,3 +488,34 @@ func (s *statsStatements) UpdateUserDailyVisits( } return err } + +func (s *statsStatements) UpsertDailyStats( + ctx context.Context, txn *sql.Tx, + serverName gomatrixserverlib.ServerName, stats types.MessageStats, + activeRooms, activeE2EERooms int64, +) error { + stmt := sqlutil.TxStmt(txn, s.upsertMessagesStmt) + timestamp := time.Now().Truncate(time.Hour * 24) + _, err := stmt.ExecContext(ctx, + gomatrixserverlib.AsTimestamp(timestamp), + serverName, + stats.Messages, stats.SentMessages, stats.MessagesE2EE, stats.SentMessagesE2EE, + activeRooms, activeE2EERooms, + ) + return err +} + +func (s *statsStatements) DailyRoomsMessages( + ctx context.Context, txn *sql.Tx, + serverName gomatrixserverlib.ServerName, +) (msgStats types.MessageStats, activeRooms, activeE2EERooms int64, err error) { + stmt := sqlutil.TxStmt(txn, s.selectDailyMessagesStmt) + timestamp := time.Now().Truncate(time.Hour * 24) + + err = stmt.QueryRowContext(ctx, serverName, gomatrixserverlib.AsTimestamp(timestamp)). + Scan(&msgStats.Messages, &msgStats.SentMessages, &msgStats.MessagesE2EE, &msgStats.SentMessagesE2EE, &activeRooms, &activeE2EERooms) + if err != nil && err != sql.ErrNoRows { + return msgStats, 0, 0, err + } + return msgStats, activeRooms, activeE2EERooms, nil +} diff --git a/userapi/storage/tables/interface.go b/userapi/storage/tables/interface.go index 1b239e442..5e1dd0971 100644 --- a/userapi/storage/tables/interface.go +++ b/userapi/storage/tables/interface.go @@ -20,6 +20,8 @@ import ( "encoding/json" "time" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/types" @@ -115,7 +117,9 @@ type NotificationTable interface { type StatsTable interface { UserStatistics(ctx context.Context, txn *sql.Tx) (*types.UserStatistics, *types.DatabaseEngine, error) + DailyRoomsMessages(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName) (msgStats types.MessageStats, activeRooms, activeE2EERooms int64, err error) UpdateUserDailyVisits(ctx context.Context, txn *sql.Tx, startTime, lastUpdate time.Time) error + UpsertDailyStats(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, stats types.MessageStats, activeRooms, activeE2EERooms int64) error } type NotificationFilter uint32 diff --git a/userapi/storage/tables/stats_table_test.go b/userapi/storage/tables/stats_table_test.go index c4aec552c..a547423bc 100644 --- a/userapi/storage/tables/stats_table_test.go +++ b/userapi/storage/tables/stats_table_test.go @@ -8,6 +8,9 @@ import ( "testing" "time" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" @@ -16,8 +19,6 @@ import ( "github.com/matrix-org/dendrite/userapi/storage/sqlite3" "github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/dendrite/userapi/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) func mustMakeDBs(t *testing.T, dbType test.DBType) ( @@ -227,7 +228,7 @@ func Test_UserStatistics(t *testing.T) { mustUserUpdateRegistered(t, ctx, db, "user4", time.Now().AddDate(0, -2, 0)) mustUpdateDeviceLastSeen(t, ctx, db, "user4", time.Now()) startTime := time.Now().AddDate(0, 0, -2) - err := statsDB.UpdateUserDailyVisits(ctx, nil, startTime, startTime.Truncate(time.Hour*24).Add(time.Hour)) + err := statsDB.UpdateUserDailyVisits(ctx, nil, startTime, startTime.Truncate(time.Hour*24)) if err != nil { t.Fatalf("unable to update daily visits stats: %v", err) } @@ -278,7 +279,7 @@ func Test_UserStatistics(t *testing.T) { mustUpdateDeviceLastSeen(t, ctx, db, "user1", time.Now().AddDate(0, 0, -i)) mustUpdateDeviceLastSeen(t, ctx, db, "user5", time.Now().AddDate(0, 0, -i)) startTime := time.Now().AddDate(0, 0, -i) - err := statsDB.UpdateUserDailyVisits(ctx, nil, startTime, startTime.Truncate(time.Hour*24).Add(time.Hour)) + err := statsDB.UpdateUserDailyVisits(ctx, nil, startTime, startTime.Truncate(time.Hour*24)) if err != nil { t.Fatalf("unable to update daily visits stats: %v", err) } diff --git a/userapi/types/statistics.go b/userapi/types/statistics.go index 09564f78f..b74e32add 100644 --- a/userapi/types/statistics.go +++ b/userapi/types/statistics.go @@ -28,3 +28,10 @@ type DatabaseEngine struct { Engine string Version string } + +type MessageStats struct { + Messages int64 + SentMessages int64 + MessagesE2EE int64 + SentMessagesE2EE int64 +} diff --git a/userapi/util/phonehomestats.go b/userapi/util/phonehomestats.go index b17f62060..6f36568c9 100644 --- a/userapi/util/phonehomestats.go +++ b/userapi/util/phonehomestats.go @@ -24,11 +24,12 @@ import ( "syscall" "time" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/storage" - "github.com/matrix-org/gomatrixserverlib" - "github.com/sirupsen/logrus" ) type phoneHomeStats struct { @@ -109,12 +110,19 @@ func (p *phoneHomeStats) collect() { } // message and room stats - // TODO: Find a solution to actually set these values + // TODO: Find a solution to actually set this value p.stats["total_room_count"] = 0 - p.stats["daily_messages"] = 0 - p.stats["daily_sent_messages"] = 0 - p.stats["daily_e2ee_messages"] = 0 - p.stats["daily_sent_e2ee_messages"] = 0 + + messageStats, activeRooms, activeE2EERooms, err := p.db.DailyRoomsMessages(ctx, p.serverName) + if err != nil { + logrus.WithError(err).Warn("unable to query message stats, using default values") + } + p.stats["daily_messages"] = messageStats.Messages + p.stats["daily_sent_messages"] = messageStats.SentMessages + p.stats["daily_e2ee_messages"] = messageStats.MessagesE2EE + p.stats["daily_sent_e2ee_messages"] = messageStats.SentMessagesE2EE + p.stats["daily_active_rooms"] = activeRooms + p.stats["daily_active_e2ee_rooms"] = activeE2EERooms // user stats and DB engine userStats, db, err := p.db.UserStatistics(ctx)