mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2026-01-16 18:43:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

subtree pull from dendrite fork for fix #2383 (#802)

Browse source 
Pull the fix for https://github.com/matrix-org/dendrite/issues/2838 into
the dendrite subtree. Includes latest dendrite main.

Signed-off-by: `Devon Hudson <devonhudson@librem.one>`
Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: Brian Meek <brian@hntlabs.com>
Signed-off-by: Austin Ellis <austin@hntlabs.com>
Signed-off-by: `Rubin Poster <rubinposter@gmail.com>`
Signed-off-by: `ash lea <example@thisismyactual.email>`
Co-authored-by: Neil Alexander <neilalexander@users.noreply.github.com>
Co-authored-by: devonh <devon.dmytro@gmail.com>
Co-authored-by: Till <2353100+S7evinK@users.noreply.github.com>
Co-authored-by: Ashley Nelson <fant@shley.email>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Brian Meek <brian@hntlabs.com>
Co-authored-by: Tak Wai Wong <takwaiw@gmail.com>
Co-authored-by: Jean Lucas <jean@4ray.co>
Co-authored-by: Kabir Kwatra <kabir@kwatra.me>
Co-authored-by: sergekh2 <gitgitgit@khorun.com>
Co-authored-by: texuf <texuf.eth@gmail.com>
Co-authored-by: kegsay <kegan@matrix.org>
Co-authored-by: Brian Meek <brian@here.video>
Co-authored-by: John Terzis <john.c.terzis@gmail.com>
Co-authored-by: John Terzis <john@hntlabs.com>
Co-authored-by: Kerem <kerem.kazan@gmail.com>
Co-authored-by: Neboer <43609792+Neboer@users.noreply.github.com>
Co-authored-by: X. Ding <dingsm@gmail.com>
Co-authored-by: dxl <dxl@plotbridge.com>
Co-authored-by: ash lea <ashkitten@users.noreply.github.com>
Co-authored-by: 0x1a8510f2 <admin@0x1a8510f2.space>
Co-authored-by: Till Faelligen <tfaelligen@gmail.com>
Co-authored-by: Till Faelligen <davidf@element.io>
This commit is contained in:
Tak Wai Wong 2022-11-02 16:28:06 -07:00 committed by GitHub
parent b4223a1a18
commit a9d3bdc058
69 changed files with 1639 additions and 371 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/PULL_REQUEST_TEMPLATE.md vendored
View file

@ -2,7 +2,7 @@
<!-- Please read https://matrix-org.github.io/dendrite/development/contributing before submitting your pull request --> <!-- Please read https://matrix-org.github.io/dendrite/development/contributing before submitting your pull request -->
* [ ] I have added tests for PR _or_ I have justified why this PR doesn't need tests. * [ ] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests
* [ ] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately * [ ] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately
Signed-off-by: `Your Name <your@email.example.org>` Signed-off-by: `Your Name <your@email.example.org>`

30
.github/workflows/dendrite.yml vendored
View file

@ -269,11 +269,18 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
include: include:
- label: SQLite - label: SQLite native
- label: SQLite, full HTTP APIs - label: SQLite Cgo
cgo: 1
- label: SQLite native, full HTTP APIs
api: full-http api: full-http
- label: SQLite Cgo, full HTTP APIs
api: full-http
cgo: 1
- label: PostgreSQL - label: PostgreSQL
postgres: postgres postgres: postgres
@ -288,8 +295,9 @@ jobs:
POSTGRES: ${{ matrix.postgres && 1}} POSTGRES: ${{ matrix.postgres && 1}}
API: ${{ matrix.api && 1 }} API: ${{ matrix.api && 1 }}
SYTEST_BRANCH: ${{ github.head_ref }} SYTEST_BRANCH: ${{ github.head_ref }}
CGO_ENABLED: ${{ matrix.cgo && 1 }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
- name: Run Sytest - name: Run Sytest
run: /bootstrap.sh dendrite run: /bootstrap.sh dendrite
working-directory: /src working-directory: /src
@ -323,11 +331,18 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
include: include:
- label: SQLite - label: SQLite native
- label: SQLite, full HTTP APIs - label: SQLite Cgo
cgo: 1
- label: SQLite native, full HTTP APIs
api: full-http api: full-http
- label: SQLite Cgo, full HTTP APIs
api: full-http
cgo: 1
- label: PostgreSQL - label: PostgreSQL
postgres: Postgres postgres: Postgres
@ -349,8 +364,8 @@ jobs:
sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
- name: Run actions/checkout@v2 for dendrite - name: Run actions/checkout@v3 for dendrite
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
path: dendrite path: dendrite
@ -393,6 +408,7 @@ jobs:
env: env:
COMPLEMENT_BASE_IMAGE: complement-dendrite:latest COMPLEMENT_BASE_IMAGE: complement-dendrite:latest
API: ${{ matrix.api && 1 }} API: ${{ matrix.api && 1 }}
CGO_ENABLED: ${{ matrix.cgo && 1 }}
working-directory: complement working-directory: complement
integration-tests-done: integration-tests-done:

184
.github/workflows/docker.yml vendored
View file

@ -26,21 +26,26 @@ jobs:
packages: write packages: write
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Get release tag - name: Get release tag & build flags
if: github.event_name == 'release' # Only for GitHub releases if: github.event_name == 'release' # Only for GitHub releases
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
- name: Set up QEMU - name: Set up QEMU
uses: docker/setup-qemu-action@v1 uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx - name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1 uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub - name: Login to Docker Hub
uses: docker/login-action@v1 uses: docker/login-action@v2
with: with:
username: ${{ env.DOCKER_HUB_USER }} username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }} password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers - name: Login to GitHub Containers
uses: docker/login-action@v1 uses: docker/login-action@v2
with: with:
registry: ghcr.io registry: ghcr.io
username: ${{ github.repository_owner }} username: ${{ github.repository_owner }}
@ -49,27 +54,41 @@ jobs:
- name: Build main monolith image - name: Build main monolith image
if: github.ref_name == 'main' if: github.ref_name == 'main'
id: docker_build_monolith id: docker_build_monolith
uses: docker/build-push-action@v2 uses: docker/build-push-action@v3
with: with:
cache-from: type=gha cache-from: type=gha
cache-to: type=gha,mode=max cache-to: type=gha,mode=max
context: . context: .
file: ./build/docker/Dockerfile.monolith build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: monolith
platforms: ${{ env.PLATFORMS }} platforms: ${{ env.PLATFORMS }}
push: true push: true
tags: | tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
format: "sarif"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v1
with:
sarif_file: "trivy-results.sarif"
- name: Build release monolith image - name: Build release monolith image
if: github.event_name == 'release' # Only for GitHub releases if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_monolith_release id: docker_build_monolith_release
uses: docker/build-push-action@v2 uses: docker/build-push-action@v3
with: with:
cache-from: type=gha cache-from: type=gha
cache-to: type=gha,mode=max cache-to: type=gha,mode=max
context: . context: .
file: ./build/docker/Dockerfile.monolith build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: monolith
platforms: ${{ env.PLATFORMS }} platforms: ${{ env.PLATFORMS }}
push: true push: true
build-args: | build-args: |
@ -88,21 +107,26 @@ jobs:
packages: write packages: write
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Get release tag - name: Get release tag & build flags
if: github.event_name == 'release' # Only for GitHub releases if: github.event_name == 'release' # Only for GitHub releases
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
- name: Set up QEMU - name: Set up QEMU
uses: docker/setup-qemu-action@v1 uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx - name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1 uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub - name: Login to Docker Hub
uses: docker/login-action@v1 uses: docker/login-action@v2
with: with:
username: ${{ env.DOCKER_HUB_USER }} username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }} password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers - name: Login to GitHub Containers
uses: docker/login-action@v1 uses: docker/login-action@v2
with: with:
registry: ghcr.io registry: ghcr.io
username: ${{ github.repository_owner }} username: ${{ github.repository_owner }}
@ -111,27 +135,41 @@ jobs:
- name: Build main polylith image - name: Build main polylith image
if: github.ref_name == 'main' if: github.ref_name == 'main'
id: docker_build_polylith id: docker_build_polylith
uses: docker/build-push-action@v2 uses: docker/build-push-action@v3
with: with:
cache-from: type=gha cache-from: type=gha
cache-to: type=gha,mode=max cache-to: type=gha,mode=max
context: . context: .
file: ./build/docker/Dockerfile.polylith build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: polylith
platforms: ${{ env.PLATFORMS }} platforms: ${{ env.PLATFORMS }}
push: true push: true
tags: | tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
format: "sarif"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v1
with:
sarif_file: "trivy-results.sarif"
- name: Build release polylith image - name: Build release polylith image
if: github.event_name == 'release' # Only for GitHub releases if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_polylith_release id: docker_build_polylith_release
uses: docker/build-push-action@v2 uses: docker/build-push-action@v3
with: with:
cache-from: type=gha cache-from: type=gha
cache-to: type=gha,mode=max cache-to: type=gha,mode=max
context: . context: .
file: ./build/docker/Dockerfile.polylith build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: polylith
platforms: ${{ env.PLATFORMS }} platforms: ${{ env.PLATFORMS }}
push: true push: true
tags: | tags: |
@ -148,54 +186,128 @@ jobs:
packages: write packages: write
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Get release tag - name: Get release tag & build flags
if: github.event_name == 'release' # Only for GitHub releases if: github.event_name == 'release' # Only for GitHub releases
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
- name: Set up QEMU - name: Set up QEMU
uses: docker/setup-qemu-action@v1 uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx - name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1 uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub - name: Login to Docker Hub
uses: docker/login-action@v1 uses: docker/login-action@v2
with: with:
username: ${{ env.DOCKER_HUB_USER }} username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }} password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers - name: Login to GitHub Containers
uses: docker/login-action@v1 uses: docker/login-action@v2
with: with:
registry: ghcr.io registry: ghcr.io
username: ${{ github.repository_owner }} username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }} password: ${{ secrets.GITHUB_TOKEN }}
- name: Build main pinecone demo image - name: Build main Pinecone demo image
if: github.ref_name == 'main' if: github.ref_name == 'main'
id: docker_build_demo_pinecone id: docker_build_demo_pinecone
uses: docker/build-push-action@v2 uses: docker/build-push-action@v3
with: with:
cache-from: type=gha cache-from: type=gha
cache-to: type=gha,mode=max cache-to: type=gha,mode=max
context: . context: .
file: ./build/docker/Dockerfile.demo-pinecone build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: demo-pinecone
platforms: ${{ env.PLATFORMS }} platforms: ${{ env.PLATFORMS }}
push: true push: true
tags: | tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }} ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }}
- name: Build release pinecone demo image - name: Build release Pinecone demo image
if: github.event_name == 'release' # Only for GitHub releases if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_demo_pinecone_release id: docker_build_demo_pinecone_release
uses: docker/build-push-action@v2 uses: docker/build-push-action@v3
with: with:
cache-from: type=gha cache-from: type=gha
cache-to: type=gha,mode=max cache-to: type=gha,mode=max
context: . context: .
file: ./build/docker/Dockerfile.demo-pinecone build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: demo-pinecone
platforms: ${{ env.PLATFORMS }} platforms: ${{ env.PLATFORMS }}
push: true push: true
tags: | tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:latest ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ env.RELEASE_VERSION }} ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:latest ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ env.RELEASE_VERSION }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}
demo-yggdrasil:
name: Yggdrasil demo image
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Get release tag & build flags
if: github.event_name == 'release' # Only for GitHub releases
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build main Yggdrasil demo image
if: github.ref_name == 'main'
id: docker_build_demo_yggdrasil
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: demo-yggdrasil
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }}
- name: Build release Yggdrasil demo image
if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_demo_yggdrasil_release
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: demo-yggdrasil
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}

14
CHANGES.md
View file

@ -1,5 +1,19 @@
# Changelog # Changelog
## Dendrite 0.10.6 (2022-11-01)
### Features
* History visibility checks have been optimised, which should speed up response times on a variety of endpoints (including `/sync`, `/messages`, `/context` and others) and reduce database load
* The built-in NATS Server has been updated to version 2.9.4
* Some other minor dependencies have been updated
### Fixes
* A panic has been fixed in the sync API PDU stream which could cause requests to fail
* The `/members` response now contains the `room_id` field, which may fix some E2EE problems with clients using the JS SDK (contributed by [ashkitten](https://github.com/ashkitten))
* The auth difference calculation in state resolution v2 has been tweaked for clarity (and moved into gomatrixserverlib with the rest of the state resolution code)
## Dendrite 0.10.5 (2022-10-31) ## Dendrite 0.10.5 (2022-10-31)
### Features ### Features

122
Dockerfile Normal file
View file

@ -0,0 +1,122 @@
#syntax=docker/dockerfile:1.2
#
# base installs required dependencies and runs go mod download to cache dependencies
#
FROM --platform=${BUILDPLATFORM} docker.io/golang:1.19-alpine AS base
RUN apk --update --no-cache add bash build-base curl
#
# build creates all needed binaries
#
FROM --platform=${BUILDPLATFORM} base AS build
WORKDIR /src
ARG TARGETOS
ARG TARGETARCH
ARG FLAGS
RUN --mount=target=. \
--mount=type=cache,target=/root/.cache/go-build \
USERARCH=`go env GOARCH` \
GOARCH="$TARGETARCH" \
GOOS="linux" \
CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0") \
go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/...
#
# The dendrite base image; mainly creates a user and switches to it
#
FROM alpine:latest AS dendrite-base
LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
LABEL org.opencontainers.image.licenses="Apache-2.0"
LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/"
LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C."
RUN addgroup dendrite && adduser dendrite -G dendrite -u 1337 -D
USER dendrite
WORKDIR /home/dendrite
#
# Builds the polylith image and only contains the polylith binary
#
FROM dendrite-base AS polylith
LABEL org.opencontainers.image.title="Dendrite (Polylith)"
COPY --from=build /out/dendrite-polylith-multi /usr/bin/
ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]
#
# Builds the monolith image and contains all required binaries
#
FROM dendrite-base AS monolith
LABEL org.opencontainers.image.title="Dendrite (Monolith)"
COPY --from=build /out/create-account /usr/bin/create-account
COPY --from=build /out/generate-config /usr/bin/generate-config
COPY --from=build /out/generate-keys /usr/bin/generate-keys
COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server
ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
EXPOSE 8008 8448
#
# Builds the Pinecone P2P demo image and contains all required binaries
#
FROM dendrite-base AS demo-pinecone
LABEL org.opencontainers.image.title="Dendrite (Pinecone P2P Demo)"
COPY --from=build /out/create-account /usr/bin/create-account
COPY --from=build /out/generate-config /usr/bin/generate-config
COPY --from=build /out/generate-keys /usr/bin/generate-keys
COPY --from=build /out/dendrite-demo-pinecone /usr/bin/dendrite-demo-pinecone
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"]
EXPOSE 8008 8448
#
# Builds the Yggdrasil P2P demo image and contains all required binaries
#
FROM dendrite-base AS demo-yggdrasil
LABEL org.opencontainers.image.title="Dendrite (Yggdrasil P2P Demo)"
COPY --from=build /out/create-account /usr/bin/create-account
COPY --from=build /out/generate-config /usr/bin/generate-config
COPY --from=build /out/generate-keys /usr/bin/generate-keys
COPY --from=build /out/dendrite-demo-yggdrasil /usr/bin/dendrite-demo-yggdrasil
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-demo-yggdrasil"]
EXPOSE 8008 8448
#
# Builds the Complement image, used for integration tests
#
FROM base AS complement
LABEL org.opencontainers.image.title="Dendrite (Complement)"
RUN apk add --no-cache sqlite openssl ca-certificates
COPY --from=build /out/generate-config /usr/bin/generate-config
COPY --from=build /out/generate-keys /usr/bin/generate-keys
COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server
WORKDIR /dendrite
RUN /usr/bin/generate-keys --private-key matrix_key.pem && \
mkdir /ca && \
openssl genrsa -out /ca/ca.key 2048 && \
openssl req -new -x509 -key /ca/ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out /ca/ca.crt
ENV SERVER_NAME=localhost
ENV API=0
EXPOSE 8008 8448
# At runtime, generate TLS cert based on the CA now mounted at /ca
# At runtime, replace the SERVER_NAME with what we are told
CMD /usr/bin/generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /ca/ca.crt --tls-authority-key /ca/ca.key && \
/usr/bin/generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
cp /ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
/usr/bin/dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}

75
appservice/api/query.go
View file

@ -19,11 +19,13 @@ package api
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/authtypes"
userapi "github.com/matrix-org/dendrite/userapi/api" userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
) )
// AppServiceInternalAPI is used to query user and room alias data from application // AppServiceInternalAPI is used to query user and room alias data from application
@ -41,6 +43,10 @@ type AppServiceInternalAPI interface {
req *UserIDExistsRequest, req *UserIDExistsRequest,
resp *UserIDExistsResponse, resp *UserIDExistsResponse,
) error ) error
Locations(ctx context.Context, req *LocationRequest, resp *LocationResponse) error
User(ctx context.Context, request *UserRequest, response *UserResponse) error
Protocols(ctx context.Context, req *ProtocolRequest, resp *ProtocolResponse) error
} }
// RoomAliasExistsRequest is a request to an application service // RoomAliasExistsRequest is a request to an application service
@ -77,6 +83,73 @@ type UserIDExistsResponse struct {
UserIDExists bool `json:"exists"` UserIDExists bool `json:"exists"`
} }
const (
ASProtocolPath = "/_matrix/app/unstable/thirdparty/protocol/"
ASUserPath = "/_matrix/app/unstable/thirdparty/user"
ASLocationPath = "/_matrix/app/unstable/thirdparty/location"
)
type ProtocolRequest struct {
Protocol string `json:"protocol,omitempty"`
}
type ProtocolResponse struct {
Protocols map[string]ASProtocolResponse `json:"protocols"`
Exists bool `json:"exists"`
}
type ASProtocolResponse struct {
FieldTypes map[string]FieldType `json:"field_types,omitempty"` // NOTSPEC: field_types is required by the spec
Icon string `json:"icon"`
Instances []ProtocolInstance `json:"instances"`
LocationFields []string `json:"location_fields"`
UserFields []string `json:"user_fields"`
}
type FieldType struct {
Placeholder string `json:"placeholder"`
Regexp string `json:"regexp"`
}
type ProtocolInstance struct {
Description string `json:"desc"`
Icon string `json:"icon,omitempty"`
NetworkID string `json:"network_id,omitempty"` // NOTSPEC: network_id is required by the spec
Fields json.RawMessage `json:"fields,omitempty"` // NOTSPEC: fields is required by the spec
}
type UserRequest struct {
Protocol string `json:"protocol"`
Params string `json:"params"`
}
type UserResponse struct {
Users []ASUserResponse `json:"users,omitempty"`
Exists bool `json:"exists,omitempty"`
}
type ASUserResponse struct {
Protocol string `json:"protocol"`
UserID string `json:"userid"`
Fields json.RawMessage `json:"fields"`
}
type LocationRequest struct {
Protocol string `json:"protocol"`
Params string `json:"params"`
}
type LocationResponse struct {
Locations []ASLocationResponse `json:"locations,omitempty"`
Exists bool `json:"exists,omitempty"`
}
type ASLocationResponse struct {
Alias string `json:"alias"`
Protocol string `json:"protocol"`
Fields json.RawMessage `json:"fields"`
}
// RetrieveUserProfile is a wrapper that queries both the local database and // RetrieveUserProfile is a wrapper that queries both the local database and
// application services for a given user's profile // application services for a given user's profile
// TODO: Remove this, it's called from federationapi and clientapi but is a pure function // TODO: Remove this, it's called from federationapi and clientapi but is a pure function

3
appservice/appservice.go
View file

@ -18,6 +18,7 @@ import (
"context" "context"
"crypto/tls" "crypto/tls"
"net/http" "net/http"
"sync"
"time" "time"
"github.com/gorilla/mux" "github.com/gorilla/mux"
@ -60,6 +61,8 @@ func NewInternalAPI(
appserviceQueryAPI := &query.AppServiceQueryAPI{ appserviceQueryAPI := &query.AppServiceQueryAPI{
HTTPClient: client, HTTPClient: client,
Cfg: &base.Cfg.AppServiceAPI, Cfg: &base.Cfg.AppServiceAPI,
ProtocolCache: map[string]appserviceAPI.ASProtocolResponse{},
CacheMu: sync.Mutex{},
} }
if len(base.Cfg.Derived.ApplicationServices) == 0 { if len(base.Cfg.Derived.ApplicationServices) == 0 {

24
appservice/inthttp/client.go
View file

@ -13,6 +13,9 @@ import (
const ( const (
AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists" AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists"
AppServiceUserIDExistsPath = "/appservice/UserIDExists" AppServiceUserIDExistsPath = "/appservice/UserIDExists"
AppServiceLocationsPath = "/appservice/locations"
AppServiceUserPath = "/appservice/users"
AppServiceProtocolsPath = "/appservice/protocols"
) )
// httpAppServiceQueryAPI contains the URL to an appservice query API and a // httpAppServiceQueryAPI contains the URL to an appservice query API and a
@ -58,3 +61,24 @@ func (h *httpAppServiceQueryAPI) UserIDExists(
h.httpClient, ctx, request, response, h.httpClient, ctx, request, response,
) )
} }
func (h *httpAppServiceQueryAPI) Locations(ctx context.Context, request *api.LocationRequest, response *api.LocationResponse) error {
return httputil.CallInternalRPCAPI(
"ASLocation", h.appserviceURL+AppServiceLocationsPath,
h.httpClient, ctx, request, response,
)
}
func (h *httpAppServiceQueryAPI) User(ctx context.Context, request *api.UserRequest, response *api.UserResponse) error {
return httputil.CallInternalRPCAPI(
"ASUser", h.appserviceURL+AppServiceUserPath,
h.httpClient, ctx, request, response,
)
}
func (h *httpAppServiceQueryAPI) Protocols(ctx context.Context, request *api.ProtocolRequest, response *api.ProtocolResponse) error {
return httputil.CallInternalRPCAPI(
"ASProtocols", h.appserviceURL+AppServiceProtocolsPath,
h.httpClient, ctx, request, response,
)
}

16
appservice/inthttp/server.go
View file

@ -2,6 +2,7 @@ package inthttp
import ( import (
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/httputil"
) )
@ -17,4 +18,19 @@ func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) {
AppServiceUserIDExistsPath, AppServiceUserIDExistsPath,
httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists), httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists),
) )
internalAPIMux.Handle(
AppServiceProtocolsPath,
httputil.MakeInternalRPCAPI("AppserviceProtocols", a.Protocols),
)
internalAPIMux.Handle(
AppServiceLocationsPath,
httputil.MakeInternalRPCAPI("AppserviceLocations", a.Locations),
)
internalAPIMux.Handle(
AppServiceUserPath,
httputil.MakeInternalRPCAPI("AppserviceUser", a.User),
)
} }

186
appservice/query/query.go
View file

@ -18,13 +18,18 @@ package query
import ( import (
"context" "context"
"encoding/json"
"io"
"net/http" "net/http"
"net/url" "net/url"
"strings"
"sync"
"github.com/opentracing/opentracing-go"
log "github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/config"
opentracing "github.com/opentracing/opentracing-go"
log "github.com/sirupsen/logrus"
) )
const roomAliasExistsPath = "/rooms/" const roomAliasExistsPath = "/rooms/"
@ -34,6 +39,8 @@ const userIDExistsPath = "/users/"
type AppServiceQueryAPI struct { type AppServiceQueryAPI struct {
HTTPClient *http.Client HTTPClient *http.Client
Cfg *config.AppServiceAPI Cfg *config.AppServiceAPI
ProtocolCache map[string]api.ASProtocolResponse
CacheMu sync.Mutex
} }
// RoomAliasExists performs a request to '/room/{roomAlias}' on all known // RoomAliasExists performs a request to '/room/{roomAlias}' on all known
@ -165,3 +172,178 @@ func (a *AppServiceQueryAPI) UserIDExists(
response.UserIDExists = false response.UserIDExists = false
return nil return nil
} }
type thirdpartyResponses interface {
api.ASProtocolResponse | []api.ASUserResponse | []api.ASLocationResponse
}
func requestDo[T thirdpartyResponses](client *http.Client, url string, response *T) (err error) {
origURL := url
// try v1 and unstable appservice endpoints
for _, version := range []string{"v1", "unstable"} {
var resp *http.Response
var body []byte
asURL := strings.Replace(origURL, "unstable", version, 1)
resp, err = client.Get(asURL)
if err != nil {
continue
}
defer resp.Body.Close() // nolint: errcheck
body, err = io.ReadAll(resp.Body)
if err != nil {
continue
}
return json.Unmarshal(body, &response)
}
return err
}
func (a *AppServiceQueryAPI) Locations(
ctx context.Context,
req *api.LocationRequest,
resp *api.LocationResponse,
) error {
params, err := url.ParseQuery(req.Params)
if err != nil {
return err
}
for _, as := range a.Cfg.Derived.ApplicationServices {
var asLocations []api.ASLocationResponse
params.Set("access_token", as.HSToken)
url := as.URL + api.ASLocationPath
if req.Protocol != "" {
url += "/" + req.Protocol
}
if err := requestDo[[]api.ASLocationResponse](a.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil {
log.WithError(err).Error("unable to get 'locations' from application service")
continue
}
resp.Locations = append(resp.Locations, asLocations...)
}
if len(resp.Locations) == 0 {
resp.Exists = false
return nil
}
resp.Exists = true
return nil
}
func (a *AppServiceQueryAPI) User(
ctx context.Context,
req *api.UserRequest,
resp *api.UserResponse,
) error {
params, err := url.ParseQuery(req.Params)
if err != nil {
return err
}
for _, as := range a.Cfg.Derived.ApplicationServices {
var asUsers []api.ASUserResponse
params.Set("access_token", as.HSToken)
url := as.URL + api.ASUserPath
if req.Protocol != "" {
url += "/" + req.Protocol
}
if err := requestDo[[]api.ASUserResponse](a.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil {
log.WithError(err).Error("unable to get 'user' from application service")
continue
}
resp.Users = append(resp.Users, asUsers...)
}
if len(resp.Users) == 0 {
resp.Exists = false
return nil
}
resp.Exists = true
return nil
}
func (a *AppServiceQueryAPI) Protocols(
ctx context.Context,
req *api.ProtocolRequest,
resp *api.ProtocolResponse,
) error {
// get a single protocol response
if req.Protocol != "" {
a.CacheMu.Lock()
defer a.CacheMu.Unlock()
if proto, ok := a.ProtocolCache[req.Protocol]; ok {
resp.Exists = true
resp.Protocols = map[string]api.ASProtocolResponse{
req.Protocol: proto,
}
return nil
}
response := api.ASProtocolResponse{}
for _, as := range a.Cfg.Derived.ApplicationServices {
var proto api.ASProtocolResponse
if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+req.Protocol, &proto); err != nil {
log.WithError(err).Error("unable to get 'protocol' from application service")
continue
}
if len(response.Instances) != 0 {
response.Instances = append(response.Instances, proto.Instances...)
} else {
response = proto
}
}
if len(response.Instances) == 0 {
resp.Exists = false
return nil
}
resp.Exists = true
resp.Protocols = map[string]api.ASProtocolResponse{
req.Protocol: response,
}
a.ProtocolCache[req.Protocol] = response
return nil
}
response := make(map[string]api.ASProtocolResponse, len(a.Cfg.Derived.ApplicationServices))
for _, as := range a.Cfg.Derived.ApplicationServices {
for _, p := range as.Protocols {
var proto api.ASProtocolResponse
if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+p, &proto); err != nil {
log.WithError(err).Error("unable to get 'protocol' from application service")
continue
}
existing, ok := response[p]
if !ok {
response[p] = proto
continue
}
existing.Instances = append(existing.Instances, proto.Instances...)
response[p] = existing
}
}
if len(response) == 0 {
resp.Exists = false
return nil
}
a.CacheMu.Lock()
defer a.CacheMu.Unlock()
a.ProtocolCache = response
resp.Exists = true
resp.Protocols = response
return nil
}

0
cmd/dendritejs-pinecone/jsServer.go → build/dendritejs-pinecone/jsServer.go
View file

0
cmd/dendritejs-pinecone/main.go → build/dendritejs-pinecone/main.go
View file

0
cmd/dendritejs-pinecone/main_noop.go → build/dendritejs-pinecone/main_noop.go
View file

0
cmd/dendritejs-pinecone/main_test.go → build/dendritejs-pinecone/main_test.go
View file

25
build/docker/Dockerfile.demo-pinecone
View file

@ -1,25 +0,0 @@
FROM docker.io/golang:1.19-alpine AS base
RUN apk --update --no-cache add bash build-base
WORKDIR /build
COPY . /build
RUN mkdir -p bin
RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-pinecone
RUN go build -trimpath -o bin/ ./cmd/create-account
RUN go build -trimpath -o bin/ ./cmd/generate-keys
FROM alpine:latest
LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)"
LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
LABEL org.opencontainers.image.licenses="Apache-2.0"
COPY --from=base /build/bin/* /usr/bin/
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"]

25
build/docker/Dockerfile.polylith
View file

@ -1,25 +0,0 @@
FROM docker.io/golang:1.19-alpine AS base
RUN apk --update --no-cache add bash build-base
WORKDIR /build
COPY . /build
RUN mkdir -p bin
RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi
RUN go build -trimpath -o bin/ ./cmd/create-account
RUN go build -trimpath -o bin/ ./cmd/generate-keys
FROM alpine:latest
LABEL org.opencontainers.image.title="Dendrite (Polylith)"
LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
LABEL org.opencontainers.image.licenses="Apache-2.0"
COPY --from=base /build/bin/* /usr/bin/
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]

15
build/docker/README.md
View file

@ -9,11 +9,16 @@ They can be found on Docker Hub:
## Dockerfiles ## Dockerfiles
The `Dockerfile` builds the base image which contains all of the Dendrite The `Dockerfile` is a multistage file which can build all four Dendrite
components. The `Dockerfile.component` file takes the given component, as images depending on the supplied `--target`. From the root of the Dendrite
specified with `--buildarg component=` from the base image and produce repository, run:
smaller component-specific images, which are substantially smaller and do
not contain the Go toolchain etc. ```
docker build . --target monolith -t matrixdotorg/dendrite-monolith
docker build . --target polylith -t matrixdotorg/dendrite-monolith
docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone
docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil
```
## Compose files ## Compose files

6
build/docker/images-build.sh
View file

@ -6,5 +6,7 @@ TAG=${1:-latest}
echo "Building tag '${TAG}'" echo "Building tag '${TAG}'"
docker build -t matrixdotorg/dendrite-monolith:${TAG} -f build/docker/Dockerfile.monolith . docker build . --target monolith -t matrixdotorg/dendrite-monolith:${TAG}
docker build -t matrixdotorg/dendrite-polylith:${TAG} -f build/docker/Dockerfile.polylith . docker build . --target polylith -t matrixdotorg/dendrite-monolith:${TAG}
docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone:${TAG}
docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil:${TAG}

48
clientapi/routing/routing.go
View file

@ -937,12 +937,50 @@ func Setup(
).Methods(http.MethodGet, http.MethodOptions) ).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/thirdparty/protocols", v3mux.Handle("/thirdparty/protocols",
httputil.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse { httputil.MakeAuthAPI("thirdparty_protocols", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
// TODO: Return the third party protcols return Protocols(req, asAPI, device, "")
return util.JSONResponse{ }),
Code: http.StatusOK, ).Methods(http.MethodGet, http.MethodOptions)
JSON: struct{}{},
v3mux.Handle("/thirdparty/protocol/{protocolID}",
httputil.MakeAuthAPI("thirdparty_protocols", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
} }
return Protocols(req, asAPI, device, vars["protocolID"])
}),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/thirdparty/user/{protocolID}",
httputil.MakeAuthAPI("thirdparty_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return User(req, asAPI, device, vars["protocolID"], req.URL.Query())
}),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/thirdparty/user",
httputil.MakeAuthAPI("thirdparty_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return User(req, asAPI, device, "", req.URL.Query())
}),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/thirdparty/location/{protocolID}",
httputil.MakeAuthAPI("thirdparty_location", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return Location(req, asAPI, device, vars["protocolID"], req.URL.Query())
}),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/thirdparty/location",
httputil.MakeAuthAPI("thirdparty_location", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return Location(req, asAPI, device, "", req.URL.Query())
}), }),
).Methods(http.MethodGet, http.MethodOptions) ).Methods(http.MethodGet, http.MethodOptions)

106
clientapi/routing/thirdparty.go Normal file
View file

@ -0,0 +1,106 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package routing
import (
"net/http"
"net/url"
"github.com/matrix-org/util"
appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/userapi/api"
)
// Protocols implements
//
// GET /_matrix/client/v3/thirdparty/protocols/{protocol}
// GET /_matrix/client/v3/thirdparty/protocols
func Protocols(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, device *api.Device, protocol string) util.JSONResponse {
resp := &appserviceAPI.ProtocolResponse{}
if err := asAPI.Protocols(req.Context(), &appserviceAPI.ProtocolRequest{Protocol: protocol}, resp); err != nil {
return jsonerror.InternalServerError()
}
if !resp.Exists {
return util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound("The protocol is unknown."),
}
}
if protocol != "" {
return util.JSONResponse{
Code: http.StatusOK,
JSON: resp.Protocols[protocol],
}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: resp.Protocols,
}
}
// User implements
//
// GET /_matrix/client/v3/thirdparty/user
// GET /_matrix/client/v3/thirdparty/user/{protocol}
func User(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, device *api.Device, protocol string, params url.Values) util.JSONResponse {
resp := &appserviceAPI.UserResponse{}
params.Del("access_token")
if err := asAPI.User(req.Context(), &appserviceAPI.UserRequest{
Protocol: protocol,
Params: params.Encode(),
}, resp); err != nil {
return jsonerror.InternalServerError()
}
if !resp.Exists {
return util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound("The Matrix User ID was not found"),
}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: resp.Users,
}
}
// Location implements
//
// GET /_matrix/client/v3/thirdparty/location
// GET /_matrix/client/v3/thirdparty/location/{protocol}
func Location(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, device *api.Device, protocol string, params url.Values) util.JSONResponse {
resp := &appserviceAPI.LocationResponse{}
params.Del("access_token")
if err := asAPI.Locations(req.Context(), &appserviceAPI.LocationRequest{
Protocol: protocol,
Params: params.Encode(),
}, resp); err != nil {
return jsonerror.InternalServerError()
}
if !resp.Exists {
return util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound("No portal rooms were found."),
}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: resp.Locations,
}
}

26
cmd/dendrite-demo-pinecone/README.md Normal file
View file

@ -0,0 +1,26 @@
# Pinecone Demo
This is the Dendrite Pinecone demo! It's easy to get started.
To run the homeserver, start at the root of the Dendrite repository and run:
```
go run ./cmd/dendrite-demo-pinecone
```
To connect to the static Pinecone peer used by the mobile demos run:
```
go run ./cmd/dendrite-demo-pinecone -peer wss://pinecone.matrix.org/public
```
The following command line arguments are accepted:
* `-peer tcp://a.b.c.d:e` to specify a static Pinecone peer to connect to - you will need to supply this if you do not have another Pinecone node on your network
* `-port 12345` to specify a port to listen on for client connections
Then point your favourite Matrix client to the homeserver URL`http://localhost:8008` (or whichever `-port` you specified), create an account and log in.
If your peering connection is operational then you should see a `Connected TCP:` line in the log output. If not then try a different peer.
Once logged in, you should be able to open the room directory or join a room by its ID.

2
cmd/dendrite-demo-pinecone/main.go
View file

@ -54,8 +54,6 @@ import (
pineconeSessions "github.com/matrix-org/pinecone/sessions" pineconeSessions "github.com/matrix-org/pinecone/sessions"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
_ "github.com/mattn/go-sqlite3"
) )
var ( var (

2
cmd/dendrite-demo-yggdrasil/main.go
View file

@ -48,8 +48,6 @@ import (
"github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/dendrite/userapi"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
_ "github.com/mattn/go-sqlite3"
) )
var ( var (

2
cmd/dendrite-monolith-server/main.go
View file

@ -30,8 +30,6 @@ import (
"github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/dendrite/userapi"
uapi "github.com/matrix-org/dendrite/userapi/api" uapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
_ "github.com/mattn/go-sqlite3"
) )
var ( var (

2
cmd/dendrite-polylith-multi/main.go
View file

@ -24,8 +24,6 @@ import (
"github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/config"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
_ "github.com/mattn/go-sqlite3"
) )
type entrypoint func(base *base.BaseDendrite, cfg *config.Dendrite) type entrypoint func(base *base.BaseDendrite, cfg *config.Dendrite)

47
go.mod
View file

@ -9,7 +9,7 @@ require (
github.com/blevesearch/bleve/v2 v2.3.4 github.com/blevesearch/bleve/v2 v2.3.4
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1
github.com/codeclysm/extract v2.2.0+incompatible github.com/codeclysm/extract v2.2.0+incompatible
github.com/dgraph-io/ristretto v0.1.1 github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d
github.com/docker/docker v20.10.19+incompatible github.com/docker/docker v20.10.19+incompatible
github.com/docker/go-connections v0.4.0 github.com/docker/go-connections v0.4.0
github.com/ethereum/go-ethereum v1.10.25 github.com/ethereum/go-ethereum v1.10.25
@ -25,12 +25,12 @@ require (
github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
github.com/matrix-org/gomatrixserverlib v0.0.0-20221031151122-0885c35ebe74 github.com/matrix-org/gomatrixserverlib v0.0.0-20221101165746-0e4a8bb6db7e
github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6 github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
github.com/mattn/go-sqlite3 v1.14.15 github.com/mattn/go-sqlite3 v1.14.15
github.com/nats-io/nats-server/v2 v2.9.3 github.com/nats-io/nats-server/v2 v2.9.4
github.com/nats-io/nats.go v1.18.0 github.com/nats-io/nats.go v1.19.0
github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79 github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79
@ -40,21 +40,22 @@ require (
github.com/prometheus/client_golang v1.13.0 github.com/prometheus/client_golang v1.13.0
github.com/sirupsen/logrus v1.9.0 github.com/sirupsen/logrus v1.9.0
github.com/spruceid/siwe-go v0.2.0 github.com/spruceid/siwe-go v0.2.0
github.com/stretchr/testify v1.8.0 github.com/stretchr/testify v1.8.1
github.com/tidwall/gjson v1.14.3 github.com/tidwall/gjson v1.14.3
github.com/tidwall/sjson v1.2.5 github.com/tidwall/sjson v1.2.5
github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-client-go v2.30.0+incompatible
github.com/uber/jaeger-lib v2.4.1+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible
github.com/yggdrasil-network/yggdrasil-go v0.4.6 github.com/yggdrasil-network/yggdrasil-go v0.4.6
go.uber.org/atomic v1.10.0 go.uber.org/atomic v1.10.0
golang.org/x/crypto v0.0.0-20221012134737-56aed061732a golang.org/x/crypto v0.1.0
golang.org/x/image v0.0.0-20220902085622-e7cb96979f69 golang.org/x/image v0.1.0
golang.org/x/mobile v0.0.0-20221012134814-c746ac228303 golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b golang.org/x/net v0.1.0
golang.org/x/term v0.0.0-20220919170432-7a66f970e087 golang.org/x/term v0.1.0
gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/h2non/bimg.v1 v1.1.9
gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v2 v2.4.0
gotest.tools/v3 v3.4.0 gotest.tools/v3 v3.4.0
modernc.org/sqlite v1.19.3
nhooyr.io/websocket v1.8.7 nhooyr.io/websocket v1.8.7
) )
@ -102,12 +103,14 @@ require (
github.com/h2non/filetype v1.1.3 // indirect github.com/h2non/filetype v1.1.3 // indirect
github.com/json-iterator/go v1.1.12 // indirect github.com/json-iterator/go v1.1.12 // indirect
github.com/juju/errors v1.0.0 // indirect github.com/juju/errors v1.0.0 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/compress v1.15.11 // indirect github.com/klauspost/compress v1.15.11 // indirect
github.com/kr/pretty v0.3.1 // indirect github.com/kr/pretty v0.3.1 // indirect
github.com/lucas-clemente/quic-go v0.29.2 // indirect github.com/lucas-clemente/quic-go v0.29.2 // indirect
github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect
github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/mattn/go-isatty v0.0.16 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/minio/highwayhash v1.0.2 // indirect github.com/minio/highwayhash v1.0.2 // indirect
github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@ -127,6 +130,7 @@ require (
github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect
github.com/relvacode/iso8601 v1.1.0 // indirect github.com/relvacode/iso8601 v1.1.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa // indirect
github.com/rjeczalik/notify v0.9.1 // indirect github.com/rjeczalik/notify v0.9.1 // indirect
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/match v1.1.1 // indirect
@ -134,17 +138,26 @@ require (
github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect
github.com/tklauser/numcpus v0.2.2 // indirect github.com/tklauser/numcpus v0.2.2 // indirect
go.etcd.io/bbolt v1.3.6 // indirect go.etcd.io/bbolt v1.3.6 // indirect
golang.org/x/exp v0.0.0-20221012211006-4de253d81b95 // indirect golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/mod v0.6.0 // indirect
golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 // indirect golang.org/x/sys v0.1.0 // indirect
golang.org/x/text v0.3.8 // indirect golang.org/x/text v0.4.0 // indirect
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect golang.org/x/time v0.1.0 // indirect
golang.org/x/tools v0.1.12 // indirect golang.org/x/tools v0.2.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/macaroon.v2 v2.1.0 // indirect gopkg.in/macaroon.v2 v2.1.0 // indirect
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/uint128 v1.2.0 // indirect
modernc.org/cc/v3 v3.40.0 // indirect
modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8 // indirect
modernc.org/libc v1.21.4 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.4.0 // indirect
modernc.org/opt v0.1.3 // indirect
modernc.org/strutil v1.1.3 // indirect
modernc.org/token v1.0.1 // indirect
) )
go 1.19 go 1.19

101
go.sum
View file

@ -175,8 +175,8 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK
github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs=
github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d h1:Wrc3UKTS+cffkOx0xRGFC+ZesNuTfn0ThvEC72N0krk=
github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d/go.mod h1:RAy2GVV4sTWVlNMavv3xhLsk18rxhfhDnombTe6EF5c=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
@ -381,6 +381,8 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8
github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5lLdR0= github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5lLdR0=
github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4= github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
@ -423,8 +425,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
github.com/matrix-org/gomatrixserverlib v0.0.0-20221031151122-0885c35ebe74 h1:I4LUlFqxZ72m3s9wIvUIV2FpprsxW28dO/0lAgepCZY= github.com/matrix-org/gomatrixserverlib v0.0.0-20221101165746-0e4a8bb6db7e h1:6I34fdyiHMRCxL6GOb/G8ZyI1WWlb6ZxCF2hIGSMSCc=
github.com/matrix-org/gomatrixserverlib v0.0.0-20221031151122-0885c35ebe74/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= github.com/matrix-org/gomatrixserverlib v0.0.0-20221101165746-0e4a8bb6db7e/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6 h1:nAT5w41Q9uWTSnpKW55/hBwP91j2IFYPDRs0jJ8TyFI= github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6 h1:nAT5w41Q9uWTSnpKW55/hBwP91j2IFYPDRs0jJ8TyFI=
github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k=
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
@ -432,13 +434,14 @@ github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/em
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/mattomatic/dijkstra v0.0.0-20130617153013-6f6d134eb237/go.mod h1:UOnLAUmVG5paym8pD3C4B9BQylUDC2vXFJJpT7JrlEA= github.com/mattomatic/dijkstra v0.0.0-20130617153013-6f6d134eb237/go.mod h1:UOnLAUmVG5paym8pD3C4B9BQylUDC2vXFJJpT7JrlEA=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
@ -465,10 +468,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI= github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI=
github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k=
github.com/nats-io/nats-server/v2 v2.9.3 h1:HrfzA7G9LNetKkm1z+jU/e9kuAe+E6uaBuuq9EB5sQQ= github.com/nats-io/nats-server/v2 v2.9.4 h1:GvRgv1936J/zYUwMg/cqtYaJ6L+bgeIOIvPslbesdow=
github.com/nats-io/nats-server/v2 v2.9.3/go.mod h1:4sq8wvrpbvSzL1n3ZfEYnH4qeUuIl5W990j3kw13rRk= github.com/nats-io/nats-server/v2 v2.9.4/go.mod h1:AB6hAnGZDlYfqb7CTAm66ZKMZy9DpfierY1/PbpvI2g=
github.com/nats-io/nats.go v1.18.0 h1:o480Ao6kuSSFyJO75rGTXCEPj7LGkY84C1Ye+Uhm4c0= github.com/nats-io/nats.go v1.19.0 h1:H6j8aBnTQFoVrTGB6Xjd903UMdE7jz6DS4YkmAqgZ9Q=
github.com/nats-io/nats.go v1.18.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= github.com/nats-io/nats.go v1.19.0/go.mod h1:tLqubohF7t4z3du1QDPYJIQQyhb4wl6DhjxEajSI7UA=
github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8=
github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
@ -552,6 +555,9 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua
github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA=
github.com/relvacode/iso8601 v1.1.0 h1:2nV8sp0eOjpoKQ2vD3xSDygsjAx37NHG2UlZiCkDH4I= github.com/relvacode/iso8601 v1.1.0 h1:2nV8sp0eOjpoKQ2vD3xSDygsjAx37NHG2UlZiCkDH4I=
github.com/relvacode/iso8601 v1.1.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= github.com/relvacode/iso8601 v1.1.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa h1:tEkEyxYeZ43TR55QU/hsIt9aRGBxbgGuz9CGykjvogY=
github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE=
github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
@ -608,8 +614,9 @@ github.com/spruceid/siwe-go v0.2.0/go.mod h1:rvV+8/z/ryBKqdw9RcexFgtcsrDlESOGR38
github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@ -617,8 +624,9 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY=
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
@ -694,8 +702,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20221012134737-56aed061732a h1:NmSIgad6KjE6VvHciPZuNRTKxGhlPfD6OA87W/PLkqg= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
golang.org/x/crypto v0.0.0-20221012134737-56aed061732a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -710,13 +718,13 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20221012211006-4de253d81b95 h1:sBdrWpxhGDdTAYNqbgBLAR+ULAPPhfgncLr1X0lyWtg= golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE=
golang.org/x/exp v0.0.0-20221012211006-4de253d81b95/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20220902085622-e7cb96979f69 h1:Lj6HJGCSn5AjxRAH2+r35Mir4icalbqku+CLUtjnvXY= golang.org/x/image v0.1.0 h1:r8Oj8ZA2Xy12/b5KZYj3tuv7NG/fBz3TwQVvpJ9l8Rk=
golang.org/x/image v0.0.0-20220902085622-e7cb96979f69/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY= golang.org/x/image v0.1.0/go.mod h1:iyPr49SD/G/TBxYVB/9RRtGUT5eNbo2u4NamWeQcD5c=
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@ -731,8 +739,8 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPI
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ=
golang.org/x/mobile v0.0.0-20221012134814-c746ac228303 h1:K4fp1rDuJBz0FCPAWzIJwnzwNEM7S6yobdZzMrZ/Zws= golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e h1:zSgtO19fpg781xknwqiQPmOHaASr6E7ZVlTseLd9Fx4=
golang.org/x/mobile v0.0.0-20221012134814-c746ac228303/go.mod h1:M32cGdzp91A8Ex9qQtyZinr19EYxzkFqDjW2oyHzTDQ= golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e/go.mod h1:aAjjkJNdrh3PMckS4B10TGS2nag27cbKR1y2BpUxsiY=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
@ -740,8 +748,9 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I=
golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -787,8 +796,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU= golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@ -811,7 +820,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220907140024-f12130a52804 h1:0SH2R3f1b1VmIMG7BXbEZCBUu2dKmHschSmjqGUrW8A= golang.org/x/sync v0.0.0-20220907140024-f12130a52804 h1:0SH2R3f1b1VmIMG7BXbEZCBUu2dKmHschSmjqGUrW8A=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -881,14 +889,14 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220730100132-1609e554cd39/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220730100132-1609e554cd39/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 h1:OK7RB6t2WQX54srQQYSXMW8dF5C6/8+oA/s5QBmmto4= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20220919170432-7a66f970e087 h1:tPwmk4vmvVCMdr98VgL4JH+qZxPL8fqlUOHnyOM8N3w= golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
golang.org/x/term v0.0.0-20220919170432-7a66f970e087/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -896,14 +904,14 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA=
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@ -955,8 +963,9 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE=
golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -1100,6 +1109,30 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI=
lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw=
modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8 h1:0+dsXf0zeLx9ixj4nilg6jKe5Bg1ilzBwSFq4kJmIUc=
modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g=
modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
modernc.org/libc v1.21.4 h1:CzTlumWeIbPV5/HVIMzYHNPCRP8uiU/CWiN2gtd/Qu8=
modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI=
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/memory v1.4.0 h1:crykUfNSnMAXaOJnnxcSzbUGMqkLWjklJKkBK2nwZwk=
modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/sqlite v1.19.3 h1:dIoagx6yIQT3V/zOSeAyZ8OqQyEr17YTgETOXTZNJMA=
modernc.org/sqlite v1.19.3/go.mod h1:xiyJD7FY8mTZXnQwE/gEL1STtFrrnDx03V8KhVQmcr8=
modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
modernc.org/tcl v1.15.0 h1:oY+JeD11qVVSgVvodMJsu7Edf8tr5E/7tuhF5cNYz34=
modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE=
nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

6
internal/sqlutil/migrate_test.go
View file

@ -7,8 +7,6 @@ import (
"reflect" "reflect"
"testing" "testing"
_ "github.com/mattn/go-sqlite3"
"github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test"
) )
@ -88,7 +86,7 @@ func Test_migrations_Up(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
driverName := "sqlite3" driverName := sqlutil.SQLITE_DRIVER_NAME
if dbType == test.DBTypePostgres { if dbType == test.DBTypePostgres {
driverName = "postgres" driverName = "postgres"
} }
@ -117,7 +115,7 @@ func Test_insertMigration(t *testing.T) {
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
conStr, close := test.PrepareDBConnectionString(t, dbType) conStr, close := test.PrepareDBConnectionString(t, dbType)
defer close() defer close()
driverName := "sqlite3" driverName := sqlutil.SQLITE_DRIVER_NAME
if dbType == test.DBTypePostgres { if dbType == test.DBTypePostgres {
driverName = "postgres" driverName = "postgres"
} }

19
internal/sqlutil/sqlite_cgo.go Normal file
View file

@ -0,0 +1,19 @@
//go:build cgo
// +build cgo
package sqlutil
import (
"github.com/mattn/go-sqlite3"
_ "github.com/mattn/go-sqlite3"
)
const SQLITE_DRIVER_NAME = "sqlite3"
func sqliteDSNExtension(dsn string) string {
return dsn
}
func sqliteDriver() *sqlite3.SQLiteDriver {
return &sqlite3.SQLiteDriver{}
}

29
internal/sqlutil/sqlite_native.go Normal file
View file

@ -0,0 +1,29 @@
//go:build !cgo
// +build !cgo
package sqlutil
import (
"modernc.org/sqlite"
"strings"
)
const SQLITE_DRIVER_NAME = "sqlite"
func sqliteDSNExtension(dsn string) string {
// add query parameters to the dsn
if strings.Contains(dsn, "?") {
dsn += "&"
} else {
dsn += "?"
}
// wait some time before erroring if the db is locked
// https://gitlab.com/cznic/sqlite/-/issues/106#note_1058094993
dsn += "_pragma=busy_timeout%3d10000"
return dsn
}
func sqliteDriver() *sqlite.Driver {
return &sqlite.Driver{}
}

5
internal/sqlutil/sqlutil.go
View file

@ -20,11 +20,12 @@ func Open(dbProperties *config.DatabaseOptions, writer Writer) (*sql.DB, error)
var driverName, dsn string var driverName, dsn string
switch { switch {
case dbProperties.ConnectionString.IsSQLite(): case dbProperties.ConnectionString.IsSQLite():
driverName = "sqlite3" driverName = SQLITE_DRIVER_NAME
dsn, err = ParseFileURI(dbProperties.ConnectionString) dsn, err = ParseFileURI(dbProperties.ConnectionString)
if err != nil { if err != nil {
return nil, fmt.Errorf("ParseFileURI: %w", err) return nil, fmt.Errorf("ParseFileURI: %w", err)
} }
dsn = sqliteDSNExtension(dsn)
case dbProperties.ConnectionString.IsPostgres(): case dbProperties.ConnectionString.IsPostgres():
driverName = "postgres" driverName = "postgres"
dsn = string(dbProperties.ConnectionString) dsn = string(dbProperties.ConnectionString)
@ -39,7 +40,7 @@ func Open(dbProperties *config.DatabaseOptions, writer Writer) (*sql.DB, error)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if driverName != "sqlite3" { if driverName != SQLITE_DRIVER_NAME {
logger := logrus.WithFields(logrus.Fields{ logger := logrus.WithFields(logrus.Fields{
"max_open_conns": dbProperties.MaxOpenConns(), "max_open_conns": dbProperties.MaxOpenConns(),
"max_idle_conns": dbProperties.MaxIdleConns(), "max_idle_conns": dbProperties.MaxIdleConns(),

3
internal/sqlutil/trace_driver.go
View file

@ -21,7 +21,6 @@ import (
"database/sql" "database/sql"
"github.com/lib/pq" "github.com/lib/pq"
sqlite "github.com/mattn/go-sqlite3"
"github.com/ngrok/sqlmw" "github.com/ngrok/sqlmw"
) )
@ -31,6 +30,6 @@ func registerDrivers() {
} }
// install the wrapped drivers // install the wrapped drivers
sql.Register("postgres-trace", sqlmw.Driver(&pq.Driver{}, new(traceInterceptor))) sql.Register("postgres-trace", sqlmw.Driver(&pq.Driver{}, new(traceInterceptor)))
sql.Register("sqlite3-trace", sqlmw.Driver(&sqlite.SQLiteDriver{}, new(traceInterceptor))) sql.Register("sqlite3-trace", sqlmw.Driver(sqliteDriver(), new(traceInterceptor)))
} }

13
internal/sqlutil/unique_constraint.go
View file

@ -12,14 +12,15 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
//go:build !wasm //go:build !wasm && !cgo
// +build !wasm // +build !wasm,!cgo
package sqlutil package sqlutil
import ( import (
"github.com/lib/pq" "github.com/lib/pq"
"github.com/mattn/go-sqlite3" "modernc.org/sqlite"
lib "modernc.org/sqlite/lib"
) )
// IsUniqueConstraintViolationErr returns true if the error is an unique_violation error // IsUniqueConstraintViolationErr returns true if the error is an unique_violation error
@ -27,10 +28,8 @@ func IsUniqueConstraintViolationErr(err error) bool {
switch e := err.(type) { switch e := err.(type) {
case *pq.Error: case *pq.Error:
return e.Code == "23505" return e.Code == "23505"
case *sqlite3.Error: case *sqlite.Error:
return e.Code == sqlite3.ErrConstraint return e.Code() == lib.SQLITE_CONSTRAINT
case sqlite3.Error:
return e.Code == sqlite3.ErrConstraint
} }
return false return false
} }

36
internal/sqlutil/unique_constraint_cgo.go Normal file
View file

@ -0,0 +1,36 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !wasm && cgo
// +build !wasm,cgo
package sqlutil
import (
"github.com/lib/pq"
"github.com/mattn/go-sqlite3"
)
// IsUniqueConstraintViolationErr returns true if the error is an unique_violation error
func IsUniqueConstraintViolationErr(err error) bool {
switch e := err.(type) {
case *pq.Error:
return e.Code == "23505"
case *sqlite3.Error:
return e.Code == sqlite3.ErrConstraint
case sqlite3.Error:
return e.Code == sqlite3.ErrConstraint
}
return false
}

11
internal/sqlutil/unique_constraint_wasm.go
View file

@ -17,15 +17,16 @@
package sqlutil package sqlutil
import "github.com/mattn/go-sqlite3" import (
"modernc.org/sqlite"
lib "modernc.org/sqlite/lib"
)
// IsUniqueConstraintViolationErr returns true if the error is an unique_violation error // IsUniqueConstraintViolationErr returns true if the error is an unique_violation error
func IsUniqueConstraintViolationErr(err error) bool { func IsUniqueConstraintViolationErr(err error) bool {
switch e := err.(type) { switch e := err.(type) {
case *sqlite3.Error: case *sqlite.Error:
return e.Code == sqlite3.ErrConstraint return e.Code() == lib.SQLITE_CONSTRAINT
case sqlite3.Error:
return e.Code == sqlite3.ErrConstraint
} }
return false return false
} }

2
internal/version.go
View file

@ -17,7 +17,7 @@ var build string
const ( const (
VersionMajor = 0 VersionMajor = 0
VersionMinor = 10 VersionMinor = 10
VersionPatch = 5 VersionPatch = 6
VersionTag = "" // example: "rc1" VersionTag = "" // example: "rc1"
) )

11
roomserver/internal/helpers/helpers.go
View file

@ -5,6 +5,7 @@ import (
"database/sql" "database/sql"
"errors" "errors"
"fmt" "fmt"
"sort"
"strings" "strings"
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
@ -159,7 +160,7 @@ func GetMembershipsAtState(
ctx context.Context, db storage.Database, stateEntries []types.StateEntry, joinedOnly bool, ctx context.Context, db storage.Database, stateEntries []types.StateEntry, joinedOnly bool,
) ([]types.Event, error) { ) ([]types.Event, error) {
var eventNIDs []types.EventNID var eventNIDs types.EventNIDs
for _, entry := range stateEntries { for _, entry := range stateEntries {
// Filter the events to retrieve to only keep the membership events // Filter the events to retrieve to only keep the membership events
if entry.EventTypeNID == types.MRoomMemberNID { if entry.EventTypeNID == types.MRoomMemberNID {
@ -167,6 +168,14 @@ func GetMembershipsAtState(
} }
} }
// There are no events to get, don't bother asking the database
if len(eventNIDs) == 0 {
return []types.Event{}, nil
}
sort.Sort(eventNIDs)
util.Unique(eventNIDs)
// Get all of the events in this state // Get all of the events in this state
stateEvents, err := db.Events(ctx, eventNIDs) stateEvents, err := db.Events(ctx, eventNIDs)
if err != nil { if err != nil {

48
roomserver/internal/input/input_missing.go
View file

@ -40,6 +40,7 @@ func (p *parsedRespState) Events() []*gomatrixserverlib.Event {
} }
type missingStateReq struct { type missingStateReq struct {
log *logrus.Entry
origin gomatrixserverlib.ServerName origin gomatrixserverlib.ServerName
db storage.Database db storage.Database
roomInfo *types.RoomInfo roomInfo *types.RoomInfo
@ -77,7 +78,7 @@ func (t *missingStateReq) processEventWithMissingState(
// event ids and then use /event to fetch the individual events. // event ids and then use /event to fetch the individual events.
// However not all version of synapse support /state_ids so you may // However not all version of synapse support /state_ids so you may
// need to fallback to /state. // need to fallback to /state.
logger := util.GetLogger(ctx).WithFields(map[string]interface{}{ t.log = util.GetLogger(ctx).WithFields(map[string]interface{}{
"txn_event": e.EventID(), "txn_event": e.EventID(),
"room_id": e.RoomID(), "room_id": e.RoomID(),
"txn_prev_events": e.PrevEventIDs(), "txn_prev_events": e.PrevEventIDs(),
@ -96,7 +97,7 @@ func (t *missingStateReq) processEventWithMissingState(
return nil, fmt.Errorf("expected to find missing events but didn't") return nil, fmt.Errorf("expected to find missing events but didn't")
} }
if isGapFilled { if isGapFilled {
logger.Infof("Gap filled by /get_missing_events, injecting %d new events", len(newEvents)) t.log.Infof("Gap filled by /get_missing_events, injecting %d new events", len(newEvents))
// we can just inject all the newEvents as new as we may have only missed 1 or 2 events and have filled // we can just inject all the newEvents as new as we may have only missed 1 or 2 events and have filled
// in the gap in the DAG // in the gap in the DAG
for _, newEvent := range newEvents { for _, newEvent := range newEvents {
@ -118,7 +119,7 @@ func (t *missingStateReq) processEventWithMissingState(
// then there's nothing else to do, we have everything we need to deal // then there's nothing else to do, we have everything we need to deal
// with the new event. // with the new event.
if isGapFilled && prevStatesKnown { if isGapFilled && prevStatesKnown {
logger.Infof("Gap filled and state found for all prev events") t.log.Infof("Gap filled and state found for all prev events")
return nil, nil return nil, nil
} }
@ -321,7 +322,7 @@ func (t *missingStateReq) lookupStateAfterEvent(ctx context.Context, roomVersion
defer span.Finish() defer span.Finish()
// try doing all this locally before we resort to querying federation // try doing all this locally before we resort to querying federation
respState := t.lookupStateAfterEventLocally(ctx, roomID, eventID) respState := t.lookupStateAfterEventLocally(ctx, eventID)
if respState != nil { if respState != nil {
return respState, true, nil return respState, true, nil
} }
@ -373,7 +374,7 @@ func (t *missingStateReq) cacheAndReturn(ev *gomatrixserverlib.Event) *gomatrixs
return ev return ev
} }
func (t *missingStateReq) lookupStateAfterEventLocally(ctx context.Context, roomID, eventID string) *parsedRespState { func (t *missingStateReq) lookupStateAfterEventLocally(ctx context.Context, eventID string) *parsedRespState {
span, ctx := opentracing.StartSpanFromContext(ctx, "lookupStateAfterEventLocally") span, ctx := opentracing.StartSpanFromContext(ctx, "lookupStateAfterEventLocally")
defer span.Finish() defer span.Finish()
@ -381,12 +382,12 @@ func (t *missingStateReq) lookupStateAfterEventLocally(ctx context.Context, room
roomState := state.NewStateResolution(t.db, t.roomInfo) roomState := state.NewStateResolution(t.db, t.roomInfo)
stateAtEvents, err := t.db.StateAtEventIDs(ctx, []string{eventID}) stateAtEvents, err := t.db.StateAtEventIDs(ctx, []string{eventID})
if err != nil { if err != nil {
util.GetLogger(ctx).WithField("room_id", roomID).WithError(err).Warnf("failed to get state after %s locally", eventID) t.log.WithError(err).Warnf("failed to get state after %s locally", eventID)
return nil return nil
} }
stateEntries, err := roomState.LoadCombinedStateAfterEvents(ctx, stateAtEvents) stateEntries, err := roomState.LoadCombinedStateAfterEvents(ctx, stateAtEvents)
if err != nil { if err != nil {
util.GetLogger(ctx).WithField("room_id", roomID).WithError(err).Warnf("failed to load combined state after %s locally", eventID) t.log.WithError(err).Warnf("failed to load combined state after %s locally", eventID)
return nil return nil
} }
stateEventNIDs := make([]types.EventNID, 0, len(stateEntries)) stateEventNIDs := make([]types.EventNID, 0, len(stateEntries))
@ -395,7 +396,7 @@ func (t *missingStateReq) lookupStateAfterEventLocally(ctx context.Context, room
} }
stateEvents, err := t.db.Events(ctx, stateEventNIDs) stateEvents, err := t.db.Events(ctx, stateEventNIDs)
if err != nil { if err != nil {
util.GetLogger(ctx).WithField("room_id", roomID).WithError(err).Warnf("failed to load state events locally") t.log.WithError(err).Warnf("failed to load state events locally")
return nil return nil
} }
res.StateEvents = make([]*gomatrixserverlib.Event, 0, len(stateEvents)) res.StateEvents = make([]*gomatrixserverlib.Event, 0, len(stateEvents))
@ -429,7 +430,7 @@ func (t *missingStateReq) lookupStateAfterEventLocally(ctx context.Context, room
for evID := range missingAuthEvents { for evID := range missingAuthEvents {
missingEventList = append(missingEventList, evID) missingEventList = append(missingEventList, evID)
} }
util.GetLogger(ctx).WithField("count", len(missingEventList)).Debugf("Fetching missing auth events") t.log.WithField("count", len(missingEventList)).Debugf("Fetching missing auth events")
events, err := t.db.EventsFromIDs(ctx, missingEventList) events, err := t.db.EventsFromIDs(ctx, missingEventList)
if err != nil { if err != nil {
return nil return nil
@ -485,7 +486,7 @@ retryAllowedState:
default: default:
return nil, fmt.Errorf("missing auth event %s and failed to look it up: %w", missing.AuthEventID, err2) return nil, fmt.Errorf("missing auth event %s and failed to look it up: %w", missing.AuthEventID, err2)
} }
util.GetLogger(ctx).Tracef("fetched event %s", missing.AuthEventID) t.log.Tracef("fetched event %s", missing.AuthEventID)
resolvedStateEvents = append(resolvedStateEvents, h) resolvedStateEvents = append(resolvedStateEvents, h)
goto retryAllowedState goto retryAllowedState
default: default:
@ -504,7 +505,7 @@ func (t *missingStateReq) getMissingEvents(ctx context.Context, e *gomatrixserve
span, ctx := opentracing.StartSpanFromContext(ctx, "getMissingEvents") span, ctx := opentracing.StartSpanFromContext(ctx, "getMissingEvents")
defer span.Finish() defer span.Finish()
logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID()) logger := t.log.WithField("event_id", e.EventID()).WithField("room_id", e.RoomID())
latest, _, _, err := t.db.LatestEventIDs(ctx, t.roomInfo.RoomNID) latest, _, _, err := t.db.LatestEventIDs(ctx, t.roomInfo.RoomNID)
if err != nil { if err != nil {
return nil, false, false, fmt.Errorf("t.DB.LatestEventIDs: %w", err) return nil, false, false, fmt.Errorf("t.DB.LatestEventIDs: %w", err)
@ -666,7 +667,7 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo
span, ctx := opentracing.StartSpanFromContext(ctx, "lookupMissingStateViaStateIDs") span, ctx := opentracing.StartSpanFromContext(ctx, "lookupMissingStateViaStateIDs")
defer span.Finish() defer span.Finish()
util.GetLogger(ctx).WithField("room_id", roomID).Infof("lookupMissingStateViaStateIDs %s", eventID) t.log.Infof("lookupMissingStateViaStateIDs %s", eventID)
// fetch the state event IDs at the time of the event // fetch the state event IDs at the time of the event
var stateIDs gomatrixserverlib.RespStateIDs var stateIDs gomatrixserverlib.RespStateIDs
var err error var err error
@ -719,16 +720,15 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo
concurrentRequests := 8 concurrentRequests := 8
missingCount := len(missing) missingCount := len(missing)
util.GetLogger(ctx).WithField("room_id", roomID).WithField("event_id", eventID).Debugf("lookupMissingStateViaStateIDs missing %d/%d events", missingCount, len(wantIDs)) t.log.WithField("event_id", eventID).Debugf("lookupMissingStateViaStateIDs missing %d/%d events", missingCount, len(wantIDs))
// If over 50% of the auth/state events from /state_ids are missing // If over 50% of the auth/state events from /state_ids are missing
// then we'll just call /state instead, otherwise we'll just end up // then we'll just call /state instead, otherwise we'll just end up
// hammering the remote side with /event requests unnecessarily. // hammering the remote side with /event requests unnecessarily.
if missingCount > concurrentRequests && missingCount > len(wantIDs)/2 { if missingCount > concurrentRequests && missingCount > len(wantIDs)/2 {
util.GetLogger(ctx).WithFields(logrus.Fields{ t.log.WithFields(logrus.Fields{
"missing": missingCount, "missing": missingCount,
"event_id": eventID, "event_id": eventID,
"room_id": roomID,
"total_state": len(stateIDs.StateEventIDs), "total_state": len(stateIDs.StateEventIDs),
"total_auth_events": len(stateIDs.AuthEventIDs), "total_auth_events": len(stateIDs.AuthEventIDs),
}).Debug("Fetching all state at event") }).Debug("Fetching all state at event")
@ -736,10 +736,9 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo
} }
if missingCount > 0 { if missingCount > 0 {
util.GetLogger(ctx).WithFields(logrus.Fields{ t.log.WithFields(logrus.Fields{
"missing": missingCount, "missing": missingCount,
"event_id": eventID, "event_id": eventID,
"room_id": roomID,
"total_state": len(stateIDs.StateEventIDs), "total_state": len(stateIDs.StateEventIDs),
"total_auth_events": len(stateIDs.AuthEventIDs), "total_auth_events": len(stateIDs.AuthEventIDs),
"concurrent_requests": concurrentRequests, "concurrent_requests": concurrentRequests,
@ -776,9 +775,8 @@ func (t *missingStateReq) lookupMissingStateViaStateIDs(ctx context.Context, roo
case nil: case nil:
break break
default: default:
util.GetLogger(ctx).WithFields(logrus.Fields{ t.log.WithFields(logrus.Fields{
"event_id": missingEventID, "missing_event_id": missingEventID,
"room_id": roomID,
}).WithError(herr).Warn("Failed to fetch missing event") }).WithError(herr).Warn("Failed to fetch missing event")
return return
} }
@ -847,7 +845,7 @@ func (t *missingStateReq) lookupEvent(ctx context.Context, roomVersion gomatrixs
// fetch from the roomserver // fetch from the roomserver
events, err := t.db.EventsFromIDs(ctx, []string{missingEventID}) events, err := t.db.EventsFromIDs(ctx, []string{missingEventID})
if err != nil { if err != nil {
util.GetLogger(ctx).Warnf("Failed to query roomserver for missing event %s: %s - falling back to remote", missingEventID, err) t.log.Warnf("Failed to query roomserver for missing event %s: %s - falling back to remote", missingEventID, err)
} else if len(events) == 1 { } else if len(events) == 1 {
return events[0].Event, nil return events[0].Event, nil
} }
@ -859,7 +857,7 @@ func (t *missingStateReq) lookupEvent(ctx context.Context, roomVersion gomatrixs
defer cancel() defer cancel()
txn, err := t.federation.GetEvent(reqctx, serverName, missingEventID) txn, err := t.federation.GetEvent(reqctx, serverName, missingEventID)
if err != nil || len(txn.PDUs) == 0 { if err != nil || len(txn.PDUs) == 0 {
util.GetLogger(ctx).WithError(err).WithField("event_id", missingEventID).Warn("Failed to get missing /event for event ID") t.log.WithError(err).WithField("missing_event_id", missingEventID).Warn("Failed to get missing /event for event ID")
if errors.Is(err, context.DeadlineExceeded) { if errors.Is(err, context.DeadlineExceeded) {
select { select {
case <-reqctx.Done(): // this server took too long case <-reqctx.Done(): // this server took too long
@ -872,18 +870,18 @@ func (t *missingStateReq) lookupEvent(ctx context.Context, roomVersion gomatrixs
} }
event, err = gomatrixserverlib.NewEventFromUntrustedJSON(txn.PDUs[0], roomVersion) event, err = gomatrixserverlib.NewEventFromUntrustedJSON(txn.PDUs[0], roomVersion)
if err != nil { if err != nil {
util.GetLogger(ctx).WithError(err).WithField("event_id", missingEventID).Warnf("Failed to parse event JSON of event returned from /event") t.log.WithError(err).WithField("missing_event_id", missingEventID).Warnf("Failed to parse event JSON of event returned from /event")
continue continue
} }
found = true found = true
break break
} }
if !found { if !found {
util.GetLogger(ctx).WithField("event_id", missingEventID).Warnf("Failed to get missing /event for event ID from %d server(s)", len(t.servers)) t.log.WithField("missing_event_id", missingEventID).Warnf("Failed to get missing /event for event ID from %d server(s)", len(t.servers))
return nil, fmt.Errorf("wasn't able to find event via %d server(s)", len(t.servers)) return nil, fmt.Errorf("wasn't able to find event via %d server(s)", len(t.servers))
} }
if err := event.VerifyEventSignatures(ctx, t.keys); err != nil { if err := event.VerifyEventSignatures(ctx, t.keys); err != nil {
util.GetLogger(ctx).WithError(err).Warnf("Couldn't validate signature of event %q from /event", event.EventID()) t.log.WithError(err).Warnf("Couldn't validate signature of event %q from /event", event.EventID())
return nil, verifySigError{event.EventID(), err} return nil, verifySigError{event.EventID(), err}
} }
return t.cacheAndReturn(event), nil return t.cacheAndReturn(event), nil

30
roomserver/internal/query/query.go
View file

@ -239,16 +239,42 @@ func (r *Queryer) QueryMembershipAtEvent(
return fmt.Errorf("unable to get state before event: %w", err) return fmt.Errorf("unable to get state before event: %w", err)
} }
// If we only have one or less state entries, we can short circuit the below
// loop and avoid hitting the database
allStateEventNIDs := make(map[types.EventNID]types.StateEntry)
for _, eventID := range request.EventIDs {
stateEntry := stateEntries[eventID]
for _, s := range stateEntry {
allStateEventNIDs[s.EventNID] = s
}
}
var canShortCircuit bool
if len(allStateEventNIDs) <= 1 {
canShortCircuit = true
}
var memberships []types.Event
for _, eventID := range request.EventIDs { for _, eventID := range request.EventIDs {
stateEntry, ok := stateEntries[eventID] stateEntry, ok := stateEntries[eventID]
if !ok { if !ok || len(stateEntry) == 0 {
response.Memberships[eventID] = []*gomatrixserverlib.HeaderedEvent{} response.Memberships[eventID] = []*gomatrixserverlib.HeaderedEvent{}
continue continue
} }
memberships, err := helpers.GetMembershipsAtState(ctx, r.DB, stateEntry, false)
// If we can short circuit, e.g. we only have 0 or 1 membership events, we only get the memberships
// once. If we have more than one membership event, we need to get the state for each state entry.
if canShortCircuit {
if len(memberships) == 0 {
memberships, err = helpers.GetMembershipsAtState(ctx, r.DB, stateEntry, false)
}
} else {
memberships, err = helpers.GetMembershipsAtState(ctx, r.DB, stateEntry, false)
}
if err != nil { if err != nil {
return fmt.Errorf("unable to get memberships at state: %w", err) return fmt.Errorf("unable to get memberships at state: %w", err)
} }
res := make([]*gomatrixserverlib.HeaderedEvent, 0, len(memberships)) res := make([]*gomatrixserverlib.HeaderedEvent, 0, len(memberships))
for i := range memberships { for i := range memberships {

94
roomserver/state/state.go
View file

@ -18,17 +18,17 @@ package state
import ( import (
"context" "context"
"database/sql"
"fmt" "fmt"
"sort" "sort"
"sync" "sync"
"time" "time"
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util" "github.com/matrix-org/util"
"github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/matrix-org/dendrite/roomserver/types"
) )
type StateResolutionStorage interface { type StateResolutionStorage interface {
@ -37,6 +37,7 @@ type StateResolutionStorage interface {
StateBlockNIDs(ctx context.Context, stateNIDs []types.StateSnapshotNID) ([]types.StateBlockNIDList, error) StateBlockNIDs(ctx context.Context, stateNIDs []types.StateSnapshotNID) ([]types.StateBlockNIDList, error)
StateEntries(ctx context.Context, stateBlockNIDs []types.StateBlockNID) ([]types.StateEntryList, error) StateEntries(ctx context.Context, stateBlockNIDs []types.StateBlockNID) ([]types.StateEntryList, error)
SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error)
BulkSelectSnapshotsFromEventIDs(ctx context.Context, eventIDs []string) (map[types.StateSnapshotNID][]string, error)
StateEntriesForTuples(ctx context.Context, stateBlockNIDs []types.StateBlockNID, stateKeyTuples []types.StateKeyTuple) ([]types.StateEntryList, error) StateEntriesForTuples(ctx context.Context, stateBlockNIDs []types.StateBlockNID, stateKeyTuples []types.StateKeyTuple) ([]types.StateEntryList, error)
StateAtEventIDs(ctx context.Context, eventIDs []string) ([]types.StateAtEvent, error) StateAtEventIDs(ctx context.Context, eventIDs []string) ([]types.StateAtEvent, error)
AddState(ctx context.Context, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, state []types.StateEntry) (types.StateSnapshotNID, error) AddState(ctx context.Context, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, state []types.StateEntry) (types.StateSnapshotNID, error)
@ -130,21 +131,10 @@ func (v *StateResolution) LoadMembershipAtEvent(
span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.LoadMembershipAtEvent") span, ctx := opentracing.StartSpanFromContext(ctx, "StateResolution.LoadMembershipAtEvent")
defer span.Finish() defer span.Finish()
// De-dupe snapshotNIDs // Get a mapping from snapshotNID -> eventIDs
snapshotNIDMap := make(map[types.StateSnapshotNID][]string) // map from snapshot NID to eventIDs snapshotNIDMap, err := v.db.BulkSelectSnapshotsFromEventIDs(ctx, eventIDs)
for i := range eventIDs { if err != nil {
eventID := eventIDs[i] return nil, err
snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID)
if err != nil && err != sql.ErrNoRows {
return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID failed for event %s : %w", eventID, err)
}
if snapshotNID == 0 {
// If we don't know a state snapshot for this event then we can't calculate
// memberships at the time of the event, so skip over it. This means that
// it isn't guaranteed that the response map will contain every single event.
continue
}
snapshotNIDMap[snapshotNID] = append(snapshotNIDMap[snapshotNID], eventID)
} }
snapshotNIDs := make([]types.StateSnapshotNID, 0, len(snapshotNIDMap)) snapshotNIDs := make([]types.StateSnapshotNID, 0, len(snapshotNIDMap))
@ -157,10 +147,12 @@ func (v *StateResolution) LoadMembershipAtEvent(
return nil, err return nil, err
} }
result := make(map[string][]types.StateEntry) var wantStateBlocks []types.StateBlockNID
for _, stateBlockNIDList := range stateBlockNIDLists { for _, x := range stateBlockNIDLists {
// Query the membership event for the user at the given stateblocks wantStateBlocks = append(wantStateBlocks, x.StateBlockNIDs...)
stateEntryLists, err := v.db.StateEntriesForTuples(ctx, stateBlockNIDList.StateBlockNIDs, []types.StateKeyTuple{ }
stateEntryLists, err := v.db.StateEntriesForTuples(ctx, uniqueStateBlockNIDs(wantStateBlocks), []types.StateKeyTuple{
{ {
EventTypeNID: types.MRoomMemberNID, EventTypeNID: types.MRoomMemberNID,
EventStateKeyNID: stateKeyNID, EventStateKeyNID: stateKeyNID,
@ -170,11 +162,30 @@ func (v *StateResolution) LoadMembershipAtEvent(
return nil, err return nil, err
} }
stateBlockNIDsMap := stateBlockNIDListMap(stateBlockNIDLists)
stateEntriesMap := stateEntryListMap(stateEntryLists)
result := make(map[string][]types.StateEntry)
for _, stateBlockNIDList := range stateBlockNIDLists {
stateBlockNIDs, ok := stateBlockNIDsMap.lookup(stateBlockNIDList.StateSnapshotNID)
if !ok {
// This should only get hit if the database is corrupt.
// It should be impossible for an event to reference a NID that doesn't exist
return nil, fmt.Errorf("corrupt DB: Missing state snapshot numeric ID %d", stateBlockNIDList.StateSnapshotNID)
}
for _, stateBlockNID := range stateBlockNIDs {
entries, ok := stateEntriesMap.lookup(stateBlockNID)
if !ok {
// This should only get hit if the database is corrupt.
// It should be impossible for an event to reference a NID that doesn't exist
return nil, fmt.Errorf("corrupt DB: Missing state block numeric ID %d", stateBlockNID)
}
evIDs := snapshotNIDMap[stateBlockNIDList.StateSnapshotNID] evIDs := snapshotNIDMap[stateBlockNIDList.StateSnapshotNID]
for _, evID := range evIDs { for _, evID := range evIDs {
for _, x := range stateEntryLists { result[evID] = append(result[evID], entries...)
result[evID] = append(result[evID], x.StateEntries...)
} }
} }
} }
@ -944,7 +955,6 @@ func (v *StateResolution) resolveConflictsV2(
authSets := make(map[string][]*gomatrixserverlib.Event, len(conflicted)) authSets := make(map[string][]*gomatrixserverlib.Event, len(conflicted))
authEvents := make([]*gomatrixserverlib.Event, 0, estimate*3) authEvents := make([]*gomatrixserverlib.Event, 0, estimate*3)
gotAuthEvents := make(map[string]struct{}, estimate*3) gotAuthEvents := make(map[string]struct{}, estimate*3)
authDifference := make([]*gomatrixserverlib.Event, 0, estimate)
knownAuthEvents := make(map[string]types.Event, estimate*3) knownAuthEvents := make(map[string]types.Event, estimate*3)
// For each conflicted event, let's try and get the needed auth events. // For each conflicted event, let's try and get the needed auth events.
@ -992,41 +1002,6 @@ func (v *StateResolution) resolveConflictsV2(
// longer need this after this point. // longer need this after this point.
gotAuthEvents = nil // nolint:ineffassign gotAuthEvents = nil // nolint:ineffassign
// This function helps us to work out whether an event exists in one of the
// auth sets.
isInAuthList := func(k string, event *gomatrixserverlib.Event) bool {
for _, e := range authSets[k] {
if e.EventID() == event.EventID() {
return true
}
}
return false
}
// This function works out if an event exists in all of the auth sets.
isInAllAuthLists := func(event *gomatrixserverlib.Event) bool {
for k := range authSets {
if !isInAuthList(k, event) {
return false
}
}
return true
}
// Look through all of the auth events that we've been given and work out if
// there are any events which don't appear in all of the auth sets. If they
// don't then we add them to the auth difference.
func() {
span, _ := opentracing.StartSpanFromContext(ctx, "isInAllAuthLists")
defer span.Finish()
for _, event := range authEvents {
if !isInAllAuthLists(event) {
authDifference = append(authDifference, event)
}
}
}()
// Resolve the conflicts. // Resolve the conflicts.
resolvedEvents := func() []*gomatrixserverlib.Event { resolvedEvents := func() []*gomatrixserverlib.Event {
span, _ := opentracing.StartSpanFromContext(ctx, "gomatrixserverlib.ResolveStateConflictsV2") span, _ := opentracing.StartSpanFromContext(ctx, "gomatrixserverlib.ResolveStateConflictsV2")
@ -1036,7 +1011,6 @@ func (v *StateResolution) resolveConflictsV2(
conflictedEvents, conflictedEvents,
nonConflictedEvents, nonConflictedEvents,
authEvents, authEvents,
authDifference,
) )
}() }()

1
roomserver/storage/interface.go
View file

@ -72,6 +72,7 @@ type Database interface {
Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error) Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error)
// Look up snapshot NID for an event ID string // Look up snapshot NID for an event ID string
SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error)
BulkSelectSnapshotsFromEventIDs(ctx context.Context, eventIDs []string) (map[types.StateSnapshotNID][]string, error)
// Stores a matrix room event in the database. Returns the room NID, the state snapshot and the redacted event ID if any, or an error. // Stores a matrix room event in the database. Returns the room NID, the state snapshot and the redacted event ID if any, or an error.
StoreEvent( StoreEvent(
ctx context.Context, event *gomatrixserverlib.Event, authEventNIDs []types.EventNID, ctx context.Context, event *gomatrixserverlib.Event, authEventNIDs []types.EventNID,

31
roomserver/storage/postgres/events_table.go
View file

@ -22,11 +22,12 @@ import (
"sort" "sort"
"github.com/lib/pq" "github.com/lib/pq"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/storage/tables"
"github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/gomatrixserverlib"
) )
const eventsSchema = ` const eventsSchema = `
@ -80,6 +81,9 @@ const insertEventSQL = "" +
const selectEventSQL = "" + const selectEventSQL = "" +
"SELECT event_nid, state_snapshot_nid FROM roomserver_events WHERE event_id = $1" "SELECT event_nid, state_snapshot_nid FROM roomserver_events WHERE event_id = $1"
const bulkSelectSnapshotsForEventIDsSQL = "" +
"SELECT event_id, state_snapshot_nid FROM roomserver_events WHERE event_id = ANY($1)"
// Bulk lookup of events by string ID. // Bulk lookup of events by string ID.
// Sort by the numeric IDs for event type and state key. // Sort by the numeric IDs for event type and state key.
// This means we can use binary search to lookup entries by type and state key. // This means we can use binary search to lookup entries by type and state key.
@ -150,6 +154,7 @@ const selectEventRejectedSQL = "" +
type eventStatements struct { type eventStatements struct {
insertEventStmt *sql.Stmt insertEventStmt *sql.Stmt
selectEventStmt *sql.Stmt selectEventStmt *sql.Stmt
bulkSelectSnapshotsForEventIDsStmt *sql.Stmt
bulkSelectStateEventByIDStmt *sql.Stmt bulkSelectStateEventByIDStmt *sql.Stmt
bulkSelectStateEventByIDExcludingRejectedStmt *sql.Stmt bulkSelectStateEventByIDExcludingRejectedStmt *sql.Stmt
bulkSelectStateEventByNIDStmt *sql.Stmt bulkSelectStateEventByNIDStmt *sql.Stmt
@ -179,6 +184,7 @@ func PrepareEventsTable(db *sql.DB) (tables.Events, error) {
return s, sqlutil.StatementList{ return s, sqlutil.StatementList{
{&s.insertEventStmt, insertEventSQL}, {&s.insertEventStmt, insertEventSQL},
{&s.selectEventStmt, selectEventSQL}, {&s.selectEventStmt, selectEventSQL},
{&s.bulkSelectSnapshotsForEventIDsStmt, bulkSelectSnapshotsForEventIDsSQL},
{&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL}, {&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL},
{&s.bulkSelectStateEventByIDExcludingRejectedStmt, bulkSelectStateEventByIDExcludingRejectedSQL}, {&s.bulkSelectStateEventByIDExcludingRejectedStmt, bulkSelectStateEventByIDExcludingRejectedSQL},
{&s.bulkSelectStateEventByNIDStmt, bulkSelectStateEventByNIDSQL}, {&s.bulkSelectStateEventByNIDStmt, bulkSelectStateEventByNIDSQL},
@ -230,6 +236,29 @@ func (s *eventStatements) SelectEvent(
return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err
} }
func (s *eventStatements) BulkSelectSnapshotsFromEventIDs(
ctx context.Context, txn *sql.Tx, eventIDs []string,
) (map[types.StateSnapshotNID][]string, error) {
stmt := sqlutil.TxStmt(txn, s.bulkSelectSnapshotsForEventIDsStmt)
rows, err := stmt.QueryContext(ctx, pq.Array(eventIDs))
if err != nil {
return nil, err
}
var eventID string
var stateNID types.StateSnapshotNID
result := make(map[types.StateSnapshotNID][]string)
for rows.Next() {
if err := rows.Scan(&eventID, &stateNID); err != nil {
return nil, err
}
result[stateNID] = append(result[stateNID], eventID)
}
return result, rows.Err()
}
// bulkSelectStateEventByID lookups a list of state events by event ID. // bulkSelectStateEventByID lookups a list of state events by event ID.
// If not excluding rejected events, and any of the requested events are missing from // If not excluding rejected events, and any of the requested events are missing from
// the database it returns a types.MissingEventError. If excluding rejected events, // the database it returns a types.MissingEventError. If excluding rejected events,

7
roomserver/storage/shared/room_updater.go
View file

@ -5,8 +5,9 @@ import (
"database/sql" "database/sql"
"fmt" "fmt"
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/roomserver/types"
) )
type RoomUpdater struct { type RoomUpdater struct {
@ -186,6 +187,10 @@ func (u *RoomUpdater) EventIDs(
return u.d.EventsTable.BulkSelectEventID(ctx, u.txn, eventNIDs) return u.d.EventsTable.BulkSelectEventID(ctx, u.txn, eventNIDs)
} }
func (u *RoomUpdater) BulkSelectSnapshotsFromEventIDs(ctx context.Context, eventIDs []string) (map[types.StateSnapshotNID][]string, error) {
return u.d.EventsTable.BulkSelectSnapshotsFromEventIDs(ctx, u.txn, eventIDs)
}
func (u *RoomUpdater) StateAtEventIDs( func (u *RoomUpdater) StateAtEventIDs(
ctx context.Context, eventIDs []string, ctx context.Context, eventIDs []string,
) ([]types.StateAtEvent, error) { ) ([]types.StateAtEvent, error) {

23
roomserver/storage/shared/storage.go
View file

@ -469,6 +469,23 @@ func (d *Database) events(
eventNIDs = append(eventNIDs, nid) eventNIDs = append(eventNIDs, nid)
} }
} }
// If we don't need to get any events from the database, short circuit now
if len(eventNIDs) == 0 {
results := make([]types.Event, 0, len(inputEventNIDs))
for _, nid := range inputEventNIDs {
event, ok := events[nid]
if !ok || event == nil {
return nil, fmt.Errorf("event %d missing", nid)
}
results = append(results, types.Event{
EventNID: nid,
Event: event,
})
}
if !redactionsArePermanent {
d.applyRedactions(results)
}
}
eventJSONs, err := d.EventJSONTable.BulkSelectEventJSON(ctx, txn, eventNIDs) eventJSONs, err := d.EventJSONTable.BulkSelectEventJSON(ctx, txn, eventNIDs)
if err != nil { if err != nil {
return nil, err return nil, err
@ -534,6 +551,12 @@ func (d *Database) events(
return results, nil return results, nil
} }
func (d *Database) BulkSelectSnapshotsFromEventIDs(
ctx context.Context, eventIDs []string,
) (map[types.StateSnapshotNID][]string, error) {
return d.EventsTable.BulkSelectSnapshotsFromEventIDs(ctx, nil, eventIDs)
}
func (d *Database) MembershipUpdater( func (d *Database) MembershipUpdater(
ctx context.Context, roomID, targetUserID string, ctx context.Context, roomID, targetUserID string,
targetLocal bool, roomVersion gomatrixserverlib.RoomVersion, targetLocal bool, roomVersion gomatrixserverlib.RoomVersion,

4
roomserver/storage/sqlite3/deltas/20221027084407_published_appservice.go
View file

@ -24,8 +24,8 @@ func UpPulishedAppservice(ctx context.Context, tx *sql.Tx) error {
_, err := tx.ExecContext(ctx, ` ALTER TABLE roomserver_published RENAME TO roomserver_published_tmp; _, err := tx.ExecContext(ctx, ` ALTER TABLE roomserver_published RENAME TO roomserver_published_tmp;
CREATE TABLE IF NOT EXISTS roomserver_published ( CREATE TABLE IF NOT EXISTS roomserver_published (
room_id TEXT NOT NULL, room_id TEXT NOT NULL,
appservice_id TEXT NOT NULL, appservice_id TEXT NOT NULL DEFAULT '',
network_id TEXT NOT NULL, network_id TEXT NOT NULL DEFAULT '',
published BOOLEAN NOT NULL DEFAULT false, published BOOLEAN NOT NULL DEFAULT false,
CONSTRAINT unique_published_idx PRIMARY KEY (room_id, appservice_id, network_id) CONSTRAINT unique_published_idx PRIMARY KEY (room_id, appservice_id, network_id)
); );

42
roomserver/storage/sqlite3/events_table.go
View file

@ -23,11 +23,12 @@ import (
"sort" "sort"
"strings" "strings"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/storage/tables"
"github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/gomatrixserverlib"
) )
const eventsSchema = ` const eventsSchema = `
@ -57,6 +58,9 @@ const insertEventSQL = `
const selectEventSQL = "" + const selectEventSQL = "" +
"SELECT event_nid, state_snapshot_nid FROM roomserver_events WHERE event_id = $1" "SELECT event_nid, state_snapshot_nid FROM roomserver_events WHERE event_id = $1"
const bulkSelectSnapshotsForEventIDsSQL = "" +
"SELECT event_id, state_snapshot_nid FROM roomserver_events WHERE event_id IN ($1)"
// Bulk lookup of events by string ID. // Bulk lookup of events by string ID.
// Sort by the numeric IDs for event type and state key. // Sort by the numeric IDs for event type and state key.
// This means we can use binary search to lookup entries by type and state key. // This means we can use binary search to lookup entries by type and state key.
@ -124,6 +128,7 @@ type eventStatements struct {
db *sql.DB db *sql.DB
insertEventStmt *sql.Stmt insertEventStmt *sql.Stmt
selectEventStmt *sql.Stmt selectEventStmt *sql.Stmt
bulkSelectSnapshotsForEventIDsStmt *sql.Stmt
bulkSelectStateEventByIDStmt *sql.Stmt bulkSelectStateEventByIDStmt *sql.Stmt
bulkSelectStateEventByIDExcludingRejectedStmt *sql.Stmt bulkSelectStateEventByIDExcludingRejectedStmt *sql.Stmt
bulkSelectStateAtEventByIDStmt *sql.Stmt bulkSelectStateAtEventByIDStmt *sql.Stmt
@ -153,6 +158,7 @@ func PrepareEventsTable(db *sql.DB) (tables.Events, error) {
return s, sqlutil.StatementList{ return s, sqlutil.StatementList{
{&s.insertEventStmt, insertEventSQL}, {&s.insertEventStmt, insertEventSQL},
{&s.selectEventStmt, selectEventSQL}, {&s.selectEventStmt, selectEventSQL},
{&s.bulkSelectSnapshotsForEventIDsStmt, bulkSelectSnapshotsForEventIDsSQL},
{&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL}, {&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL},
{&s.bulkSelectStateEventByIDExcludingRejectedStmt, bulkSelectStateEventByIDExcludingRejectedSQL}, {&s.bulkSelectStateEventByIDExcludingRejectedStmt, bulkSelectStateEventByIDExcludingRejectedSQL},
{&s.bulkSelectStateAtEventByIDStmt, bulkSelectStateAtEventByIDSQL}, {&s.bulkSelectStateAtEventByIDStmt, bulkSelectStateAtEventByIDSQL},
@ -203,6 +209,40 @@ func (s *eventStatements) SelectEvent(
return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err
} }
func (s *eventStatements) BulkSelectSnapshotsFromEventIDs(
ctx context.Context, txn *sql.Tx, eventIDs []string,
) (map[types.StateSnapshotNID][]string, error) {
qry := strings.Replace(bulkSelectSnapshotsForEventIDsSQL, "($1)", sqlutil.QueryVariadic(len(eventIDs)), 1)
stmt, err := s.db.Prepare(qry)
if err != nil {
return nil, err
}
defer internal.CloseAndLogIfError(ctx, stmt, "BulkSelectSnapshotsFromEventIDs: stmt.close() failed")
params := make([]interface{}, len(eventIDs))
for i := range eventIDs {
params[i] = eventIDs[i]
}
rows, err := stmt.QueryContext(ctx, params...)
if err != nil {
return nil, err
}
defer internal.CloseAndLogIfError(ctx, rows, "BulkSelectSnapshotsFromEventIDs: rows.close() failed")
var eventID string
var stateNID types.StateSnapshotNID
result := make(map[types.StateSnapshotNID][]string)
for rows.Next() {
if err := rows.Scan(&eventID, &stateNID); err != nil {
return nil, err
}
result[stateNID] = append(result[stateNID], eventID)
}
return result, rows.Err()
}
// bulkSelectStateEventByID lookups a list of state events by event ID. // bulkSelectStateEventByID lookups a list of state events by event ID.
// If not excluding rejected events, and any of the requested events are missing from // If not excluding rejected events, and any of the requested events are missing from
// the database it returns a types.MissingEventError. If excluding rejected events, // the database it returns a types.MissingEventError. If excluding rejected events,

1
roomserver/storage/tables/interface.go
View file

@ -44,6 +44,7 @@ type Events interface {
referenceSHA256 []byte, authEventNIDs []types.EventNID, depth int64, isRejected bool, referenceSHA256 []byte, authEventNIDs []types.EventNID, depth int64, isRejected bool,
) (types.EventNID, types.StateSnapshotNID, error) ) (types.EventNID, types.StateSnapshotNID, error)
SelectEvent(ctx context.Context, txn *sql.Tx, eventID string) (types.EventNID, types.StateSnapshotNID, error) SelectEvent(ctx context.Context, txn *sql.Tx, eventID string) (types.EventNID, types.StateSnapshotNID, error)
BulkSelectSnapshotsFromEventIDs(ctx context.Context, txn *sql.Tx, eventIDs []string) (map[types.StateSnapshotNID][]string, error)
// bulkSelectStateEventByID lookups a list of state events by event ID. // bulkSelectStateEventByID lookups a list of state events by event ID.
// If any of the requested events are missing from the database it returns a types.MissingEventError // If any of the requested events are missing from the database it returns a types.MissingEventError
BulkSelectStateEventByID(ctx context.Context, txn *sql.Tx, eventIDs []string, excludeRejected bool) ([]types.StateEntry, error) BulkSelectStateEventByID(ctx context.Context, txn *sql.Tx, eventIDs []string, excludeRejected bool) ([]types.StateEntry, error)

1
syncapi/routing/context.go
View file

@ -93,7 +93,6 @@ func Context(
} }
stateFilter := gomatrixserverlib.StateFilter{ stateFilter := gomatrixserverlib.StateFilter{
Limit: 100,
NotSenders: filter.NotSenders, NotSenders: filter.NotSenders,
NotTypes: filter.NotTypes, NotTypes: filter.NotTypes,
Senders: filter.Senders, Senders: filter.Senders,

2
syncapi/routing/memberships.go
View file

@ -135,6 +135,6 @@ func GetMemberships(
} }
return util.JSONResponse{ return util.JSONResponse{
Code: http.StatusOK, Code: http.StatusOK,
JSON: getMembershipResponse{gomatrixserverlib.HeaderedToClientEvents(result, gomatrixserverlib.FormatSync)}, JSON: getMembershipResponse{gomatrixserverlib.HeaderedToClientEvents(result, gomatrixserverlib.FormatAll)},
} }
} }

2
syncapi/routing/search.go
View file

@ -294,7 +294,7 @@ type SearchRequest struct {
BeforeLimit int `json:"before_limit,omitempty"` BeforeLimit int `json:"before_limit,omitempty"`
IncludeProfile bool `json:"include_profile,omitempty"` IncludeProfile bool `json:"include_profile,omitempty"`
} `json:"event_context"` } `json:"event_context"`
Filter gomatrixserverlib.StateFilter `json:"filter"` Filter gomatrixserverlib.RoomEventFilter `json:"filter"`
Groupings struct { Groupings struct {
GroupBy []struct { GroupBy []struct {
Key string `json:"key"` Key string `json:"key"`

4
syncapi/storage/postgres/current_room_state_table.go
View file

@ -91,8 +91,7 @@ const selectCurrentStateSQL = "" +
" AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" + " AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" +
" AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" + " AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" +
" AND ( $6::bool IS NULL OR contains_url = $6 )" + " AND ( $6::bool IS NULL OR contains_url = $6 )" +
" AND (event_id = ANY($7)) IS NOT TRUE" + " AND (event_id = ANY($7)) IS NOT TRUE"
" LIMIT $8"
const selectJoinedUsersSQL = "" + const selectJoinedUsersSQL = "" +
"SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'"
@ -290,7 +289,6 @@ func (s *currentRoomStateStatements) SelectCurrentState(
pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.NotTypes)), pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.NotTypes)),
stateFilter.ContainsURL, stateFilter.ContainsURL,
pq.StringArray(excludeEventIDs), pq.StringArray(excludeEventIDs),
stateFilter.Limit,
) )
if err != nil { if err != nil {
return nil, err return nil, err

9
syncapi/storage/postgres/output_room_events_table.go
View file

@ -144,17 +144,16 @@ const selectStateInRangeFilteredSQL = "" +
" AND ( $6::text[] IS NULL OR type LIKE ANY($6) )" + " AND ( $6::text[] IS NULL OR type LIKE ANY($6) )" +
" AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" + " AND ( $7::text[] IS NULL OR NOT(type LIKE ANY($7)) )" +
" AND ( $8::bool IS NULL OR contains_url = $8 )" + " AND ( $8::bool IS NULL OR contains_url = $8 )" +
" ORDER BY id ASC" + " ORDER BY id ASC"
" LIMIT $9"
// In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id).
// In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). // In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id).
const selectStateInRangeSQL = "" + const selectStateInRangeSQL = "" +
"SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids, history_visibility" + "SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids, history_visibility" +
" FROM syncapi_output_room_events" + " FROM syncapi_output_room_events" +
" WHERE (id > $1 AND id <= $2) AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + " WHERE (id > $1 AND id <= $2) AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" +
" AND room_id = ANY($3)" + " AND room_id = ANY($3)" +
" ORDER BY id ASC" + " ORDER BY id ASC"
" LIMIT $4"
const deleteEventsForRoomSQL = "" + const deleteEventsForRoomSQL = "" +
"DELETE FROM syncapi_output_room_events WHERE room_id = $1" "DELETE FROM syncapi_output_room_events WHERE room_id = $1"
@ -264,13 +263,11 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.Types)), pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.Types)),
pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.NotTypes)), pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.NotTypes)),
stateFilter.ContainsURL, stateFilter.ContainsURL,
stateFilter.Limit,
) )
} else { } else {
stmt := sqlutil.TxStmt(txn, s.selectStateInRangeStmt) stmt := sqlutil.TxStmt(txn, s.selectStateInRangeStmt)
rows, err = stmt.QueryContext( rows, err = stmt.QueryContext(
ctx, r.Low(), r.High(), pq.StringArray(roomIDs), ctx, r.Low(), r.High(), pq.StringArray(roomIDs),
r.High()-r.Low(),
) )
} }

3
syncapi/storage/sqlite3/current_room_state_table.go
View file

@ -277,7 +277,8 @@ func (s *currentRoomStateStatements) SelectCurrentState(
}, },
stateFilter.Senders, stateFilter.NotSenders, stateFilter.Senders, stateFilter.NotSenders,
stateFilter.Types, stateFilter.NotTypes, stateFilter.Types, stateFilter.NotTypes,
excludeEventIDs, stateFilter.ContainsURL, stateFilter.Limit, FilterOrderNone, excludeEventIDs, stateFilter.ContainsURL, 0,
FilterOrderNone,
) )
if err != nil { if err != nil {
return nil, fmt.Errorf("s.prepareWithFilters: %w", err) return nil, fmt.Errorf("s.prepareWithFilters: %w", err)

2
syncapi/storage/sqlite3/filtering.go
View file

@ -84,8 +84,10 @@ func prepareWithFilters(
case FilterOrderDesc: case FilterOrderDesc:
query += " ORDER BY id DESC" query += " ORDER BY id DESC"
} }
if limit > 0 {
query += fmt.Sprintf(" LIMIT $%d", offset+1) query += fmt.Sprintf(" LIMIT $%d", offset+1)
params = append(params, limit) params = append(params, limit)
}
var stmt *sql.Stmt var stmt *sql.Stmt
var err error var err error

2
syncapi/storage/sqlite3/output_room_events_table.go
View file

@ -200,7 +200,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange(
s.db, txn, stmtSQL, inputParams, s.db, txn, stmtSQL, inputParams,
stateFilter.Senders, stateFilter.NotSenders, stateFilter.Senders, stateFilter.NotSenders,
stateFilter.Types, stateFilter.NotTypes, stateFilter.Types, stateFilter.NotTypes,
nil, stateFilter.ContainsURL, stateFilter.Limit, FilterOrderAsc, nil, stateFilter.ContainsURL, 0, FilterOrderAsc,
) )
} else { } else {
stmt, params, err = prepareWithFilters( stmt, params, err = prepareWithFilters(

26
syncapi/streams/stream_invite.go
View file

@ -4,6 +4,7 @@ import (
"context" "context"
"crypto/sha256" "crypto/sha256"
"encoding/base64" "encoding/base64"
"math"
"strconv" "strconv"
"time" "time"
@ -73,19 +74,14 @@ func (p *InviteStreamProvider) IncrementalSync(
return to return to
} }
for roomID := range retiredInvites { for roomID := range retiredInvites {
if req.Response.Rooms.Invite[roomID] != nil { membership, _, err := snapshot.SelectMembershipForUser(ctx, roomID, req.Device.UserID, math.MaxInt64)
continue // Skip if the user is an existing member of the room.
} // Otherwise, the NewLeaveResponse will eject the user from the room unintentionally
if req.Response.Rooms.Join[roomID] != nil { if membership == gomatrixserverlib.Join ||
err != nil {
continue continue
} }
joinedUsers, err := snapshot.AllJoinedUsersInRoom(ctx, []string{roomID})
if err != nil {
continue
}
if !contains(joinedUsers[roomID], req.Device.UserID) {
lr := types.NewLeaveResponse() lr := types.NewLeaveResponse()
h := sha256.Sum256(append([]byte(roomID), []byte(strconv.FormatInt(int64(to), 10))...)) h := sha256.Sum256(append([]byte(roomID), []byte(strconv.FormatInt(int64(to), 10))...))
lr.Timeline.Events = append(lr.Timeline.Events, gomatrixserverlib.ClientEvent{ lr.Timeline.Events = append(lr.Timeline.Events, gomatrixserverlib.ClientEvent{
@ -100,16 +96,6 @@ func (p *InviteStreamProvider) IncrementalSync(
}) })
req.Response.Rooms.Leave[roomID] = lr req.Response.Rooms.Leave[roomID] = lr
} }
}
return maxID return maxID
} }
func contains(values []string, findVal string) bool {
for _, v := range values {
if v == findVal {
return true
}
}
return false
}

22
syncapi/streams/stream_pdu.go
View file

@ -301,7 +301,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
} }
// Applies the history visibility rules // Applies the history visibility rules
events, err := applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, delta.RoomID, device.UserID, eventFilter.Limit, recentEvents) events, err := applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, delta.RoomID, device.UserID, recentEvents)
if err != nil { if err != nil {
logrus.WithError(err).Error("unable to apply history visibility filter") logrus.WithError(err).Error("unable to apply history visibility filter")
} }
@ -315,11 +315,6 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
// Now that we've filtered the timeline, work out which state events are still // Now that we've filtered the timeline, work out which state events are still
// left. Anything that appears in the filtered timeline will be removed from the // left. Anything that appears in the filtered timeline will be removed from the
// "state" section and kept in "timeline". // "state" section and kept in "timeline".
// Don't remove the state events if this is a newly joined room. The member
// needs the full room state. For example, to figure out if the room is a space
// or not, it needs the m.room.create event which may be chronologically outside
// the filter limit.
if !delta.NewlyJoined { if !delta.NewlyJoined {
delta.StateEvents = gomatrixserverlib.HeaderedReverseTopologicalOrdering( delta.StateEvents = gomatrixserverlib.HeaderedReverseTopologicalOrdering(
removeDuplicates(delta.StateEvents, recentEvents), removeDuplicates(delta.StateEvents, recentEvents),
@ -328,10 +323,14 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
} }
if len(delta.StateEvents) > 0 { if len(delta.StateEvents) > 0 {
updateLatestPosition(delta.StateEvents[len(delta.StateEvents)-1].EventID()) if last := delta.StateEvents[len(delta.StateEvents)-1]; last != nil {
updateLatestPosition(last.EventID())
}
} }
if len(events) > 0 { if len(events) > 0 {
updateLatestPosition(events[len(events)-1].EventID()) if last := events[len(events)-1]; last != nil {
updateLatestPosition(last.EventID())
}
} }
switch delta.Membership { switch delta.Membership {
@ -381,12 +380,12 @@ func applyHistoryVisibilityFilter(
snapshot storage.DatabaseTransaction, snapshot storage.DatabaseTransaction,
rsAPI roomserverAPI.SyncRoomserverAPI, rsAPI roomserverAPI.SyncRoomserverAPI,
roomID, userID string, roomID, userID string,
limit int,
recentEvents []*gomatrixserverlib.HeaderedEvent, recentEvents []*gomatrixserverlib.HeaderedEvent,
) ([]*gomatrixserverlib.HeaderedEvent, error) { ) ([]*gomatrixserverlib.HeaderedEvent, error) {
// We need to make sure we always include the latest states events, if they are in the timeline. // We need to make sure we always include the latest states events, if they are in the timeline.
// We grep at least limit * 2 events, to ensure we really get the needed events. // We grep at least limit * 2 events, to ensure we really get the needed events.
stateEvents, err := snapshot.CurrentState(ctx, roomID, &gomatrixserverlib.StateFilter{Limit: limit * 2}, nil) filter := gomatrixserverlib.DefaultStateFilter()
stateEvents, err := snapshot.CurrentState(ctx, roomID, &filter, nil)
if err != nil { if err != nil {
// Not a fatal error, we can continue without the stateEvents, // Not a fatal error, we can continue without the stateEvents,
// they are only needed if there are state events in the timeline. // they are only needed if there are state events in the timeline.
@ -524,7 +523,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync(
events := recentEvents events := recentEvents
// Only apply history visibility checks if the response is for joined rooms // Only apply history visibility checks if the response is for joined rooms
if !isPeek { if !isPeek {
events, err = applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, roomID, device.UserID, eventFilter.Limit, recentEvents) events, err = applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, roomID, device.UserID, recentEvents)
if err != nil { if err != nil {
logrus.WithError(err).Error("unable to apply history visibility filter") logrus.WithError(err).Error("unable to apply history visibility filter")
} }
@ -604,7 +603,6 @@ func (p *PDUStreamProvider) lazyLoadMembers(
} }
// Query missing membership events // Query missing membership events
filter := gomatrixserverlib.DefaultStateFilter() filter := gomatrixserverlib.DefaultStateFilter()
filter.Limit = stateFilter.Limit
filter.Senders = &wantUsers filter.Senders = &wantUsers
filter.Types = &[]string{gomatrixserverlib.MRoomMember} filter.Types = &[]string{gomatrixserverlib.MRoomMember}
memberships, err := snapshot.GetStateEventsForRoom(ctx, roomID, &filter) memberships, err := snapshot.GetStateEventsForRoom(ctx, roomID, &filter)

1
syncapi/sync/request.go
View file

@ -79,7 +79,6 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat
// for the rest of the data to trickle down. // for the rest of the data to trickle down.
filter.AccountData.Limit = math.MaxInt32 filter.AccountData.Limit = math.MaxInt32
filter.Room.AccountData.Limit = math.MaxInt32 filter.Room.AccountData.Limit = math.MaxInt32
filter.Room.State.Limit = math.MaxInt32
} }
logger := util.GetLogger(req.Context()).WithFields(logrus.Fields{ logger := util.GetLogger(req.Context()).WithFields(logrus.Fields{

2
test/db.go
View file

@ -176,7 +176,7 @@ func WithAllDatabases(t *testing.T, testFn func(t *testing.T, db DBType)) {
for dbName, dbType := range dbs { for dbName, dbType := range dbs {
dbt := dbType dbt := dbType
t.Run(dbName, func(tt *testing.T) { t.Run(dbName, func(tt *testing.T) {
tt.Parallel() //tt.Parallel()
testFn(tt, dbt) testFn(tt, dbt)
}) })
} }

78
userapi/consumers/roomserver.go
View file

@ -5,6 +5,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"strings" "strings"
"sync"
"time" "time"
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
@ -23,6 +24,7 @@ import (
"github.com/matrix-org/dendrite/userapi/producers" "github.com/matrix-org/dendrite/userapi/producers"
"github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/dendrite/userapi/storage"
"github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/dendrite/userapi/storage/tables"
userAPITypes "github.com/matrix-org/dendrite/userapi/types"
"github.com/matrix-org/dendrite/userapi/util" "github.com/matrix-org/dendrite/userapi/util"
) )
@ -36,6 +38,11 @@ type OutputRoomEventConsumer struct {
topic string topic string
pgClient pushgateway.Client pgClient pushgateway.Client
syncProducer *producers.SyncAPI syncProducer *producers.SyncAPI
msgCounts map[gomatrixserverlib.ServerName]userAPITypes.MessageStats
roomCounts map[gomatrixserverlib.ServerName]map[string]bool // map from serverName to map from rommID to "isEncrypted"
lastUpdate time.Time
countsLock sync.Mutex
serverName gomatrixserverlib.ServerName
} }
func NewOutputRoomEventConsumer( func NewOutputRoomEventConsumer(
@ -57,6 +64,11 @@ func NewOutputRoomEventConsumer(
pgClient: pgClient, pgClient: pgClient,
rsAPI: rsAPI, rsAPI: rsAPI,
syncProducer: syncProducer, syncProducer: syncProducer,
msgCounts: map[gomatrixserverlib.ServerName]userAPITypes.MessageStats{},
roomCounts: map[gomatrixserverlib.ServerName]map[string]bool{},
lastUpdate: time.Now(),
countsLock: sync.Mutex{},
serverName: cfg.Matrix.ServerName,
} }
} }
@ -88,6 +100,10 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms
return true return true
} }
if s.cfg.Matrix.ReportStats.Enabled {
go s.storeMessageStats(ctx, event.Type(), event.Sender(), event.RoomID())
}
log.WithFields(log.Fields{ log.WithFields(log.Fields{
"event_id": event.EventID(), "event_id": event.EventID(),
"event_type": event.Type(), "event_type": event.Type(),
@ -107,6 +123,68 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms
return true return true
} }
func (s *OutputRoomEventConsumer) storeMessageStats(ctx context.Context, eventType, eventSender, roomID string) {
s.countsLock.Lock()
defer s.countsLock.Unlock()
// reset the roomCounts on a day change
if s.lastUpdate.Day() != time.Now().Day() {
s.roomCounts[s.serverName] = make(map[string]bool)
s.lastUpdate = time.Now()
}
_, sender, err := gomatrixserverlib.SplitID('@', eventSender)
if err != nil {
return
}
msgCount := s.msgCounts[s.serverName]
roomCount := s.roomCounts[s.serverName]
if roomCount == nil {
roomCount = make(map[string]bool)
}
switch eventType {
case "m.room.message":
roomCount[roomID] = false
msgCount.Messages++
if sender == s.serverName {
msgCount.SentMessages++
}
case "m.room.encrypted":
roomCount[roomID] = true
msgCount.MessagesE2EE++
if sender == s.serverName {
msgCount.SentMessagesE2EE++
}
default:
return
}
s.msgCounts[s.serverName] = msgCount
s.roomCounts[s.serverName] = roomCount
for serverName, stats := range s.msgCounts {
var normalRooms, encryptedRooms int64 = 0, 0
for _, isEncrypted := range s.roomCounts[s.serverName] {
if isEncrypted {
encryptedRooms++
} else {
normalRooms++
}
}
err := s.db.UpsertDailyRoomsMessages(ctx, serverName, stats, normalRooms, encryptedRooms)
if err != nil {
log.WithError(err).Errorf("failed to upsert daily messages")
}
// Clear stats if we successfully stored it
if err == nil {
stats.Messages = 0
stats.SentMessages = 0
stats.MessagesE2EE = 0
stats.SentMessagesE2EE = 0
s.msgCounts[serverName] = stats
}
}
}
func (s *OutputRoomEventConsumer) processMessage(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, streamPos uint64) error { func (s *OutputRoomEventConsumer) processMessage(ctx context.Context, event *gomatrixserverlib.HeaderedEvent, streamPos uint64) error {
members, roomSize, err := s.localRoomMembers(ctx, event.RoomID()) members, roomSize, err := s.localRoomMembers(ctx, event.RoomID())
if err != nil { if err != nil {

123
userapi/consumers/roomserver_test.go
View file

@ -2,7 +2,10 @@ package consumers
import ( import (
"context" "context"
"reflect"
"sync"
"testing" "testing"
"time"
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@ -12,6 +15,7 @@ import (
"github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/test/testrig"
"github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/dendrite/userapi/storage"
userAPITypes "github.com/matrix-org/dendrite/userapi/types"
) )
func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
@ -132,3 +136,122 @@ func Test_evaluatePushRules(t *testing.T) {
} }
}) })
} }
func TestMessageStats(t *testing.T) {
type args struct {
eventType string
eventSender string
roomID string
}
tests := []struct {
name string
args args
ourServer gomatrixserverlib.ServerName
lastUpdate time.Time
initRoomCounts map[gomatrixserverlib.ServerName]map[string]bool
wantStats userAPITypes.MessageStats
}{
{
name: "m.room.create does not count as a message",
ourServer: "localhost",
args: args{
eventType: "m.room.create",
eventSender: "@alice:localhost",
},
},
{
name: "our server - message",
ourServer: "localhost",
args: args{
eventType: "m.room.message",
eventSender: "@alice:localhost",
roomID: "normalRoom",
},
wantStats: userAPITypes.MessageStats{Messages: 1, SentMessages: 1},
},
{
name: "our server - E2EE message",
ourServer: "localhost",
args: args{
eventType: "m.room.encrypted",
eventSender: "@alice:localhost",
roomID: "encryptedRoom",
},
wantStats: userAPITypes.MessageStats{Messages: 1, SentMessages: 1, MessagesE2EE: 1, SentMessagesE2EE: 1},
},
{
name: "remote server - message",
ourServer: "localhost",
args: args{
eventType: "m.room.message",
eventSender: "@alice:remote",
roomID: "normalRoom",
},
wantStats: userAPITypes.MessageStats{Messages: 2, SentMessages: 1, MessagesE2EE: 1, SentMessagesE2EE: 1},
},
{
name: "remote server - E2EE message",
ourServer: "localhost",
args: args{
eventType: "m.room.encrypted",
eventSender: "@alice:remote",
roomID: "encryptedRoom",
},
wantStats: userAPITypes.MessageStats{Messages: 2, SentMessages: 1, MessagesE2EE: 2, SentMessagesE2EE: 1},
},
{
name: "day change creates a new room map",
ourServer: "localhost",
lastUpdate: time.Now().Add(-time.Hour * 24),
initRoomCounts: map[gomatrixserverlib.ServerName]map[string]bool{
"localhost": {"encryptedRoom": true},
},
args: args{
eventType: "m.room.encrypted",
eventSender: "@alice:remote",
roomID: "someOtherRoom",
},
wantStats: userAPITypes.MessageStats{Messages: 2, SentMessages: 1, MessagesE2EE: 3, SentMessagesE2EE: 1},
},
}
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
db, close := mustCreateDatabase(t, dbType)
defer close()
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.lastUpdate.IsZero() {
tt.lastUpdate = time.Now()
}
if tt.initRoomCounts == nil {
tt.initRoomCounts = map[gomatrixserverlib.ServerName]map[string]bool{}
}
s := &OutputRoomEventConsumer{
db: db,
msgCounts: map[gomatrixserverlib.ServerName]userAPITypes.MessageStats{},
roomCounts: tt.initRoomCounts,
countsLock: sync.Mutex{},
lastUpdate: tt.lastUpdate,
serverName: tt.ourServer,
}
s.storeMessageStats(context.Background(), tt.args.eventType, tt.args.eventSender, tt.args.roomID)
t.Logf("%+v", s.roomCounts)
gotStats, activeRooms, activeE2EERooms, err := db.DailyRoomsMessages(context.Background(), tt.ourServer)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if !reflect.DeepEqual(gotStats, tt.wantStats) {
t.Fatalf("expected %+v, got %+v", tt.wantStats, gotStats)
}
if tt.args.eventType == "m.room.encrypted" && activeE2EERooms != 1 {
t.Fatalf("expected room to be activeE2EE")
}
if tt.args.eventType == "m.room.message" && activeRooms != 1 {
t.Fatalf("expected room to be active")
}
})
}
})
}

4
userapi/storage/interface.go
View file

@ -19,6 +19,8 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/internal/pushrules" "github.com/matrix-org/dendrite/internal/pushrules"
"github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/api"
@ -144,6 +146,8 @@ type Database interface {
type Statistics interface { type Statistics interface {
UserStatistics(ctx context.Context) (*types.UserStatistics, *types.DatabaseEngine, error) UserStatistics(ctx context.Context) (*types.UserStatistics, *types.DatabaseEngine, error)
DailyRoomsMessages(ctx context.Context, serverName gomatrixserverlib.ServerName) (stats types.MessageStats, activeRooms, activeE2EERooms int64, err error)
UpsertDailyRoomsMessages(ctx context.Context, serverName gomatrixserverlib.ServerName, stats types.MessageStats, activeRooms, activeE2EERooms int64) error
} }
// Err3PIDInUse is the error returned when trying to save an association involving // Err3PIDInUse is the error returned when trying to save an association involving

73
userapi/storage/postgres/stats_table.go
View file

@ -20,13 +20,14 @@ import (
"time" "time"
"github.com/lib/pq" "github.com/lib/pq"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/dendrite/userapi/storage/tables"
"github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/dendrite/userapi/types"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
) )
const userDailyVisitsSchema = ` const userDailyVisitsSchema = `
@ -43,6 +44,35 @@ CREATE INDEX IF NOT EXISTS userapi_daily_visits_timestamp_idx ON userapi_daily_v
CREATE INDEX IF NOT EXISTS userapi_daily_visits_localpart_timestamp_idx ON userapi_daily_visits(localpart, timestamp); CREATE INDEX IF NOT EXISTS userapi_daily_visits_localpart_timestamp_idx ON userapi_daily_visits(localpart, timestamp);
` `
const messagesDailySchema = `
CREATE TABLE IF NOT EXISTS userapi_daily_stats (
timestamp BIGINT NOT NULL,
server_name TEXT NOT NULL,
messages BIGINT NOT NULL,
sent_messages BIGINT NOT NULL,
e2ee_messages BIGINT NOT NULL,
sent_e2ee_messages BIGINT NOT NULL,
active_rooms BIGINT NOT NULL,
active_e2ee_rooms BIGINT NOT NULL,
CONSTRAINT daily_stats_unique UNIQUE (timestamp, server_name)
);
`
const upsertDailyMessagesSQL = `
INSERT INTO userapi_daily_stats AS u (timestamp, server_name, messages, sent_messages, e2ee_messages, sent_e2ee_messages, active_rooms, active_e2ee_rooms)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT ON CONSTRAINT daily_stats_unique
DO UPDATE SET
messages=u.messages+excluded.messages, sent_messages=u.sent_messages+excluded.sent_messages,
e2ee_messages=u.e2ee_messages+excluded.e2ee_messages, sent_e2ee_messages=u.sent_e2ee_messages+excluded.sent_e2ee_messages,
active_rooms=GREATEST($7, u.active_rooms), active_e2ee_rooms=GREATEST($8, u.active_e2ee_rooms)
`
const selectDailyMessagesSQL = `
SELECT messages, sent_messages, e2ee_messages, sent_e2ee_messages, active_rooms, active_e2ee_rooms
FROM userapi_daily_stats
WHERE server_name = $1 AND timestamp = $2;
`
const countUsersLastSeenAfterSQL = "" + const countUsersLastSeenAfterSQL = "" +
"SELECT COUNT(*) FROM (" + "SELECT COUNT(*) FROM (" +
" SELECT localpart FROM userapi_devices WHERE last_seen_ts > $1 " + " SELECT localpart FROM userapi_devices WHERE last_seen_ts > $1 " +
@ -170,6 +200,8 @@ type statsStatements struct {
countUserByAccountTypeStmt *sql.Stmt countUserByAccountTypeStmt *sql.Stmt
countRegisteredUserByTypeStmt *sql.Stmt countRegisteredUserByTypeStmt *sql.Stmt
dbEngineVersionStmt *sql.Stmt dbEngineVersionStmt *sql.Stmt
upsertMessagesStmt *sql.Stmt
selectDailyMessagesStmt *sql.Stmt
} }
func NewPostgresStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (tables.StatsTable, error) { func NewPostgresStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (tables.StatsTable, error) {
@ -182,6 +214,10 @@ func NewPostgresStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName)
if err != nil { if err != nil {
return nil, err return nil, err
} }
_, err = db.Exec(messagesDailySchema)
if err != nil {
return nil, err
}
go s.startTimers() go s.startTimers()
return s, sqlutil.StatementList{ return s, sqlutil.StatementList{
{&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL}, {&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL},
@ -191,6 +227,8 @@ func NewPostgresStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName)
{&s.countUserByAccountTypeStmt, countUserByAccountTypeSQL}, {&s.countUserByAccountTypeStmt, countUserByAccountTypeSQL},
{&s.countRegisteredUserByTypeStmt, countRegisteredUserByTypeStmt}, {&s.countRegisteredUserByTypeStmt, countRegisteredUserByTypeStmt},
{&s.dbEngineVersionStmt, queryDBEngineVersion}, {&s.dbEngineVersionStmt, queryDBEngineVersion},
{&s.upsertMessagesStmt, upsertDailyMessagesSQL},
{&s.selectDailyMessagesStmt, selectDailyMessagesSQL},
}.Prepare(db) }.Prepare(db)
} }
@ -435,3 +473,34 @@ func (s *statsStatements) UpdateUserDailyVisits(
} }
return err return err
} }
func (s *statsStatements) UpsertDailyStats(
ctx context.Context, txn *sql.Tx,
serverName gomatrixserverlib.ServerName, stats types.MessageStats,
activeRooms, activeE2EERooms int64,
) error {
stmt := sqlutil.TxStmt(txn, s.upsertMessagesStmt)
timestamp := time.Now().Truncate(time.Hour * 24)
_, err := stmt.ExecContext(ctx,
gomatrixserverlib.AsTimestamp(timestamp),
serverName,
stats.Messages, stats.SentMessages, stats.MessagesE2EE, stats.SentMessagesE2EE,
activeRooms, activeE2EERooms,
)
return err
}
func (s *statsStatements) DailyRoomsMessages(
ctx context.Context, txn *sql.Tx,
serverName gomatrixserverlib.ServerName,
) (msgStats types.MessageStats, activeRooms, activeE2EERooms int64, err error) {
stmt := sqlutil.TxStmt(txn, s.selectDailyMessagesStmt)
timestamp := time.Now().Truncate(time.Hour * 24)
err = stmt.QueryRowContext(ctx, serverName, gomatrixserverlib.AsTimestamp(timestamp)).
Scan(&msgStats.Messages, &msgStats.SentMessages, &msgStats.MessagesE2EE, &msgStats.SentMessagesE2EE, &activeRooms, &activeE2EERooms)
if err != nil && err != sql.ErrNoRows {
return msgStats, 0, 0, err
}
return msgStats, activeRooms, activeE2EERooms, nil
}

15
userapi/storage/shared/storage.go
View file

@ -29,13 +29,12 @@ import (
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
"golang.org/x/crypto/bcrypt" "golang.org/x/crypto/bcrypt"
"github.com/matrix-org/dendrite/userapi/types"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/internal/pushrules" "github.com/matrix-org/dendrite/internal/pushrules"
"github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/dendrite/userapi/storage/tables"
"github.com/matrix-org/dendrite/userapi/types"
) )
// Database represents an account database // Database represents an account database
@ -808,3 +807,15 @@ func (d *Database) RemovePushers(
func (d *Database) UserStatistics(ctx context.Context) (*types.UserStatistics, *types.DatabaseEngine, error) { func (d *Database) UserStatistics(ctx context.Context) (*types.UserStatistics, *types.DatabaseEngine, error) {
return d.Stats.UserStatistics(ctx, nil) return d.Stats.UserStatistics(ctx, nil)
} }
func (d *Database) UpsertDailyRoomsMessages(ctx context.Context, serverName gomatrixserverlib.ServerName, stats types.MessageStats, activeRooms, activeE2EERooms int64) error {
return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
return d.Stats.UpsertDailyStats(ctx, txn, serverName, stats, activeRooms, activeE2EERooms)
})
}
func (d *Database) DailyRoomsMessages(
ctx context.Context, serverName gomatrixserverlib.ServerName,
) (stats types.MessageStats, activeRooms, activeE2EERooms int64, err error) {
return d.Stats.DailyRoomsMessages(ctx, nil, serverName)
}

68
userapi/storage/sqlite3/stats_table.go
View file

@ -44,6 +44,35 @@ CREATE INDEX IF NOT EXISTS userapi_daily_visits_timestamp_idx ON userapi_daily_v
CREATE INDEX IF NOT EXISTS userapi_daily_visits_localpart_timestamp_idx ON userapi_daily_visits(localpart, timestamp); CREATE INDEX IF NOT EXISTS userapi_daily_visits_localpart_timestamp_idx ON userapi_daily_visits(localpart, timestamp);
` `
const messagesDailySchema = `
CREATE TABLE IF NOT EXISTS userapi_daily_stats (
timestamp BIGINT NOT NULL,
server_name TEXT NOT NULL,
messages BIGINT NOT NULL,
sent_messages BIGINT NOT NULL,
e2ee_messages BIGINT NOT NULL,
sent_e2ee_messages BIGINT NOT NULL,
active_rooms BIGINT NOT NULL,
active_e2ee_rooms BIGINT NOT NULL,
CONSTRAINT daily_stats_unique UNIQUE (timestamp, server_name)
);
`
const upsertDailyMessagesSQL = `
INSERT INTO userapi_daily_stats (timestamp, server_name, messages, sent_messages, e2ee_messages, sent_e2ee_messages, active_rooms, active_e2ee_rooms)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (timestamp, server_name)
DO UPDATE SET
messages=messages+excluded.messages, sent_messages=sent_messages+excluded.sent_messages,
e2ee_messages=e2ee_messages+excluded.e2ee_messages, sent_e2ee_messages=sent_e2ee_messages+excluded.sent_e2ee_messages,
active_rooms=MAX($7, active_rooms), active_e2ee_rooms=MAX($8, active_e2ee_rooms)
`
const selectDailyMessagesSQL = `
SELECT messages, sent_messages, e2ee_messages, sent_e2ee_messages, active_rooms, active_e2ee_rooms
FROM userapi_daily_stats
WHERE server_name = $1 AND timestamp = $2;
`
const countUsersLastSeenAfterSQL = "" + const countUsersLastSeenAfterSQL = "" +
"SELECT COUNT(*) FROM (" + "SELECT COUNT(*) FROM (" +
" SELECT localpart FROM userapi_devices WHERE last_seen_ts > $1 " + " SELECT localpart FROM userapi_devices WHERE last_seen_ts > $1 " +
@ -176,6 +205,8 @@ type statsStatements struct {
countUserByAccountTypeStmt *sql.Stmt countUserByAccountTypeStmt *sql.Stmt
countRegisteredUserByTypeStmt *sql.Stmt countRegisteredUserByTypeStmt *sql.Stmt
dbEngineVersionStmt *sql.Stmt dbEngineVersionStmt *sql.Stmt
upsertMessagesStmt *sql.Stmt
selectDailyMessagesStmt *sql.Stmt
} }
func NewSQLiteStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (tables.StatsTable, error) { func NewSQLiteStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (tables.StatsTable, error) {
@ -189,6 +220,10 @@ func NewSQLiteStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (t
if err != nil { if err != nil {
return nil, err return nil, err
} }
_, err = db.Exec(messagesDailySchema)
if err != nil {
return nil, err
}
go s.startTimers() go s.startTimers()
return s, sqlutil.StatementList{ return s, sqlutil.StatementList{
{&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL}, {&s.countUsersLastSeenAfterStmt, countUsersLastSeenAfterSQL},
@ -198,6 +233,8 @@ func NewSQLiteStatsTable(db *sql.DB, serverName gomatrixserverlib.ServerName) (t
{&s.countUserByAccountTypeStmt, countUserByAccountTypeSQL}, {&s.countUserByAccountTypeStmt, countUserByAccountTypeSQL},
{&s.countRegisteredUserByTypeStmt, countRegisteredUserByTypeSQL}, {&s.countRegisteredUserByTypeStmt, countRegisteredUserByTypeSQL},
{&s.dbEngineVersionStmt, queryDBEngineVersion}, {&s.dbEngineVersionStmt, queryDBEngineVersion},
{&s.upsertMessagesStmt, upsertDailyMessagesSQL},
{&s.selectDailyMessagesStmt, selectDailyMessagesSQL},
}.Prepare(db) }.Prepare(db)
} }
@ -451,3 +488,34 @@ func (s *statsStatements) UpdateUserDailyVisits(
} }
return err return err
} }
func (s *statsStatements) UpsertDailyStats(
ctx context.Context, txn *sql.Tx,
serverName gomatrixserverlib.ServerName, stats types.MessageStats,
activeRooms, activeE2EERooms int64,
) error {
stmt := sqlutil.TxStmt(txn, s.upsertMessagesStmt)
timestamp := time.Now().Truncate(time.Hour * 24)
_, err := stmt.ExecContext(ctx,
gomatrixserverlib.AsTimestamp(timestamp),
serverName,
stats.Messages, stats.SentMessages, stats.MessagesE2EE, stats.SentMessagesE2EE,
activeRooms, activeE2EERooms,
)
return err
}
func (s *statsStatements) DailyRoomsMessages(
ctx context.Context, txn *sql.Tx,
serverName gomatrixserverlib.ServerName,
) (msgStats types.MessageStats, activeRooms, activeE2EERooms int64, err error) {
stmt := sqlutil.TxStmt(txn, s.selectDailyMessagesStmt)
timestamp := time.Now().Truncate(time.Hour * 24)
err = stmt.QueryRowContext(ctx, serverName, gomatrixserverlib.AsTimestamp(timestamp)).
Scan(&msgStats.Messages, &msgStats.SentMessages, &msgStats.MessagesE2EE, &msgStats.SentMessagesE2EE, &activeRooms, &activeE2EERooms)
if err != nil && err != sql.ErrNoRows {
return msgStats, 0, 0, err
}
return msgStats, activeRooms, activeE2EERooms, nil
}

4
userapi/storage/tables/interface.go
View file

@ -20,6 +20,8 @@ import (
"encoding/json" "encoding/json"
"time" "time"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/dendrite/userapi/types"
@ -115,7 +117,9 @@ type NotificationTable interface {
type StatsTable interface { type StatsTable interface {
UserStatistics(ctx context.Context, txn *sql.Tx) (*types.UserStatistics, *types.DatabaseEngine, error) UserStatistics(ctx context.Context, txn *sql.Tx) (*types.UserStatistics, *types.DatabaseEngine, error)
DailyRoomsMessages(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName) (msgStats types.MessageStats, activeRooms, activeE2EERooms int64, err error)
UpdateUserDailyVisits(ctx context.Context, txn *sql.Tx, startTime, lastUpdate time.Time) error UpdateUserDailyVisits(ctx context.Context, txn *sql.Tx, startTime, lastUpdate time.Time) error
UpsertDailyStats(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, stats types.MessageStats, activeRooms, activeE2EERooms int64) error
} }
type NotificationFilter uint32 type NotificationFilter uint32

9
userapi/storage/tables/stats_table_test.go
View file

@ -8,6 +8,9 @@ import (
"testing" "testing"
"time" "time"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test"
@ -16,8 +19,6 @@ import (
"github.com/matrix-org/dendrite/userapi/storage/sqlite3" "github.com/matrix-org/dendrite/userapi/storage/sqlite3"
"github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/dendrite/userapi/storage/tables"
"github.com/matrix-org/dendrite/userapi/types" "github.com/matrix-org/dendrite/userapi/types"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
) )
func mustMakeDBs(t *testing.T, dbType test.DBType) ( func mustMakeDBs(t *testing.T, dbType test.DBType) (
@ -227,7 +228,7 @@ func Test_UserStatistics(t *testing.T) {
mustUserUpdateRegistered(t, ctx, db, "user4", time.Now().AddDate(0, -2, 0)) mustUserUpdateRegistered(t, ctx, db, "user4", time.Now().AddDate(0, -2, 0))
mustUpdateDeviceLastSeen(t, ctx, db, "user4", time.Now()) mustUpdateDeviceLastSeen(t, ctx, db, "user4", time.Now())
startTime := time.Now().AddDate(0, 0, -2) startTime := time.Now().AddDate(0, 0, -2)
err := statsDB.UpdateUserDailyVisits(ctx, nil, startTime, startTime.Truncate(time.Hour*24).Add(time.Hour)) err := statsDB.UpdateUserDailyVisits(ctx, nil, startTime, startTime.Truncate(time.Hour*24))
if err != nil { if err != nil {
t.Fatalf("unable to update daily visits stats: %v", err) t.Fatalf("unable to update daily visits stats: %v", err)
} }
@ -278,7 +279,7 @@ func Test_UserStatistics(t *testing.T) {
mustUpdateDeviceLastSeen(t, ctx, db, "user1", time.Now().AddDate(0, 0, -i)) mustUpdateDeviceLastSeen(t, ctx, db, "user1", time.Now().AddDate(0, 0, -i))
mustUpdateDeviceLastSeen(t, ctx, db, "user5", time.Now().AddDate(0, 0, -i)) mustUpdateDeviceLastSeen(t, ctx, db, "user5", time.Now().AddDate(0, 0, -i))
startTime := time.Now().AddDate(0, 0, -i) startTime := time.Now().AddDate(0, 0, -i)
err := statsDB.UpdateUserDailyVisits(ctx, nil, startTime, startTime.Truncate(time.Hour*24).Add(time.Hour)) err := statsDB.UpdateUserDailyVisits(ctx, nil, startTime, startTime.Truncate(time.Hour*24))
if err != nil { if err != nil {
t.Fatalf("unable to update daily visits stats: %v", err) t.Fatalf("unable to update daily visits stats: %v", err)
} }

7
userapi/types/statistics.go
View file

@ -28,3 +28,10 @@ type DatabaseEngine struct {
Engine string Engine string
Version string Version string
} }
type MessageStats struct {
Messages int64
SentMessages int64
MessagesE2EE int64
SentMessagesE2EE int64
}

22
userapi/util/phonehomestats.go
View file

@ -24,11 +24,12 @@ import (
"syscall" "syscall"
"time" "time"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/dendrite/userapi/storage"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
) )
type phoneHomeStats struct { type phoneHomeStats struct {
@ -109,12 +110,19 @@ func (p *phoneHomeStats) collect() {
} }
// message and room stats // message and room stats
// TODO: Find a solution to actually set these values // TODO: Find a solution to actually set this value
p.stats["total_room_count"] = 0 p.stats["total_room_count"] = 0
p.stats["daily_messages"] = 0
p.stats["daily_sent_messages"] = 0 messageStats, activeRooms, activeE2EERooms, err := p.db.DailyRoomsMessages(ctx, p.serverName)
p.stats["daily_e2ee_messages"] = 0 if err != nil {
p.stats["daily_sent_e2ee_messages"] = 0 logrus.WithError(err).Warn("unable to query message stats, using default values")
}
p.stats["daily_messages"] = messageStats.Messages
p.stats["daily_sent_messages"] = messageStats.SentMessages
p.stats["daily_e2ee_messages"] = messageStats.MessagesE2EE
p.stats["daily_sent_e2ee_messages"] = messageStats.SentMessagesE2EE
p.stats["daily_active_rooms"] = activeRooms
p.stats["daily_active_e2ee_rooms"] = activeE2EERooms
// user stats and DB engine // user stats and DB engine
userStats, db, err := p.db.UserStatistics(ctx) userStats, db, err := p.db.UserStatistics(ctx)