mirror of
https://github.com/matrix-org/dendrite.git
synced 2025-12-06 22:43:10 -06:00
Merge remote-tracking branch 'origin' into release/upstream-0.10.7
This commit is contained in:
commit
5c07b88b0d
12
.cloudbuild/dev.yaml
Normal file
12
.cloudbuild/dev.yaml
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
steps:
|
||||
- name: gcr.io/cloud-builders/docker
|
||||
args: ['build', '-t', 'gcr.io/$PROJECT_ID/dendrite-monolith:$COMMIT_SHA', '-f', 'build/docker/Dockerfile.monolith', '.']
|
||||
- name: gcr.io/cloud-builders/kubectl
|
||||
args: ['-n', 'dendrite', 'set', 'image', 'deployment/dendrite', 'dendrite=gcr.io/$PROJECT_ID/dendrite-monolith:$COMMIT_SHA']
|
||||
env:
|
||||
- CLOUDSDK_CORE_PROJECT=globekeeper-development
|
||||
- CLOUDSDK_COMPUTE_ZONE=europe-west2-a
|
||||
- CLOUDSDK_CONTAINER_CLUSTER=synapse
|
||||
images:
|
||||
- gcr.io/$PROJECT_ID/dendrite-monolith:$COMMIT_SHA
|
||||
timeout: 480s
|
||||
12
.cloudbuild/prod.yaml
Normal file
12
.cloudbuild/prod.yaml
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
steps:
|
||||
- name: gcr.io/cloud-builders/docker
|
||||
args: ['build', '-t', 'gcr.io/$PROJECT_ID/dendrite-monolith:$TAG_NAME', '-f', 'build/docker/Dockerfile.monolith', '.']
|
||||
- name: gcr.io/cloud-builders/kubectl
|
||||
args: ['set', 'image', 'deployment/dendrite', 'dendrite=gcr.io/$PROJECT_ID/dendrite-monolith:$TAG_NAME']
|
||||
env:
|
||||
- CLOUDSDK_CORE_PROJECT=globekeeper-production
|
||||
- CLOUDSDK_COMPUTE_ZONE=europe-west2-a
|
||||
- CLOUDSDK_CONTAINER_CLUSTER=synapse-production
|
||||
images:
|
||||
- gcr.io/$PROJECT_ID/dendrite-monolith:$TAG_NAME
|
||||
timeout: 480s
|
||||
185
.github/workflows/dendrite.yml
vendored
185
.github/workflows/dendrite.yml
vendored
|
|
@ -14,51 +14,6 @@ concurrency:
|
|||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
wasm:
|
||||
name: WASM build test
|
||||
timeout-minutes: 5
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ false }} # disable for now
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: 1.18
|
||||
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-go-wasm-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-go-wasm
|
||||
|
||||
- name: Install Node
|
||||
uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: 14
|
||||
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-node-
|
||||
|
||||
- name: Reconfigure Git to use HTTPS auth for repo packages
|
||||
run: >
|
||||
git config --global url."https://github.com/".insteadOf
|
||||
ssh://git@github.com/
|
||||
|
||||
- name: Install test dependencies
|
||||
working-directory: ./test/wasm
|
||||
run: npm ci
|
||||
|
||||
- name: Test
|
||||
run: ./test-dendritejs.sh
|
||||
|
||||
# Run golangci-lint
|
||||
lint:
|
||||
|
|
@ -74,7 +29,7 @@ jobs:
|
|||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v3
|
||||
|
||||
# run go test with different go versions
|
||||
# run go test with go 1.19
|
||||
test:
|
||||
timeout-minutes: 5
|
||||
name: Unit tests (Go ${{ matrix.go }})
|
||||
|
|
@ -102,7 +57,7 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
go: ["1.18", "1.19"]
|
||||
go: ["1.19"]
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Setup go
|
||||
|
|
@ -129,7 +84,7 @@ jobs:
|
|||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: dendrite
|
||||
|
||||
# build Dendrite for linux with different architectures and go versions
|
||||
# build Dendrite for linux amd64 with go 1.18
|
||||
build:
|
||||
name: Build for Linux
|
||||
timeout-minutes: 10
|
||||
|
|
@ -137,9 +92,9 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
go: ["1.18", "1.19"]
|
||||
go: ["1.19"]
|
||||
goos: ["linux"]
|
||||
goarch: ["amd64", "386"]
|
||||
goarch: ["amd64"]
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Setup go
|
||||
|
|
@ -164,43 +119,10 @@ jobs:
|
|||
CGO_CFLAGS: -fno-stack-protector
|
||||
run: go build -trimpath -v -o "bin/" ./cmd/...
|
||||
|
||||
# build for Windows 64-bit
|
||||
build_windows:
|
||||
name: Build for Windows
|
||||
timeout-minutes: 10
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
go: ["1.18", "1.19"]
|
||||
goos: ["windows"]
|
||||
goarch: ["amd64"]
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Setup Go ${{ matrix.go }}
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: ${{ matrix.go }}
|
||||
- name: Install dependencies
|
||||
run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }}-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }}
|
||||
- env:
|
||||
GOOS: ${{ matrix.goos }}
|
||||
GOARCH: ${{ matrix.goarch }}
|
||||
CGO_ENABLED: 1
|
||||
CC: "/usr/bin/x86_64-w64-mingw32-gcc"
|
||||
run: go build -trimpath -v -o "bin/" ./cmd/...
|
||||
|
||||
# Dummy step to gate other tests on without repeating the whole list
|
||||
initial-tests-done:
|
||||
name: Initial tests passed
|
||||
needs: [lint, test, build, build_windows]
|
||||
needs: [lint, test, build]
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
|
||||
steps:
|
||||
|
|
@ -209,56 +131,6 @@ jobs:
|
|||
with:
|
||||
jobs: ${{ toJSON(needs) }}
|
||||
|
||||
# run database upgrade tests
|
||||
upgrade_test:
|
||||
name: Upgrade tests
|
||||
timeout-minutes: 20
|
||||
needs: initial-tests-done
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Setup go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: "1.18"
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-go-upgrade
|
||||
- name: Build upgrade-tests
|
||||
run: go build ./cmd/dendrite-upgrade-tests
|
||||
- name: Test upgrade
|
||||
run: ./dendrite-upgrade-tests --head .
|
||||
|
||||
# run database upgrade tests, skipping over one version
|
||||
upgrade_test_direct:
|
||||
name: Upgrade tests from HEAD-2
|
||||
timeout-minutes: 20
|
||||
needs: initial-tests-done
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Setup go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: "1.18"
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-go-upgrade
|
||||
- name: Build upgrade-tests
|
||||
run: go build ./cmd/dendrite-upgrade-tests
|
||||
- name: Test upgrade
|
||||
run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
|
||||
|
||||
# run Sytest in different variations
|
||||
sytest:
|
||||
timeout-minutes: 20
|
||||
|
|
@ -269,18 +141,6 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- label: SQLite native
|
||||
|
||||
- label: SQLite Cgo
|
||||
cgo: 1
|
||||
|
||||
- label: SQLite native, full HTTP APIs
|
||||
api: full-http
|
||||
|
||||
- label: SQLite Cgo, full HTTP APIs
|
||||
api: full-http
|
||||
cgo: 1
|
||||
|
||||
- label: PostgreSQL
|
||||
postgres: postgres
|
||||
|
||||
|
|
@ -313,13 +173,12 @@ jobs:
|
|||
run: /src/are-we-synapse-yet.py /logs/results.tap -v
|
||||
continue-on-error: true # not fatal
|
||||
- name: Upload Sytest logs
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
|
||||
path: |
|
||||
/logs/results.tap
|
||||
/logs/**/*.log*
|
||||
/logs
|
||||
|
||||
# run Complement
|
||||
complement:
|
||||
|
|
@ -331,18 +190,6 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- label: SQLite native
|
||||
|
||||
- label: SQLite Cgo
|
||||
cgo: 1
|
||||
|
||||
- label: SQLite native, full HTTP APIs
|
||||
api: full-http
|
||||
|
||||
- label: SQLite Cgo, full HTTP APIs
|
||||
api: full-http
|
||||
cgo: 1
|
||||
|
||||
- label: PostgreSQL
|
||||
postgres: Postgres
|
||||
|
||||
|
|
@ -390,7 +237,7 @@ jobs:
|
|||
continue
|
||||
fi
|
||||
|
||||
(wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
|
||||
(wget -O - "https://github.com/globekeeper/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
|
||||
done
|
||||
|
||||
# Build initial Dendrite image
|
||||
|
|
@ -416,8 +263,6 @@ jobs:
|
|||
needs:
|
||||
[
|
||||
initial-tests-done,
|
||||
upgrade_test,
|
||||
upgrade_test_direct,
|
||||
sytest,
|
||||
complement,
|
||||
]
|
||||
|
|
@ -428,15 +273,3 @@ jobs:
|
|||
uses: re-actors/alls-green@release/v1
|
||||
with:
|
||||
jobs: ${{ toJSON(needs) }}
|
||||
|
||||
update-docker-images:
|
||||
name: Update Docker images
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
security-events: write # To upload Trivy sarif files
|
||||
if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main'
|
||||
needs: [integration-tests-done]
|
||||
uses: matrix-org/dendrite/.github/workflows/docker.yml@main
|
||||
secrets:
|
||||
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
|
|
|||
313
.github/workflows/docker.yml
vendored
313
.github/workflows/docker.yml
vendored
|
|
@ -1,313 +0,0 @@
|
|||
# Based on https://github.com/docker/build-push-action
|
||||
|
||||
name: "Docker"
|
||||
|
||||
on:
|
||||
release: # A GitHub release was published
|
||||
types: [published]
|
||||
workflow_dispatch: # A build was manually requested
|
||||
workflow_call: # Another pipeline called us
|
||||
secrets:
|
||||
DOCKER_TOKEN:
|
||||
required: true
|
||||
|
||||
env:
|
||||
DOCKER_NAMESPACE: matrixdotorg
|
||||
DOCKER_HUB_USER: dendritegithub
|
||||
GHCR_NAMESPACE: matrix-org
|
||||
PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7
|
||||
|
||||
jobs:
|
||||
monolith:
|
||||
name: Monolith image
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
security-events: write # To upload Trivy sarif files
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
- name: Get release tag & build flags
|
||||
if: github.event_name == 'release' # Only for GitHub releases
|
||||
run: |
|
||||
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
|
||||
echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
|
||||
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
|
||||
[ ${BRANCH} == "main" ] && BRANCH=""
|
||||
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ env.DOCKER_HUB_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
- name: Login to GitHub Containers
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build main monolith image
|
||||
if: github.ref_name == 'main'
|
||||
id: docker_build_monolith
|
||||
uses: docker/build-push-action@v3
|
||||
with:
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
context: .
|
||||
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
|
||||
target: monolith
|
||||
platforms: ${{ env.PLATFORMS }}
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
|
||||
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
|
||||
|
||||
- name: Run Trivy vulnerability scanner
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
|
||||
format: "sarif"
|
||||
output: "trivy-results.sarif"
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security tab
|
||||
uses: github/codeql-action/upload-sarif@v2
|
||||
with:
|
||||
sarif_file: "trivy-results.sarif"
|
||||
|
||||
- name: Build release monolith image
|
||||
if: github.event_name == 'release' # Only for GitHub releases
|
||||
id: docker_build_monolith_release
|
||||
uses: docker/build-push-action@v3
|
||||
with:
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
context: .
|
||||
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
|
||||
target: monolith
|
||||
platforms: ${{ env.PLATFORMS }}
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:latest
|
||||
${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
|
||||
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:latest
|
||||
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
|
||||
|
||||
polylith:
|
||||
name: Polylith image
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
security-events: write # To upload Trivy sarif files
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
- name: Get release tag & build flags
|
||||
if: github.event_name == 'release' # Only for GitHub releases
|
||||
run: |
|
||||
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
|
||||
echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
|
||||
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
|
||||
[ ${BRANCH} == "main" ] && BRANCH=""
|
||||
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ env.DOCKER_HUB_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
- name: Login to GitHub Containers
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build main polylith image
|
||||
if: github.ref_name == 'main'
|
||||
id: docker_build_polylith
|
||||
uses: docker/build-push-action@v3
|
||||
with:
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
context: .
|
||||
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
|
||||
target: polylith
|
||||
platforms: ${{ env.PLATFORMS }}
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
|
||||
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
|
||||
|
||||
- name: Run Trivy vulnerability scanner
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
|
||||
format: "sarif"
|
||||
output: "trivy-results.sarif"
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security tab
|
||||
uses: github/codeql-action/upload-sarif@v2
|
||||
with:
|
||||
sarif_file: "trivy-results.sarif"
|
||||
|
||||
- name: Build release polylith image
|
||||
if: github.event_name == 'release' # Only for GitHub releases
|
||||
id: docker_build_polylith_release
|
||||
uses: docker/build-push-action@v3
|
||||
with:
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
context: .
|
||||
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
|
||||
target: polylith
|
||||
platforms: ${{ env.PLATFORMS }}
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest
|
||||
${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
|
||||
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest
|
||||
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
|
||||
|
||||
demo-pinecone:
|
||||
name: Pinecone demo image
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
- name: Get release tag & build flags
|
||||
if: github.event_name == 'release' # Only for GitHub releases
|
||||
run: |
|
||||
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
|
||||
echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
|
||||
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
|
||||
[ ${BRANCH} == "main" ] && BRANCH=""
|
||||
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ env.DOCKER_HUB_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
- name: Login to GitHub Containers
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build main Pinecone demo image
|
||||
if: github.ref_name == 'main'
|
||||
id: docker_build_demo_pinecone
|
||||
uses: docker/build-push-action@v3
|
||||
with:
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
context: .
|
||||
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
|
||||
file: ./build/docker/Dockerfile.demo-pinecone
|
||||
platforms: ${{ env.PLATFORMS }}
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }}
|
||||
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }}
|
||||
|
||||
- name: Build release Pinecone demo image
|
||||
if: github.event_name == 'release' # Only for GitHub releases
|
||||
id: docker_build_demo_pinecone_release
|
||||
uses: docker/build-push-action@v3
|
||||
with:
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
context: .
|
||||
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
|
||||
file: ./build/docker/Dockerfile.demo-pinecone
|
||||
platforms: ${{ env.PLATFORMS }}
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest
|
||||
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}
|
||||
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest
|
||||
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}
|
||||
|
||||
demo-yggdrasil:
|
||||
name: Yggdrasil demo image
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
- name: Get release tag & build flags
|
||||
if: github.event_name == 'release' # Only for GitHub releases
|
||||
run: |
|
||||
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
|
||||
echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
|
||||
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
|
||||
[ ${BRANCH} == "main" ] && BRANCH=""
|
||||
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ env.DOCKER_HUB_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
- name: Login to GitHub Containers
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build main Yggdrasil demo image
|
||||
if: github.ref_name == 'main'
|
||||
id: docker_build_demo_yggdrasil
|
||||
uses: docker/build-push-action@v3
|
||||
with:
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
context: .
|
||||
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
|
||||
file: ./build/docker/Dockerfile.demo-yggdrasil
|
||||
platforms: ${{ env.PLATFORMS }}
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }}
|
||||
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }}
|
||||
|
||||
- name: Build release Yggdrasil demo image
|
||||
if: github.event_name == 'release' # Only for GitHub releases
|
||||
id: docker_build_demo_yggdrasil_release
|
||||
uses: docker/build-push-action@v3
|
||||
with:
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
context: .
|
||||
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
|
||||
file: ./build/docker/Dockerfile.demo-yggdrasil
|
||||
platforms: ${{ env.PLATFORMS }}
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest
|
||||
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}
|
||||
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest
|
||||
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}
|
||||
6
.gitignore
vendored
6
.gitignore
vendored
|
|
@ -2,6 +2,8 @@
|
|||
|
||||
# Hidden files
|
||||
.*
|
||||
!.vscode
|
||||
!.cloudbuild
|
||||
|
||||
# Allow GitHub config
|
||||
!.github
|
||||
|
|
@ -73,3 +75,7 @@ complement/
|
|||
docs/_site
|
||||
|
||||
media_store/
|
||||
|
||||
__debug_bin
|
||||
|
||||
cmd/dendrite-monolith-server/dendrite-monolith-server
|
||||
|
|
@ -179,7 +179,6 @@ linters-settings:
|
|||
|
||||
linters:
|
||||
enable:
|
||||
- deadcode
|
||||
- errcheck
|
||||
- goconst
|
||||
- gocyclo
|
||||
|
|
@ -191,10 +190,8 @@ linters:
|
|||
- misspell # Check code comments, whereas misspell in CI checks *.md files
|
||||
- nakedret
|
||||
- staticcheck
|
||||
- structcheck
|
||||
- unparam
|
||||
- unused
|
||||
- varcheck
|
||||
enable-all: false
|
||||
disable:
|
||||
- bodyclose
|
||||
|
|
|
|||
16
.vscode/launch.json
vendored
Normal file
16
.vscode/launch.json
vendored
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
{
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Launch Package",
|
||||
"type": "go",
|
||||
"request": "launch",
|
||||
"mode": "auto",
|
||||
"program": "${workspaceFolder}/cmd/dendrite-monolith-server",
|
||||
"args": [
|
||||
"-really-enable-open-registration",
|
||||
"-config",
|
||||
"../../../adminas/.ci/config/dendrite-local/dendrite.yaml"
|
||||
],
|
||||
}
|
||||
]
|
||||
}
|
||||
9
.vscode/settings.json
vendored
Normal file
9
.vscode/settings.json
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"go.lintTool": "golangci-lint",
|
||||
"go.testEnvVars": {
|
||||
"POSTGRES_HOST": "localhost",
|
||||
"POSTGRES_USER": "postgres",
|
||||
"POSTGRES_PASSWORD": "foobar",
|
||||
"POSTGRES_DB": "postgres"
|
||||
}
|
||||
}
|
||||
|
|
@ -34,13 +34,16 @@ type JSServer struct {
|
|||
|
||||
// OnRequestFromJS is the function that JS will invoke when there is a new request.
|
||||
// The JS function signature is:
|
||||
// function(reqString: string): Promise<{result: string, error: string}>
|
||||
//
|
||||
// function(reqString: string): Promise<{result: string, error: string}>
|
||||
//
|
||||
// Usage is like:
|
||||
// const res = await global._go_js_server.fetch(reqString);
|
||||
// if (res.error) {
|
||||
// // handle error: this is a 'network' error, not a non-2xx error.
|
||||
// }
|
||||
// const rawHttpResponse = res.result;
|
||||
//
|
||||
// const res = await global._go_js_server.fetch(reqString);
|
||||
// if (res.error) {
|
||||
// // handle error: this is a 'network' error, not a non-2xx error.
|
||||
// }
|
||||
// const rawHttpResponse = res.result;
|
||||
func (h *JSServer) OnRequestFromJS(this js.Value, args []js.Value) interface{} {
|
||||
// we HAVE to spawn a new goroutine and return immediately or else Go will deadlock
|
||||
// if this request blocks at all e.g for /sync calls
|
||||
|
|
|
|||
|
|
@ -11,4 +11,6 @@ const (
|
|||
LoginTypeRecaptcha = "m.login.recaptcha"
|
||||
LoginTypeApplicationService = "m.login.application_service"
|
||||
LoginTypeToken = "m.login.token"
|
||||
LoginTypeJwt = "org.matrix.login.jwt"
|
||||
LoginTypeEmail = "m.login.email.identity"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import (
|
|||
|
||||
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
|
||||
"github.com/matrix-org/dendrite/clientapi/jsonerror"
|
||||
"github.com/matrix-org/dendrite/clientapi/ratelimit"
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
uapi "github.com/matrix-org/dendrite/userapi/api"
|
||||
"github.com/matrix-org/util"
|
||||
|
|
@ -32,7 +33,7 @@ import (
|
|||
// called after authorization has completed, with the result of the authorization.
|
||||
// If the final return value is non-nil, an error occurred and the cleanup function
|
||||
// is nil.
|
||||
func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserLoginAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) {
|
||||
func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.ClientUserAPI, cfg *config.ClientAPI, rt *ratelimit.RtFailedLogin) (*Login, LoginCleanupFunc, *util.JSONResponse) {
|
||||
reqBytes, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
err := &util.JSONResponse{
|
||||
|
|
@ -43,7 +44,8 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U
|
|||
}
|
||||
|
||||
var header struct {
|
||||
Type string `json:"type"`
|
||||
Type string `json:"type"`
|
||||
InhibitDevice bool `json:"inhibit_device"`
|
||||
}
|
||||
if err := json.Unmarshal(reqBytes, &header); err != nil {
|
||||
err := &util.JSONResponse{
|
||||
|
|
@ -57,14 +59,20 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U
|
|||
switch header.Type {
|
||||
case authtypes.LoginTypePassword:
|
||||
typ = &LoginTypePassword{
|
||||
GetAccountByPassword: useraccountAPI.QueryAccountByPassword,
|
||||
Config: cfg,
|
||||
UserApi: useraccountAPI,
|
||||
Config: cfg,
|
||||
Rt: rt,
|
||||
InhibitDevice: header.InhibitDevice,
|
||||
}
|
||||
case authtypes.LoginTypeToken:
|
||||
typ = &LoginTypeToken{
|
||||
UserAPI: userAPI,
|
||||
UserAPI: useraccountAPI,
|
||||
Config: cfg,
|
||||
}
|
||||
case authtypes.LoginTypeJwt:
|
||||
typ = &LoginTypeTokenJwt{
|
||||
Config: cfg,
|
||||
}
|
||||
default:
|
||||
err := util.JSONResponse{
|
||||
Code: http.StatusBadRequest,
|
||||
|
|
|
|||
74
clientapi/auth/login_jwt.go
Normal file
74
clientapi/auth/login_jwt.go
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/golang-jwt/jwt/v4"
|
||||
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
|
||||
"github.com/matrix-org/dendrite/clientapi/httputil"
|
||||
"github.com/matrix-org/dendrite/clientapi/jsonerror"
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
"github.com/matrix-org/util"
|
||||
)
|
||||
|
||||
// LoginTypeToken describes how to authenticate with a login token.
|
||||
type LoginTypeTokenJwt struct {
|
||||
// UserAPI uapi.LoginTokenInternalAPI
|
||||
Config *config.ClientAPI
|
||||
}
|
||||
|
||||
// Name implements Type.
|
||||
func (t *LoginTypeTokenJwt) Name() string {
|
||||
return authtypes.LoginTypeJwt
|
||||
}
|
||||
|
||||
type Claims struct {
|
||||
jwt.StandardClaims
|
||||
}
|
||||
|
||||
const mIdUser = "m.id.user"
|
||||
|
||||
// LoginFromJSON implements Type. The cleanup function deletes the token from
|
||||
// the database on success.
|
||||
func (t *LoginTypeTokenJwt) LoginFromJSON(ctx context.Context, reqBytes []byte) (*Login, LoginCleanupFunc, *util.JSONResponse) {
|
||||
var r loginTokenRequest
|
||||
if err := httputil.UnmarshalJSON(reqBytes, &r); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if r.Token == "" {
|
||||
return nil, nil, &util.JSONResponse{
|
||||
Code: http.StatusForbidden,
|
||||
JSON: jsonerror.Forbidden("Token field for JWT is missing"),
|
||||
}
|
||||
}
|
||||
c := &Claims{}
|
||||
token, err := jwt.ParseWithClaims(r.Token, c, func(token *jwt.Token) (interface{}, error) {
|
||||
if _, ok := token.Method.(*jwt.SigningMethodEd25519); !ok {
|
||||
return nil, fmt.Errorf("unexpected signing method: %v", token.Method.Alg())
|
||||
}
|
||||
return t.Config.JwtConfig.SecretKey, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
util.GetLogger(ctx).WithError(err).Error("jwt.ParseWithClaims failed")
|
||||
return nil, nil, &util.JSONResponse{
|
||||
Code: http.StatusForbidden,
|
||||
JSON: jsonerror.Forbidden("Couldn't parse JWT"),
|
||||
}
|
||||
}
|
||||
|
||||
if !token.Valid {
|
||||
return nil, nil, &util.JSONResponse{
|
||||
Code: http.StatusForbidden,
|
||||
JSON: jsonerror.Forbidden("Invalid JWT"),
|
||||
}
|
||||
}
|
||||
|
||||
r.Login.Identifier.User = c.Subject
|
||||
r.Login.Identifier.Type = mIdUser
|
||||
|
||||
return &r.Login, func(context.Context, *util.JSONResponse) {}, nil
|
||||
}
|
||||
|
|
@ -22,6 +22,7 @@ import (
|
|||
"testing"
|
||||
|
||||
"github.com/matrix-org/dendrite/clientapi/jsonerror"
|
||||
"github.com/matrix-org/dendrite/clientapi/ratelimit"
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
uapi "github.com/matrix-org/dendrite/userapi/api"
|
||||
"github.com/matrix-org/util"
|
||||
|
|
@ -68,8 +69,11 @@ func TestLoginFromJSONReader(t *testing.T) {
|
|||
Matrix: &config.Global{
|
||||
ServerName: serverName,
|
||||
},
|
||||
RtFailedLogin: ratelimit.RtFailedLoginConfig{
|
||||
Enabled: false,
|
||||
},
|
||||
}
|
||||
login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
|
||||
login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, cfg, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("LoginFromJSONReader failed: %+v", err)
|
||||
}
|
||||
|
|
@ -147,7 +151,7 @@ func TestBadLoginFromJSONReader(t *testing.T) {
|
|||
ServerName: serverName,
|
||||
},
|
||||
}
|
||||
_, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
|
||||
_, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, cfg, nil)
|
||||
if errRes == nil {
|
||||
cleanup(ctx, nil)
|
||||
t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode)
|
||||
|
|
@ -159,6 +163,7 @@ func TestBadLoginFromJSONReader(t *testing.T) {
|
|||
}
|
||||
|
||||
type fakeUserInternalAPI struct {
|
||||
uapi.ClientUserAPI
|
||||
UserInternalAPIForLogin
|
||||
DeletedTokens []string
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ func (t *LoginTypeToken) LoginFromJSON(ctx context.Context, reqBytes []byte) (*L
|
|||
}
|
||||
}
|
||||
|
||||
r.Login.Identifier.Type = "m.id.user"
|
||||
r.Login.Identifier.Type = mIdUser
|
||||
r.Login.Identifier.User = res.Data.UserID
|
||||
|
||||
cleanup := func(ctx context.Context, authRes *util.JSONResponse) {
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import (
|
|||
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
|
||||
"github.com/matrix-org/dendrite/clientapi/httputil"
|
||||
"github.com/matrix-org/dendrite/clientapi/jsonerror"
|
||||
"github.com/matrix-org/dendrite/clientapi/ratelimit"
|
||||
"github.com/matrix-org/dendrite/clientapi/userutil"
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
"github.com/matrix-org/dendrite/userapi/api"
|
||||
|
|
@ -33,12 +34,18 @@ type GetAccountByPassword func(ctx context.Context, req *api.QueryAccountByPassw
|
|||
type PasswordRequest struct {
|
||||
Login
|
||||
Password string `json:"password"`
|
||||
Address string `json:"address"`
|
||||
Medium string `json:"medium"`
|
||||
}
|
||||
|
||||
const email = "email"
|
||||
|
||||
// LoginTypePassword implements https://matrix.org/docs/spec/client_server/r0.6.1#password-based
|
||||
type LoginTypePassword struct {
|
||||
GetAccountByPassword GetAccountByPassword
|
||||
Config *config.ClientAPI
|
||||
UserApi api.ClientUserAPI
|
||||
Config *config.ClientAPI
|
||||
Rt *ratelimit.RtFailedLogin
|
||||
InhibitDevice bool
|
||||
}
|
||||
|
||||
func (t *LoginTypePassword) Name() string {
|
||||
|
|
@ -55,13 +62,42 @@ func (t *LoginTypePassword) LoginFromJSON(ctx context.Context, reqBytes []byte)
|
|||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
login.InhibitDevice = t.InhibitDevice
|
||||
|
||||
return login, func(context.Context, *util.JSONResponse) {}, nil
|
||||
}
|
||||
|
||||
func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) {
|
||||
r := req.(*PasswordRequest)
|
||||
username := strings.ToLower(r.Username())
|
||||
if r.Identifier.Address != "" {
|
||||
r.Address = r.Identifier.Address
|
||||
}
|
||||
if r.Identifier.Medium != "" {
|
||||
r.Medium = r.Identifier.Medium
|
||||
}
|
||||
var username string
|
||||
if r.Medium == email && r.Address != "" {
|
||||
r.Address = strings.ToLower(r.Address)
|
||||
res := api.QueryLocalpartForThreePIDResponse{}
|
||||
err := t.UserApi.QueryLocalpartForThreePID(ctx, &api.QueryLocalpartForThreePIDRequest{
|
||||
ThreePID: r.Address,
|
||||
Medium: email,
|
||||
}, &res)
|
||||
if err != nil {
|
||||
util.GetLogger(ctx).WithError(err).Error("userApi.QueryLocalpartForThreePID failed")
|
||||
resp := jsonerror.InternalServerError()
|
||||
return nil, &resp
|
||||
}
|
||||
username = res.Localpart
|
||||
if username == "" {
|
||||
return nil, &util.JSONResponse{
|
||||
Code: http.StatusUnauthorized,
|
||||
JSON: jsonerror.Forbidden("Invalid username or password"),
|
||||
}
|
||||
}
|
||||
} else {
|
||||
username = strings.ToLower(r.Username())
|
||||
}
|
||||
if username == "" {
|
||||
return nil, &util.JSONResponse{
|
||||
Code: http.StatusUnauthorized,
|
||||
|
|
@ -83,7 +119,17 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
|
|||
}
|
||||
// Squash username to all lowercase letters
|
||||
res := &api.QueryAccountByPasswordResponse{}
|
||||
err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{Localpart: strings.ToLower(localpart), PlaintextPassword: r.Password}, res)
|
||||
localpart = strings.ToLower(localpart)
|
||||
if t.Rt != nil {
|
||||
ok, retryIn := t.Rt.CanAct(localpart)
|
||||
if !ok {
|
||||
return nil, &util.JSONResponse{
|
||||
Code: http.StatusTooManyRequests,
|
||||
JSON: jsonerror.LimitExceeded("Too Many Requests", retryIn.Milliseconds()),
|
||||
}
|
||||
}
|
||||
}
|
||||
err = t.UserApi.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{Localpart: localpart, PlaintextPassword: r.Password}, res)
|
||||
if err != nil {
|
||||
return nil, &util.JSONResponse{
|
||||
Code: http.StatusInternalServerError,
|
||||
|
|
@ -92,7 +138,7 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
|
|||
}
|
||||
|
||||
if !res.Exists {
|
||||
err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
|
||||
err = t.UserApi.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
|
||||
Localpart: localpart,
|
||||
PlaintextPassword: r.Password,
|
||||
}, res)
|
||||
|
|
@ -105,11 +151,15 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
|
|||
// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
|
||||
// but that would leak the existence of the user.
|
||||
if !res.Exists {
|
||||
if t.Rt != nil {
|
||||
t.Rt.Act(localpart)
|
||||
}
|
||||
return nil, &util.JSONResponse{
|
||||
Code: http.StatusForbidden,
|
||||
JSON: jsonerror.Forbidden("The username or password was incorrect or the account does not exist."),
|
||||
JSON: jsonerror.Forbidden("Invalid username or password"),
|
||||
}
|
||||
}
|
||||
}
|
||||
r.Login.User = username
|
||||
return &r.Login, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ type LoginIdentifier struct {
|
|||
type Login struct {
|
||||
LoginIdentifier // Flat fields deprecated in favour of `identifier`.
|
||||
Identifier LoginIdentifier `json:"identifier"`
|
||||
InhibitDevice bool `json:"inhibit_device,omitempty"`
|
||||
|
||||
// Both DeviceID and InitialDisplayName can be omitted, or empty strings ("")
|
||||
// Thus a pointer is needed to differentiate between the two
|
||||
|
|
@ -75,7 +76,7 @@ type Login struct {
|
|||
|
||||
// Username returns the user localpart/user_id in this request, if it exists.
|
||||
func (r *Login) Username() string {
|
||||
if r.Identifier.Type == "m.id.user" {
|
||||
if r.Identifier.Type == mIdUser {
|
||||
return r.Identifier.User
|
||||
}
|
||||
// deprecated but without it Element iOS won't log in
|
||||
|
|
@ -88,8 +89,8 @@ func (r *Login) ThirdPartyID() (medium, address string) {
|
|||
return r.Identifier.Medium, r.Identifier.Address
|
||||
}
|
||||
// deprecated
|
||||
if r.Medium == "email" {
|
||||
return "email", r.Address
|
||||
if r.Medium == email {
|
||||
return email, r.Address
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
|
@ -111,10 +112,10 @@ type UserInteractive struct {
|
|||
Sessions map[string][]string
|
||||
}
|
||||
|
||||
func NewUserInteractive(userAccountAPI api.UserLoginAPI, cfg *config.ClientAPI) *UserInteractive {
|
||||
func NewUserInteractive(userAccountAPI api.ClientUserAPI, cfg *config.ClientAPI) *UserInteractive {
|
||||
typePassword := &LoginTypePassword{
|
||||
GetAccountByPassword: userAccountAPI.QueryAccountByPassword,
|
||||
Config: cfg,
|
||||
UserApi: userAccountAPI,
|
||||
Config: cfg,
|
||||
}
|
||||
return &UserInteractive{
|
||||
Flows: []userInteractiveFlow{
|
||||
|
|
|
|||
|
|
@ -24,7 +24,9 @@ var (
|
|||
}
|
||||
)
|
||||
|
||||
type fakeAccountDatabase struct{}
|
||||
type fakeAccountDatabase struct {
|
||||
api.ClientUserAPI
|
||||
}
|
||||
|
||||
func (d *fakeAccountDatabase) PerformPasswordUpdate(ctx context.Context, req *api.PerformPasswordUpdateRequest, res *api.PerformPasswordUpdateResponse) error {
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ func AddPublicRoutes(
|
|||
TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
|
||||
TopicTypingEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
|
||||
TopicPresenceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
|
||||
TopicMultiRoomCast: cfg.Matrix.JetStream.Prefixed(jetstream.OutputMultiRoomCast),
|
||||
UserAPI: userAPI,
|
||||
ServerName: cfg.Matrix.ServerName,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ type SyncAPIProducer struct {
|
|||
TopicSendToDeviceEvent string
|
||||
TopicTypingEvent string
|
||||
TopicPresenceEvent string
|
||||
TopicMultiRoomCast string
|
||||
JetStream nats.JetStreamContext
|
||||
ServerName gomatrixserverlib.ServerName
|
||||
UserAPI userapi.ClientUserAPI
|
||||
|
|
@ -159,3 +160,14 @@ func (p *SyncAPIProducer) SendPresence(
|
|||
_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *SyncAPIProducer) SendMultiroom(
|
||||
ctx context.Context, userID string, dataType string, message []byte,
|
||||
) error {
|
||||
m := nats.NewMsg(p.TopicMultiRoomCast)
|
||||
m.Header.Set(jetstream.UserID, userID)
|
||||
m.Header.Set("type", dataType)
|
||||
m.Data = message
|
||||
_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
117
clientapi/ratelimit/rt_failed_login.go
Normal file
117
clientapi/ratelimit/rt_failed_login.go
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
package ratelimit
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type rateLimit struct {
|
||||
cfg *RtFailedLoginConfig
|
||||
times *list.List
|
||||
}
|
||||
|
||||
type RtFailedLogin struct {
|
||||
cfg *RtFailedLoginConfig
|
||||
mtx sync.RWMutex
|
||||
rts map[string]*rateLimit
|
||||
}
|
||||
|
||||
type RtFailedLoginConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Limit int `yaml:"burst"`
|
||||
Interval time.Duration `yaml:"interval"`
|
||||
}
|
||||
|
||||
// New creates a new rate limiter for the limit and interval.
|
||||
func NewRtFailedLogin(cfg *RtFailedLoginConfig) *RtFailedLogin {
|
||||
if !cfg.Enabled {
|
||||
return nil
|
||||
}
|
||||
rt := &RtFailedLogin{
|
||||
cfg: cfg,
|
||||
mtx: sync.RWMutex{},
|
||||
rts: make(map[string]*rateLimit),
|
||||
}
|
||||
go rt.clean()
|
||||
return rt
|
||||
}
|
||||
|
||||
// CanAct is expected to be called before Act
|
||||
func (r *RtFailedLogin) CanAct(key string) (ok bool, remaining time.Duration) {
|
||||
r.mtx.RLock()
|
||||
rt, ok := r.rts[key]
|
||||
if !ok {
|
||||
r.mtx.RUnlock()
|
||||
return true, 0
|
||||
}
|
||||
ok, remaining = rt.canAct()
|
||||
r.mtx.RUnlock()
|
||||
return
|
||||
}
|
||||
|
||||
// Act can be called after CanAct returns true.
|
||||
func (r *RtFailedLogin) Act(key string) {
|
||||
r.mtx.Lock()
|
||||
rt, ok := r.rts[key]
|
||||
if !ok {
|
||||
rt = &rateLimit{
|
||||
cfg: r.cfg,
|
||||
times: list.New(),
|
||||
}
|
||||
r.rts[key] = rt
|
||||
}
|
||||
rt.act()
|
||||
r.mtx.Unlock()
|
||||
}
|
||||
|
||||
func (r *RtFailedLogin) clean() {
|
||||
for {
|
||||
r.mtx.Lock()
|
||||
for k, v := range r.rts {
|
||||
if v.empty() {
|
||||
delete(r.rts, k)
|
||||
}
|
||||
}
|
||||
r.mtx.Unlock()
|
||||
time.Sleep(time.Hour)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *rateLimit) empty() bool {
|
||||
back := r.times.Back()
|
||||
if back == nil {
|
||||
return true
|
||||
}
|
||||
v := back.Value
|
||||
b := v.(time.Time)
|
||||
now := time.Now()
|
||||
return now.Sub(b) > r.cfg.Interval
|
||||
}
|
||||
|
||||
func (r *rateLimit) canAct() (ok bool, remaining time.Duration) {
|
||||
now := time.Now()
|
||||
l := r.times.Len()
|
||||
if l < r.cfg.Limit {
|
||||
return true, 0
|
||||
}
|
||||
frnt := r.times.Front()
|
||||
t := frnt.Value.(time.Time)
|
||||
diff := now.Sub(t)
|
||||
if diff < r.cfg.Interval {
|
||||
return false, r.cfg.Interval - diff
|
||||
}
|
||||
return true, 0
|
||||
}
|
||||
|
||||
func (r *rateLimit) act() {
|
||||
now := time.Now()
|
||||
l := r.times.Len()
|
||||
if l < r.cfg.Limit {
|
||||
r.times.PushBack(now)
|
||||
return
|
||||
}
|
||||
frnt := r.times.Front()
|
||||
frnt.Value = now
|
||||
r.times.MoveToBack(frnt)
|
||||
}
|
||||
40
clientapi/ratelimit/rt_failed_login_test.go
Normal file
40
clientapi/ratelimit/rt_failed_login_test.go
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
package ratelimit
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/matryer/is"
|
||||
)
|
||||
|
||||
func TestRtFailedLogin(t *testing.T) {
|
||||
is := is.New(t)
|
||||
rtfl := NewRtFailedLogin(&RtFailedLoginConfig{
|
||||
Enabled: true,
|
||||
Limit: 3,
|
||||
Interval: 10 * time.Millisecond,
|
||||
})
|
||||
var (
|
||||
can bool
|
||||
remaining time.Duration
|
||||
remainingB time.Duration
|
||||
)
|
||||
for i := 0; i < 3; i++ {
|
||||
can, remaining = rtfl.CanAct("foo")
|
||||
is.True(can)
|
||||
is.Equal(remaining, time.Duration(0))
|
||||
rtfl.Act("foo")
|
||||
}
|
||||
can, remaining = rtfl.CanAct("foo")
|
||||
is.True(!can)
|
||||
is.True(remaining > time.Millisecond*9)
|
||||
can, remainingB = rtfl.CanAct("bar")
|
||||
is.True(can)
|
||||
is.Equal(remainingB, time.Duration(0))
|
||||
rtfl.Act("bar")
|
||||
rtfl.Act("bar")
|
||||
time.Sleep(remaining + time.Millisecond)
|
||||
can, remaining = rtfl.CanAct("foo")
|
||||
is.True(can)
|
||||
is.Equal(remaining, time.Duration(0))
|
||||
}
|
||||
|
|
@ -27,13 +27,17 @@ func Deactivate(
|
|||
JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()),
|
||||
}
|
||||
}
|
||||
|
||||
login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, deviceAPI)
|
||||
if errRes != nil {
|
||||
return *errRes
|
||||
var userId string
|
||||
if deviceAPI.AccountType != api.AccountTypeAppService {
|
||||
login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, deviceAPI)
|
||||
if errRes != nil {
|
||||
return *errRes
|
||||
}
|
||||
userId = login.Username()
|
||||
} else {
|
||||
userId = deviceAPI.UserID
|
||||
}
|
||||
|
||||
localpart, _, err := gomatrixserverlib.SplitID('@', login.Username())
|
||||
localpart, _, err := gomatrixserverlib.SplitID('@', userId)
|
||||
if err != nil {
|
||||
util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
|
||||
return jsonerror.InternalServerError()
|
||||
|
|
|
|||
|
|
@ -63,8 +63,8 @@ func UploadCrossSigningDeviceKeys(
|
|||
}
|
||||
}
|
||||
typePassword := auth.LoginTypePassword{
|
||||
GetAccountByPassword: accountAPI.QueryAccountByPassword,
|
||||
Config: cfg,
|
||||
UserApi: accountAPI,
|
||||
Config: cfg,
|
||||
}
|
||||
if _, authErr := typePassword.Login(req.Context(), &uploadReq.Auth.PasswordRequest); authErr != nil {
|
||||
return *authErr
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import (
|
|||
|
||||
"github.com/matrix-org/dendrite/clientapi/auth"
|
||||
"github.com/matrix-org/dendrite/clientapi/jsonerror"
|
||||
"github.com/matrix-org/dendrite/clientapi/ratelimit"
|
||||
"github.com/matrix-org/dendrite/clientapi/userutil"
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
userapi "github.com/matrix-org/dendrite/userapi/api"
|
||||
|
|
@ -55,6 +56,7 @@ func passwordLogin() flows {
|
|||
func Login(
|
||||
req *http.Request, userAPI userapi.ClientUserAPI,
|
||||
cfg *config.ClientAPI,
|
||||
rt *ratelimit.RtFailedLogin,
|
||||
) util.JSONResponse {
|
||||
if req.Method == http.MethodGet {
|
||||
// TODO: support other forms of login other than password, depending on config options
|
||||
|
|
@ -63,10 +65,21 @@ func Login(
|
|||
JSON: passwordLogin(),
|
||||
}
|
||||
} else if req.Method == http.MethodPost {
|
||||
login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, userAPI, userAPI, cfg)
|
||||
login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, userAPI, cfg, rt)
|
||||
if authErr != nil {
|
||||
return *authErr
|
||||
}
|
||||
if login.InhibitDevice {
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusOK,
|
||||
JSON: loginResponse{
|
||||
UserID: userutil.MakeUserID(login.Username(), cfg.Matrix.ServerName),
|
||||
AccessToken: "",
|
||||
HomeServer: cfg.Matrix.ServerName,
|
||||
DeviceID: "",
|
||||
},
|
||||
}
|
||||
}
|
||||
// make a device/access token
|
||||
authErr2 := completeAuth(req.Context(), cfg.Matrix, userAPI, login, req.RemoteAddr, req.UserAgent())
|
||||
cleanup(req.Context(), &authErr2)
|
||||
|
|
|
|||
48
clientapi/routing/multiroom.go
Normal file
48
clientapi/routing/multiroom.go
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
package routing
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/matrix-org/dendrite/clientapi/jsonerror"
|
||||
"github.com/matrix-org/dendrite/clientapi/producers"
|
||||
"github.com/matrix-org/dendrite/userapi/api"
|
||||
"github.com/matrix-org/gomatrixserverlib"
|
||||
"github.com/matrix-org/util"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func PostMultiroom(
|
||||
req *http.Request,
|
||||
device *api.Device,
|
||||
producer *producers.SyncAPIProducer,
|
||||
dataType string,
|
||||
) util.JSONResponse {
|
||||
b, err := io.ReadAll(req.Body)
|
||||
if err != nil {
|
||||
log.WithError(err).Errorf("failed to read request body")
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusInternalServerError,
|
||||
JSON: jsonerror.InternalServerError(),
|
||||
}
|
||||
}
|
||||
canonicalB, err := gomatrixserverlib.CanonicalJSON(b)
|
||||
if err != nil {
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusBadRequest,
|
||||
JSON: jsonerror.BadJSON("The request body is not valid canonical JSON." + err.Error()),
|
||||
}
|
||||
}
|
||||
err = producer.SendMultiroom(req.Context(), device.UserID, dataType, canonicalB)
|
||||
if err != nil {
|
||||
log.WithError(err).Errorf("failed to send multiroomcast")
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusInternalServerError,
|
||||
JSON: jsonerror.InternalServerError(),
|
||||
}
|
||||
}
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusOK,
|
||||
JSON: struct{}{},
|
||||
}
|
||||
}
|
||||
|
|
@ -1,12 +1,14 @@
|
|||
package routing
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/matrix-org/dendrite/clientapi/auth"
|
||||
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
|
||||
"github.com/matrix-org/dendrite/clientapi/httputil"
|
||||
"github.com/matrix-org/dendrite/clientapi/jsonerror"
|
||||
"github.com/matrix-org/dendrite/clientapi/threepid"
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
"github.com/matrix-org/dendrite/userapi/api"
|
||||
"github.com/matrix-org/gomatrixserverlib"
|
||||
|
|
@ -24,6 +26,7 @@ type newPasswordAuth struct {
|
|||
Type string `json:"type"`
|
||||
Session string `json:"session"`
|
||||
auth.PasswordRequest
|
||||
ThreePidCreds threepid.Credentials `json:"threepid_creds"`
|
||||
}
|
||||
|
||||
func Password(
|
||||
|
|
@ -33,13 +36,17 @@ func Password(
|
|||
cfg *config.ClientAPI,
|
||||
) util.JSONResponse {
|
||||
// Check that the existing password is right.
|
||||
var fields logrus.Fields
|
||||
if device != nil {
|
||||
fields = logrus.Fields{
|
||||
"sessionId": device.SessionID,
|
||||
"userId": device.UserID,
|
||||
}
|
||||
}
|
||||
var r newPasswordRequest
|
||||
r.LogoutDevices = true
|
||||
|
||||
logrus.WithFields(logrus.Fields{
|
||||
"sessionId": device.SessionID,
|
||||
"userId": device.UserID,
|
||||
}).Debug("Changing password")
|
||||
logrus.WithFields(fields).Debug("Changing password")
|
||||
|
||||
// Unmarshal the request.
|
||||
resErr := httputil.UnmarshalJSONRequest(req, &r)
|
||||
|
|
@ -53,45 +60,95 @@ func Password(
|
|||
// Generate a new, random session ID
|
||||
sessionID = util.RandomString(sessionIDLength)
|
||||
}
|
||||
|
||||
// Require password auth to change the password.
|
||||
if r.Auth.Type != authtypes.LoginTypePassword {
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusUnauthorized,
|
||||
JSON: newUserInteractiveResponse(
|
||||
sessionID,
|
||||
[]authtypes.Flow{
|
||||
{
|
||||
Stages: []authtypes.LoginType{authtypes.LoginTypePassword},
|
||||
},
|
||||
var localpart string
|
||||
switch r.Auth.Type {
|
||||
case authtypes.LoginTypePassword:
|
||||
// Check if the existing password is correct.
|
||||
typePassword := auth.LoginTypePassword{
|
||||
UserApi: userAPI,
|
||||
Config: cfg,
|
||||
}
|
||||
if _, authErr := typePassword.Login(req.Context(), &r.Auth.PasswordRequest); authErr != nil {
|
||||
return *authErr
|
||||
}
|
||||
// Get the local part.
|
||||
var err error
|
||||
localpart, _, err = gomatrixserverlib.SplitID('@', device.UserID)
|
||||
if err != nil {
|
||||
util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
|
||||
return jsonerror.InternalServerError()
|
||||
}
|
||||
sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypePassword)
|
||||
case authtypes.LoginTypeEmail:
|
||||
threePid := &authtypes.ThreePID{}
|
||||
r.Auth.ThreePidCreds.IDServer = cfg.ThreePidDelegate
|
||||
var (
|
||||
bound bool
|
||||
err error
|
||||
)
|
||||
bound, threePid.Address, threePid.Medium, err = threepid.CheckAssociation(req.Context(), r.Auth.ThreePidCreds, cfg)
|
||||
if err != nil {
|
||||
util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAssociation failed")
|
||||
return jsonerror.InternalServerError()
|
||||
}
|
||||
if !bound {
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusBadRequest,
|
||||
JSON: jsonerror.MatrixError{
|
||||
ErrCode: "M_THREEPID_AUTH_FAILED",
|
||||
Err: "Failed to auth 3pid",
|
||||
},
|
||||
nil,
|
||||
),
|
||||
}
|
||||
}
|
||||
var res api.QueryLocalpartForThreePIDResponse
|
||||
err = userAPI.QueryLocalpartForThreePID(req.Context(), &api.QueryLocalpartForThreePIDRequest{
|
||||
Medium: threePid.Medium,
|
||||
ThreePID: threePid.Address,
|
||||
}, &res)
|
||||
if err != nil {
|
||||
util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryLocalpartForThreePID failed")
|
||||
return jsonerror.InternalServerError()
|
||||
}
|
||||
if res.Localpart == "" {
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusBadRequest,
|
||||
JSON: jsonerror.MatrixError{
|
||||
ErrCode: "M_THREEPID_NOT_FOUND",
|
||||
Err: "3pid is not bound to any account",
|
||||
},
|
||||
}
|
||||
}
|
||||
localpart = res.Localpart
|
||||
sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeEmail)
|
||||
default:
|
||||
flows := []authtypes.Flow{
|
||||
{
|
||||
Stages: []authtypes.LoginType{authtypes.LoginTypePassword},
|
||||
},
|
||||
}
|
||||
if cfg.ThreePidDelegate != "" {
|
||||
flows = append(flows, authtypes.Flow{
|
||||
Stages: []authtypes.LoginType{authtypes.LoginTypeEmail},
|
||||
})
|
||||
}
|
||||
// Require password auth to change the password.
|
||||
if r.Auth.Type == authtypes.LoginTypePassword {
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusUnauthorized,
|
||||
JSON: newUserInteractiveResponse(
|
||||
sessionID,
|
||||
flows,
|
||||
nil,
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if the existing password is correct.
|
||||
typePassword := auth.LoginTypePassword{
|
||||
GetAccountByPassword: userAPI.QueryAccountByPassword,
|
||||
Config: cfg,
|
||||
}
|
||||
if _, authErr := typePassword.Login(req.Context(), &r.Auth.PasswordRequest); authErr != nil {
|
||||
return *authErr
|
||||
}
|
||||
sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypePassword)
|
||||
|
||||
// Check the new password strength.
|
||||
if resErr = validatePassword(r.NewPassword); resErr != nil {
|
||||
return *resErr
|
||||
}
|
||||
|
||||
// Get the local part.
|
||||
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
|
||||
if err != nil {
|
||||
util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
|
||||
return jsonerror.InternalServerError()
|
||||
}
|
||||
|
||||
// Ask the user API to perform the password change.
|
||||
passwordReq := &api.PerformPasswordUpdateRequest{
|
||||
Localpart: localpart,
|
||||
|
|
@ -109,11 +166,23 @@ func Password(
|
|||
|
||||
// If the request asks us to log out all other devices then
|
||||
// ask the user API to do that.
|
||||
|
||||
if r.LogoutDevices {
|
||||
logoutReq := &api.PerformDeviceDeletionRequest{
|
||||
UserID: device.UserID,
|
||||
DeviceIDs: nil,
|
||||
ExceptDeviceID: device.ID,
|
||||
var logoutReq *api.PerformDeviceDeletionRequest
|
||||
var sessionId int64
|
||||
if device == nil {
|
||||
logoutReq = &api.PerformDeviceDeletionRequest{
|
||||
UserID: fmt.Sprintf("@%s:%s", localpart, cfg.Matrix.ServerName),
|
||||
DeviceIDs: []string{},
|
||||
}
|
||||
sessionId = 0
|
||||
} else {
|
||||
logoutReq = &api.PerformDeviceDeletionRequest{
|
||||
UserID: device.UserID,
|
||||
DeviceIDs: nil,
|
||||
ExceptDeviceID: device.ID,
|
||||
}
|
||||
sessionId = device.SessionID
|
||||
}
|
||||
logoutRes := &api.PerformDeviceDeletionResponse{}
|
||||
if err := userAPI.PerformDeviceDeletion(req.Context(), logoutReq, logoutRes); err != nil {
|
||||
|
|
@ -123,7 +192,7 @@ func Password(
|
|||
|
||||
pushersReq := &api.PerformPusherDeletionRequest{
|
||||
Localpart: localpart,
|
||||
SessionID: device.SessionID,
|
||||
SessionID: sessionId,
|
||||
}
|
||||
if err := userAPI.PerformPusherDeletion(req.Context(), pushersReq, &struct{}{}); err != nil {
|
||||
util.GetLogger(req.Context()).WithError(err).Error("PerformPusherDeletion failed")
|
||||
|
|
|
|||
|
|
@ -106,12 +106,6 @@ func SetAvatarURL(
|
|||
if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
|
||||
return *resErr
|
||||
}
|
||||
if r.AvatarURL == "" {
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusBadRequest,
|
||||
JSON: jsonerror.BadJSON("'avatar_url' must be supplied."),
|
||||
}
|
||||
}
|
||||
|
||||
localpart, domain, err := gomatrixserverlib.SplitID('@', userID)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -86,8 +86,8 @@ func SetPusher(
|
|||
if err != nil {
|
||||
return invalidParam("malformed url passed")
|
||||
}
|
||||
if pushUrl.Scheme != "https" {
|
||||
return invalidParam("only https scheme is allowed")
|
||||
if pushUrl.Scheme != "https" && pushUrl.Scheme != "http" {
|
||||
return invalidParam("only https and http schemes are allowed")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ import (
|
|||
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
|
||||
"github.com/matrix-org/dendrite/clientapi/httputil"
|
||||
"github.com/matrix-org/dendrite/clientapi/jsonerror"
|
||||
"github.com/matrix-org/dendrite/clientapi/threepid"
|
||||
"github.com/matrix-org/dendrite/clientapi/userutil"
|
||||
userapi "github.com/matrix-org/dendrite/userapi/api"
|
||||
)
|
||||
|
|
@ -238,6 +239,7 @@ type authDict struct {
|
|||
// Recaptcha
|
||||
Response string `json:"response"`
|
||||
// TODO: Lots of custom keys depending on the type
|
||||
ThreePidCreds threepid.Credentials `json:"threepid_creds"`
|
||||
}
|
||||
|
||||
// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#user-interactive-authentication-api
|
||||
|
|
@ -747,6 +749,7 @@ func handleRegistrationFlow(
|
|||
}
|
||||
}
|
||||
|
||||
var threePid *authtypes.ThreePID
|
||||
switch r.Auth.Type {
|
||||
case authtypes.LoginTypeRecaptcha:
|
||||
// Check given captcha response
|
||||
|
|
@ -763,6 +766,29 @@ func handleRegistrationFlow(
|
|||
// Add Dummy to the list of completed registration stages
|
||||
sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeDummy)
|
||||
|
||||
case authtypes.LoginTypeEmail:
|
||||
threePid = &authtypes.ThreePID{}
|
||||
r.Auth.ThreePidCreds.IDServer = cfg.ThreePidDelegate
|
||||
var (
|
||||
bound bool
|
||||
err error
|
||||
)
|
||||
bound, threePid.Address, threePid.Medium, err = threepid.CheckAssociation(req.Context(), r.Auth.ThreePidCreds, cfg)
|
||||
if err != nil {
|
||||
util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAssociation failed")
|
||||
return jsonerror.InternalServerError()
|
||||
}
|
||||
if !bound {
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusBadRequest,
|
||||
JSON: jsonerror.MatrixError{
|
||||
ErrCode: "M_THREEPID_AUTH_FAILED",
|
||||
Err: "Failed to auth 3pid",
|
||||
},
|
||||
}
|
||||
}
|
||||
sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeEmail)
|
||||
|
||||
case "":
|
||||
// An empty auth type means that we want to fetch the available
|
||||
// flows. It can also mean that we want to register as an appservice
|
||||
|
|
@ -778,7 +804,7 @@ func handleRegistrationFlow(
|
|||
// A response with current registration flow and remaining available methods
|
||||
// will be returned if a flow has not been successfully completed yet
|
||||
return checkAndCompleteFlow(sessions.getCompletedStages(sessionID),
|
||||
req, r, sessionID, cfg, userAPI)
|
||||
req, r, sessionID, cfg, userAPI, threePid)
|
||||
}
|
||||
|
||||
// handleApplicationServiceRegistration handles the registration of an
|
||||
|
|
@ -820,7 +846,7 @@ func handleApplicationServiceRegistration(
|
|||
// application service registration is entirely separate.
|
||||
return completeRegistration(
|
||||
req.Context(), userAPI, r.Username, "", appserviceID, req.RemoteAddr, req.UserAgent(), r.Auth.Session,
|
||||
r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeAppService,
|
||||
r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeAppService, nil,
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -834,12 +860,13 @@ func checkAndCompleteFlow(
|
|||
sessionID string,
|
||||
cfg *config.ClientAPI,
|
||||
userAPI userapi.ClientUserAPI,
|
||||
threePid *authtypes.ThreePID,
|
||||
) util.JSONResponse {
|
||||
if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
|
||||
// This flow was completed, registration can continue
|
||||
return completeRegistration(
|
||||
req.Context(), userAPI, r.Username, r.Password, "", req.RemoteAddr, req.UserAgent(), sessionID,
|
||||
r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeUser,
|
||||
r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeUser, threePid,
|
||||
)
|
||||
}
|
||||
sessions.addParams(sessionID, r)
|
||||
|
|
@ -865,6 +892,7 @@ func completeRegistration(
|
|||
inhibitLogin eventutil.WeakBoolean,
|
||||
displayName, deviceID *string,
|
||||
accType userapi.AccountType,
|
||||
threePid *authtypes.ThreePID,
|
||||
) util.JSONResponse {
|
||||
if username == "" {
|
||||
return util.JSONResponse{
|
||||
|
|
@ -903,6 +931,21 @@ func completeRegistration(
|
|||
// Increment prometheus counter for created users
|
||||
amtRegUsers.Inc()
|
||||
|
||||
// TODO-entry refuse register if threepid is already bound to account.
|
||||
if threePid != nil {
|
||||
err = userAPI.PerformSaveThreePIDAssociation(ctx, &userapi.PerformSaveThreePIDAssociationRequest{
|
||||
Medium: threePid.Medium,
|
||||
ThreePID: threePid.Address,
|
||||
Localpart: accRes.Account.Localpart,
|
||||
}, &struct{}{})
|
||||
if err != nil {
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusInternalServerError,
|
||||
JSON: jsonerror.Unknown("Failed to save 3PID association: " + err.Error()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check whether inhibit_login option is set. If so, don't create an access
|
||||
// token or a device for this user
|
||||
if inhibitLogin {
|
||||
|
|
@ -1094,5 +1137,5 @@ func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.Clien
|
|||
if ssrr.Admin {
|
||||
accType = userapi.AccountTypeAdmin
|
||||
}
|
||||
return completeRegistration(req.Context(), userAPI, ssrr.User, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType)
|
||||
return completeRegistration(req.Context(), userAPI, ssrr.User, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType, nil)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import (
|
|||
clientutil "github.com/matrix-org/dendrite/clientapi/httputil"
|
||||
"github.com/matrix-org/dendrite/clientapi/jsonerror"
|
||||
"github.com/matrix-org/dendrite/clientapi/producers"
|
||||
"github.com/matrix-org/dendrite/clientapi/ratelimit"
|
||||
federationAPI "github.com/matrix-org/dendrite/federationapi/api"
|
||||
"github.com/matrix-org/dendrite/internal/httputil"
|
||||
"github.com/matrix-org/dendrite/internal/transactions"
|
||||
|
|
@ -66,6 +67,7 @@ func Setup(
|
|||
prometheus.MustRegister(amtRegUsers, sendEventDuration)
|
||||
|
||||
rateLimits := httputil.NewRateLimits(&cfg.RateLimiting)
|
||||
rateLimitsFailedLogin := ratelimit.NewRtFailedLogin(&cfg.RtFailedLogin)
|
||||
userInteractiveAuth := auth.NewUserInteractive(userAPI, cfg)
|
||||
|
||||
unstableFeatures := map[string]bool{
|
||||
|
|
@ -434,6 +436,17 @@ func Setup(
|
|||
}),
|
||||
).Methods(http.MethodPut, http.MethodOptions)
|
||||
|
||||
v3mux.Handle("/multiroom/{dataType}",
|
||||
httputil.MakeAuthAPI("send_multiroom", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
|
||||
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
|
||||
if err != nil {
|
||||
return util.ErrorResponse(err)
|
||||
}
|
||||
dataType := vars["dataType"]
|
||||
return PostMultiroom(req, device, syncProducer, dataType)
|
||||
}),
|
||||
).Methods(http.MethodPost, http.MethodOptions)
|
||||
|
||||
v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
|
||||
if r := rateLimits.Limit(req, nil); r != nil {
|
||||
return *r
|
||||
|
|
@ -602,7 +615,7 @@ func Setup(
|
|||
).Methods(http.MethodGet, http.MethodOptions)
|
||||
|
||||
v3mux.Handle("/account/password",
|
||||
httputil.MakeAuthAPI("password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
|
||||
httputil.MakeConditionalAuthAPI("password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
|
||||
if r := rateLimits.Limit(req, device); r != nil {
|
||||
return *r
|
||||
}
|
||||
|
|
@ -626,7 +639,7 @@ func Setup(
|
|||
if r := rateLimits.Limit(req, nil); r != nil {
|
||||
return *r
|
||||
}
|
||||
return Login(req, userAPI, cfg)
|
||||
return Login(req, userAPI, cfg, rateLimitsFailedLogin)
|
||||
}),
|
||||
).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
|
||||
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
|
|||
}
|
||||
// If the user has never been in the room then stop at this point.
|
||||
// We won't tell the user about a room they have never joined.
|
||||
if !membershipRes.HasBeenInRoom {
|
||||
if !membershipRes.HasBeenInRoom && membershipRes.Membership != gomatrixserverlib.Invite {
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusForbidden,
|
||||
JSON: jsonerror.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)),
|
||||
|
|
@ -241,7 +241,7 @@ func OnIncomingStateTypeRequest(
|
|||
}
|
||||
// If the user has never been in the room then stop at this point.
|
||||
// We won't tell the user about a room they have never joined.
|
||||
if !membershipRes.HasBeenInRoom || membershipRes.Membership == gomatrixserverlib.Ban {
|
||||
if !membershipRes.HasBeenInRoom && membershipRes.Membership != gomatrixserverlib.Invite || membershipRes.Membership == gomatrixserverlib.Ban {
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusForbidden,
|
||||
JSON: jsonerror.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)),
|
||||
|
|
|
|||
|
|
@ -103,11 +103,8 @@ func CreateSession(
|
|||
func CheckAssociation(
|
||||
ctx context.Context, creds Credentials, cfg *config.ClientAPI,
|
||||
) (bool, string, string, error) {
|
||||
if err := isTrusted(creds.IDServer, cfg); err != nil {
|
||||
return false, "", "", err
|
||||
}
|
||||
|
||||
requestURL := fmt.Sprintf("https://%s/_matrix/identity/api/v1/3pid/getValidated3pid?sid=%s&client_secret=%s", creds.IDServer, creds.SID, creds.Secret)
|
||||
requestURL := fmt.Sprintf("%s/_matrix/identity/api/v1/3pid/getValidated3pid?sid=%s&client_secret=%s", cfg.ThreePidDelegate, creds.SID, creds.Secret)
|
||||
req, err := http.NewRequest(http.MethodGet, requestURL, nil)
|
||||
if err != nil {
|
||||
return false, "", "", err
|
||||
|
|
|
|||
8
cmd/dendrite-monolith-server/Dockerfile.dev
Normal file
8
cmd/dendrite-monolith-server/Dockerfile.dev
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
FROM alpine:latest
|
||||
|
||||
COPY dendrite-monolith-server /usr/bin/
|
||||
|
||||
VOLUME /etc/dendrite
|
||||
WORKDIR /etc/dendrite
|
||||
|
||||
ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
|
||||
12
cmd/dendrite-monolith-server/build_dev.sh
Executable file
12
cmd/dendrite-monolith-server/build_dev.sh
Executable file
|
|
@ -0,0 +1,12 @@
|
|||
set -xe
|
||||
if [ -z "$(git status --porcelain)" ]; then
|
||||
CGO_ENABLED=0 go build .
|
||||
TAG=$(git rev-parse --short HEAD)
|
||||
docker build -f Dockerfile.dev -t gcr.io/globekeeper-development/dendrite-monolith:$TAG -t gcr.io/globekeeper-development/dendrite-monolith -t gcr.io/globekeeper-production/dendrite-monolith:$TAG .
|
||||
docker push gcr.io/globekeeper-development/dendrite-monolith:$TAG
|
||||
docker push gcr.io/globekeeper-production/dendrite-monolith:$TAG
|
||||
docker push gcr.io/globekeeper-development/dendrite-monolith
|
||||
else
|
||||
echo "Please commit changes"
|
||||
exit 0
|
||||
fi
|
||||
|
|
@ -16,6 +16,7 @@ package main
|
|||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/matrix-org/dendrite/appservice"
|
||||
|
|
@ -45,6 +46,16 @@ var (
|
|||
func main() {
|
||||
cfg := setup.ParseFlags(true)
|
||||
httpAddr := config.HTTPAddress("http://" + *httpBindAddr)
|
||||
for _, logging := range cfg.Logging {
|
||||
if logging.Type == "std" {
|
||||
level, err := logrus.ParseLevel(logging.Level)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
logrus.SetLevel(level)
|
||||
logrus.SetFormatter(&logrus.JSONFormatter{})
|
||||
}
|
||||
}
|
||||
httpsAddr := config.HTTPAddress("https://" + *httpsBindAddr)
|
||||
httpAPIAddr := httpAddr
|
||||
options := []basepkg.BaseDendriteOptions{}
|
||||
|
|
|
|||
71
docs/installation/10_optimisation.md
Normal file
71
docs/installation/10_optimisation.md
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
---
|
||||
title: Optimise your installation
|
||||
parent: Installation
|
||||
has_toc: true
|
||||
nav_order: 10
|
||||
permalink: /installation/start/optimisation
|
||||
---
|
||||
|
||||
# Optimise your installation
|
||||
|
||||
Now that you have Dendrite running, the following tweaks will improve the reliability
|
||||
and performance of your installation.
|
||||
|
||||
## File descriptor limit
|
||||
|
||||
Most platforms have a limit on how many file descriptors a single process can open. All
|
||||
connections made by Dendrite consume file descriptors — this includes database connections
|
||||
and network requests to remote homeservers. When participating in large federated rooms
|
||||
where Dendrite must talk to many remote servers, it is often very easy to exhaust default
|
||||
limits which are quite low.
|
||||
|
||||
We currently recommend setting the file descriptor limit to 65535 to avoid such
|
||||
issues. Dendrite will log immediately after startup if the file descriptor limit is too low:
|
||||
|
||||
```
|
||||
level=warning msg="IMPORTANT: Process file descriptor limit is currently 1024, it is recommended to raise the limit for Dendrite to at least 65535 to avoid issues"
|
||||
```
|
||||
|
||||
UNIX systems have two limits: a hard limit and a soft limit. You can view the soft limit
|
||||
by running `ulimit -Sn` and the hard limit with `ulimit -Hn`:
|
||||
|
||||
```bash
|
||||
$ ulimit -Hn
|
||||
1048576
|
||||
|
||||
$ ulimit -Sn
|
||||
1024
|
||||
```
|
||||
|
||||
Increase the soft limit before starting Dendrite:
|
||||
|
||||
```bash
|
||||
ulimit -Sn 65535
|
||||
```
|
||||
|
||||
The log line at startup should no longer appear if the limit is sufficient.
|
||||
|
||||
If you are running under a systemd service, you can instead add `LimitNOFILE=65535` option
|
||||
to the `[Service]` section of your service unit file.
|
||||
|
||||
## DNS caching
|
||||
|
||||
Dendrite has a built-in DNS cache which significantly reduces the load that Dendrite will
|
||||
place on your DNS resolver. This may also speed up outbound federation.
|
||||
|
||||
Consider enabling the DNS cache by modifying the `global` section of your configuration file:
|
||||
|
||||
```yaml
|
||||
dns_cache:
|
||||
enabled: true
|
||||
cache_size: 4096
|
||||
cache_lifetime: 600s
|
||||
```
|
||||
|
||||
## Time synchronisation
|
||||
|
||||
Matrix relies heavily on TLS which requires the system time to be correct. If the clock
|
||||
drifts then you may find that federation no works reliably (or at all) and clients may
|
||||
struggle to connect to your Dendrite server.
|
||||
|
||||
Ensure that the time is synchronised on your system by enabling NTP sync.
|
||||
6
go.mod
6
go.mod
|
|
@ -4,7 +4,6 @@ require (
|
|||
github.com/Arceliar/ironwood v0.0.0-20221025225125-45b4281814c2
|
||||
github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.0
|
||||
github.com/MFAshby/stdemuxerhook v1.0.0
|
||||
github.com/Masterminds/semver/v3 v3.1.1
|
||||
github.com/blevesearch/bleve/v2 v2.3.4
|
||||
github.com/codeclysm/extract v2.2.0+incompatible
|
||||
|
|
@ -12,6 +11,7 @@ require (
|
|||
github.com/docker/docker v20.10.19+incompatible
|
||||
github.com/docker/go-connections v0.4.0
|
||||
github.com/getsentry/sentry-go v0.14.0
|
||||
github.com/golang-jwt/jwt/v4 v4.4.1
|
||||
github.com/gologme/log v1.3.0
|
||||
github.com/google/go-cmp v0.5.9
|
||||
github.com/google/uuid v1.3.0
|
||||
|
|
@ -55,6 +55,8 @@ require (
|
|||
nhooyr.io/websocket v1.8.7
|
||||
)
|
||||
|
||||
require github.com/matryer/is v1.4.0
|
||||
|
||||
require (
|
||||
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
|
||||
github.com/Microsoft/go-winio v0.5.2 // indirect
|
||||
|
|
@ -82,7 +84,7 @@ require (
|
|||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.0 // indirect
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/gogo/protobuf v1.1.1 // indirect
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
|
||||
github.com/golang/glog v1.0.0 // indirect
|
||||
github.com/golang/mock v1.6.0 // indirect
|
||||
|
|
|
|||
12
go.sum
12
go.sum
|
|
@ -43,8 +43,6 @@ github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20O
|
|||
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
|
||||
github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM=
|
||||
github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
|
||||
github.com/MFAshby/stdemuxerhook v1.0.0 h1:1XFGzakrsHMv76AeanPDL26NOgwjPl/OUxbGhJthwMc=
|
||||
github.com/MFAshby/stdemuxerhook v1.0.0/go.mod h1:nLMI9FUf9Hz98n+yAXsTMUR4RZQy28uCTLG1Fzvj/uY=
|
||||
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
|
||||
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
|
||||
github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
|
||||
|
|
@ -202,9 +200,10 @@ github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm
|
|||
github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
|
||||
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
|
||||
github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
|
||||
github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ=
|
||||
github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo=
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
|
||||
|
|
@ -316,7 +315,6 @@ github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5l
|
|||
github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4=
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c=
|
||||
|
|
@ -354,6 +352,8 @@ github.com/matrix-org/pinecone v0.0.0-20221103125849-37f2e9b9ba37 h1:CQWFrgH9TJO
|
|||
github.com/matrix-org/pinecone v0.0.0-20221103125849-37f2e9b9ba37/go.mod h1:F3GHppRuHCTDeoOmmgjZMeJdbql91+RSGGsATWfC7oc=
|
||||
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
|
||||
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
|
||||
github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE=
|
||||
github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
|
|
@ -757,11 +757,9 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY
|
|||
golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
|
|
|
|||
|
|
@ -87,6 +87,57 @@ func MakeAuthAPI(
|
|||
return MakeExternalAPI(metricsName, h)
|
||||
}
|
||||
|
||||
// MakeConditionalAuthAPI turns a util.JSONRequestHandler function into an http.Handler which authenticates the request.
|
||||
// It passes nil device if header is not provided.
|
||||
func MakeConditionalAuthAPI(
|
||||
metricsName string, userAPI userapi.QueryAcccessTokenAPI,
|
||||
f func(*http.Request, *userapi.Device) util.JSONResponse,
|
||||
) http.Handler {
|
||||
h := func(req *http.Request) util.JSONResponse {
|
||||
var (
|
||||
jsonRes util.JSONResponse
|
||||
dev *userapi.Device
|
||||
)
|
||||
if _, err := auth.ExtractAccessToken(req); err != nil {
|
||||
dev = nil
|
||||
} else {
|
||||
logger := util.GetLogger(req.Context())
|
||||
var err *util.JSONResponse
|
||||
dev, err = auth.VerifyUserFromRequest(req, userAPI)
|
||||
if err != nil {
|
||||
logger.Debugf("VerifyUserFromRequest %s -> HTTP %d", req.RemoteAddr, err.Code)
|
||||
return *err
|
||||
}
|
||||
// add the user ID to the logger
|
||||
logger = logger.WithField("user_id", dev.UserID)
|
||||
req = req.WithContext(util.ContextWithLogger(req.Context(), logger))
|
||||
}
|
||||
// add the user to Sentry, if enabled
|
||||
hub := sentry.GetHubFromContext(req.Context())
|
||||
if hub != nil {
|
||||
hub.Scope().SetTag("user_id", dev.UserID)
|
||||
hub.Scope().SetTag("device_id", dev.ID)
|
||||
}
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
if hub != nil {
|
||||
hub.CaptureException(fmt.Errorf("%s panicked", req.URL.Path))
|
||||
}
|
||||
// re-panic to return the 500
|
||||
panic(r)
|
||||
}
|
||||
}()
|
||||
jsonRes = f(req, dev)
|
||||
// do not log 4xx as errors as they are client fails, not server fails
|
||||
if hub != nil && jsonRes.Code >= 500 {
|
||||
hub.Scope().SetExtra("response", jsonRes)
|
||||
hub.CaptureException(fmt.Errorf("%s returned HTTP %d", req.URL.Path, jsonRes.Code))
|
||||
}
|
||||
return jsonRes
|
||||
}
|
||||
return MakeExternalAPI(metricsName, h)
|
||||
}
|
||||
|
||||
// MakeAdminAPI is a wrapper around MakeAuthAPI which enforces that the request can only be
|
||||
// completed by a user that is a server administrator.
|
||||
func MakeAdminAPI(
|
||||
|
|
|
|||
|
|
@ -18,10 +18,8 @@
|
|||
package internal
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log/syslog"
|
||||
|
||||
"github.com/MFAshby/stdemuxerhook"
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
"github.com/sirupsen/logrus"
|
||||
lSyslog "github.com/sirupsen/logrus/hooks/syslog"
|
||||
|
|
@ -31,7 +29,6 @@ import (
|
|||
// If something fails here it means that the logging was improperly configured,
|
||||
// so we just exit with the error
|
||||
func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
|
||||
stdLogAdded := false
|
||||
for _, hook := range hooks {
|
||||
// Check we received a proper logging level
|
||||
level, err := logrus.ParseLevel(hook.Level)
|
||||
|
|
@ -39,12 +36,6 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
|
|||
logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err)
|
||||
}
|
||||
|
||||
// Perform a first filter on the logs according to the lowest level of all
|
||||
// (Eg: If we have hook for info and above, prevent logrus from processing debug logs)
|
||||
if logrus.GetLevel() < level {
|
||||
logrus.SetLevel(level)
|
||||
}
|
||||
|
||||
switch hook.Type {
|
||||
case "file":
|
||||
checkFileHookParams(hook.Params)
|
||||
|
|
@ -53,17 +44,10 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
|
|||
checkSyslogHookParams(hook.Params)
|
||||
setupSyslogHook(hook, level, componentName)
|
||||
case "std":
|
||||
setupStdLogHook(level)
|
||||
stdLogAdded = true
|
||||
default:
|
||||
logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type)
|
||||
}
|
||||
}
|
||||
if !stdLogAdded {
|
||||
setupStdLogHook(logrus.InfoLevel)
|
||||
}
|
||||
// Hooks are now configured for stdout/err, so throw away the default logger output
|
||||
logrus.SetOutput(io.Discard)
|
||||
}
|
||||
|
||||
func checkSyslogHookParams(params map[string]interface{}) {
|
||||
|
|
@ -87,10 +71,6 @@ func checkSyslogHookParams(params map[string]interface{}) {
|
|||
|
||||
}
|
||||
|
||||
func setupStdLogHook(level logrus.Level) {
|
||||
logrus.AddHook(&logLevelHook{level, stdemuxerhook.New(logrus.StandardLogger())})
|
||||
}
|
||||
|
||||
func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) {
|
||||
syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, componentName)
|
||||
if err == nil {
|
||||
|
|
|
|||
|
|
@ -140,7 +140,7 @@ func (r *Inviter) PerformInvite(
|
|||
|
||||
var isAlreadyJoined bool
|
||||
if info != nil {
|
||||
_, isAlreadyJoined, _, err = r.DB.GetMembership(ctx, info.RoomNID, *event.StateKey())
|
||||
_, _, isAlreadyJoined, _, err = r.DB.GetMembership(ctx, info.RoomNID, *event.StateKey())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("r.DB.GetMembership: %w", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import (
|
|||
"github.com/matrix-org/dendrite/roomserver/internal/helpers"
|
||||
"github.com/matrix-org/dendrite/roomserver/state"
|
||||
"github.com/matrix-org/dendrite/roomserver/storage"
|
||||
"github.com/matrix-org/dendrite/roomserver/storage/tables"
|
||||
"github.com/matrix-org/dendrite/roomserver/types"
|
||||
"github.com/matrix-org/dendrite/roomserver/version"
|
||||
)
|
||||
|
|
@ -180,11 +181,16 @@ func (r *Queryer) QueryMembershipForUser(
|
|||
}
|
||||
response.RoomExists = true
|
||||
|
||||
membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.UserID)
|
||||
membershipEventNID, membershipState, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.UserID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if membershipState == tables.MembershipStateInvite {
|
||||
response.Membership = gomatrixserverlib.Invite
|
||||
response.IsInRoom = true
|
||||
}
|
||||
|
||||
response.IsRoomForgotten = isRoomforgotten
|
||||
|
||||
if membershipEventNID == 0 {
|
||||
|
|
@ -327,7 +333,7 @@ func (r *Queryer) QueryMembershipsForRoom(
|
|||
return nil
|
||||
}
|
||||
|
||||
membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.Sender)
|
||||
membershipEventNID, _, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.Sender)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -942,7 +948,7 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.Query
|
|||
}
|
||||
// At this point we're happy that we are in the room, so now let's
|
||||
// see if the target user is in the room.
|
||||
_, isIn, _, err = r.DB.GetMembership(ctx, targetRoomInfo.RoomNID, req.UserID)
|
||||
_, _, isIn, _, err = r.DB.GetMembership(ctx, targetRoomInfo.RoomNID, req.UserID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ type Database interface {
|
|||
// in this room, along a boolean set to true if the user is still in this room,
|
||||
// false if not.
|
||||
// Returns an error if there was a problem talking to the database.
|
||||
GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom, isRoomForgotten bool, err error)
|
||||
GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, membershipNID tables.MembershipState, stillInRoom, isRoomForgotten bool, err error)
|
||||
// Lookup the membership event numeric IDs for all user that are or have
|
||||
// been members of a given room. Only lookup events of "join" membership if
|
||||
// joinOnly is set to true.
|
||||
|
|
|
|||
|
|
@ -400,14 +400,14 @@ func (d *Database) RemoveRoomAlias(ctx context.Context, alias string) error {
|
|||
})
|
||||
}
|
||||
|
||||
func (d *Database) GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom, isRoomforgotten bool, err error) {
|
||||
func (d *Database) GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, membershipState tables.MembershipState, stillInRoom, isRoomforgotten bool, err error) {
|
||||
var requestSenderUserNID types.EventStateKeyNID
|
||||
err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
|
||||
requestSenderUserNID, err = d.assignStateKeyNID(ctx, txn, requestSenderUserID)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return 0, false, false, fmt.Errorf("d.assignStateKeyNID: %w", err)
|
||||
return 0, 0, false, false, fmt.Errorf("d.assignStateKeyNID: %w", err)
|
||||
}
|
||||
|
||||
senderMembershipEventNID, senderMembership, isRoomforgotten, err :=
|
||||
|
|
@ -416,12 +416,12 @@ func (d *Database) GetMembership(ctx context.Context, roomNID types.RoomNID, req
|
|||
)
|
||||
if err == sql.ErrNoRows {
|
||||
// The user has never been a member of that room
|
||||
return 0, false, false, nil
|
||||
return 0, 0, false, false, nil
|
||||
} else if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return senderMembershipEventNID, senderMembership == tables.MembershipStateJoin, isRoomforgotten, nil
|
||||
return senderMembershipEventNID, senderMembership, senderMembership == tables.MembershipStateJoin, isRoomforgotten, nil
|
||||
}
|
||||
|
||||
func (d *Database) GetMembershipEventNIDsForRoom(
|
||||
|
|
|
|||
|
|
@ -137,7 +137,6 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
|
|||
logrus.Fatalf("Failed to start due to configuration errors")
|
||||
}
|
||||
|
||||
internal.SetupStdLogging()
|
||||
internal.SetupHookLogging(cfg.Logging, componentName)
|
||||
internal.SetupPprof()
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ package config
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"io"
|
||||
|
|
@ -266,6 +267,15 @@ func loadConfig(
|
|||
return nil, fmt.Errorf("either specify a 'private_key' path or supply both 'public_key' and 'key_id'")
|
||||
}
|
||||
}
|
||||
if c.ClientAPI.JwtConfig.Enabled {
|
||||
pubPki, _ := pem.Decode([]byte(c.ClientAPI.JwtConfig.Secret))
|
||||
var pub interface{}
|
||||
pub, err = x509.ParsePKIXPublicKey(pubPki.Bytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.ClientAPI.JwtConfig.SecretKey = pub.(ed25519.PublicKey)
|
||||
}
|
||||
|
||||
c.MediaAPI.AbsBasePath = Path(absPath(basePath, c.MediaAPI.BasePath))
|
||||
|
||||
|
|
@ -305,7 +315,10 @@ func (config *Dendrite) Derive() error {
|
|||
config.Derived.Registration.Flows = append(config.Derived.Registration.Flows,
|
||||
authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeDummy}})
|
||||
}
|
||||
|
||||
if config.ClientAPI.ThreePidDelegate != "" {
|
||||
config.Derived.Registration.Flows = append(config.Derived.Registration.Flows,
|
||||
authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeEmail}})
|
||||
}
|
||||
// Load application service configuration files
|
||||
if err := loadAppServices(&config.AppServiceAPI, &config.Derived); err != nil {
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import (
|
|||
"regexp"
|
||||
"strings"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
yaml "gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
|
|
@ -346,11 +345,11 @@ func checkErrors(config *AppServiceAPI, derived *Derived) (err error) {
|
|||
|
||||
// TODO: Remove once rate_limited is implemented
|
||||
if appservice.RateLimited {
|
||||
log.Warn("WARNING: Application service option rate_limited is currently unimplemented")
|
||||
// log.Warn("WARNING: Application service option rate_limited is currently unimplemented")
|
||||
}
|
||||
// TODO: Remove once protocols is implemented
|
||||
if len(appservice.Protocols) > 0 {
|
||||
log.Warn("WARNING: Application service option protocols is currently unimplemented")
|
||||
// log.Warn("WARNING: Application service option protocols is currently unimplemented")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -376,7 +375,7 @@ func validateNamespace(
|
|||
// Check if GroupID for the users namespace is in the correct format
|
||||
if key == "users" && namespace.GroupID != "" {
|
||||
// TODO: Remove once group_id is implemented
|
||||
log.Warn("WARNING: Application service option group_id is currently unimplemented")
|
||||
// log.Warn("WARNING: Application service option group_id is currently unimplemented")
|
||||
|
||||
correctFormat := groupIDRegexp.MatchString(namespace.GroupID)
|
||||
if !correctFormat {
|
||||
|
|
|
|||
|
|
@ -3,6 +3,9 @@ package config
|
|||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/matrix-org/dendrite/clientapi/ratelimit"
|
||||
"golang.org/x/crypto/ed25519"
|
||||
)
|
||||
|
||||
type ClientAPI struct {
|
||||
|
|
@ -52,9 +55,23 @@ type ClientAPI struct {
|
|||
TURN TURN `yaml:"turn"`
|
||||
|
||||
// Rate-limiting options
|
||||
RateLimiting RateLimiting `yaml:"rate_limiting"`
|
||||
RateLimiting RateLimiting `yaml:"rate_limiting"`
|
||||
RtFailedLogin ratelimit.RtFailedLoginConfig `yaml:"rate_limiting_failed_login"`
|
||||
|
||||
MSCs *MSCs `yaml:"-"`
|
||||
|
||||
ThreePidDelegate string `yaml:"three_pid_delegate"`
|
||||
|
||||
JwtConfig JwtConfig `yaml:"jwt_config"`
|
||||
}
|
||||
|
||||
type JwtConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Algorithm string `yaml:"algorithm"`
|
||||
Issuer string `yaml:"issuer"`
|
||||
Secret string `yaml:"secret"`
|
||||
SecretKey ed25519.PublicKey
|
||||
Audiences []string `yaml:"audiences"`
|
||||
}
|
||||
|
||||
func (c *ClientAPI) Defaults(opts DefaultOpts) {
|
||||
|
|
|
|||
|
|
@ -8,13 +8,13 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
"github.com/matrix-org/dendrite/setup/process"
|
||||
|
||||
natsserver "github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
natsclient "github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
|
|
@ -36,7 +36,7 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS
|
|||
defer natsLock.Unlock()
|
||||
// check if we need an in-process NATS Server
|
||||
if len(cfg.Addresses) != 0 {
|
||||
return setupNATS(process, cfg, nil)
|
||||
return setupNATS(cfg, nil)
|
||||
}
|
||||
if s.Server == nil {
|
||||
var err error
|
||||
|
|
@ -72,13 +72,34 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS
|
|||
if err != nil {
|
||||
logrus.Fatalln("Failed to create NATS client")
|
||||
}
|
||||
return setupNATS(process, cfg, nc)
|
||||
return setupNATS(cfg, nc)
|
||||
}
|
||||
|
||||
func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) {
|
||||
func setupNATS(cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) {
|
||||
var s nats.JetStreamContext
|
||||
var err error
|
||||
if nc == nil {
|
||||
var err error
|
||||
opts := []natsclient.Option{}
|
||||
opts := []natsclient.Option{
|
||||
natsclient.DisconnectErrHandler(func(c *natsclient.Conn, err error) {
|
||||
logrus.WithError(err).Error("nats connection: disconnected")
|
||||
}),
|
||||
natsclient.ReconnectHandler(func(_ *natsclient.Conn) {
|
||||
logrus.Info("nats connection: client reconnected")
|
||||
for _, stream := range []*nats.StreamConfig{
|
||||
streams[6],
|
||||
streams[10],
|
||||
} {
|
||||
err = configureStream(stream, cfg, s)
|
||||
if err != nil {
|
||||
logrus.WithError(err).WithField("stream", stream.Name).Error("unable to configure a stream")
|
||||
}
|
||||
|
||||
}
|
||||
}),
|
||||
natsclient.ClosedHandler(func(_ *natsclient.Conn) {
|
||||
logrus.Info("nats connection: client closed")
|
||||
}),
|
||||
}
|
||||
if cfg.DisableTLSValidation {
|
||||
opts = append(opts, natsclient.Secure(&tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
|
|
@ -91,89 +112,16 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc
|
|||
}
|
||||
}
|
||||
|
||||
s, err := nc.JetStream()
|
||||
s, err = nc.JetStream()
|
||||
if err != nil {
|
||||
logrus.WithError(err).Panic("Unable to get JetStream context")
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
for _, stream := range streams { // streams are defined in streams.go
|
||||
name := cfg.Prefixed(stream.Name)
|
||||
info, err := s.StreamInfo(name)
|
||||
if err != nil && err != natsclient.ErrStreamNotFound {
|
||||
logrus.WithError(err).Fatal("Unable to get stream info")
|
||||
}
|
||||
subjects := stream.Subjects
|
||||
if len(subjects) == 0 {
|
||||
// By default we want each stream to listen for the subjects
|
||||
// that are either an exact match for the stream name, or where
|
||||
// the first part of the subject is the stream name. ">" is a
|
||||
// wildcard in NATS for one or more subject tokens. In the case
|
||||
// that the stream is called "Foo", this will match any message
|
||||
// with the subject "Foo", "Foo.Bar" or "Foo.Bar.Baz" etc.
|
||||
subjects = []string{name, name + ".>"}
|
||||
}
|
||||
if info != nil {
|
||||
switch {
|
||||
case !reflect.DeepEqual(info.Config.Subjects, subjects):
|
||||
fallthrough
|
||||
case info.Config.Retention != stream.Retention:
|
||||
fallthrough
|
||||
case info.Config.Storage != stream.Storage:
|
||||
if err = s.DeleteStream(name); err != nil {
|
||||
logrus.WithError(err).Fatal("Unable to delete stream")
|
||||
}
|
||||
info = nil
|
||||
}
|
||||
}
|
||||
if info == nil {
|
||||
// If we're trying to keep everything in memory (e.g. unit tests)
|
||||
// then overwrite the storage policy.
|
||||
if cfg.InMemory {
|
||||
stream.Storage = natsclient.MemoryStorage
|
||||
}
|
||||
|
||||
// Namespace the streams without modifying the original streams
|
||||
// array, otherwise we end up with namespaces on namespaces.
|
||||
namespaced := *stream
|
||||
namespaced.Name = name
|
||||
namespaced.Subjects = subjects
|
||||
if _, err = s.AddStream(&namespaced); err != nil {
|
||||
logger := logrus.WithError(err).WithFields(logrus.Fields{
|
||||
"stream": namespaced.Name,
|
||||
"subjects": namespaced.Subjects,
|
||||
})
|
||||
|
||||
// If the stream was supposed to be in-memory to begin with
|
||||
// then an error here is fatal so we'll give up.
|
||||
if namespaced.Storage == natsclient.MemoryStorage {
|
||||
logger.WithError(err).Fatal("Unable to add in-memory stream")
|
||||
}
|
||||
|
||||
// The stream was supposed to be on disk. Let's try starting
|
||||
// Dendrite with the stream in-memory instead. That'll mean that
|
||||
// we can't recover anything that was queued on the disk but we
|
||||
// will still be able to start and run hopefully in the meantime.
|
||||
logger.WithError(err).Error("Unable to add stream")
|
||||
sentry.CaptureException(fmt.Errorf("Unable to add stream %q: %w", namespaced.Name, err))
|
||||
|
||||
namespaced.Storage = natsclient.MemoryStorage
|
||||
if _, err = s.AddStream(&namespaced); err != nil {
|
||||
// We tried to add the stream in-memory instead but something
|
||||
// went wrong. That's an unrecoverable situation so we will
|
||||
// give up at this point.
|
||||
logger.WithError(err).Fatal("Unable to add in-memory stream")
|
||||
}
|
||||
|
||||
if stream.Storage != namespaced.Storage {
|
||||
// We've managed to add the stream in memory. What's on the
|
||||
// disk will be left alone, but our ability to recover from a
|
||||
// future crash will be limited. Yell about it.
|
||||
err := fmt.Errorf("Stream %q is running in-memory; this may be due to data corruption in the JetStream storage directory", namespaced.Name)
|
||||
sentry.CaptureException(err)
|
||||
process.Degraded(err)
|
||||
}
|
||||
}
|
||||
err = configureStream(stream, cfg, s)
|
||||
if err != nil {
|
||||
logrus.WithError(err).WithField("stream", stream.Name).Fatal("unable to configure a stream")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -203,3 +151,52 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc
|
|||
|
||||
return s, nc
|
||||
}
|
||||
|
||||
func configureStream(stream *nats.StreamConfig, cfg *config.JetStream, s nats.JetStreamContext) error {
|
||||
name := cfg.Prefixed(stream.Name)
|
||||
info, err := s.StreamInfo(name)
|
||||
if err != nil && err != natsclient.ErrStreamNotFound {
|
||||
return fmt.Errorf("get stream info: %w", err)
|
||||
}
|
||||
subjects := stream.Subjects
|
||||
if len(subjects) == 0 {
|
||||
// By default we want each stream to listen for the subjects
|
||||
// that are either an exact match for the stream name, or where
|
||||
// the first part of the subject is the stream name. ">" is a
|
||||
// wildcard in NATS for one or more subject tokens. In the case
|
||||
// that the stream is called "Foo", this will match any message
|
||||
// with the subject "Foo", "Foo.Bar" or "Foo.Bar.Baz" etc.
|
||||
subjects = []string{name, name + ".>"}
|
||||
}
|
||||
if info != nil {
|
||||
switch {
|
||||
case !reflect.DeepEqual(info.Config.Subjects, subjects):
|
||||
fallthrough
|
||||
case info.Config.Retention != stream.Retention:
|
||||
fallthrough
|
||||
case info.Config.Storage != stream.Storage:
|
||||
if err = s.DeleteStream(name); err != nil {
|
||||
return fmt.Errorf("delete stream: %w", err)
|
||||
}
|
||||
info = nil
|
||||
}
|
||||
}
|
||||
if info == nil {
|
||||
// If we're trying to keep everything in memory (e.g. unit tests)
|
||||
// then overwrite the storage policy.
|
||||
if cfg.InMemory {
|
||||
stream.Storage = natsclient.MemoryStorage
|
||||
}
|
||||
|
||||
// Namespace the streams without modifying the original streams
|
||||
// array, otherwise we end up with namespaces on namespaces.
|
||||
namespaced := *stream
|
||||
namespaced.Name = name
|
||||
namespaced.Subjects = subjects
|
||||
if _, err = s.AddStream(&namespaced); err != nil {
|
||||
return fmt.Errorf("add stream: %w", err)
|
||||
}
|
||||
logrus.Infof("stream created: %s", stream.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ var (
|
|||
RequestPresence = "GetPresence"
|
||||
OutputPresenceEvent = "OutputPresenceEvent"
|
||||
InputFulltextReindex = "InputFulltextReindex"
|
||||
OutputMultiRoomCast = "OutputMultiRoomCast"
|
||||
)
|
||||
|
||||
var safeCharacters = regexp.MustCompile("[^A-Za-z0-9$]+")
|
||||
|
|
@ -101,4 +102,9 @@ var streams = []*nats.StreamConfig{
|
|||
Storage: nats.MemoryStorage,
|
||||
MaxAge: time.Minute * 5,
|
||||
},
|
||||
{
|
||||
Name: OutputMultiRoomCast,
|
||||
Retention: nats.InterestPolicy,
|
||||
Storage: nats.FileStorage,
|
||||
},
|
||||
}
|
||||
|
|
|
|||
113
syncapi/consumers/multiroomdata.go
Normal file
113
syncapi/consumers/multiroomdata.go
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
// Copyright 2017 Vector Creations Ltd
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package consumers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
"github.com/nats-io/nats.go"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
"github.com/matrix-org/dendrite/setup/jetstream"
|
||||
"github.com/matrix-org/dendrite/setup/process"
|
||||
"github.com/matrix-org/dendrite/syncapi/notifier"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage/mrd"
|
||||
"github.com/matrix-org/dendrite/syncapi/streams"
|
||||
"github.com/matrix-org/dendrite/syncapi/types"
|
||||
)
|
||||
|
||||
// OutputMultiRoomDataConsumer consumes events that originated in the client API server.
|
||||
type OutputMultiRoomDataConsumer struct {
|
||||
ctx context.Context
|
||||
jetstream nats.JetStreamContext
|
||||
durable string
|
||||
topic string
|
||||
db *mrd.Queries
|
||||
stream streams.StreamProvider
|
||||
notifier *notifier.Notifier
|
||||
}
|
||||
|
||||
// NewOutputMultiRoomDataConsumer creates a new OutputMultiRoomDataConsumer consumer. Call Start() to begin consuming from room servers.
|
||||
func NewOutputMultiRoomDataConsumer(
|
||||
process *process.ProcessContext,
|
||||
cfg *config.SyncAPI,
|
||||
js nats.JetStreamContext,
|
||||
q *mrd.Queries,
|
||||
notifier *notifier.Notifier,
|
||||
stream streams.StreamProvider,
|
||||
) *OutputMultiRoomDataConsumer {
|
||||
return &OutputMultiRoomDataConsumer{
|
||||
ctx: process.Context(),
|
||||
jetstream: js,
|
||||
topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputMultiRoomCast),
|
||||
durable: cfg.Matrix.JetStream.Durable("SyncAPIMultiRoomDataConsumer"),
|
||||
db: q,
|
||||
notifier: notifier,
|
||||
stream: stream,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *OutputMultiRoomDataConsumer) Start() error {
|
||||
return jetstream.JetStreamConsumer(
|
||||
s.ctx, s.jetstream, s.topic, s.durable, 1,
|
||||
s.onMessage, nats.DeliverAll(), nats.ManualAck(),
|
||||
)
|
||||
}
|
||||
|
||||
func (s *OutputMultiRoomDataConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
|
||||
msg := msgs[0]
|
||||
userID := msg.Header.Get(jetstream.UserID)
|
||||
dataType := msg.Header.Get("type")
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"type": dataType,
|
||||
"user_id": userID,
|
||||
}).Debug("Received multiroom data from client API server")
|
||||
|
||||
pos, err := s.db.InsertMultiRoomData(ctx, mrd.InsertMultiRoomDataParams{
|
||||
UserID: userID,
|
||||
Type: dataType,
|
||||
Data: msg.Data,
|
||||
})
|
||||
if err != nil {
|
||||
sentry.CaptureException(err)
|
||||
log.WithFields(log.Fields{
|
||||
"type": dataType,
|
||||
"user_id": userID,
|
||||
}).WithError(err).Errorf("could not insert multi room data")
|
||||
return false
|
||||
}
|
||||
|
||||
rooms, err := s.db.SelectMultiRoomVisibilityRooms(ctx, mrd.SelectMultiRoomVisibilityRoomsParams{
|
||||
UserID: userID,
|
||||
ExpireTs: time.Now().Unix(),
|
||||
})
|
||||
if err != nil {
|
||||
sentry.CaptureException(err)
|
||||
log.WithFields(log.Fields{
|
||||
"type": dataType,
|
||||
"user_id": userID,
|
||||
}).WithError(err).Errorf("failed to select multi room visibility")
|
||||
return false
|
||||
}
|
||||
|
||||
s.stream.Advance(types.StreamPosition(pos))
|
||||
s.notifier.OnNewMultiRoomData(types.StreamingToken{MultiRoomDataPosition: types.StreamPosition(pos)}, rooms)
|
||||
|
||||
return true
|
||||
}
|
||||
|
|
@ -280,6 +280,32 @@ func (n *Notifier) _sharedUsers(userID string) []string {
|
|||
return sharedUsers
|
||||
}
|
||||
|
||||
func (n *Notifier) OnNewMultiRoomData(
|
||||
posUpdate types.StreamingToken, roomIds []string,
|
||||
) {
|
||||
n.lock.Lock()
|
||||
defer n.lock.Unlock()
|
||||
|
||||
n.currPos.ApplyUpdates(posUpdate)
|
||||
usersInRoom := n._usersInRooms(roomIds)
|
||||
|
||||
n._wakeupUsers(usersInRoom, nil, n.currPos)
|
||||
}
|
||||
|
||||
func (n *Notifier) _usersInRooms(roomIds []string) []string {
|
||||
for i := range roomIds {
|
||||
for _, userID := range n._joinedUsers(roomIds[i]) {
|
||||
n._sharedUserMap[userID] = struct{}{}
|
||||
}
|
||||
}
|
||||
usersInRooms := make([]string, 0, len(n._sharedUserMap)+1)
|
||||
for userID := range n._sharedUserMap {
|
||||
usersInRooms = append(usersInRooms, userID)
|
||||
delete(n._sharedUserMap, userID)
|
||||
}
|
||||
return usersInRooms
|
||||
}
|
||||
|
||||
func (n *Notifier) IsSharedUser(userA, userB string) bool {
|
||||
n.lock.RLock()
|
||||
defer n.lock.RUnlock()
|
||||
|
|
|
|||
|
|
@ -109,6 +109,7 @@ type DatabaseTransaction interface {
|
|||
GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error)
|
||||
PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error)
|
||||
RelationsFor(ctx context.Context, roomID, eventID, relType, eventType string, from, to types.StreamPosition, backwards bool, limit int) (events []types.StreamEvent, prevBatch, nextBatch string, err error)
|
||||
SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string) (types.MultiRoom, error)
|
||||
}
|
||||
|
||||
type Database interface {
|
||||
|
|
@ -188,6 +189,10 @@ type Database interface {
|
|||
type Presence interface {
|
||||
GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error)
|
||||
UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error)
|
||||
PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error)
|
||||
MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error)
|
||||
ExpirePresence(ctx context.Context) ([]types.PresenceNotify, error)
|
||||
UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error
|
||||
}
|
||||
|
||||
type SharedUsers interface {
|
||||
|
|
|
|||
3
syncapi/storage/mrd/README.md
Normal file
3
syncapi/storage/mrd/README.md
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
## Multiroom storage
|
||||
|
||||
please install `sqlc`: `go install github.com/kyleconroy/sqlc/cmd/sqlc@latest`. Then run `sqlc -f sqlc.yaml generate` in this directory after changing `queries.sql` or `../postgres/schema.sql` files.
|
||||
138
syncapi/storage/mrd/db.go
Normal file
138
syncapi/storage/mrd/db.go
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.15.0
|
||||
|
||||
package mrd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type DBTX interface {
|
||||
ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
|
||||
PrepareContext(context.Context, string) (*sql.Stmt, error)
|
||||
QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
|
||||
QueryRowContext(context.Context, string, ...interface{}) *sql.Row
|
||||
}
|
||||
|
||||
func New(db DBTX) *Queries {
|
||||
return &Queries{db: db}
|
||||
}
|
||||
|
||||
func Prepare(ctx context.Context, db DBTX) (*Queries, error) {
|
||||
q := Queries{db: db}
|
||||
var err error
|
||||
if q.deleteMultiRoomVisibilityStmt, err = db.PrepareContext(ctx, deleteMultiRoomVisibility); err != nil {
|
||||
return nil, fmt.Errorf("error preparing query DeleteMultiRoomVisibility: %w", err)
|
||||
}
|
||||
if q.deleteMultiRoomVisibilityByExpireTSStmt, err = db.PrepareContext(ctx, deleteMultiRoomVisibilityByExpireTS); err != nil {
|
||||
return nil, fmt.Errorf("error preparing query DeleteMultiRoomVisibilityByExpireTS: %w", err)
|
||||
}
|
||||
if q.insertMultiRoomDataStmt, err = db.PrepareContext(ctx, insertMultiRoomData); err != nil {
|
||||
return nil, fmt.Errorf("error preparing query InsertMultiRoomData: %w", err)
|
||||
}
|
||||
if q.insertMultiRoomVisibilityStmt, err = db.PrepareContext(ctx, insertMultiRoomVisibility); err != nil {
|
||||
return nil, fmt.Errorf("error preparing query InsertMultiRoomVisibility: %w", err)
|
||||
}
|
||||
if q.selectMaxIdStmt, err = db.PrepareContext(ctx, selectMaxId); err != nil {
|
||||
return nil, fmt.Errorf("error preparing query SelectMaxId: %w", err)
|
||||
}
|
||||
if q.selectMultiRoomVisibilityRoomsStmt, err = db.PrepareContext(ctx, selectMultiRoomVisibilityRooms); err != nil {
|
||||
return nil, fmt.Errorf("error preparing query SelectMultiRoomVisibilityRooms: %w", err)
|
||||
}
|
||||
return &q, nil
|
||||
}
|
||||
|
||||
func (q *Queries) Close() error {
|
||||
var err error
|
||||
if q.deleteMultiRoomVisibilityStmt != nil {
|
||||
if cerr := q.deleteMultiRoomVisibilityStmt.Close(); cerr != nil {
|
||||
err = fmt.Errorf("error closing deleteMultiRoomVisibilityStmt: %w", cerr)
|
||||
}
|
||||
}
|
||||
if q.deleteMultiRoomVisibilityByExpireTSStmt != nil {
|
||||
if cerr := q.deleteMultiRoomVisibilityByExpireTSStmt.Close(); cerr != nil {
|
||||
err = fmt.Errorf("error closing deleteMultiRoomVisibilityByExpireTSStmt: %w", cerr)
|
||||
}
|
||||
}
|
||||
if q.insertMultiRoomDataStmt != nil {
|
||||
if cerr := q.insertMultiRoomDataStmt.Close(); cerr != nil {
|
||||
err = fmt.Errorf("error closing insertMultiRoomDataStmt: %w", cerr)
|
||||
}
|
||||
}
|
||||
if q.insertMultiRoomVisibilityStmt != nil {
|
||||
if cerr := q.insertMultiRoomVisibilityStmt.Close(); cerr != nil {
|
||||
err = fmt.Errorf("error closing insertMultiRoomVisibilityStmt: %w", cerr)
|
||||
}
|
||||
}
|
||||
if q.selectMaxIdStmt != nil {
|
||||
if cerr := q.selectMaxIdStmt.Close(); cerr != nil {
|
||||
err = fmt.Errorf("error closing selectMaxIdStmt: %w", cerr)
|
||||
}
|
||||
}
|
||||
if q.selectMultiRoomVisibilityRoomsStmt != nil {
|
||||
if cerr := q.selectMultiRoomVisibilityRoomsStmt.Close(); cerr != nil {
|
||||
err = fmt.Errorf("error closing selectMultiRoomVisibilityRoomsStmt: %w", cerr)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (q *Queries) exec(ctx context.Context, stmt *sql.Stmt, query string, args ...interface{}) (sql.Result, error) {
|
||||
switch {
|
||||
case stmt != nil && q.tx != nil:
|
||||
return q.tx.StmtContext(ctx, stmt).ExecContext(ctx, args...)
|
||||
case stmt != nil:
|
||||
return stmt.ExecContext(ctx, args...)
|
||||
default:
|
||||
return q.db.ExecContext(ctx, query, args...)
|
||||
}
|
||||
}
|
||||
|
||||
func (q *Queries) query(ctx context.Context, stmt *sql.Stmt, query string, args ...interface{}) (*sql.Rows, error) {
|
||||
switch {
|
||||
case stmt != nil && q.tx != nil:
|
||||
return q.tx.StmtContext(ctx, stmt).QueryContext(ctx, args...)
|
||||
case stmt != nil:
|
||||
return stmt.QueryContext(ctx, args...)
|
||||
default:
|
||||
return q.db.QueryContext(ctx, query, args...)
|
||||
}
|
||||
}
|
||||
|
||||
func (q *Queries) queryRow(ctx context.Context, stmt *sql.Stmt, query string, args ...interface{}) *sql.Row {
|
||||
switch {
|
||||
case stmt != nil && q.tx != nil:
|
||||
return q.tx.StmtContext(ctx, stmt).QueryRowContext(ctx, args...)
|
||||
case stmt != nil:
|
||||
return stmt.QueryRowContext(ctx, args...)
|
||||
default:
|
||||
return q.db.QueryRowContext(ctx, query, args...)
|
||||
}
|
||||
}
|
||||
|
||||
type Queries struct {
|
||||
db DBTX
|
||||
tx *sql.Tx
|
||||
deleteMultiRoomVisibilityStmt *sql.Stmt
|
||||
deleteMultiRoomVisibilityByExpireTSStmt *sql.Stmt
|
||||
insertMultiRoomDataStmt *sql.Stmt
|
||||
insertMultiRoomVisibilityStmt *sql.Stmt
|
||||
selectMaxIdStmt *sql.Stmt
|
||||
selectMultiRoomVisibilityRoomsStmt *sql.Stmt
|
||||
}
|
||||
|
||||
func (q *Queries) WithTx(tx *sql.Tx) *Queries {
|
||||
return &Queries{
|
||||
db: tx,
|
||||
tx: tx,
|
||||
deleteMultiRoomVisibilityStmt: q.deleteMultiRoomVisibilityStmt,
|
||||
deleteMultiRoomVisibilityByExpireTSStmt: q.deleteMultiRoomVisibilityByExpireTSStmt,
|
||||
insertMultiRoomDataStmt: q.insertMultiRoomDataStmt,
|
||||
insertMultiRoomVisibilityStmt: q.insertMultiRoomVisibilityStmt,
|
||||
selectMaxIdStmt: q.selectMaxIdStmt,
|
||||
selectMultiRoomVisibilityRoomsStmt: q.selectMultiRoomVisibilityRoomsStmt,
|
||||
}
|
||||
}
|
||||
24
syncapi/storage/mrd/models.go
Normal file
24
syncapi/storage/mrd/models.go
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.15.0
|
||||
|
||||
package mrd
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type SyncapiMultiroomDatum struct {
|
||||
ID int64 `json:"id"`
|
||||
UserID string `json:"user_id"`
|
||||
Type string `json:"type"`
|
||||
Data []byte `json:"data"`
|
||||
Ts time.Time `json:"ts"`
|
||||
}
|
||||
|
||||
type SyncapiMultiroomVisibility struct {
|
||||
UserID string `json:"user_id"`
|
||||
Type string `json:"type"`
|
||||
RoomID string `json:"room_id"`
|
||||
ExpireTs int64 `json:"expire_ts"`
|
||||
}
|
||||
44
syncapi/storage/mrd/queries.sql
Normal file
44
syncapi/storage/mrd/queries.sql
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
-- name: InsertMultiRoomData :one
|
||||
INSERT INTO syncapi_multiroom_data (
|
||||
user_id,
|
||||
type,
|
||||
data
|
||||
) VALUES (
|
||||
$1,
|
||||
$2,
|
||||
$3
|
||||
) ON CONFLICT (user_id, type) DO UPDATE SET id = nextval('syncapi_multiroom_id'), data = $3, ts = current_timestamp
|
||||
RETURNING id;
|
||||
|
||||
|
||||
-- name: InsertMultiRoomVisibility :exec
|
||||
INSERT INTO syncapi_multiroom_visibility (
|
||||
user_id,
|
||||
type,
|
||||
room_id,
|
||||
expire_ts
|
||||
) VALUES (
|
||||
$1,
|
||||
$2,
|
||||
$3,
|
||||
$4
|
||||
) ON CONFLICT (user_id, type, room_id) DO UPDATE SET expire_ts = $4;
|
||||
|
||||
-- name: SelectMultiRoomVisibilityRooms :many
|
||||
SELECT room_id FROM syncapi_multiroom_visibility
|
||||
WHERE user_id = $1
|
||||
AND expire_ts > $2;
|
||||
|
||||
|
||||
-- name: SelectMaxId :one
|
||||
SELECT MAX(id) FROM syncapi_multiroom_data;
|
||||
|
||||
-- name: DeleteMultiRoomVisibility :exec
|
||||
DELETE FROM syncapi_multiroom_visibility
|
||||
WHERE user_id = $1
|
||||
AND type = $2
|
||||
AND room_id = $3;
|
||||
|
||||
-- name: DeleteMultiRoomVisibilityByExpireTS :execrows
|
||||
DELETE FROM syncapi_multiroom_visibility
|
||||
WHERE expire_ts <= $1;
|
||||
143
syncapi/storage/mrd/queries.sql.go
Normal file
143
syncapi/storage/mrd/queries.sql.go
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.15.0
|
||||
// source: queries.sql
|
||||
|
||||
package mrd
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
const deleteMultiRoomVisibility = `-- name: DeleteMultiRoomVisibility :exec
|
||||
DELETE FROM syncapi_multiroom_visibility
|
||||
WHERE user_id = $1
|
||||
AND type = $2
|
||||
AND room_id = $3
|
||||
`
|
||||
|
||||
type DeleteMultiRoomVisibilityParams struct {
|
||||
UserID string `json:"user_id"`
|
||||
Type string `json:"type"`
|
||||
RoomID string `json:"room_id"`
|
||||
}
|
||||
|
||||
func (q *Queries) DeleteMultiRoomVisibility(ctx context.Context, arg DeleteMultiRoomVisibilityParams) error {
|
||||
_, err := q.exec(ctx, q.deleteMultiRoomVisibilityStmt, deleteMultiRoomVisibility, arg.UserID, arg.Type, arg.RoomID)
|
||||
return err
|
||||
}
|
||||
|
||||
const deleteMultiRoomVisibilityByExpireTS = `-- name: DeleteMultiRoomVisibilityByExpireTS :execrows
|
||||
DELETE FROM syncapi_multiroom_visibility
|
||||
WHERE expire_ts <= $1
|
||||
`
|
||||
|
||||
func (q *Queries) DeleteMultiRoomVisibilityByExpireTS(ctx context.Context, expireTs int64) (int64, error) {
|
||||
result, err := q.exec(ctx, q.deleteMultiRoomVisibilityByExpireTSStmt, deleteMultiRoomVisibilityByExpireTS, expireTs)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected()
|
||||
}
|
||||
|
||||
const insertMultiRoomData = `-- name: InsertMultiRoomData :one
|
||||
INSERT INTO syncapi_multiroom_data (
|
||||
user_id,
|
||||
type,
|
||||
data
|
||||
) VALUES (
|
||||
$1,
|
||||
$2,
|
||||
$3
|
||||
) ON CONFLICT (user_id, type) DO UPDATE SET id = nextval('syncapi_multiroom_id'), data = $3, ts = current_timestamp
|
||||
RETURNING id
|
||||
`
|
||||
|
||||
type InsertMultiRoomDataParams struct {
|
||||
UserID string `json:"user_id"`
|
||||
Type string `json:"type"`
|
||||
Data []byte `json:"data"`
|
||||
}
|
||||
|
||||
func (q *Queries) InsertMultiRoomData(ctx context.Context, arg InsertMultiRoomDataParams) (int64, error) {
|
||||
row := q.queryRow(ctx, q.insertMultiRoomDataStmt, insertMultiRoomData, arg.UserID, arg.Type, arg.Data)
|
||||
var id int64
|
||||
err := row.Scan(&id)
|
||||
return id, err
|
||||
}
|
||||
|
||||
const insertMultiRoomVisibility = `-- name: InsertMultiRoomVisibility :exec
|
||||
INSERT INTO syncapi_multiroom_visibility (
|
||||
user_id,
|
||||
type,
|
||||
room_id,
|
||||
expire_ts
|
||||
) VALUES (
|
||||
$1,
|
||||
$2,
|
||||
$3,
|
||||
$4
|
||||
) ON CONFLICT (user_id, type, room_id) DO UPDATE SET expire_ts = $4
|
||||
`
|
||||
|
||||
type InsertMultiRoomVisibilityParams struct {
|
||||
UserID string `json:"user_id"`
|
||||
Type string `json:"type"`
|
||||
RoomID string `json:"room_id"`
|
||||
ExpireTs int64 `json:"expire_ts"`
|
||||
}
|
||||
|
||||
func (q *Queries) InsertMultiRoomVisibility(ctx context.Context, arg InsertMultiRoomVisibilityParams) error {
|
||||
_, err := q.exec(ctx, q.insertMultiRoomVisibilityStmt, insertMultiRoomVisibility,
|
||||
arg.UserID,
|
||||
arg.Type,
|
||||
arg.RoomID,
|
||||
arg.ExpireTs,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const selectMaxId = `-- name: SelectMaxId :one
|
||||
SELECT MAX(id) FROM syncapi_multiroom_data
|
||||
`
|
||||
|
||||
func (q *Queries) SelectMaxId(ctx context.Context) (interface{}, error) {
|
||||
row := q.queryRow(ctx, q.selectMaxIdStmt, selectMaxId)
|
||||
var max interface{}
|
||||
err := row.Scan(&max)
|
||||
return max, err
|
||||
}
|
||||
|
||||
const selectMultiRoomVisibilityRooms = `-- name: SelectMultiRoomVisibilityRooms :many
|
||||
SELECT room_id FROM syncapi_multiroom_visibility
|
||||
WHERE user_id = $1
|
||||
AND expire_ts > $2
|
||||
`
|
||||
|
||||
type SelectMultiRoomVisibilityRoomsParams struct {
|
||||
UserID string `json:"user_id"`
|
||||
ExpireTs int64 `json:"expire_ts"`
|
||||
}
|
||||
|
||||
func (q *Queries) SelectMultiRoomVisibilityRooms(ctx context.Context, arg SelectMultiRoomVisibilityRoomsParams) ([]string, error) {
|
||||
rows, err := q.query(ctx, q.selectMultiRoomVisibilityRoomsStmt, selectMultiRoomVisibilityRooms, arg.UserID, arg.ExpireTs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []string
|
||||
for rows.Next() {
|
||||
var room_id string
|
||||
if err := rows.Scan(&room_id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, room_id)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
8
syncapi/storage/mrd/sqlc.yaml
Normal file
8
syncapi/storage/mrd/sqlc.yaml
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
version: 1
|
||||
packages:
|
||||
- path: ../mrd
|
||||
engine: postgresql
|
||||
schema: ../postgres/schema.sql
|
||||
queries: queries.sql
|
||||
emit_json_tags: true
|
||||
emit_prepared_queries: true
|
||||
6
syncapi/storage/mrd/types.go
Normal file
6
syncapi/storage/mrd/types.go
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
package mrd
|
||||
|
||||
type StateEvent struct {
|
||||
Hidden bool `json:"hidden"`
|
||||
ExpireTs int `json:"expire_ts"`
|
||||
}
|
||||
61
syncapi/storage/postgres/multiroomcast_table.go
Normal file
61
syncapi/storage/postgres/multiroomcast_table.go
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
"github.com/matrix-org/dendrite/internal"
|
||||
"github.com/matrix-org/dendrite/internal/sqlutil"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage/tables"
|
||||
"github.com/matrix-org/dendrite/syncapi/types"
|
||||
)
|
||||
|
||||
//go:embed schema.sql
|
||||
var schema string
|
||||
|
||||
var selectMultiRoomCastSQL = `SELECT d.user_id, d.type, d.data, d.ts FROM syncapi_multiroom_data AS d
|
||||
JOIN syncapi_multiroom_visibility AS v
|
||||
ON d.user_id = v.user_id
|
||||
AND d.type = v.type
|
||||
WHERE v.room_id = ANY($1)
|
||||
AND id > $2
|
||||
AND id <= $3`
|
||||
|
||||
type multiRoomStatements struct {
|
||||
selectMultiRoomCast *sql.Stmt
|
||||
}
|
||||
|
||||
func NewPostgresMultiRoomCastTable(db *sql.DB) (tables.MultiRoom, error) {
|
||||
r := &multiRoomStatements{}
|
||||
_, err := db.Exec(schema)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r, sqlutil.StatementList{
|
||||
{&r.selectMultiRoomCast, selectMultiRoomCastSQL},
|
||||
}.Prepare(db)
|
||||
}
|
||||
|
||||
func (s *multiRoomStatements) SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string, txn *sql.Tx) ([]*types.MultiRoomDataRow, error) {
|
||||
rows, err := sqlutil.TxStmt(txn, s.selectMultiRoomCast).QueryContext(ctx, pq.StringArray(joinedRooms), r.Low(), r.High())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data := make([]*types.MultiRoomDataRow, 0)
|
||||
defer internal.CloseAndLogIfError(ctx, rows, "SelectMultiRoomData: rows.close() failed")
|
||||
var t time.Time
|
||||
for rows.Next() {
|
||||
r := types.MultiRoomDataRow{}
|
||||
err = rows.Scan(&r.UserId, &r.Type, &r.Data, &t)
|
||||
r.Timestamp = t.Unix()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rows scan: %w", err)
|
||||
}
|
||||
data = append(data, &r)
|
||||
}
|
||||
return data, rows.Err()
|
||||
}
|
||||
|
|
@ -62,6 +62,10 @@ const upsertPresenceFromSyncSQL = "" +
|
|||
" presence = $2, last_active_ts = $3" +
|
||||
" RETURNING id"
|
||||
|
||||
const updateLastActiveSQL = `UPDATE syncapi_presence
|
||||
SET last_active_ts = $1
|
||||
WHERE user_id = $2`
|
||||
|
||||
const selectPresenceForUserSQL = "" +
|
||||
"SELECT presence, status_msg, last_active_ts" +
|
||||
" FROM syncapi_presence" +
|
||||
|
|
@ -76,12 +80,24 @@ const selectPresenceAfter = "" +
|
|||
" WHERE id > $1 AND last_active_ts >= $2" +
|
||||
" ORDER BY id ASC LIMIT $3"
|
||||
|
||||
const expirePresenceSQL = `UPDATE syncapi_presence SET
|
||||
id = nextval('syncapi_presence_id'),
|
||||
presence = 3
|
||||
WHERE
|
||||
to_timestamp(last_active_ts / 1000) < NOW() - INTERVAL` + types.PresenceExpire + `
|
||||
AND
|
||||
presence != 3
|
||||
RETURNING id, user_id
|
||||
`
|
||||
|
||||
type presenceStatements struct {
|
||||
upsertPresenceStmt *sql.Stmt
|
||||
upsertPresenceFromSyncStmt *sql.Stmt
|
||||
selectPresenceForUsersStmt *sql.Stmt
|
||||
selectMaxPresenceStmt *sql.Stmt
|
||||
selectPresenceAfterStmt *sql.Stmt
|
||||
expirePresenceStmt *sql.Stmt
|
||||
updateLastActiveStmt *sql.Stmt
|
||||
}
|
||||
|
||||
func NewPostgresPresenceTable(db *sql.DB) (*presenceStatements, error) {
|
||||
|
|
@ -96,6 +112,8 @@ func NewPostgresPresenceTable(db *sql.DB) (*presenceStatements, error) {
|
|||
{&s.selectPresenceForUsersStmt, selectPresenceForUserSQL},
|
||||
{&s.selectMaxPresenceStmt, selectMaxPresenceSQL},
|
||||
{&s.selectPresenceAfterStmt, selectPresenceAfter},
|
||||
{&s.expirePresenceStmt, expirePresenceSQL},
|
||||
{&s.updateLastActiveStmt, updateLastActiveSQL},
|
||||
}.Prepare(db)
|
||||
}
|
||||
|
||||
|
|
@ -166,3 +184,28 @@ func (p *presenceStatements) GetPresenceAfter(
|
|||
}
|
||||
return presences, rows.Err()
|
||||
}
|
||||
|
||||
func (p *presenceStatements) ExpirePresence(
|
||||
ctx context.Context,
|
||||
) ([]types.PresenceNotify, error) {
|
||||
rows, err := p.expirePresenceStmt.QueryContext(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
presences := make([]types.PresenceNotify, 0)
|
||||
i := 0
|
||||
for rows.Next() {
|
||||
presences = append(presences, types.PresenceNotify{})
|
||||
err = rows.Scan(&presences[i].StreamPos, &presences[i].UserID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
i++
|
||||
}
|
||||
return presences, err
|
||||
}
|
||||
|
||||
func (p *presenceStatements) UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error {
|
||||
_, err := p.updateLastActiveStmt.Exec(&lastActiveTs, &userId)
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
19
syncapi/storage/postgres/schema.sql
Normal file
19
syncapi/storage/postgres/schema.sql
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
CREATE SEQUENCE IF NOT EXISTS syncapi_multiroom_id;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS syncapi_multiroom_data (
|
||||
id BIGINT PRIMARY KEY DEFAULT nextval('syncapi_multiroom_id'),
|
||||
user_id TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
data BYTEA NOT NULL,
|
||||
ts TIMESTAMP NOT NULL DEFAULT current_timestamp
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS syncapi_multiroom_data_user_id_type_idx ON syncapi_multiroom_data(user_id, type);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS syncapi_multiroom_visibility (
|
||||
user_id TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
room_id TEXT NOT NULL,
|
||||
expire_ts BIGINT NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY(user_id, type, room_id)
|
||||
)
|
||||
|
|
@ -23,6 +23,7 @@ import (
|
|||
"github.com/matrix-org/dendrite/internal/sqlutil"
|
||||
"github.com/matrix-org/dendrite/setup/base"
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage/mrd"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage/postgres/deltas"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage/shared"
|
||||
)
|
||||
|
|
@ -102,6 +103,11 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mr, err := NewPostgresMultiRoomCastTable(d.db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mrq := mrd.New(d.db)
|
||||
|
||||
// apply migrations which need multiple tables
|
||||
m := sqlutil.NewMigrator(d.db)
|
||||
|
|
@ -134,6 +140,8 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
|
|||
Ignores: ignores,
|
||||
Presence: presence,
|
||||
Relations: relations,
|
||||
MultiRoom: mr,
|
||||
MultiRoomQ: mrq,
|
||||
}
|
||||
return &d, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import (
|
|||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/tidwall/gjson"
|
||||
|
||||
|
|
@ -30,6 +31,7 @@ import (
|
|||
"github.com/matrix-org/dendrite/internal/eventutil"
|
||||
"github.com/matrix-org/dendrite/internal/sqlutil"
|
||||
"github.com/matrix-org/dendrite/roomserver/api"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage/mrd"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage/tables"
|
||||
"github.com/matrix-org/dendrite/syncapi/types"
|
||||
)
|
||||
|
|
@ -54,6 +56,8 @@ type Database struct {
|
|||
Ignores tables.Ignores
|
||||
Presence tables.Presence
|
||||
Relations tables.Relations
|
||||
MultiRoomQ *mrd.Queries
|
||||
MultiRoom tables.MultiRoom
|
||||
}
|
||||
|
||||
func (d *Database) NewDatabaseSnapshot(ctx context.Context) (*DatabaseTransaction, error) {
|
||||
|
|
@ -336,6 +340,13 @@ func (d *Database) updateRoomState(
|
|||
}
|
||||
}
|
||||
|
||||
if strings.HasPrefix(event.Type(), "connect.mrd") {
|
||||
err := d.UpdateMultiRoomVisibility(ctx, event)
|
||||
if err != nil {
|
||||
logrus.WithError(err).WithField("event_id", event.EventID()).Error("failed to update multi room visibility")
|
||||
}
|
||||
}
|
||||
|
||||
if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membership, pduPosition); err != nil {
|
||||
return fmt.Errorf("d.CurrentRoomState.UpsertRoomState: %w", err)
|
||||
}
|
||||
|
|
@ -625,3 +636,49 @@ func (d *Database) SelectMemberships(
|
|||
) (eventIDs []string, err error) {
|
||||
return d.Memberships.SelectMemberships(ctx, nil, roomID, pos, membership, notMembership)
|
||||
}
|
||||
|
||||
func (s *Database) ExpirePresence(ctx context.Context) ([]types.PresenceNotify, error) {
|
||||
return s.Presence.ExpirePresence(ctx)
|
||||
}
|
||||
|
||||
func (d *Database) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) {
|
||||
return d.Presence.GetMaxPresenceID(ctx, nil)
|
||||
}
|
||||
|
||||
func (d *Database) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) {
|
||||
return d.Presence.GetPresenceAfter(ctx, nil, after, filter)
|
||||
}
|
||||
|
||||
func (s *Database) UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error {
|
||||
return s.Presence.UpdateLastActive(ctx, userId, lastActiveTs)
|
||||
}
|
||||
|
||||
func (d *Database) UpdateMultiRoomVisibility(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error {
|
||||
var mrdEv mrd.StateEvent
|
||||
err := json.Unmarshal(event.Content(), &mrdEv)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unmarshal multiroom visibility failed: %w", err)
|
||||
}
|
||||
if mrdEv.Hidden {
|
||||
err = d.MultiRoomQ.DeleteMultiRoomVisibility(ctx, mrd.DeleteMultiRoomVisibilityParams{
|
||||
UserID: event.Sender(),
|
||||
Type: event.Type(),
|
||||
RoomID: event.RoomID(),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete multiroom visibility failed: %w", err)
|
||||
}
|
||||
}
|
||||
if mrdEv.ExpireTs > 0 {
|
||||
err = d.MultiRoomQ.InsertMultiRoomVisibility(ctx, mrd.InsertMultiRoomVisibilityParams{
|
||||
UserID: event.Sender(),
|
||||
Type: event.Type(),
|
||||
RoomID: event.RoomID(),
|
||||
ExpireTs: int64(mrdEv.ExpireTs),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert multiroom visibility failed: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -688,3 +688,22 @@ func (d *DatabaseTransaction) RelationsFor(ctx context.Context, roomID, eventID,
|
|||
|
||||
return events, prevBatch, nextBatch, nil
|
||||
}
|
||||
|
||||
func (d *DatabaseTransaction) SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string) (types.MultiRoom, error) {
|
||||
rows, err := d.MultiRoom.SelectMultiRoomData(ctx, r, joinedRooms, d.txn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("select multi room data: %w", err)
|
||||
}
|
||||
mr := make(types.MultiRoom, 3)
|
||||
for _, row := range rows {
|
||||
if mr[row.UserId] == nil {
|
||||
mr[row.UserId] = make(map[string]types.MultiRoomData)
|
||||
}
|
||||
mr[row.UserId][row.Type] = types.MultiRoomData{
|
||||
Content: row.Data,
|
||||
Timestamp: row.Timestamp,
|
||||
}
|
||||
}
|
||||
return mr, nil
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -180,3 +180,15 @@ func (p *presenceStatements) GetPresenceAfter(
|
|||
}
|
||||
return presences, rows.Err()
|
||||
}
|
||||
|
||||
func (p *presenceStatements) ExpirePresence(
|
||||
ctx context.Context,
|
||||
) ([]types.PresenceNotify, error) {
|
||||
// TODO implement
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *presenceStatements) UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error {
|
||||
// TODO implement
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,18 +22,26 @@ import (
|
|||
|
||||
"github.com/matrix-org/dendrite/setup/base"
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage/mrd"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage/postgres"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage/sqlite3"
|
||||
)
|
||||
|
||||
// NewSyncServerDatasource opens a database connection.
|
||||
func NewSyncServerDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) {
|
||||
func NewSyncServerDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, *mrd.Queries, error) {
|
||||
switch {
|
||||
case dbProperties.ConnectionString.IsSQLite():
|
||||
return sqlite3.NewDatabase(base, dbProperties)
|
||||
ds, err := sqlite3.NewDatabase(base, dbProperties)
|
||||
return ds, nil, err
|
||||
|
||||
case dbProperties.ConnectionString.IsPostgres():
|
||||
return postgres.NewDatabase(base, dbProperties)
|
||||
ds, err := postgres.NewDatabase(base, dbProperties)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
mrq := mrd.New(ds.DB)
|
||||
return ds, mrq, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected database type")
|
||||
return nil, nil, fmt.Errorf("unexpected database type")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ var ctx = context.Background()
|
|||
func MustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func(), func()) {
|
||||
connStr, close := test.PrepareDBConnectionString(t, dbType)
|
||||
base, closeBase := testrig.CreateBaseDendrite(t, dbType)
|
||||
db, err := storage.NewSyncServerDatasource(base, &config.DatabaseOptions{
|
||||
db, _, err := storage.NewSyncServerDatasource(base, &config.DatabaseOptions{
|
||||
ConnectionString: config.DataSource(connStr),
|
||||
})
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -210,6 +210,8 @@ type Presence interface {
|
|||
GetPresenceForUser(ctx context.Context, txn *sql.Tx, userID string) (presence *types.PresenceInternal, err error)
|
||||
GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error)
|
||||
GetPresenceAfter(ctx context.Context, txn *sql.Tx, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (presences map[string]*types.PresenceInternal, err error)
|
||||
ExpirePresence(ctx context.Context) ([]types.PresenceNotify, error)
|
||||
UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error
|
||||
}
|
||||
|
||||
type Relations interface {
|
||||
|
|
@ -230,3 +232,7 @@ type Relations interface {
|
|||
// "from" or want to work forwards and don't have a "to").
|
||||
SelectMaxRelationID(ctx context.Context, txn *sql.Tx) (id int64, err error)
|
||||
}
|
||||
|
||||
type MultiRoom interface {
|
||||
SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string, txn *sql.Tx) ([]*types.MultiRoomDataRow, error)
|
||||
}
|
||||
|
|
|
|||
72
syncapi/streams/stream_multiroomdata.go
Normal file
72
syncapi/streams/stream_multiroomdata.go
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
// Copyright 2022 The Matrix.org Foundation C.I.C.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package streams
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
|
||||
"github.com/matrix-org/dendrite/syncapi/notifier"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage/mrd"
|
||||
"github.com/matrix-org/dendrite/syncapi/types"
|
||||
)
|
||||
|
||||
type MultiRoomDataStreamProvider struct {
|
||||
DefaultStreamProvider
|
||||
notifier *notifier.Notifier
|
||||
mrdDb *mrd.Queries
|
||||
}
|
||||
|
||||
func (p *MultiRoomDataStreamProvider) Setup(ctx context.Context, snapshot storage.DatabaseTransaction) {
|
||||
p.DefaultStreamProvider.Setup(ctx, snapshot)
|
||||
|
||||
id, err := p.mrdDb.SelectMaxId(context.Background())
|
||||
if err != nil && err != sql.ErrNoRows {
|
||||
panic(err)
|
||||
}
|
||||
p.latestMutex.Lock()
|
||||
defer p.latestMutex.Unlock()
|
||||
if id == nil {
|
||||
p.latest = types.StreamPosition(0)
|
||||
} else {
|
||||
p.latest = types.StreamPosition(id.(int64))
|
||||
}
|
||||
}
|
||||
|
||||
func (p *MultiRoomDataStreamProvider) CompleteSync(
|
||||
ctx context.Context,
|
||||
snapshot storage.DatabaseTransaction,
|
||||
req *types.SyncRequest,
|
||||
) types.StreamPosition {
|
||||
return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx))
|
||||
}
|
||||
|
||||
func (p *MultiRoomDataStreamProvider) IncrementalSync(
|
||||
ctx context.Context,
|
||||
snapshot storage.DatabaseTransaction,
|
||||
req *types.SyncRequest,
|
||||
from, to types.StreamPosition,
|
||||
) types.StreamPosition {
|
||||
mr, err := snapshot.SelectMultiRoomData(ctx, &types.Range{From: from, To: to}, req.JoinedRooms)
|
||||
if err != nil {
|
||||
req.Log.WithError(err).Error("GetUserUnreadNotificationCountsForRooms failed")
|
||||
return from
|
||||
}
|
||||
req.Log.Tracef("MultiRoomDataStreamProvider IncrementalSync: %+v", mr)
|
||||
req.Response.MultiRoom = mr
|
||||
return to
|
||||
|
||||
}
|
||||
|
|
@ -77,6 +77,7 @@ func (p *PDUStreamProvider) CompleteSync(
|
|||
req.Log.WithError(err).Error("p.DB.RoomIDsWithMembership failed")
|
||||
return from
|
||||
}
|
||||
req.JoinedRooms = joinedRoomIDs
|
||||
|
||||
stateFilter := req.Filter.Room.State
|
||||
eventFilter := req.Filter.Room.Timeline
|
||||
|
|
@ -170,6 +171,7 @@ func (p *PDUStreamProvider) IncrementalSync(
|
|||
for _, roomID := range syncJoinedRooms {
|
||||
req.Rooms[roomID] = gomatrixserverlib.Join
|
||||
}
|
||||
req.JoinedRooms = syncJoinedRooms
|
||||
|
||||
if len(stateDeltas) == 0 {
|
||||
return to
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ package streams
|
|||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
|
||||
"github.com/matrix-org/gomatrixserverlib"
|
||||
"github.com/tidwall/gjson"
|
||||
|
|
@ -29,8 +28,6 @@ import (
|
|||
|
||||
type PresenceStreamProvider struct {
|
||||
DefaultStreamProvider
|
||||
// cache contains previously sent presence updates to avoid unneeded updates
|
||||
cache sync.Map
|
||||
notifier *notifier.Notifier
|
||||
}
|
||||
|
||||
|
|
@ -114,19 +111,6 @@ func (p *PresenceStreamProvider) IncrementalSync(
|
|||
if req.Device.UserID != presence.UserID && !p.notifier.IsSharedUser(req.Device.UserID, presence.UserID) {
|
||||
continue
|
||||
}
|
||||
cacheKey := req.Device.UserID + req.Device.ID + presence.UserID
|
||||
pres, ok := p.cache.Load(cacheKey)
|
||||
if ok {
|
||||
// skip already sent presence
|
||||
prevPresence := pres.(*types.PresenceInternal)
|
||||
currentlyActive := prevPresence.CurrentlyActive()
|
||||
skip := prevPresence.Equals(presence) && currentlyActive && req.Device.UserID != presence.UserID
|
||||
_, membershipChange := req.MembershipChanges[presence.UserID]
|
||||
if skip && !membershipChange {
|
||||
req.Log.Tracef("Skipping presence, no change (%s)", presence.UserID)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if _, known := types.PresenceFromString(presence.ClientFields.Presence); known {
|
||||
presence.ClientFields.LastActiveAgo = presence.LastActiveAgo()
|
||||
|
|
@ -154,7 +138,6 @@ func (p *PresenceStreamProvider) IncrementalSync(
|
|||
if len(req.Response.Presence.Events) == req.Filter.Presence.Limit {
|
||||
break
|
||||
}
|
||||
p.cache.Store(cacheKey, presence)
|
||||
}
|
||||
|
||||
if len(req.Response.Presence.Events) == 0 {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
rsapi "github.com/matrix-org/dendrite/roomserver/api"
|
||||
"github.com/matrix-org/dendrite/syncapi/notifier"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage/mrd"
|
||||
"github.com/matrix-org/dendrite/syncapi/types"
|
||||
userapi "github.com/matrix-org/dendrite/userapi/api"
|
||||
)
|
||||
|
|
@ -23,12 +24,14 @@ type Streams struct {
|
|||
DeviceListStreamProvider StreamProvider
|
||||
NotificationDataStreamProvider StreamProvider
|
||||
PresenceStreamProvider StreamProvider
|
||||
MultiRoomStreamProvider StreamProvider
|
||||
}
|
||||
|
||||
func NewSyncStreamProviders(
|
||||
d storage.Database, userAPI userapi.SyncUserAPI,
|
||||
rsAPI rsapi.SyncRoomserverAPI, keyAPI keyapi.SyncKeyAPI,
|
||||
eduCache *caching.EDUCache, lazyLoadCache caching.LazyLoadCache, notifier *notifier.Notifier,
|
||||
mrdb *mrd.Queries,
|
||||
) *Streams {
|
||||
streams := &Streams{
|
||||
PDUStreamProvider: &PDUStreamProvider{
|
||||
|
|
@ -66,6 +69,11 @@ func NewSyncStreamProviders(
|
|||
DefaultStreamProvider: DefaultStreamProvider{DB: d},
|
||||
notifier: notifier,
|
||||
},
|
||||
MultiRoomStreamProvider: &MultiRoomDataStreamProvider{
|
||||
DefaultStreamProvider: DefaultStreamProvider{DB: d},
|
||||
notifier: notifier,
|
||||
mrdDb: mrdb,
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.TODO()
|
||||
|
|
@ -85,6 +93,7 @@ func NewSyncStreamProviders(
|
|||
streams.NotificationDataStreamProvider.Setup(ctx, snapshot)
|
||||
streams.DeviceListStreamProvider.Setup(ctx, snapshot)
|
||||
streams.PresenceStreamProvider.Setup(ctx, snapshot)
|
||||
streams.MultiRoomStreamProvider.Setup(ctx, snapshot)
|
||||
|
||||
succeeded = true
|
||||
return streams
|
||||
|
|
|
|||
|
|
@ -80,6 +80,8 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat
|
|||
filter.AccountData.Limit = math.MaxInt32
|
||||
filter.Room.AccountData.Limit = math.MaxInt32
|
||||
}
|
||||
// Ignore state limit filter so that e.g. correct room name appears on clients.
|
||||
filter.Room.State.Limit = math.MaxInt32
|
||||
|
||||
logger := util.GetLogger(req.Context()).WithFields(logrus.Fields{
|
||||
"user_id": device.UserID,
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ type RequestPool struct {
|
|||
keyAPI keyapi.SyncKeyAPI
|
||||
rsAPI roomserverAPI.SyncRoomserverAPI
|
||||
lastseen *sync.Map
|
||||
presence *sync.Map
|
||||
Presence *sync.Map
|
||||
streams *streams.Streams
|
||||
Notifier *notifier.Notifier
|
||||
producer PresencePublisher
|
||||
|
|
@ -86,14 +86,14 @@ func NewRequestPool(
|
|||
keyAPI: keyAPI,
|
||||
rsAPI: rsAPI,
|
||||
lastseen: &sync.Map{},
|
||||
presence: &sync.Map{},
|
||||
Presence: &sync.Map{},
|
||||
streams: streams,
|
||||
Notifier: notifier,
|
||||
producer: producer,
|
||||
consumer: consumer,
|
||||
}
|
||||
go rp.cleanLastSeen()
|
||||
go rp.cleanPresence(db, time.Minute*5)
|
||||
// go rp.cleanPresence(db, time.Minute*5)
|
||||
return rp
|
||||
}
|
||||
|
||||
|
|
@ -112,11 +112,11 @@ func (rp *RequestPool) cleanPresence(db storage.Presence, cleanupTime time.Durat
|
|||
return
|
||||
}
|
||||
for {
|
||||
rp.presence.Range(func(key interface{}, v interface{}) bool {
|
||||
rp.Presence.Range(func(key interface{}, v interface{}) bool {
|
||||
p := v.(types.PresenceInternal)
|
||||
if time.Since(p.LastActiveTS.Time()) > cleanupTime {
|
||||
rp.updatePresence(db, types.PresenceUnavailable.String(), p.UserID)
|
||||
rp.presence.Delete(key)
|
||||
rp.Presence.Delete(key)
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
|
@ -154,13 +154,22 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user
|
|||
}
|
||||
newPresence.ClientFields.Presence = presenceID.String()
|
||||
|
||||
defer rp.presence.Store(userID, newPresence)
|
||||
defer rp.Presence.Store(userID, newPresence)
|
||||
// avoid spamming presence updates when syncing
|
||||
existingPresence, ok := rp.presence.LoadOrStore(userID, newPresence)
|
||||
existingPresence, ok := rp.Presence.LoadOrStore(userID, newPresence)
|
||||
if ok {
|
||||
p := existingPresence.(types.PresenceInternal)
|
||||
if p.ClientFields.Presence == newPresence.ClientFields.Presence {
|
||||
return
|
||||
if dbPresence != nil {
|
||||
if p.Presence == newPresence.Presence && newPresence.LastActiveTS-dbPresence.LastActiveTS < types.PresenceNoOpMs {
|
||||
return
|
||||
}
|
||||
if dbPresence.Presence == types.PresenceOnline && presenceID == types.PresenceOnline && newPresence.LastActiveTS-dbPresence.LastActiveTS >= types.PresenceNoOpMs {
|
||||
err := db.UpdateLastActive(context.Background(), userID, uint64(newPresence.LastActiveTS))
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error("failed to update last active")
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -248,7 +257,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
|
|||
defer activeSyncRequests.Dec()
|
||||
|
||||
rp.updateLastSeen(req, device)
|
||||
rp.updatePresence(rp.db, req.FormValue("set_presence"), device.UserID)
|
||||
rp.updatePresence(rp.db, "", device.UserID)
|
||||
|
||||
waitingSyncRequests.Inc()
|
||||
defer waitingSyncRequests.Dec()
|
||||
|
|
@ -398,6 +407,14 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
|
|||
)
|
||||
},
|
||||
),
|
||||
MultiRoomDataPosition: withTransaction(
|
||||
syncReq.Since.MultiRoomDataPosition,
|
||||
func(txn storage.DatabaseTransaction) types.StreamPosition {
|
||||
return rp.streams.MultiRoomStreamProvider.CompleteSync(
|
||||
syncReq.Context, txn, syncReq,
|
||||
)
|
||||
},
|
||||
),
|
||||
}
|
||||
} else {
|
||||
// Incremental sync
|
||||
|
|
@ -483,6 +500,15 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
|
|||
)
|
||||
},
|
||||
),
|
||||
MultiRoomDataPosition: withTransaction(
|
||||
syncReq.Since.MultiRoomDataPosition,
|
||||
func(snapshot storage.DatabaseTransaction) types.StreamPosition {
|
||||
return rp.streams.MultiRoomStreamProvider.IncrementalSync(
|
||||
syncReq.Context, snapshot, syncReq,
|
||||
syncReq.Since.MultiRoomDataPosition, rp.Notifier.CurrentPosition().MultiRoomDataPosition,
|
||||
)
|
||||
},
|
||||
),
|
||||
}
|
||||
// it's possible for there to be no updates for this user even though since < current pos,
|
||||
// e.g busy servers with a quiet user. In this scenario, we don't want to return a no-op
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/matrix-org/dendrite/setup/config"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage"
|
||||
"github.com/matrix-org/dendrite/syncapi/types"
|
||||
"github.com/matrix-org/gomatrixserverlib"
|
||||
)
|
||||
|
|
@ -23,7 +24,9 @@ func (d *dummyPublisher) SendPresence(userID string, presence types.Presence, st
|
|||
return nil
|
||||
}
|
||||
|
||||
type dummyDB struct{}
|
||||
type dummyDB struct {
|
||||
storage.Database
|
||||
}
|
||||
|
||||
func (d dummyDB) UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) {
|
||||
return 0, nil
|
||||
|
|
@ -109,7 +112,7 @@ func TestRequestPool_updatePresence(t *testing.T) {
|
|||
},
|
||||
}
|
||||
rp := &RequestPool{
|
||||
presence: &syncMap,
|
||||
Presence: &syncMap,
|
||||
producer: publisher,
|
||||
consumer: consumer,
|
||||
cfg: &config.SyncAPI{
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ package syncapi
|
|||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
|
|
@ -34,6 +35,7 @@ import (
|
|||
"github.com/matrix-org/dendrite/syncapi/storage"
|
||||
"github.com/matrix-org/dendrite/syncapi/streams"
|
||||
"github.com/matrix-org/dendrite/syncapi/sync"
|
||||
"github.com/matrix-org/dendrite/syncapi/types"
|
||||
)
|
||||
|
||||
// AddPublicRoutes sets up and registers HTTP handlers for the SyncAPI
|
||||
|
|
@ -48,14 +50,26 @@ func AddPublicRoutes(
|
|||
|
||||
js, natsClient := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
|
||||
|
||||
syncDB, err := storage.NewSyncServerDatasource(base, &cfg.Database)
|
||||
syncDB, mrq, err := storage.NewSyncServerDatasource(base, &cfg.Database)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Panicf("failed to connect to sync db")
|
||||
}
|
||||
|
||||
go func() {
|
||||
var affected int64
|
||||
for {
|
||||
affected, err = mrq.DeleteMultiRoomVisibilityByExpireTS(context.Background(), time.Now().Unix())
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error("failed to expire multiroom visibility")
|
||||
}
|
||||
logrus.WithField("rows", affected).Info("expired multiroom visibility")
|
||||
time.Sleep(time.Minute)
|
||||
}
|
||||
}()
|
||||
|
||||
eduCache := caching.NewTypingCache()
|
||||
notifier := notifier.NewNotifier()
|
||||
streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache, base.Caches, notifier)
|
||||
streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache, base.Caches, notifier, mrq)
|
||||
notifier.SetCurrentPosition(streams.Latest(context.Background()))
|
||||
if err = notifier.Load(context.Background(), syncDB); err != nil {
|
||||
logrus.WithError(err).Panicf("failed to load notifier ")
|
||||
|
|
@ -130,8 +144,35 @@ func AddPublicRoutes(
|
|||
logrus.WithError(err).Panicf("failed to start receipts consumer")
|
||||
}
|
||||
|
||||
multiRoomConsumer := consumers.NewOutputMultiRoomDataConsumer(
|
||||
base.ProcessContext, cfg, js, mrq, notifier, streams.MultiRoomStreamProvider,
|
||||
)
|
||||
if err = multiRoomConsumer.Start(); err != nil {
|
||||
logrus.WithError(err).Panicf("failed to start multiroom consumer")
|
||||
}
|
||||
|
||||
routing.Setup(
|
||||
base.PublicClientAPIMux, requestPool, syncDB, userAPI,
|
||||
rsAPI, cfg, base.Caches, base.Fulltext,
|
||||
)
|
||||
|
||||
go func() {
|
||||
ctx := context.Background()
|
||||
for {
|
||||
notify, err := syncDB.ExpirePresence(ctx)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Error("failed to expire presence")
|
||||
}
|
||||
for i := range notify {
|
||||
requestPool.Presence.Store(notify[i].UserID, types.PresenceInternal{
|
||||
Presence: types.PresenceOffline,
|
||||
})
|
||||
notifier.OnNewPresence(types.StreamingToken{
|
||||
PresencePosition: notify[i].StreamPos,
|
||||
}, notify[i].UserID)
|
||||
|
||||
}
|
||||
time.Sleep(types.PresenceExpireInterval)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
|
|
|||
21
syncapi/types/multiroom.go
Normal file
21
syncapi/types/multiroom.go
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
package types
|
||||
|
||||
type MultiRoom map[string]map[string]MultiRoomData
|
||||
|
||||
type MultiRoomContent []byte
|
||||
|
||||
type MultiRoomData struct {
|
||||
Content MultiRoomContent `json:"content"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
}
|
||||
|
||||
func (d MultiRoomContent) MarshalJSON() ([]byte, error) {
|
||||
return d, nil
|
||||
}
|
||||
|
||||
type MultiRoomDataRow struct {
|
||||
Data []byte
|
||||
Type string
|
||||
UserId string
|
||||
Timestamp int64
|
||||
}
|
||||
21
syncapi/types/multiroom_test.go
Normal file
21
syncapi/types/multiroom_test.go
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
package types
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/matryer/is"
|
||||
)
|
||||
|
||||
func TestMarshallMultiRoom(t *testing.T) {
|
||||
is := is.New(t)
|
||||
m, err := json.Marshal(
|
||||
MultiRoom{
|
||||
"@3:example.com": map[string]MultiRoomData{
|
||||
"location": {
|
||||
Content: MultiRoomContent(`{"foo":"bar"}`),
|
||||
Timestamp: 123,
|
||||
}}})
|
||||
is.NoErr(err)
|
||||
is.Equal(m, []byte(`{"@3:example.com":{"location":{"content":{"foo":"bar"},"timestamp":123}}}`))
|
||||
}
|
||||
|
|
@ -21,6 +21,12 @@ import (
|
|||
"github.com/matrix-org/gomatrixserverlib"
|
||||
)
|
||||
|
||||
const (
|
||||
PresenceNoOpMs = 60_000
|
||||
PresenceExpire = "'4 minutes'"
|
||||
PresenceExpireInterval = time.Second * 30
|
||||
)
|
||||
|
||||
type Presence uint8
|
||||
|
||||
const (
|
||||
|
|
@ -66,6 +72,11 @@ type PresenceInternal struct {
|
|||
Presence Presence `json:"-"`
|
||||
}
|
||||
|
||||
type PresenceNotify struct {
|
||||
StreamPos StreamPosition
|
||||
UserID string
|
||||
}
|
||||
|
||||
// Equals compares p1 with p2.
|
||||
func (p1 *PresenceInternal) Equals(p2 *PresenceInternal) bool {
|
||||
return p1.ClientFields.Presence == p2.ClientFields.Presence &&
|
||||
|
|
|
|||
|
|
@ -21,7 +21,8 @@ type SyncRequest struct {
|
|||
WantFullState bool
|
||||
|
||||
// Updated by the PDU stream.
|
||||
Rooms map[string]string
|
||||
Rooms map[string]string
|
||||
JoinedRooms []string
|
||||
// Updated by the PDU stream.
|
||||
MembershipChanges map[string]struct{}
|
||||
// Updated by the PDU stream.
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/matrix-org/gomatrixserverlib"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/tidwall/gjson"
|
||||
|
||||
"github.com/matrix-org/dendrite/roomserver/api"
|
||||
|
|
@ -114,6 +115,7 @@ type StreamingToken struct {
|
|||
DeviceListPosition StreamPosition
|
||||
NotificationDataPosition StreamPosition
|
||||
PresencePosition StreamPosition
|
||||
MultiRoomDataPosition StreamPosition
|
||||
}
|
||||
|
||||
// This will be used as a fallback by json.Marshal.
|
||||
|
|
@ -129,12 +131,12 @@ func (s *StreamingToken) UnmarshalText(text []byte) (err error) {
|
|||
|
||||
func (t StreamingToken) String() string {
|
||||
posStr := fmt.Sprintf(
|
||||
"s%d_%d_%d_%d_%d_%d_%d_%d_%d",
|
||||
"s%d_%d_%d_%d_%d_%d_%d_%d_%d_%d",
|
||||
t.PDUPosition, t.TypingPosition,
|
||||
t.ReceiptPosition, t.SendToDevicePosition,
|
||||
t.InvitePosition, t.AccountDataPosition,
|
||||
t.DeviceListPosition, t.NotificationDataPosition,
|
||||
t.PresencePosition,
|
||||
t.PresencePosition, t.MultiRoomDataPosition,
|
||||
)
|
||||
return posStr
|
||||
}
|
||||
|
|
@ -160,12 +162,14 @@ func (t *StreamingToken) IsAfter(other StreamingToken) bool {
|
|||
return true
|
||||
case t.PresencePosition > other.PresencePosition:
|
||||
return true
|
||||
case t.MultiRoomDataPosition > other.MultiRoomDataPosition:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (t *StreamingToken) IsEmpty() bool {
|
||||
return t == nil || t.PDUPosition+t.TypingPosition+t.ReceiptPosition+t.SendToDevicePosition+t.InvitePosition+t.AccountDataPosition+t.DeviceListPosition+t.NotificationDataPosition+t.PresencePosition == 0
|
||||
return t == nil || t.PDUPosition+t.TypingPosition+t.ReceiptPosition+t.SendToDevicePosition+t.InvitePosition+t.AccountDataPosition+t.DeviceListPosition+t.NotificationDataPosition+t.PresencePosition+t.MultiRoomDataPosition == 0
|
||||
}
|
||||
|
||||
// WithUpdates returns a copy of the StreamingToken with updates applied from another StreamingToken.
|
||||
|
|
@ -209,6 +213,9 @@ func (t *StreamingToken) ApplyUpdates(other StreamingToken) {
|
|||
if other.PresencePosition > t.PresencePosition {
|
||||
t.PresencePosition = other.PresencePosition
|
||||
}
|
||||
if other.MultiRoomDataPosition > t.MultiRoomDataPosition {
|
||||
t.MultiRoomDataPosition = other.MultiRoomDataPosition
|
||||
}
|
||||
}
|
||||
|
||||
type TopologyToken struct {
|
||||
|
|
@ -294,9 +301,11 @@ func NewTopologyTokenFromString(tok string) (token TopologyToken, err error) {
|
|||
func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
|
||||
if len(tok) < 1 {
|
||||
err = ErrMalformedSyncToken
|
||||
logrus.WithField("token", tok).Info("invalid stream token: bad length")
|
||||
return
|
||||
}
|
||||
if tok[0] != SyncTokenTypeStream[0] {
|
||||
logrus.WithField("token", tok).Info("invalid stream token: not starting from s")
|
||||
err = ErrMalformedSyncToken
|
||||
return
|
||||
}
|
||||
|
|
@ -304,7 +313,7 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
|
|||
// s478_0_0_0_0_13.dl-0-2 but we have now removed partitioned stream positions
|
||||
tok = strings.Split(tok, ".")[0]
|
||||
parts := strings.Split(tok[1:], "_")
|
||||
var positions [9]StreamPosition
|
||||
var positions [10]StreamPosition
|
||||
for i, p := range parts {
|
||||
if i >= len(positions) {
|
||||
break
|
||||
|
|
@ -312,6 +321,7 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
|
|||
var pos int
|
||||
pos, err = strconv.Atoi(p)
|
||||
if err != nil {
|
||||
logrus.WithField("token", tok).Info("invalid stream token: strconv")
|
||||
err = ErrMalformedSyncToken
|
||||
return
|
||||
}
|
||||
|
|
@ -327,6 +337,7 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
|
|||
DeviceListPosition: positions[6],
|
||||
NotificationDataPosition: positions[7],
|
||||
PresencePosition: positions[8],
|
||||
MultiRoomDataPosition: positions[9],
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
|
@ -363,6 +374,7 @@ type Response struct {
|
|||
ToDevice *ToDeviceResponse `json:"to_device,omitempty"`
|
||||
DeviceLists *DeviceLists `json:"device_lists,omitempty"`
|
||||
DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"`
|
||||
MultiRoom MultiRoom `json:"multiroom,omitempty"`
|
||||
}
|
||||
|
||||
func (r Response) MarshalJSON() ([]byte, error) {
|
||||
|
|
@ -401,7 +413,8 @@ func (r *Response) HasUpdates() bool {
|
|||
len(r.Rooms.Peek) > 0 ||
|
||||
len(r.ToDevice.Events) > 0 ||
|
||||
len(r.DeviceLists.Changed) > 0 ||
|
||||
len(r.DeviceLists.Left) > 0)
|
||||
len(r.DeviceLists.Left) > 0) ||
|
||||
len(r.MultiRoom) > 0
|
||||
}
|
||||
|
||||
// NewResponse creates an empty response with initialised maps.
|
||||
|
|
|
|||
|
|
@ -9,10 +9,10 @@ import (
|
|||
|
||||
func TestSyncTokens(t *testing.T) {
|
||||
shouldPass := map[string]string{
|
||||
"s4_0_0_0_0_0_0_0_3": StreamingToken{4, 0, 0, 0, 0, 0, 0, 0, 3}.String(),
|
||||
"s3_1_0_0_0_0_2_0_5": StreamingToken{3, 1, 0, 0, 0, 0, 2, 0, 5}.String(),
|
||||
"s3_1_2_3_5_0_0_0_6": StreamingToken{3, 1, 2, 3, 5, 0, 0, 0, 6}.String(),
|
||||
"t3_1": TopologyToken{3, 1}.String(),
|
||||
"s4_0_0_0_0_0_0_0_3_0": StreamingToken{4, 0, 0, 0, 0, 0, 0, 0, 3, 0}.String(),
|
||||
"s3_1_0_0_0_0_2_0_5_1": StreamingToken{3, 1, 0, 0, 0, 0, 2, 0, 5, 1}.String(),
|
||||
"s3_1_2_3_5_0_0_0_6_2": StreamingToken{3, 1, 2, 3, 5, 0, 0, 0, 6, 2}.String(),
|
||||
"t3_1": TopologyToken{3, 1}.String(),
|
||||
}
|
||||
|
||||
for a, b := range shouldPass {
|
||||
|
|
|
|||
|
|
@ -49,3 +49,18 @@ Leaves are present in non-gapped incremental syncs
|
|||
|
||||
# Below test was passing for the wrong reason, failing correctly since #2858
|
||||
New federated private chats get full presence information (SYN-115)
|
||||
If a device list update goes missing, the server resyncs on the next one
|
||||
# You'll be shocked to discover this is flakey too
|
||||
|
||||
Inbound /v1/send_join rejects joins from other servers
|
||||
|
||||
# For notifications extension on iOS
|
||||
|
||||
/event/ does not allow access to events before the user joined
|
||||
|
||||
# Failing after recent updates with presence
|
||||
|
||||
Newly joined room includes presence in incremental sync
|
||||
User sees their own presence in a sync
|
||||
User is offline if they set_presence=offline in their sync
|
||||
User sees updates to presence from other users in the incremental sync.
|
||||
|
|
|
|||
|
|
@ -204,7 +204,6 @@ Deleted tags appear in an incremental v2 /sync
|
|||
/event/ on non world readable room does not work
|
||||
Outbound federation can query profile data
|
||||
/event/ on joined room works
|
||||
/event/ does not allow access to events before the user joined
|
||||
Federation key API allows unsigned requests for keys
|
||||
GET /publicRooms lists rooms
|
||||
GET /publicRooms includes avatar URLs
|
||||
|
|
|
|||
|
|
@ -171,7 +171,6 @@ func PrepareDBConnectionString(t *testing.T, dbType DBType) (connStr string, clo
|
|||
func WithAllDatabases(t *testing.T, testFn func(t *testing.T, db DBType)) {
|
||||
dbs := map[string]DBType{
|
||||
"postgres": DBTypePostgres,
|
||||
"sqlite": DBTypeSQLite,
|
||||
}
|
||||
for dbName, dbType := range dbs {
|
||||
dbt := dbType
|
||||
|
|
|
|||
|
|
@ -533,7 +533,7 @@ type PerformPusherSetRequest struct {
|
|||
|
||||
type PerformPusherDeletionRequest struct {
|
||||
Localpart string
|
||||
SessionID int64
|
||||
SessionID int64 // Pusher corresponding to this SessionID will not be deleted
|
||||
}
|
||||
|
||||
// Pusher represents a push notification subscriber
|
||||
|
|
|
|||
6
userapi/api/api_multicast.go
Normal file
6
userapi/api/api_multicast.go
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
package api
|
||||
|
||||
type MulticastMetadata struct {
|
||||
ExpireMs int
|
||||
ExcludeRoomIds []string
|
||||
}
|
||||
|
|
@ -613,14 +613,25 @@ func (a *UserInternalAPI) PerformAccountDeactivation(ctx context.Context, req *a
|
|||
return err
|
||||
}
|
||||
|
||||
threepids, err := a.DB.GetThreePIDsForLocalpart(ctx, req.Localpart)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := 0; i < len(threepids); i++ {
|
||||
err = a.DB.RemoveThreePIDAssociation(ctx, threepids[i].Address, threepids[i].Medium)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
pusherReq := &api.PerformPusherDeletionRequest{
|
||||
Localpart: req.Localpart,
|
||||
}
|
||||
if err := a.PerformPusherDeletion(ctx, pusherReq, &struct{}{}); err != nil {
|
||||
if err = a.PerformPusherDeletion(ctx, pusherReq, &struct{}{}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err := a.DB.DeactivateAccount(ctx, req.Localpart)
|
||||
err = a.DB.DeactivateAccount(ctx, req.Localpart)
|
||||
res.AccountDeactivated = err == nil
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -98,6 +98,11 @@ func NewPostgresAccountsTable(db *sql.DB, serverName gomatrixserverlib.ServerNam
|
|||
Up: deltas.UpAddAccountType,
|
||||
Down: deltas.DownAddAccountType,
|
||||
},
|
||||
{
|
||||
Version: "userapi: no guests",
|
||||
Up: deltas.UpNoGuests,
|
||||
Down: deltas.DownNoGuests,
|
||||
},
|
||||
}...)
|
||||
err = m.Up(context.Background())
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
package deltas
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func UpNoGuests(ctx context.Context, tx *sql.Tx) error {
|
||||
// AddAccountType introduced a bug where each user that had was registered as a regular user, but without user_id, became a guest.
|
||||
_, err := tx.ExecContext(ctx, "UPDATE userapi_accounts SET account_type = 1 WHERE account_type = 2;")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to execute upgrade: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func DownNoGuests(ctx context.Context, tx *sql.Tx) error {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -72,7 +72,7 @@ const selectNotificationSQL = "" +
|
|||
") AND NOT read ORDER BY localpart, id LIMIT $4"
|
||||
|
||||
const selectNotificationCountSQL = "" +
|
||||
"SELECT COUNT(*) FROM userapi_notifications WHERE localpart = $1 AND (" +
|
||||
"SELECT COUNT(DISTINCT(room_id)) FROM userapi_notifications WHERE localpart = $1 AND (" +
|
||||
"(($2 & 1) <> 0 AND highlight) OR (($2 & 2) <> 0 AND NOT highlight)" +
|
||||
") AND NOT read"
|
||||
|
||||
|
|
|
|||
|
|
@ -565,7 +565,7 @@ func (d *Database) CreateDevice(
|
|||
ctx context.Context, localpart string, deviceID *string, accessToken string,
|
||||
displayName *string, ipAddr, userAgent string,
|
||||
) (dev *api.Device, returnErr error) {
|
||||
if deviceID != nil {
|
||||
if deviceID != nil && *deviceID != "" {
|
||||
returnErr = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
|
||||
var err error
|
||||
// Revoke existing tokens for this device
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ const selectNotificationSQL = "" +
|
|||
") AND NOT read ORDER BY localpart, id LIMIT $4"
|
||||
|
||||
const selectNotificationCountSQL = "" +
|
||||
"SELECT COUNT(*) FROM userapi_notifications WHERE localpart = $1 AND (" +
|
||||
"SELECT COUNT(DISTINCT(room_id)) FROM userapi_notifications WHERE localpart = $1 AND (" +
|
||||
"(($2 & 1) <> 0 AND highlight) OR (($2 & 2) <> 0 AND NOT highlight)" +
|
||||
") AND NOT read"
|
||||
|
||||
|
|
|
|||
|
|
@ -128,6 +128,11 @@ func Test_Accounts(t *testing.T) {
|
|||
_, err = db.GetAccountByPassword(ctx, aliceLocalpart, "newPassword")
|
||||
assert.Error(t, err, "expected an error, got none")
|
||||
|
||||
// This should return an empty slice, as the account is deactivated and the 3pid is unbound
|
||||
threepids, err := db.GetThreePIDsForLocalpart(ctx, aliceLocalpart)
|
||||
assert.NoError(t, err, "failed to get 3pid for account")
|
||||
assert.Equal(t, len(threepids), 0)
|
||||
|
||||
_, err = db.GetAccountByLocalpart(ctx, "unusename")
|
||||
assert.Error(t, err, "expected an error for non existent localpart")
|
||||
|
||||
|
|
@ -533,7 +538,7 @@ func Test_Notification(t *testing.T) {
|
|||
// get notifications
|
||||
count, err := db.GetNotificationCount(ctx, aliceLocalpart, tables.AllNotifications)
|
||||
assert.NoError(t, err, "unable to get notification count")
|
||||
assert.Equal(t, int64(10), count)
|
||||
assert.Equal(t, int64(2), count)
|
||||
notifs, count, err := db.GetNotifications(ctx, aliceLocalpart, 0, 15, tables.AllNotifications)
|
||||
assert.NoError(t, err, "unable to get notifications")
|
||||
assert.Equal(t, int64(10), count)
|
||||
|
|
|
|||
Loading…
Reference in a new issue