Merge remote-tracking branch 'origin' into release/upstream-0.10.7

This commit is contained in:
danielaloni 2022-11-07 10:50:36 +02:00
commit 5c07b88b0d
96 changed files with 2018 additions and 772 deletions

12
.cloudbuild/dev.yaml Normal file
View file

@ -0,0 +1,12 @@
steps:
- name: gcr.io/cloud-builders/docker
args: ['build', '-t', 'gcr.io/$PROJECT_ID/dendrite-monolith:$COMMIT_SHA', '-f', 'build/docker/Dockerfile.monolith', '.']
- name: gcr.io/cloud-builders/kubectl
args: ['-n', 'dendrite', 'set', 'image', 'deployment/dendrite', 'dendrite=gcr.io/$PROJECT_ID/dendrite-monolith:$COMMIT_SHA']
env:
- CLOUDSDK_CORE_PROJECT=globekeeper-development
- CLOUDSDK_COMPUTE_ZONE=europe-west2-a
- CLOUDSDK_CONTAINER_CLUSTER=synapse
images:
- gcr.io/$PROJECT_ID/dendrite-monolith:$COMMIT_SHA
timeout: 480s

12
.cloudbuild/prod.yaml Normal file
View file

@ -0,0 +1,12 @@
steps:
- name: gcr.io/cloud-builders/docker
args: ['build', '-t', 'gcr.io/$PROJECT_ID/dendrite-monolith:$TAG_NAME', '-f', 'build/docker/Dockerfile.monolith', '.']
- name: gcr.io/cloud-builders/kubectl
args: ['set', 'image', 'deployment/dendrite', 'dendrite=gcr.io/$PROJECT_ID/dendrite-monolith:$TAG_NAME']
env:
- CLOUDSDK_CORE_PROJECT=globekeeper-production
- CLOUDSDK_COMPUTE_ZONE=europe-west2-a
- CLOUDSDK_CONTAINER_CLUSTER=synapse-production
images:
- gcr.io/$PROJECT_ID/dendrite-monolith:$TAG_NAME
timeout: 480s

View file

@ -14,51 +14,6 @@ concurrency:
cancel-in-progress: true
jobs:
wasm:
name: WASM build test
timeout-minutes: 5
runs-on: ubuntu-latest
if: ${{ false }} # disable for now
steps:
- uses: actions/checkout@v3
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: 1.18
- uses: actions/cache@v2
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-wasm-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-wasm
- name: Install Node
uses: actions/setup-node@v2
with:
node-version: 14
- uses: actions/cache@v2
with:
path: ~/.npm
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-
- name: Reconfigure Git to use HTTPS auth for repo packages
run: >
git config --global url."https://github.com/".insteadOf
ssh://git@github.com/
- name: Install test dependencies
working-directory: ./test/wasm
run: npm ci
- name: Test
run: ./test-dendritejs.sh
# Run golangci-lint
lint:
@ -74,7 +29,7 @@ jobs:
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
# run go test with different go versions
# run go test with go 1.19
test:
timeout-minutes: 5
name: Unit tests (Go ${{ matrix.go }})
@ -102,7 +57,7 @@ jobs:
strategy:
fail-fast: false
matrix:
go: ["1.18", "1.19"]
go: ["1.19"]
steps:
- uses: actions/checkout@v3
- name: Setup go
@ -129,7 +84,7 @@ jobs:
POSTGRES_PASSWORD: postgres
POSTGRES_DB: dendrite
# build Dendrite for linux with different architectures and go versions
# build Dendrite for linux amd64 with go 1.18
build:
name: Build for Linux
timeout-minutes: 10
@ -137,9 +92,9 @@ jobs:
strategy:
fail-fast: false
matrix:
go: ["1.18", "1.19"]
go: ["1.19"]
goos: ["linux"]
goarch: ["amd64", "386"]
goarch: ["amd64"]
steps:
- uses: actions/checkout@v3
- name: Setup go
@ -164,43 +119,10 @@ jobs:
CGO_CFLAGS: -fno-stack-protector
run: go build -trimpath -v -o "bin/" ./cmd/...
# build for Windows 64-bit
build_windows:
name: Build for Windows
timeout-minutes: 10
runs-on: ubuntu-latest
strategy:
matrix:
go: ["1.18", "1.19"]
goos: ["windows"]
goarch: ["amd64"]
steps:
- uses: actions/checkout@v3
- name: Setup Go ${{ matrix.go }}
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- name: Install dependencies
run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }}-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }}
- env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: 1
CC: "/usr/bin/x86_64-w64-mingw32-gcc"
run: go build -trimpath -v -o "bin/" ./cmd/...
# Dummy step to gate other tests on without repeating the whole list
initial-tests-done:
name: Initial tests passed
needs: [lint, test, build, build_windows]
needs: [lint, test, build]
runs-on: ubuntu-latest
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
steps:
@ -209,56 +131,6 @@ jobs:
with:
jobs: ${{ toJSON(needs) }}
# run database upgrade tests
upgrade_test:
name: Upgrade tests
timeout-minutes: 20
needs: initial-tests-done
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: "1.18"
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-upgrade
- name: Build upgrade-tests
run: go build ./cmd/dendrite-upgrade-tests
- name: Test upgrade
run: ./dendrite-upgrade-tests --head .
# run database upgrade tests, skipping over one version
upgrade_test_direct:
name: Upgrade tests from HEAD-2
timeout-minutes: 20
needs: initial-tests-done
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: "1.18"
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-upgrade
- name: Build upgrade-tests
run: go build ./cmd/dendrite-upgrade-tests
- name: Test upgrade
run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
# run Sytest in different variations
sytest:
timeout-minutes: 20
@ -269,18 +141,6 @@ jobs:
fail-fast: false
matrix:
include:
- label: SQLite native
- label: SQLite Cgo
cgo: 1
- label: SQLite native, full HTTP APIs
api: full-http
- label: SQLite Cgo, full HTTP APIs
api: full-http
cgo: 1
- label: PostgreSQL
postgres: postgres
@ -313,13 +173,12 @@ jobs:
run: /src/are-we-synapse-yet.py /logs/results.tap -v
continue-on-error: true # not fatal
- name: Upload Sytest logs
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
if: ${{ always() }}
with:
name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
path: |
/logs/results.tap
/logs/**/*.log*
/logs
# run Complement
complement:
@ -331,18 +190,6 @@ jobs:
fail-fast: false
matrix:
include:
- label: SQLite native
- label: SQLite Cgo
cgo: 1
- label: SQLite native, full HTTP APIs
api: full-http
- label: SQLite Cgo, full HTTP APIs
api: full-http
cgo: 1
- label: PostgreSQL
postgres: Postgres
@ -390,7 +237,7 @@ jobs:
continue
fi
(wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
(wget -O - "https://github.com/globekeeper/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
done
# Build initial Dendrite image
@ -416,8 +263,6 @@ jobs:
needs:
[
initial-tests-done,
upgrade_test,
upgrade_test_direct,
sytest,
complement,
]
@ -428,15 +273,3 @@ jobs:
uses: re-actors/alls-green@release/v1
with:
jobs: ${{ toJSON(needs) }}
update-docker-images:
name: Update Docker images
permissions:
packages: write
contents: read
security-events: write # To upload Trivy sarif files
if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main'
needs: [integration-tests-done]
uses: matrix-org/dendrite/.github/workflows/docker.yml@main
secrets:
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}

View file

@ -1,313 +0,0 @@
# Based on https://github.com/docker/build-push-action
name: "Docker"
on:
release: # A GitHub release was published
types: [published]
workflow_dispatch: # A build was manually requested
workflow_call: # Another pipeline called us
secrets:
DOCKER_TOKEN:
required: true
env:
DOCKER_NAMESPACE: matrixdotorg
DOCKER_HUB_USER: dendritegithub
GHCR_NAMESPACE: matrix-org
PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7
jobs:
monolith:
name: Monolith image
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
security-events: write # To upload Trivy sarif files
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Get release tag & build flags
if: github.event_name == 'release' # Only for GitHub releases
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build main monolith image
if: github.ref_name == 'main'
id: docker_build_monolith
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: monolith
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
format: "sarif"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
- name: Build release monolith image
if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_monolith_release
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: monolith
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:latest
${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
polylith:
name: Polylith image
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
security-events: write # To upload Trivy sarif files
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Get release tag & build flags
if: github.event_name == 'release' # Only for GitHub releases
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build main polylith image
if: github.ref_name == 'main'
id: docker_build_polylith
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: polylith
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
format: "sarif"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
- name: Build release polylith image
if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_polylith_release
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: polylith
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest
${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
demo-pinecone:
name: Pinecone demo image
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Get release tag & build flags
if: github.event_name == 'release' # Only for GitHub releases
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build main Pinecone demo image
if: github.ref_name == 'main'
id: docker_build_demo_pinecone
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
file: ./build/docker/Dockerfile.demo-pinecone
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }}
- name: Build release Pinecone demo image
if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_demo_pinecone_release
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
file: ./build/docker/Dockerfile.demo-pinecone
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}
demo-yggdrasil:
name: Yggdrasil demo image
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Get release tag & build flags
if: github.event_name == 'release' # Only for GitHub releases
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build main Yggdrasil demo image
if: github.ref_name == 'main'
id: docker_build_demo_yggdrasil
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
file: ./build/docker/Dockerfile.demo-yggdrasil
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }}
- name: Build release Yggdrasil demo image
if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_demo_yggdrasil_release
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
file: ./build/docker/Dockerfile.demo-yggdrasil
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}

6
.gitignore vendored
View file

@ -2,6 +2,8 @@
# Hidden files
.*
!.vscode
!.cloudbuild
# Allow GitHub config
!.github
@ -73,3 +75,7 @@ complement/
docs/_site
media_store/
__debug_bin
cmd/dendrite-monolith-server/dendrite-monolith-server

View file

@ -179,7 +179,6 @@ linters-settings:
linters:
enable:
- deadcode
- errcheck
- goconst
- gocyclo
@ -191,10 +190,8 @@ linters:
- misspell # Check code comments, whereas misspell in CI checks *.md files
- nakedret
- staticcheck
- structcheck
- unparam
- unused
- varcheck
enable-all: false
disable:
- bodyclose

16
.vscode/launch.json vendored Normal file
View file

@ -0,0 +1,16 @@
{
"configurations": [
{
"name": "Launch Package",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/cmd/dendrite-monolith-server",
"args": [
"-really-enable-open-registration",
"-config",
"../../../adminas/.ci/config/dendrite-local/dendrite.yaml"
],
}
]
}

9
.vscode/settings.json vendored Normal file
View file

@ -0,0 +1,9 @@
{
"go.lintTool": "golangci-lint",
"go.testEnvVars": {
"POSTGRES_HOST": "localhost",
"POSTGRES_USER": "postgres",
"POSTGRES_PASSWORD": "foobar",
"POSTGRES_DB": "postgres"
}
}

View file

@ -34,13 +34,16 @@ type JSServer struct {
// OnRequestFromJS is the function that JS will invoke when there is a new request.
// The JS function signature is:
// function(reqString: string): Promise<{result: string, error: string}>
//
// function(reqString: string): Promise<{result: string, error: string}>
//
// Usage is like:
// const res = await global._go_js_server.fetch(reqString);
// if (res.error) {
// // handle error: this is a 'network' error, not a non-2xx error.
// }
// const rawHttpResponse = res.result;
//
// const res = await global._go_js_server.fetch(reqString);
// if (res.error) {
// // handle error: this is a 'network' error, not a non-2xx error.
// }
// const rawHttpResponse = res.result;
func (h *JSServer) OnRequestFromJS(this js.Value, args []js.Value) interface{} {
// we HAVE to spawn a new goroutine and return immediately or else Go will deadlock
// if this request blocks at all e.g for /sync calls

View file

@ -11,4 +11,6 @@ const (
LoginTypeRecaptcha = "m.login.recaptcha"
LoginTypeApplicationService = "m.login.application_service"
LoginTypeToken = "m.login.token"
LoginTypeJwt = "org.matrix.login.jwt"
LoginTypeEmail = "m.login.email.identity"
)

View file

@ -22,6 +22,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/ratelimit"
"github.com/matrix-org/dendrite/setup/config"
uapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/util"
@ -32,7 +33,7 @@ import (
// called after authorization has completed, with the result of the authorization.
// If the final return value is non-nil, an error occurred and the cleanup function
// is nil.
func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserLoginAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) {
func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.ClientUserAPI, cfg *config.ClientAPI, rt *ratelimit.RtFailedLogin) (*Login, LoginCleanupFunc, *util.JSONResponse) {
reqBytes, err := io.ReadAll(r)
if err != nil {
err := &util.JSONResponse{
@ -43,7 +44,8 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U
}
var header struct {
Type string `json:"type"`
Type string `json:"type"`
InhibitDevice bool `json:"inhibit_device"`
}
if err := json.Unmarshal(reqBytes, &header); err != nil {
err := &util.JSONResponse{
@ -57,14 +59,20 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U
switch header.Type {
case authtypes.LoginTypePassword:
typ = &LoginTypePassword{
GetAccountByPassword: useraccountAPI.QueryAccountByPassword,
Config: cfg,
UserApi: useraccountAPI,
Config: cfg,
Rt: rt,
InhibitDevice: header.InhibitDevice,
}
case authtypes.LoginTypeToken:
typ = &LoginTypeToken{
UserAPI: userAPI,
UserAPI: useraccountAPI,
Config: cfg,
}
case authtypes.LoginTypeJwt:
typ = &LoginTypeTokenJwt{
Config: cfg,
}
default:
err := util.JSONResponse{
Code: http.StatusBadRequest,

View file

@ -0,0 +1,74 @@
package auth
import (
"context"
"fmt"
"net/http"
"github.com/golang-jwt/jwt/v4"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/util"
)
// LoginTypeToken describes how to authenticate with a login token.
type LoginTypeTokenJwt struct {
// UserAPI uapi.LoginTokenInternalAPI
Config *config.ClientAPI
}
// Name implements Type.
func (t *LoginTypeTokenJwt) Name() string {
return authtypes.LoginTypeJwt
}
type Claims struct {
jwt.StandardClaims
}
const mIdUser = "m.id.user"
// LoginFromJSON implements Type. The cleanup function deletes the token from
// the database on success.
func (t *LoginTypeTokenJwt) LoginFromJSON(ctx context.Context, reqBytes []byte) (*Login, LoginCleanupFunc, *util.JSONResponse) {
var r loginTokenRequest
if err := httputil.UnmarshalJSON(reqBytes, &r); err != nil {
return nil, nil, err
}
if r.Token == "" {
return nil, nil, &util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Token field for JWT is missing"),
}
}
c := &Claims{}
token, err := jwt.ParseWithClaims(r.Token, c, func(token *jwt.Token) (interface{}, error) {
if _, ok := token.Method.(*jwt.SigningMethodEd25519); !ok {
return nil, fmt.Errorf("unexpected signing method: %v", token.Method.Alg())
}
return t.Config.JwtConfig.SecretKey, nil
})
if err != nil {
util.GetLogger(ctx).WithError(err).Error("jwt.ParseWithClaims failed")
return nil, nil, &util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Couldn't parse JWT"),
}
}
if !token.Valid {
return nil, nil, &util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Invalid JWT"),
}
}
r.Login.Identifier.User = c.Subject
r.Login.Identifier.Type = mIdUser
return &r.Login, func(context.Context, *util.JSONResponse) {}, nil
}

View file

@ -22,6 +22,7 @@ import (
"testing"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/ratelimit"
"github.com/matrix-org/dendrite/setup/config"
uapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/util"
@ -68,8 +69,11 @@ func TestLoginFromJSONReader(t *testing.T) {
Matrix: &config.Global{
ServerName: serverName,
},
RtFailedLogin: ratelimit.RtFailedLoginConfig{
Enabled: false,
},
}
login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, cfg, nil)
if err != nil {
t.Fatalf("LoginFromJSONReader failed: %+v", err)
}
@ -147,7 +151,7 @@ func TestBadLoginFromJSONReader(t *testing.T) {
ServerName: serverName,
},
}
_, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
_, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, cfg, nil)
if errRes == nil {
cleanup(ctx, nil)
t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode)
@ -159,6 +163,7 @@ func TestBadLoginFromJSONReader(t *testing.T) {
}
type fakeUserInternalAPI struct {
uapi.ClientUserAPI
UserInternalAPIForLogin
DeletedTokens []string
}

View file

@ -58,7 +58,7 @@ func (t *LoginTypeToken) LoginFromJSON(ctx context.Context, reqBytes []byte) (*L
}
}
r.Login.Identifier.Type = "m.id.user"
r.Login.Identifier.Type = mIdUser
r.Login.Identifier.User = res.Data.UserID
cleanup := func(ctx context.Context, authRes *util.JSONResponse) {

View file

@ -22,6 +22,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/ratelimit"
"github.com/matrix-org/dendrite/clientapi/userutil"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/api"
@ -33,12 +34,18 @@ type GetAccountByPassword func(ctx context.Context, req *api.QueryAccountByPassw
type PasswordRequest struct {
Login
Password string `json:"password"`
Address string `json:"address"`
Medium string `json:"medium"`
}
const email = "email"
// LoginTypePassword implements https://matrix.org/docs/spec/client_server/r0.6.1#password-based
type LoginTypePassword struct {
GetAccountByPassword GetAccountByPassword
Config *config.ClientAPI
UserApi api.ClientUserAPI
Config *config.ClientAPI
Rt *ratelimit.RtFailedLogin
InhibitDevice bool
}
func (t *LoginTypePassword) Name() string {
@ -55,13 +62,42 @@ func (t *LoginTypePassword) LoginFromJSON(ctx context.Context, reqBytes []byte)
if err != nil {
return nil, nil, err
}
login.InhibitDevice = t.InhibitDevice
return login, func(context.Context, *util.JSONResponse) {}, nil
}
func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) {
r := req.(*PasswordRequest)
username := strings.ToLower(r.Username())
if r.Identifier.Address != "" {
r.Address = r.Identifier.Address
}
if r.Identifier.Medium != "" {
r.Medium = r.Identifier.Medium
}
var username string
if r.Medium == email && r.Address != "" {
r.Address = strings.ToLower(r.Address)
res := api.QueryLocalpartForThreePIDResponse{}
err := t.UserApi.QueryLocalpartForThreePID(ctx, &api.QueryLocalpartForThreePIDRequest{
ThreePID: r.Address,
Medium: email,
}, &res)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("userApi.QueryLocalpartForThreePID failed")
resp := jsonerror.InternalServerError()
return nil, &resp
}
username = res.Localpart
if username == "" {
return nil, &util.JSONResponse{
Code: http.StatusUnauthorized,
JSON: jsonerror.Forbidden("Invalid username or password"),
}
}
} else {
username = strings.ToLower(r.Username())
}
if username == "" {
return nil, &util.JSONResponse{
Code: http.StatusUnauthorized,
@ -83,7 +119,17 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
}
// Squash username to all lowercase letters
res := &api.QueryAccountByPasswordResponse{}
err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{Localpart: strings.ToLower(localpart), PlaintextPassword: r.Password}, res)
localpart = strings.ToLower(localpart)
if t.Rt != nil {
ok, retryIn := t.Rt.CanAct(localpart)
if !ok {
return nil, &util.JSONResponse{
Code: http.StatusTooManyRequests,
JSON: jsonerror.LimitExceeded("Too Many Requests", retryIn.Milliseconds()),
}
}
}
err = t.UserApi.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{Localpart: localpart, PlaintextPassword: r.Password}, res)
if err != nil {
return nil, &util.JSONResponse{
Code: http.StatusInternalServerError,
@ -92,7 +138,7 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
}
if !res.Exists {
err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
err = t.UserApi.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
Localpart: localpart,
PlaintextPassword: r.Password,
}, res)
@ -105,11 +151,15 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
// but that would leak the existence of the user.
if !res.Exists {
if t.Rt != nil {
t.Rt.Act(localpart)
}
return nil, &util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("The username or password was incorrect or the account does not exist."),
JSON: jsonerror.Forbidden("Invalid username or password"),
}
}
}
r.Login.User = username
return &r.Login, nil
}

View file

@ -66,6 +66,7 @@ type LoginIdentifier struct {
type Login struct {
LoginIdentifier // Flat fields deprecated in favour of `identifier`.
Identifier LoginIdentifier `json:"identifier"`
InhibitDevice bool `json:"inhibit_device,omitempty"`
// Both DeviceID and InitialDisplayName can be omitted, or empty strings ("")
// Thus a pointer is needed to differentiate between the two
@ -75,7 +76,7 @@ type Login struct {
// Username returns the user localpart/user_id in this request, if it exists.
func (r *Login) Username() string {
if r.Identifier.Type == "m.id.user" {
if r.Identifier.Type == mIdUser {
return r.Identifier.User
}
// deprecated but without it Element iOS won't log in
@ -88,8 +89,8 @@ func (r *Login) ThirdPartyID() (medium, address string) {
return r.Identifier.Medium, r.Identifier.Address
}
// deprecated
if r.Medium == "email" {
return "email", r.Address
if r.Medium == email {
return email, r.Address
}
return "", ""
}
@ -111,10 +112,10 @@ type UserInteractive struct {
Sessions map[string][]string
}
func NewUserInteractive(userAccountAPI api.UserLoginAPI, cfg *config.ClientAPI) *UserInteractive {
func NewUserInteractive(userAccountAPI api.ClientUserAPI, cfg *config.ClientAPI) *UserInteractive {
typePassword := &LoginTypePassword{
GetAccountByPassword: userAccountAPI.QueryAccountByPassword,
Config: cfg,
UserApi: userAccountAPI,
Config: cfg,
}
return &UserInteractive{
Flows: []userInteractiveFlow{

View file

@ -24,7 +24,9 @@ var (
}
)
type fakeAccountDatabase struct{}
type fakeAccountDatabase struct {
api.ClientUserAPI
}
func (d *fakeAccountDatabase) PerformPasswordUpdate(ctx context.Context, req *api.PerformPasswordUpdateRequest, res *api.PerformPasswordUpdateResponse) error {
return nil

View file

@ -52,6 +52,7 @@ func AddPublicRoutes(
TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
TopicTypingEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
TopicPresenceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
TopicMultiRoomCast: cfg.Matrix.JetStream.Prefixed(jetstream.OutputMultiRoomCast),
UserAPI: userAPI,
ServerName: cfg.Matrix.ServerName,
}

View file

@ -36,6 +36,7 @@ type SyncAPIProducer struct {
TopicSendToDeviceEvent string
TopicTypingEvent string
TopicPresenceEvent string
TopicMultiRoomCast string
JetStream nats.JetStreamContext
ServerName gomatrixserverlib.ServerName
UserAPI userapi.ClientUserAPI
@ -159,3 +160,14 @@ func (p *SyncAPIProducer) SendPresence(
_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
return err
}
func (p *SyncAPIProducer) SendMultiroom(
ctx context.Context, userID string, dataType string, message []byte,
) error {
m := nats.NewMsg(p.TopicMultiRoomCast)
m.Header.Set(jetstream.UserID, userID)
m.Header.Set("type", dataType)
m.Data = message
_, err := p.JetStream.PublishMsg(m, nats.Context(ctx))
return err
}

View file

@ -0,0 +1,117 @@
package ratelimit
import (
"container/list"
"sync"
"time"
)
type rateLimit struct {
cfg *RtFailedLoginConfig
times *list.List
}
type RtFailedLogin struct {
cfg *RtFailedLoginConfig
mtx sync.RWMutex
rts map[string]*rateLimit
}
type RtFailedLoginConfig struct {
Enabled bool `yaml:"enabled"`
Limit int `yaml:"burst"`
Interval time.Duration `yaml:"interval"`
}
// New creates a new rate limiter for the limit and interval.
func NewRtFailedLogin(cfg *RtFailedLoginConfig) *RtFailedLogin {
if !cfg.Enabled {
return nil
}
rt := &RtFailedLogin{
cfg: cfg,
mtx: sync.RWMutex{},
rts: make(map[string]*rateLimit),
}
go rt.clean()
return rt
}
// CanAct is expected to be called before Act
func (r *RtFailedLogin) CanAct(key string) (ok bool, remaining time.Duration) {
r.mtx.RLock()
rt, ok := r.rts[key]
if !ok {
r.mtx.RUnlock()
return true, 0
}
ok, remaining = rt.canAct()
r.mtx.RUnlock()
return
}
// Act can be called after CanAct returns true.
func (r *RtFailedLogin) Act(key string) {
r.mtx.Lock()
rt, ok := r.rts[key]
if !ok {
rt = &rateLimit{
cfg: r.cfg,
times: list.New(),
}
r.rts[key] = rt
}
rt.act()
r.mtx.Unlock()
}
func (r *RtFailedLogin) clean() {
for {
r.mtx.Lock()
for k, v := range r.rts {
if v.empty() {
delete(r.rts, k)
}
}
r.mtx.Unlock()
time.Sleep(time.Hour)
}
}
func (r *rateLimit) empty() bool {
back := r.times.Back()
if back == nil {
return true
}
v := back.Value
b := v.(time.Time)
now := time.Now()
return now.Sub(b) > r.cfg.Interval
}
func (r *rateLimit) canAct() (ok bool, remaining time.Duration) {
now := time.Now()
l := r.times.Len()
if l < r.cfg.Limit {
return true, 0
}
frnt := r.times.Front()
t := frnt.Value.(time.Time)
diff := now.Sub(t)
if diff < r.cfg.Interval {
return false, r.cfg.Interval - diff
}
return true, 0
}
func (r *rateLimit) act() {
now := time.Now()
l := r.times.Len()
if l < r.cfg.Limit {
r.times.PushBack(now)
return
}
frnt := r.times.Front()
frnt.Value = now
r.times.MoveToBack(frnt)
}

View file

@ -0,0 +1,40 @@
package ratelimit
import (
"testing"
"time"
"github.com/matryer/is"
)
func TestRtFailedLogin(t *testing.T) {
is := is.New(t)
rtfl := NewRtFailedLogin(&RtFailedLoginConfig{
Enabled: true,
Limit: 3,
Interval: 10 * time.Millisecond,
})
var (
can bool
remaining time.Duration
remainingB time.Duration
)
for i := 0; i < 3; i++ {
can, remaining = rtfl.CanAct("foo")
is.True(can)
is.Equal(remaining, time.Duration(0))
rtfl.Act("foo")
}
can, remaining = rtfl.CanAct("foo")
is.True(!can)
is.True(remaining > time.Millisecond*9)
can, remainingB = rtfl.CanAct("bar")
is.True(can)
is.Equal(remainingB, time.Duration(0))
rtfl.Act("bar")
rtfl.Act("bar")
time.Sleep(remaining + time.Millisecond)
can, remaining = rtfl.CanAct("foo")
is.True(can)
is.Equal(remaining, time.Duration(0))
}

View file

@ -27,13 +27,17 @@ func Deactivate(
JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()),
}
}
login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, deviceAPI)
if errRes != nil {
return *errRes
var userId string
if deviceAPI.AccountType != api.AccountTypeAppService {
login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, deviceAPI)
if errRes != nil {
return *errRes
}
userId = login.Username()
} else {
userId = deviceAPI.UserID
}
localpart, _, err := gomatrixserverlib.SplitID('@', login.Username())
localpart, _, err := gomatrixserverlib.SplitID('@', userId)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
return jsonerror.InternalServerError()

View file

@ -63,8 +63,8 @@ func UploadCrossSigningDeviceKeys(
}
}
typePassword := auth.LoginTypePassword{
GetAccountByPassword: accountAPI.QueryAccountByPassword,
Config: cfg,
UserApi: accountAPI,
Config: cfg,
}
if _, authErr := typePassword.Login(req.Context(), &uploadReq.Auth.PasswordRequest); authErr != nil {
return *authErr

View file

@ -20,6 +20,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/auth"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/ratelimit"
"github.com/matrix-org/dendrite/clientapi/userutil"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
@ -55,6 +56,7 @@ func passwordLogin() flows {
func Login(
req *http.Request, userAPI userapi.ClientUserAPI,
cfg *config.ClientAPI,
rt *ratelimit.RtFailedLogin,
) util.JSONResponse {
if req.Method == http.MethodGet {
// TODO: support other forms of login other than password, depending on config options
@ -63,10 +65,21 @@ func Login(
JSON: passwordLogin(),
}
} else if req.Method == http.MethodPost {
login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, userAPI, userAPI, cfg)
login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, userAPI, cfg, rt)
if authErr != nil {
return *authErr
}
if login.InhibitDevice {
return util.JSONResponse{
Code: http.StatusOK,
JSON: loginResponse{
UserID: userutil.MakeUserID(login.Username(), cfg.Matrix.ServerName),
AccessToken: "",
HomeServer: cfg.Matrix.ServerName,
DeviceID: "",
},
}
}
// make a device/access token
authErr2 := completeAuth(req.Context(), cfg.Matrix, userAPI, login, req.RemoteAddr, req.UserAgent())
cleanup(req.Context(), &authErr2)

View file

@ -0,0 +1,48 @@
package routing
import (
"io"
"net/http"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
log "github.com/sirupsen/logrus"
)
func PostMultiroom(
req *http.Request,
device *api.Device,
producer *producers.SyncAPIProducer,
dataType string,
) util.JSONResponse {
b, err := io.ReadAll(req.Body)
if err != nil {
log.WithError(err).Errorf("failed to read request body")
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: jsonerror.InternalServerError(),
}
}
canonicalB, err := gomatrixserverlib.CanonicalJSON(b)
if err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON("The request body is not valid canonical JSON." + err.Error()),
}
}
err = producer.SendMultiroom(req.Context(), device.UserID, dataType, canonicalB)
if err != nil {
log.WithError(err).Errorf("failed to send multiroomcast")
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: jsonerror.InternalServerError(),
}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
}
}

View file

@ -1,12 +1,14 @@
package routing
import (
"fmt"
"net/http"
"github.com/matrix-org/dendrite/clientapi/auth"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/threepid"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
@ -24,6 +26,7 @@ type newPasswordAuth struct {
Type string `json:"type"`
Session string `json:"session"`
auth.PasswordRequest
ThreePidCreds threepid.Credentials `json:"threepid_creds"`
}
func Password(
@ -33,13 +36,17 @@ func Password(
cfg *config.ClientAPI,
) util.JSONResponse {
// Check that the existing password is right.
var fields logrus.Fields
if device != nil {
fields = logrus.Fields{
"sessionId": device.SessionID,
"userId": device.UserID,
}
}
var r newPasswordRequest
r.LogoutDevices = true
logrus.WithFields(logrus.Fields{
"sessionId": device.SessionID,
"userId": device.UserID,
}).Debug("Changing password")
logrus.WithFields(fields).Debug("Changing password")
// Unmarshal the request.
resErr := httputil.UnmarshalJSONRequest(req, &r)
@ -53,45 +60,95 @@ func Password(
// Generate a new, random session ID
sessionID = util.RandomString(sessionIDLength)
}
// Require password auth to change the password.
if r.Auth.Type != authtypes.LoginTypePassword {
return util.JSONResponse{
Code: http.StatusUnauthorized,
JSON: newUserInteractiveResponse(
sessionID,
[]authtypes.Flow{
{
Stages: []authtypes.LoginType{authtypes.LoginTypePassword},
},
var localpart string
switch r.Auth.Type {
case authtypes.LoginTypePassword:
// Check if the existing password is correct.
typePassword := auth.LoginTypePassword{
UserApi: userAPI,
Config: cfg,
}
if _, authErr := typePassword.Login(req.Context(), &r.Auth.PasswordRequest); authErr != nil {
return *authErr
}
// Get the local part.
var err error
localpart, _, err = gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
return jsonerror.InternalServerError()
}
sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypePassword)
case authtypes.LoginTypeEmail:
threePid := &authtypes.ThreePID{}
r.Auth.ThreePidCreds.IDServer = cfg.ThreePidDelegate
var (
bound bool
err error
)
bound, threePid.Address, threePid.Medium, err = threepid.CheckAssociation(req.Context(), r.Auth.ThreePidCreds, cfg)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAssociation failed")
return jsonerror.InternalServerError()
}
if !bound {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.MatrixError{
ErrCode: "M_THREEPID_AUTH_FAILED",
Err: "Failed to auth 3pid",
},
nil,
),
}
}
var res api.QueryLocalpartForThreePIDResponse
err = userAPI.QueryLocalpartForThreePID(req.Context(), &api.QueryLocalpartForThreePIDRequest{
Medium: threePid.Medium,
ThreePID: threePid.Address,
}, &res)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryLocalpartForThreePID failed")
return jsonerror.InternalServerError()
}
if res.Localpart == "" {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.MatrixError{
ErrCode: "M_THREEPID_NOT_FOUND",
Err: "3pid is not bound to any account",
},
}
}
localpart = res.Localpart
sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeEmail)
default:
flows := []authtypes.Flow{
{
Stages: []authtypes.LoginType{authtypes.LoginTypePassword},
},
}
if cfg.ThreePidDelegate != "" {
flows = append(flows, authtypes.Flow{
Stages: []authtypes.LoginType{authtypes.LoginTypeEmail},
})
}
// Require password auth to change the password.
if r.Auth.Type == authtypes.LoginTypePassword {
return util.JSONResponse{
Code: http.StatusUnauthorized,
JSON: newUserInteractiveResponse(
sessionID,
flows,
nil,
),
}
}
}
// Check if the existing password is correct.
typePassword := auth.LoginTypePassword{
GetAccountByPassword: userAPI.QueryAccountByPassword,
Config: cfg,
}
if _, authErr := typePassword.Login(req.Context(), &r.Auth.PasswordRequest); authErr != nil {
return *authErr
}
sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypePassword)
// Check the new password strength.
if resErr = validatePassword(r.NewPassword); resErr != nil {
return *resErr
}
// Get the local part.
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
return jsonerror.InternalServerError()
}
// Ask the user API to perform the password change.
passwordReq := &api.PerformPasswordUpdateRequest{
Localpart: localpart,
@ -109,11 +166,23 @@ func Password(
// If the request asks us to log out all other devices then
// ask the user API to do that.
if r.LogoutDevices {
logoutReq := &api.PerformDeviceDeletionRequest{
UserID: device.UserID,
DeviceIDs: nil,
ExceptDeviceID: device.ID,
var logoutReq *api.PerformDeviceDeletionRequest
var sessionId int64
if device == nil {
logoutReq = &api.PerformDeviceDeletionRequest{
UserID: fmt.Sprintf("@%s:%s", localpart, cfg.Matrix.ServerName),
DeviceIDs: []string{},
}
sessionId = 0
} else {
logoutReq = &api.PerformDeviceDeletionRequest{
UserID: device.UserID,
DeviceIDs: nil,
ExceptDeviceID: device.ID,
}
sessionId = device.SessionID
}
logoutRes := &api.PerformDeviceDeletionResponse{}
if err := userAPI.PerformDeviceDeletion(req.Context(), logoutReq, logoutRes); err != nil {
@ -123,7 +192,7 @@ func Password(
pushersReq := &api.PerformPusherDeletionRequest{
Localpart: localpart,
SessionID: device.SessionID,
SessionID: sessionId,
}
if err := userAPI.PerformPusherDeletion(req.Context(), pushersReq, &struct{}{}); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("PerformPusherDeletion failed")

View file

@ -106,12 +106,6 @@ func SetAvatarURL(
if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
return *resErr
}
if r.AvatarURL == "" {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON("'avatar_url' must be supplied."),
}
}
localpart, domain, err := gomatrixserverlib.SplitID('@', userID)
if err != nil {

View file

@ -86,8 +86,8 @@ func SetPusher(
if err != nil {
return invalidParam("malformed url passed")
}
if pushUrl.Scheme != "https" {
return invalidParam("only https scheme is allowed")
if pushUrl.Scheme != "https" && pushUrl.Scheme != "http" {
return invalidParam("only https and http schemes are allowed")
}
}

View file

@ -45,6 +45,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/threepid"
"github.com/matrix-org/dendrite/clientapi/userutil"
userapi "github.com/matrix-org/dendrite/userapi/api"
)
@ -238,6 +239,7 @@ type authDict struct {
// Recaptcha
Response string `json:"response"`
// TODO: Lots of custom keys depending on the type
ThreePidCreds threepid.Credentials `json:"threepid_creds"`
}
// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#user-interactive-authentication-api
@ -747,6 +749,7 @@ func handleRegistrationFlow(
}
}
var threePid *authtypes.ThreePID
switch r.Auth.Type {
case authtypes.LoginTypeRecaptcha:
// Check given captcha response
@ -763,6 +766,29 @@ func handleRegistrationFlow(
// Add Dummy to the list of completed registration stages
sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeDummy)
case authtypes.LoginTypeEmail:
threePid = &authtypes.ThreePID{}
r.Auth.ThreePidCreds.IDServer = cfg.ThreePidDelegate
var (
bound bool
err error
)
bound, threePid.Address, threePid.Medium, err = threepid.CheckAssociation(req.Context(), r.Auth.ThreePidCreds, cfg)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAssociation failed")
return jsonerror.InternalServerError()
}
if !bound {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.MatrixError{
ErrCode: "M_THREEPID_AUTH_FAILED",
Err: "Failed to auth 3pid",
},
}
}
sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeEmail)
case "":
// An empty auth type means that we want to fetch the available
// flows. It can also mean that we want to register as an appservice
@ -778,7 +804,7 @@ func handleRegistrationFlow(
// A response with current registration flow and remaining available methods
// will be returned if a flow has not been successfully completed yet
return checkAndCompleteFlow(sessions.getCompletedStages(sessionID),
req, r, sessionID, cfg, userAPI)
req, r, sessionID, cfg, userAPI, threePid)
}
// handleApplicationServiceRegistration handles the registration of an
@ -820,7 +846,7 @@ func handleApplicationServiceRegistration(
// application service registration is entirely separate.
return completeRegistration(
req.Context(), userAPI, r.Username, "", appserviceID, req.RemoteAddr, req.UserAgent(), r.Auth.Session,
r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeAppService,
r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeAppService, nil,
)
}
@ -834,12 +860,13 @@ func checkAndCompleteFlow(
sessionID string,
cfg *config.ClientAPI,
userAPI userapi.ClientUserAPI,
threePid *authtypes.ThreePID,
) util.JSONResponse {
if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
// This flow was completed, registration can continue
return completeRegistration(
req.Context(), userAPI, r.Username, r.Password, "", req.RemoteAddr, req.UserAgent(), sessionID,
r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeUser,
r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeUser, threePid,
)
}
sessions.addParams(sessionID, r)
@ -865,6 +892,7 @@ func completeRegistration(
inhibitLogin eventutil.WeakBoolean,
displayName, deviceID *string,
accType userapi.AccountType,
threePid *authtypes.ThreePID,
) util.JSONResponse {
if username == "" {
return util.JSONResponse{
@ -903,6 +931,21 @@ func completeRegistration(
// Increment prometheus counter for created users
amtRegUsers.Inc()
// TODO-entry refuse register if threepid is already bound to account.
if threePid != nil {
err = userAPI.PerformSaveThreePIDAssociation(ctx, &userapi.PerformSaveThreePIDAssociationRequest{
Medium: threePid.Medium,
ThreePID: threePid.Address,
Localpart: accRes.Account.Localpart,
}, &struct{}{})
if err != nil {
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: jsonerror.Unknown("Failed to save 3PID association: " + err.Error()),
}
}
}
// Check whether inhibit_login option is set. If so, don't create an access
// token or a device for this user
if inhibitLogin {
@ -1094,5 +1137,5 @@ func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.Clien
if ssrr.Admin {
accType = userapi.AccountTypeAdmin
}
return completeRegistration(req.Context(), userAPI, ssrr.User, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType)
return completeRegistration(req.Context(), userAPI, ssrr.User, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType, nil)
}

View file

@ -32,6 +32,7 @@ import (
clientutil "github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/dendrite/clientapi/ratelimit"
federationAPI "github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/transactions"
@ -66,6 +67,7 @@ func Setup(
prometheus.MustRegister(amtRegUsers, sendEventDuration)
rateLimits := httputil.NewRateLimits(&cfg.RateLimiting)
rateLimitsFailedLogin := ratelimit.NewRtFailedLogin(&cfg.RtFailedLogin)
userInteractiveAuth := auth.NewUserInteractive(userAPI, cfg)
unstableFeatures := map[string]bool{
@ -434,6 +436,17 @@ func Setup(
}),
).Methods(http.MethodPut, http.MethodOptions)
v3mux.Handle("/multiroom/{dataType}",
httputil.MakeAuthAPI("send_multiroom", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
dataType := vars["dataType"]
return PostMultiroom(req, device, syncProducer, dataType)
}),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
if r := rateLimits.Limit(req, nil); r != nil {
return *r
@ -602,7 +615,7 @@ func Setup(
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/account/password",
httputil.MakeAuthAPI("password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
httputil.MakeConditionalAuthAPI("password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
@ -626,7 +639,7 @@ func Setup(
if r := rateLimits.Limit(req, nil); r != nil {
return *r
}
return Login(req, userAPI, cfg)
return Login(req, userAPI, cfg, rateLimitsFailedLogin)
}),
).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)

View file

@ -101,7 +101,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
}
// If the user has never been in the room then stop at this point.
// We won't tell the user about a room they have never joined.
if !membershipRes.HasBeenInRoom {
if !membershipRes.HasBeenInRoom && membershipRes.Membership != gomatrixserverlib.Invite {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)),
@ -241,7 +241,7 @@ func OnIncomingStateTypeRequest(
}
// If the user has never been in the room then stop at this point.
// We won't tell the user about a room they have never joined.
if !membershipRes.HasBeenInRoom || membershipRes.Membership == gomatrixserverlib.Ban {
if !membershipRes.HasBeenInRoom && membershipRes.Membership != gomatrixserverlib.Invite || membershipRes.Membership == gomatrixserverlib.Ban {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)),

View file

@ -103,11 +103,8 @@ func CreateSession(
func CheckAssociation(
ctx context.Context, creds Credentials, cfg *config.ClientAPI,
) (bool, string, string, error) {
if err := isTrusted(creds.IDServer, cfg); err != nil {
return false, "", "", err
}
requestURL := fmt.Sprintf("https://%s/_matrix/identity/api/v1/3pid/getValidated3pid?sid=%s&client_secret=%s", creds.IDServer, creds.SID, creds.Secret)
requestURL := fmt.Sprintf("%s/_matrix/identity/api/v1/3pid/getValidated3pid?sid=%s&client_secret=%s", cfg.ThreePidDelegate, creds.SID, creds.Secret)
req, err := http.NewRequest(http.MethodGet, requestURL, nil)
if err != nil {
return false, "", "", err

View file

@ -0,0 +1,8 @@
FROM alpine:latest
COPY dendrite-monolith-server /usr/bin/
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]

View file

@ -0,0 +1,12 @@
set -xe
if [ -z "$(git status --porcelain)" ]; then
CGO_ENABLED=0 go build .
TAG=$(git rev-parse --short HEAD)
docker build -f Dockerfile.dev -t gcr.io/globekeeper-development/dendrite-monolith:$TAG -t gcr.io/globekeeper-development/dendrite-monolith -t gcr.io/globekeeper-production/dendrite-monolith:$TAG .
docker push gcr.io/globekeeper-development/dendrite-monolith:$TAG
docker push gcr.io/globekeeper-production/dendrite-monolith:$TAG
docker push gcr.io/globekeeper-development/dendrite-monolith
else
echo "Please commit changes"
exit 0
fi

View file

@ -16,6 +16,7 @@ package main
import (
"flag"
"log"
"os"
"github.com/matrix-org/dendrite/appservice"
@ -45,6 +46,16 @@ var (
func main() {
cfg := setup.ParseFlags(true)
httpAddr := config.HTTPAddress("http://" + *httpBindAddr)
for _, logging := range cfg.Logging {
if logging.Type == "std" {
level, err := logrus.ParseLevel(logging.Level)
if err != nil {
log.Fatal(err)
}
logrus.SetLevel(level)
logrus.SetFormatter(&logrus.JSONFormatter{})
}
}
httpsAddr := config.HTTPAddress("https://" + *httpsBindAddr)
httpAPIAddr := httpAddr
options := []basepkg.BaseDendriteOptions{}

View file

@ -0,0 +1,71 @@
---
title: Optimise your installation
parent: Installation
has_toc: true
nav_order: 10
permalink: /installation/start/optimisation
---
# Optimise your installation
Now that you have Dendrite running, the following tweaks will improve the reliability
and performance of your installation.
## File descriptor limit
Most platforms have a limit on how many file descriptors a single process can open. All
connections made by Dendrite consume file descriptors — this includes database connections
and network requests to remote homeservers. When participating in large federated rooms
where Dendrite must talk to many remote servers, it is often very easy to exhaust default
limits which are quite low.
We currently recommend setting the file descriptor limit to 65535 to avoid such
issues. Dendrite will log immediately after startup if the file descriptor limit is too low:
```
level=warning msg="IMPORTANT: Process file descriptor limit is currently 1024, it is recommended to raise the limit for Dendrite to at least 65535 to avoid issues"
```
UNIX systems have two limits: a hard limit and a soft limit. You can view the soft limit
by running `ulimit -Sn` and the hard limit with `ulimit -Hn`:
```bash
$ ulimit -Hn
1048576
$ ulimit -Sn
1024
```
Increase the soft limit before starting Dendrite:
```bash
ulimit -Sn 65535
```
The log line at startup should no longer appear if the limit is sufficient.
If you are running under a systemd service, you can instead add `LimitNOFILE=65535` option
to the `[Service]` section of your service unit file.
## DNS caching
Dendrite has a built-in DNS cache which significantly reduces the load that Dendrite will
place on your DNS resolver. This may also speed up outbound federation.
Consider enabling the DNS cache by modifying the `global` section of your configuration file:
```yaml
dns_cache:
enabled: true
cache_size: 4096
cache_lifetime: 600s
```
## Time synchronisation
Matrix relies heavily on TLS which requires the system time to be correct. If the clock
drifts then you may find that federation no works reliably (or at all) and clients may
struggle to connect to your Dendrite server.
Ensure that the time is synchronised on your system by enabling NTP sync.

6
go.mod
View file

@ -4,7 +4,6 @@ require (
github.com/Arceliar/ironwood v0.0.0-20221025225125-45b4281814c2
github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/MFAshby/stdemuxerhook v1.0.0
github.com/Masterminds/semver/v3 v3.1.1
github.com/blevesearch/bleve/v2 v2.3.4
github.com/codeclysm/extract v2.2.0+incompatible
@ -12,6 +11,7 @@ require (
github.com/docker/docker v20.10.19+incompatible
github.com/docker/go-connections v0.4.0
github.com/getsentry/sentry-go v0.14.0
github.com/golang-jwt/jwt/v4 v4.4.1
github.com/gologme/log v1.3.0
github.com/google/go-cmp v0.5.9
github.com/google/uuid v1.3.0
@ -55,6 +55,8 @@ require (
nhooyr.io/websocket v1.8.7
)
require github.com/matryer/is v1.4.0
require (
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
@ -82,7 +84,7 @@ require (
github.com/docker/go-units v0.5.0 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/gogo/protobuf v1.1.1 // indirect
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
github.com/golang/glog v1.0.0 // indirect
github.com/golang/mock v1.6.0 // indirect

12
go.sum
View file

@ -43,8 +43,6 @@ github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20O
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM=
github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
github.com/MFAshby/stdemuxerhook v1.0.0 h1:1XFGzakrsHMv76AeanPDL26NOgwjPl/OUxbGhJthwMc=
github.com/MFAshby/stdemuxerhook v1.0.0/go.mod h1:nLMI9FUf9Hz98n+yAXsTMUR4RZQy28uCTLG1Fzvj/uY=
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
@ -202,9 +200,10 @@ github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm
github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ=
github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo=
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
@ -316,7 +315,6 @@ github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5l
github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c=
@ -354,6 +352,8 @@ github.com/matrix-org/pinecone v0.0.0-20221103125849-37f2e9b9ba37 h1:CQWFrgH9TJO
github.com/matrix-org/pinecone v0.0.0-20221103125849-37f2e9b9ba37/go.mod h1:F3GHppRuHCTDeoOmmgjZMeJdbql91+RSGGsATWfC7oc=
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE=
github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@ -757,11 +757,9 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY
golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

View file

@ -87,6 +87,57 @@ func MakeAuthAPI(
return MakeExternalAPI(metricsName, h)
}
// MakeConditionalAuthAPI turns a util.JSONRequestHandler function into an http.Handler which authenticates the request.
// It passes nil device if header is not provided.
func MakeConditionalAuthAPI(
metricsName string, userAPI userapi.QueryAcccessTokenAPI,
f func(*http.Request, *userapi.Device) util.JSONResponse,
) http.Handler {
h := func(req *http.Request) util.JSONResponse {
var (
jsonRes util.JSONResponse
dev *userapi.Device
)
if _, err := auth.ExtractAccessToken(req); err != nil {
dev = nil
} else {
logger := util.GetLogger(req.Context())
var err *util.JSONResponse
dev, err = auth.VerifyUserFromRequest(req, userAPI)
if err != nil {
logger.Debugf("VerifyUserFromRequest %s -> HTTP %d", req.RemoteAddr, err.Code)
return *err
}
// add the user ID to the logger
logger = logger.WithField("user_id", dev.UserID)
req = req.WithContext(util.ContextWithLogger(req.Context(), logger))
}
// add the user to Sentry, if enabled
hub := sentry.GetHubFromContext(req.Context())
if hub != nil {
hub.Scope().SetTag("user_id", dev.UserID)
hub.Scope().SetTag("device_id", dev.ID)
}
defer func() {
if r := recover(); r != nil {
if hub != nil {
hub.CaptureException(fmt.Errorf("%s panicked", req.URL.Path))
}
// re-panic to return the 500
panic(r)
}
}()
jsonRes = f(req, dev)
// do not log 4xx as errors as they are client fails, not server fails
if hub != nil && jsonRes.Code >= 500 {
hub.Scope().SetExtra("response", jsonRes)
hub.CaptureException(fmt.Errorf("%s returned HTTP %d", req.URL.Path, jsonRes.Code))
}
return jsonRes
}
return MakeExternalAPI(metricsName, h)
}
// MakeAdminAPI is a wrapper around MakeAuthAPI which enforces that the request can only be
// completed by a user that is a server administrator.
func MakeAdminAPI(

View file

@ -18,10 +18,8 @@
package internal
import (
"io"
"log/syslog"
"github.com/MFAshby/stdemuxerhook"
"github.com/matrix-org/dendrite/setup/config"
"github.com/sirupsen/logrus"
lSyslog "github.com/sirupsen/logrus/hooks/syslog"
@ -31,7 +29,6 @@ import (
// If something fails here it means that the logging was improperly configured,
// so we just exit with the error
func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
stdLogAdded := false
for _, hook := range hooks {
// Check we received a proper logging level
level, err := logrus.ParseLevel(hook.Level)
@ -39,12 +36,6 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err)
}
// Perform a first filter on the logs according to the lowest level of all
// (Eg: If we have hook for info and above, prevent logrus from processing debug logs)
if logrus.GetLevel() < level {
logrus.SetLevel(level)
}
switch hook.Type {
case "file":
checkFileHookParams(hook.Params)
@ -53,17 +44,10 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
checkSyslogHookParams(hook.Params)
setupSyslogHook(hook, level, componentName)
case "std":
setupStdLogHook(level)
stdLogAdded = true
default:
logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type)
}
}
if !stdLogAdded {
setupStdLogHook(logrus.InfoLevel)
}
// Hooks are now configured for stdout/err, so throw away the default logger output
logrus.SetOutput(io.Discard)
}
func checkSyslogHookParams(params map[string]interface{}) {
@ -87,10 +71,6 @@ func checkSyslogHookParams(params map[string]interface{}) {
}
func setupStdLogHook(level logrus.Level) {
logrus.AddHook(&logLevelHook{level, stdemuxerhook.New(logrus.StandardLogger())})
}
func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) {
syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, componentName)
if err == nil {

View file

@ -140,7 +140,7 @@ func (r *Inviter) PerformInvite(
var isAlreadyJoined bool
if info != nil {
_, isAlreadyJoined, _, err = r.DB.GetMembership(ctx, info.RoomNID, *event.StateKey())
_, _, isAlreadyJoined, _, err = r.DB.GetMembership(ctx, info.RoomNID, *event.StateKey())
if err != nil {
return nil, fmt.Errorf("r.DB.GetMembership: %w", err)
}

View file

@ -32,6 +32,7 @@ import (
"github.com/matrix-org/dendrite/roomserver/internal/helpers"
"github.com/matrix-org/dendrite/roomserver/state"
"github.com/matrix-org/dendrite/roomserver/storage"
"github.com/matrix-org/dendrite/roomserver/storage/tables"
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/dendrite/roomserver/version"
)
@ -180,11 +181,16 @@ func (r *Queryer) QueryMembershipForUser(
}
response.RoomExists = true
membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.UserID)
membershipEventNID, membershipState, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.UserID)
if err != nil {
return err
}
if membershipState == tables.MembershipStateInvite {
response.Membership = gomatrixserverlib.Invite
response.IsInRoom = true
}
response.IsRoomForgotten = isRoomforgotten
if membershipEventNID == 0 {
@ -327,7 +333,7 @@ func (r *Queryer) QueryMembershipsForRoom(
return nil
}
membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.Sender)
membershipEventNID, _, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.Sender)
if err != nil {
return err
}
@ -942,7 +948,7 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.Query
}
// At this point we're happy that we are in the room, so now let's
// see if the target user is in the room.
_, isIn, _, err = r.DB.GetMembership(ctx, targetRoomInfo.RoomNID, req.UserID)
_, _, isIn, _, err = r.DB.GetMembership(ctx, targetRoomInfo.RoomNID, req.UserID)
if err != nil {
continue
}

View file

@ -129,7 +129,7 @@ type Database interface {
// in this room, along a boolean set to true if the user is still in this room,
// false if not.
// Returns an error if there was a problem talking to the database.
GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom, isRoomForgotten bool, err error)
GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, membershipNID tables.MembershipState, stillInRoom, isRoomForgotten bool, err error)
// Lookup the membership event numeric IDs for all user that are or have
// been members of a given room. Only lookup events of "join" membership if
// joinOnly is set to true.

View file

@ -400,14 +400,14 @@ func (d *Database) RemoveRoomAlias(ctx context.Context, alias string) error {
})
}
func (d *Database) GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom, isRoomforgotten bool, err error) {
func (d *Database) GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, membershipState tables.MembershipState, stillInRoom, isRoomforgotten bool, err error) {
var requestSenderUserNID types.EventStateKeyNID
err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
requestSenderUserNID, err = d.assignStateKeyNID(ctx, txn, requestSenderUserID)
return err
})
if err != nil {
return 0, false, false, fmt.Errorf("d.assignStateKeyNID: %w", err)
return 0, 0, false, false, fmt.Errorf("d.assignStateKeyNID: %w", err)
}
senderMembershipEventNID, senderMembership, isRoomforgotten, err :=
@ -416,12 +416,12 @@ func (d *Database) GetMembership(ctx context.Context, roomNID types.RoomNID, req
)
if err == sql.ErrNoRows {
// The user has never been a member of that room
return 0, false, false, nil
return 0, 0, false, false, nil
} else if err != nil {
return
}
return senderMembershipEventNID, senderMembership == tables.MembershipStateJoin, isRoomforgotten, nil
return senderMembershipEventNID, senderMembership, senderMembership == tables.MembershipStateJoin, isRoomforgotten, nil
}
func (d *Database) GetMembershipEventNIDsForRoom(

View file

@ -137,7 +137,6 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base
logrus.Fatalf("Failed to start due to configuration errors")
}
internal.SetupStdLogging()
internal.SetupHookLogging(cfg.Logging, componentName)
internal.SetupPprof()

View file

@ -16,6 +16,7 @@ package config
import (
"bytes"
"crypto/x509"
"encoding/pem"
"fmt"
"io"
@ -266,6 +267,15 @@ func loadConfig(
return nil, fmt.Errorf("either specify a 'private_key' path or supply both 'public_key' and 'key_id'")
}
}
if c.ClientAPI.JwtConfig.Enabled {
pubPki, _ := pem.Decode([]byte(c.ClientAPI.JwtConfig.Secret))
var pub interface{}
pub, err = x509.ParsePKIXPublicKey(pubPki.Bytes)
if err != nil {
return nil, err
}
c.ClientAPI.JwtConfig.SecretKey = pub.(ed25519.PublicKey)
}
c.MediaAPI.AbsBasePath = Path(absPath(basePath, c.MediaAPI.BasePath))
@ -305,7 +315,10 @@ func (config *Dendrite) Derive() error {
config.Derived.Registration.Flows = append(config.Derived.Registration.Flows,
authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeDummy}})
}
if config.ClientAPI.ThreePidDelegate != "" {
config.Derived.Registration.Flows = append(config.Derived.Registration.Flows,
authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeEmail}})
}
// Load application service configuration files
if err := loadAppServices(&config.AppServiceAPI, &config.Derived); err != nil {
return err

View file

@ -21,7 +21,6 @@ import (
"regexp"
"strings"
log "github.com/sirupsen/logrus"
yaml "gopkg.in/yaml.v2"
)
@ -346,11 +345,11 @@ func checkErrors(config *AppServiceAPI, derived *Derived) (err error) {
// TODO: Remove once rate_limited is implemented
if appservice.RateLimited {
log.Warn("WARNING: Application service option rate_limited is currently unimplemented")
// log.Warn("WARNING: Application service option rate_limited is currently unimplemented")
}
// TODO: Remove once protocols is implemented
if len(appservice.Protocols) > 0 {
log.Warn("WARNING: Application service option protocols is currently unimplemented")
// log.Warn("WARNING: Application service option protocols is currently unimplemented")
}
}
@ -376,7 +375,7 @@ func validateNamespace(
// Check if GroupID for the users namespace is in the correct format
if key == "users" && namespace.GroupID != "" {
// TODO: Remove once group_id is implemented
log.Warn("WARNING: Application service option group_id is currently unimplemented")
// log.Warn("WARNING: Application service option group_id is currently unimplemented")
correctFormat := groupIDRegexp.MatchString(namespace.GroupID)
if !correctFormat {

View file

@ -3,6 +3,9 @@ package config
import (
"fmt"
"time"
"github.com/matrix-org/dendrite/clientapi/ratelimit"
"golang.org/x/crypto/ed25519"
)
type ClientAPI struct {
@ -52,9 +55,23 @@ type ClientAPI struct {
TURN TURN `yaml:"turn"`
// Rate-limiting options
RateLimiting RateLimiting `yaml:"rate_limiting"`
RateLimiting RateLimiting `yaml:"rate_limiting"`
RtFailedLogin ratelimit.RtFailedLoginConfig `yaml:"rate_limiting_failed_login"`
MSCs *MSCs `yaml:"-"`
ThreePidDelegate string `yaml:"three_pid_delegate"`
JwtConfig JwtConfig `yaml:"jwt_config"`
}
type JwtConfig struct {
Enabled bool `yaml:"enabled"`
Algorithm string `yaml:"algorithm"`
Issuer string `yaml:"issuer"`
Secret string `yaml:"secret"`
SecretKey ed25519.PublicKey
Audiences []string `yaml:"audiences"`
}
func (c *ClientAPI) Defaults(opts DefaultOpts) {

View file

@ -8,13 +8,13 @@ import (
"sync"
"time"
"github.com/getsentry/sentry-go"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/process"
natsserver "github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
natsclient "github.com/nats-io/nats.go"
)
@ -36,7 +36,7 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS
defer natsLock.Unlock()
// check if we need an in-process NATS Server
if len(cfg.Addresses) != 0 {
return setupNATS(process, cfg, nil)
return setupNATS(cfg, nil)
}
if s.Server == nil {
var err error
@ -72,13 +72,34 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS
if err != nil {
logrus.Fatalln("Failed to create NATS client")
}
return setupNATS(process, cfg, nc)
return setupNATS(cfg, nc)
}
func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) {
func setupNATS(cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) {
var s nats.JetStreamContext
var err error
if nc == nil {
var err error
opts := []natsclient.Option{}
opts := []natsclient.Option{
natsclient.DisconnectErrHandler(func(c *natsclient.Conn, err error) {
logrus.WithError(err).Error("nats connection: disconnected")
}),
natsclient.ReconnectHandler(func(_ *natsclient.Conn) {
logrus.Info("nats connection: client reconnected")
for _, stream := range []*nats.StreamConfig{
streams[6],
streams[10],
} {
err = configureStream(stream, cfg, s)
if err != nil {
logrus.WithError(err).WithField("stream", stream.Name).Error("unable to configure a stream")
}
}
}),
natsclient.ClosedHandler(func(_ *natsclient.Conn) {
logrus.Info("nats connection: client closed")
}),
}
if cfg.DisableTLSValidation {
opts = append(opts, natsclient.Secure(&tls.Config{
InsecureSkipVerify: true,
@ -91,89 +112,16 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc
}
}
s, err := nc.JetStream()
s, err = nc.JetStream()
if err != nil {
logrus.WithError(err).Panic("Unable to get JetStream context")
return nil, nil
}
for _, stream := range streams { // streams are defined in streams.go
name := cfg.Prefixed(stream.Name)
info, err := s.StreamInfo(name)
if err != nil && err != natsclient.ErrStreamNotFound {
logrus.WithError(err).Fatal("Unable to get stream info")
}
subjects := stream.Subjects
if len(subjects) == 0 {
// By default we want each stream to listen for the subjects
// that are either an exact match for the stream name, or where
// the first part of the subject is the stream name. ">" is a
// wildcard in NATS for one or more subject tokens. In the case
// that the stream is called "Foo", this will match any message
// with the subject "Foo", "Foo.Bar" or "Foo.Bar.Baz" etc.
subjects = []string{name, name + ".>"}
}
if info != nil {
switch {
case !reflect.DeepEqual(info.Config.Subjects, subjects):
fallthrough
case info.Config.Retention != stream.Retention:
fallthrough
case info.Config.Storage != stream.Storage:
if err = s.DeleteStream(name); err != nil {
logrus.WithError(err).Fatal("Unable to delete stream")
}
info = nil
}
}
if info == nil {
// If we're trying to keep everything in memory (e.g. unit tests)
// then overwrite the storage policy.
if cfg.InMemory {
stream.Storage = natsclient.MemoryStorage
}
// Namespace the streams without modifying the original streams
// array, otherwise we end up with namespaces on namespaces.
namespaced := *stream
namespaced.Name = name
namespaced.Subjects = subjects
if _, err = s.AddStream(&namespaced); err != nil {
logger := logrus.WithError(err).WithFields(logrus.Fields{
"stream": namespaced.Name,
"subjects": namespaced.Subjects,
})
// If the stream was supposed to be in-memory to begin with
// then an error here is fatal so we'll give up.
if namespaced.Storage == natsclient.MemoryStorage {
logger.WithError(err).Fatal("Unable to add in-memory stream")
}
// The stream was supposed to be on disk. Let's try starting
// Dendrite with the stream in-memory instead. That'll mean that
// we can't recover anything that was queued on the disk but we
// will still be able to start and run hopefully in the meantime.
logger.WithError(err).Error("Unable to add stream")
sentry.CaptureException(fmt.Errorf("Unable to add stream %q: %w", namespaced.Name, err))
namespaced.Storage = natsclient.MemoryStorage
if _, err = s.AddStream(&namespaced); err != nil {
// We tried to add the stream in-memory instead but something
// went wrong. That's an unrecoverable situation so we will
// give up at this point.
logger.WithError(err).Fatal("Unable to add in-memory stream")
}
if stream.Storage != namespaced.Storage {
// We've managed to add the stream in memory. What's on the
// disk will be left alone, but our ability to recover from a
// future crash will be limited. Yell about it.
err := fmt.Errorf("Stream %q is running in-memory; this may be due to data corruption in the JetStream storage directory", namespaced.Name)
sentry.CaptureException(err)
process.Degraded(err)
}
}
err = configureStream(stream, cfg, s)
if err != nil {
logrus.WithError(err).WithField("stream", stream.Name).Fatal("unable to configure a stream")
}
}
@ -203,3 +151,52 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc
return s, nc
}
func configureStream(stream *nats.StreamConfig, cfg *config.JetStream, s nats.JetStreamContext) error {
name := cfg.Prefixed(stream.Name)
info, err := s.StreamInfo(name)
if err != nil && err != natsclient.ErrStreamNotFound {
return fmt.Errorf("get stream info: %w", err)
}
subjects := stream.Subjects
if len(subjects) == 0 {
// By default we want each stream to listen for the subjects
// that are either an exact match for the stream name, or where
// the first part of the subject is the stream name. ">" is a
// wildcard in NATS for one or more subject tokens. In the case
// that the stream is called "Foo", this will match any message
// with the subject "Foo", "Foo.Bar" or "Foo.Bar.Baz" etc.
subjects = []string{name, name + ".>"}
}
if info != nil {
switch {
case !reflect.DeepEqual(info.Config.Subjects, subjects):
fallthrough
case info.Config.Retention != stream.Retention:
fallthrough
case info.Config.Storage != stream.Storage:
if err = s.DeleteStream(name); err != nil {
return fmt.Errorf("delete stream: %w", err)
}
info = nil
}
}
if info == nil {
// If we're trying to keep everything in memory (e.g. unit tests)
// then overwrite the storage policy.
if cfg.InMemory {
stream.Storage = natsclient.MemoryStorage
}
// Namespace the streams without modifying the original streams
// array, otherwise we end up with namespaces on namespaces.
namespaced := *stream
namespaced.Name = name
namespaced.Subjects = subjects
if _, err = s.AddStream(&namespaced); err != nil {
return fmt.Errorf("add stream: %w", err)
}
logrus.Infof("stream created: %s", stream.Name)
}
return nil
}

View file

@ -31,6 +31,7 @@ var (
RequestPresence = "GetPresence"
OutputPresenceEvent = "OutputPresenceEvent"
InputFulltextReindex = "InputFulltextReindex"
OutputMultiRoomCast = "OutputMultiRoomCast"
)
var safeCharacters = regexp.MustCompile("[^A-Za-z0-9$]+")
@ -101,4 +102,9 @@ var streams = []*nats.StreamConfig{
Storage: nats.MemoryStorage,
MaxAge: time.Minute * 5,
},
{
Name: OutputMultiRoomCast,
Retention: nats.InterestPolicy,
Storage: nats.FileStorage,
},
}

View file

@ -0,0 +1,113 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package consumers
import (
"context"
"time"
"github.com/getsentry/sentry-go"
"github.com/nats-io/nats.go"
log "github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/jetstream"
"github.com/matrix-org/dendrite/setup/process"
"github.com/matrix-org/dendrite/syncapi/notifier"
"github.com/matrix-org/dendrite/syncapi/storage/mrd"
"github.com/matrix-org/dendrite/syncapi/streams"
"github.com/matrix-org/dendrite/syncapi/types"
)
// OutputMultiRoomDataConsumer consumes events that originated in the client API server.
type OutputMultiRoomDataConsumer struct {
ctx context.Context
jetstream nats.JetStreamContext
durable string
topic string
db *mrd.Queries
stream streams.StreamProvider
notifier *notifier.Notifier
}
// NewOutputMultiRoomDataConsumer creates a new OutputMultiRoomDataConsumer consumer. Call Start() to begin consuming from room servers.
func NewOutputMultiRoomDataConsumer(
process *process.ProcessContext,
cfg *config.SyncAPI,
js nats.JetStreamContext,
q *mrd.Queries,
notifier *notifier.Notifier,
stream streams.StreamProvider,
) *OutputMultiRoomDataConsumer {
return &OutputMultiRoomDataConsumer{
ctx: process.Context(),
jetstream: js,
topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputMultiRoomCast),
durable: cfg.Matrix.JetStream.Durable("SyncAPIMultiRoomDataConsumer"),
db: q,
notifier: notifier,
stream: stream,
}
}
func (s *OutputMultiRoomDataConsumer) Start() error {
return jetstream.JetStreamConsumer(
s.ctx, s.jetstream, s.topic, s.durable, 1,
s.onMessage, nats.DeliverAll(), nats.ManualAck(),
)
}
func (s *OutputMultiRoomDataConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
msg := msgs[0]
userID := msg.Header.Get(jetstream.UserID)
dataType := msg.Header.Get("type")
log.WithFields(log.Fields{
"type": dataType,
"user_id": userID,
}).Debug("Received multiroom data from client API server")
pos, err := s.db.InsertMultiRoomData(ctx, mrd.InsertMultiRoomDataParams{
UserID: userID,
Type: dataType,
Data: msg.Data,
})
if err != nil {
sentry.CaptureException(err)
log.WithFields(log.Fields{
"type": dataType,
"user_id": userID,
}).WithError(err).Errorf("could not insert multi room data")
return false
}
rooms, err := s.db.SelectMultiRoomVisibilityRooms(ctx, mrd.SelectMultiRoomVisibilityRoomsParams{
UserID: userID,
ExpireTs: time.Now().Unix(),
})
if err != nil {
sentry.CaptureException(err)
log.WithFields(log.Fields{
"type": dataType,
"user_id": userID,
}).WithError(err).Errorf("failed to select multi room visibility")
return false
}
s.stream.Advance(types.StreamPosition(pos))
s.notifier.OnNewMultiRoomData(types.StreamingToken{MultiRoomDataPosition: types.StreamPosition(pos)}, rooms)
return true
}

View file

@ -280,6 +280,32 @@ func (n *Notifier) _sharedUsers(userID string) []string {
return sharedUsers
}
func (n *Notifier) OnNewMultiRoomData(
posUpdate types.StreamingToken, roomIds []string,
) {
n.lock.Lock()
defer n.lock.Unlock()
n.currPos.ApplyUpdates(posUpdate)
usersInRoom := n._usersInRooms(roomIds)
n._wakeupUsers(usersInRoom, nil, n.currPos)
}
func (n *Notifier) _usersInRooms(roomIds []string) []string {
for i := range roomIds {
for _, userID := range n._joinedUsers(roomIds[i]) {
n._sharedUserMap[userID] = struct{}{}
}
}
usersInRooms := make([]string, 0, len(n._sharedUserMap)+1)
for userID := range n._sharedUserMap {
usersInRooms = append(usersInRooms, userID)
delete(n._sharedUserMap, userID)
}
return usersInRooms
}
func (n *Notifier) IsSharedUser(userA, userB string) bool {
n.lock.RLock()
defer n.lock.RUnlock()

View file

@ -109,6 +109,7 @@ type DatabaseTransaction interface {
GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error)
PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error)
RelationsFor(ctx context.Context, roomID, eventID, relType, eventType string, from, to types.StreamPosition, backwards bool, limit int) (events []types.StreamEvent, prevBatch, nextBatch string, err error)
SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string) (types.MultiRoom, error)
}
type Database interface {
@ -188,6 +189,10 @@ type Database interface {
type Presence interface {
GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error)
UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error)
PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error)
MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error)
ExpirePresence(ctx context.Context) ([]types.PresenceNotify, error)
UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error
}
type SharedUsers interface {

View file

@ -0,0 +1,3 @@
## Multiroom storage
please install `sqlc`: `go install github.com/kyleconroy/sqlc/cmd/sqlc@latest`. Then run `sqlc -f sqlc.yaml generate` in this directory after changing `queries.sql` or `../postgres/schema.sql` files.

138
syncapi/storage/mrd/db.go Normal file
View file

@ -0,0 +1,138 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.15.0
package mrd
import (
"context"
"database/sql"
"fmt"
)
type DBTX interface {
ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
PrepareContext(context.Context, string) (*sql.Stmt, error)
QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
QueryRowContext(context.Context, string, ...interface{}) *sql.Row
}
func New(db DBTX) *Queries {
return &Queries{db: db}
}
func Prepare(ctx context.Context, db DBTX) (*Queries, error) {
q := Queries{db: db}
var err error
if q.deleteMultiRoomVisibilityStmt, err = db.PrepareContext(ctx, deleteMultiRoomVisibility); err != nil {
return nil, fmt.Errorf("error preparing query DeleteMultiRoomVisibility: %w", err)
}
if q.deleteMultiRoomVisibilityByExpireTSStmt, err = db.PrepareContext(ctx, deleteMultiRoomVisibilityByExpireTS); err != nil {
return nil, fmt.Errorf("error preparing query DeleteMultiRoomVisibilityByExpireTS: %w", err)
}
if q.insertMultiRoomDataStmt, err = db.PrepareContext(ctx, insertMultiRoomData); err != nil {
return nil, fmt.Errorf("error preparing query InsertMultiRoomData: %w", err)
}
if q.insertMultiRoomVisibilityStmt, err = db.PrepareContext(ctx, insertMultiRoomVisibility); err != nil {
return nil, fmt.Errorf("error preparing query InsertMultiRoomVisibility: %w", err)
}
if q.selectMaxIdStmt, err = db.PrepareContext(ctx, selectMaxId); err != nil {
return nil, fmt.Errorf("error preparing query SelectMaxId: %w", err)
}
if q.selectMultiRoomVisibilityRoomsStmt, err = db.PrepareContext(ctx, selectMultiRoomVisibilityRooms); err != nil {
return nil, fmt.Errorf("error preparing query SelectMultiRoomVisibilityRooms: %w", err)
}
return &q, nil
}
func (q *Queries) Close() error {
var err error
if q.deleteMultiRoomVisibilityStmt != nil {
if cerr := q.deleteMultiRoomVisibilityStmt.Close(); cerr != nil {
err = fmt.Errorf("error closing deleteMultiRoomVisibilityStmt: %w", cerr)
}
}
if q.deleteMultiRoomVisibilityByExpireTSStmt != nil {
if cerr := q.deleteMultiRoomVisibilityByExpireTSStmt.Close(); cerr != nil {
err = fmt.Errorf("error closing deleteMultiRoomVisibilityByExpireTSStmt: %w", cerr)
}
}
if q.insertMultiRoomDataStmt != nil {
if cerr := q.insertMultiRoomDataStmt.Close(); cerr != nil {
err = fmt.Errorf("error closing insertMultiRoomDataStmt: %w", cerr)
}
}
if q.insertMultiRoomVisibilityStmt != nil {
if cerr := q.insertMultiRoomVisibilityStmt.Close(); cerr != nil {
err = fmt.Errorf("error closing insertMultiRoomVisibilityStmt: %w", cerr)
}
}
if q.selectMaxIdStmt != nil {
if cerr := q.selectMaxIdStmt.Close(); cerr != nil {
err = fmt.Errorf("error closing selectMaxIdStmt: %w", cerr)
}
}
if q.selectMultiRoomVisibilityRoomsStmt != nil {
if cerr := q.selectMultiRoomVisibilityRoomsStmt.Close(); cerr != nil {
err = fmt.Errorf("error closing selectMultiRoomVisibilityRoomsStmt: %w", cerr)
}
}
return err
}
func (q *Queries) exec(ctx context.Context, stmt *sql.Stmt, query string, args ...interface{}) (sql.Result, error) {
switch {
case stmt != nil && q.tx != nil:
return q.tx.StmtContext(ctx, stmt).ExecContext(ctx, args...)
case stmt != nil:
return stmt.ExecContext(ctx, args...)
default:
return q.db.ExecContext(ctx, query, args...)
}
}
func (q *Queries) query(ctx context.Context, stmt *sql.Stmt, query string, args ...interface{}) (*sql.Rows, error) {
switch {
case stmt != nil && q.tx != nil:
return q.tx.StmtContext(ctx, stmt).QueryContext(ctx, args...)
case stmt != nil:
return stmt.QueryContext(ctx, args...)
default:
return q.db.QueryContext(ctx, query, args...)
}
}
func (q *Queries) queryRow(ctx context.Context, stmt *sql.Stmt, query string, args ...interface{}) *sql.Row {
switch {
case stmt != nil && q.tx != nil:
return q.tx.StmtContext(ctx, stmt).QueryRowContext(ctx, args...)
case stmt != nil:
return stmt.QueryRowContext(ctx, args...)
default:
return q.db.QueryRowContext(ctx, query, args...)
}
}
type Queries struct {
db DBTX
tx *sql.Tx
deleteMultiRoomVisibilityStmt *sql.Stmt
deleteMultiRoomVisibilityByExpireTSStmt *sql.Stmt
insertMultiRoomDataStmt *sql.Stmt
insertMultiRoomVisibilityStmt *sql.Stmt
selectMaxIdStmt *sql.Stmt
selectMultiRoomVisibilityRoomsStmt *sql.Stmt
}
func (q *Queries) WithTx(tx *sql.Tx) *Queries {
return &Queries{
db: tx,
tx: tx,
deleteMultiRoomVisibilityStmt: q.deleteMultiRoomVisibilityStmt,
deleteMultiRoomVisibilityByExpireTSStmt: q.deleteMultiRoomVisibilityByExpireTSStmt,
insertMultiRoomDataStmt: q.insertMultiRoomDataStmt,
insertMultiRoomVisibilityStmt: q.insertMultiRoomVisibilityStmt,
selectMaxIdStmt: q.selectMaxIdStmt,
selectMultiRoomVisibilityRoomsStmt: q.selectMultiRoomVisibilityRoomsStmt,
}
}

View file

@ -0,0 +1,24 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.15.0
package mrd
import (
"time"
)
type SyncapiMultiroomDatum struct {
ID int64 `json:"id"`
UserID string `json:"user_id"`
Type string `json:"type"`
Data []byte `json:"data"`
Ts time.Time `json:"ts"`
}
type SyncapiMultiroomVisibility struct {
UserID string `json:"user_id"`
Type string `json:"type"`
RoomID string `json:"room_id"`
ExpireTs int64 `json:"expire_ts"`
}

View file

@ -0,0 +1,44 @@
-- name: InsertMultiRoomData :one
INSERT INTO syncapi_multiroom_data (
user_id,
type,
data
) VALUES (
$1,
$2,
$3
) ON CONFLICT (user_id, type) DO UPDATE SET id = nextval('syncapi_multiroom_id'), data = $3, ts = current_timestamp
RETURNING id;
-- name: InsertMultiRoomVisibility :exec
INSERT INTO syncapi_multiroom_visibility (
user_id,
type,
room_id,
expire_ts
) VALUES (
$1,
$2,
$3,
$4
) ON CONFLICT (user_id, type, room_id) DO UPDATE SET expire_ts = $4;
-- name: SelectMultiRoomVisibilityRooms :many
SELECT room_id FROM syncapi_multiroom_visibility
WHERE user_id = $1
AND expire_ts > $2;
-- name: SelectMaxId :one
SELECT MAX(id) FROM syncapi_multiroom_data;
-- name: DeleteMultiRoomVisibility :exec
DELETE FROM syncapi_multiroom_visibility
WHERE user_id = $1
AND type = $2
AND room_id = $3;
-- name: DeleteMultiRoomVisibilityByExpireTS :execrows
DELETE FROM syncapi_multiroom_visibility
WHERE expire_ts <= $1;

View file

@ -0,0 +1,143 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.15.0
// source: queries.sql
package mrd
import (
"context"
)
const deleteMultiRoomVisibility = `-- name: DeleteMultiRoomVisibility :exec
DELETE FROM syncapi_multiroom_visibility
WHERE user_id = $1
AND type = $2
AND room_id = $3
`
type DeleteMultiRoomVisibilityParams struct {
UserID string `json:"user_id"`
Type string `json:"type"`
RoomID string `json:"room_id"`
}
func (q *Queries) DeleteMultiRoomVisibility(ctx context.Context, arg DeleteMultiRoomVisibilityParams) error {
_, err := q.exec(ctx, q.deleteMultiRoomVisibilityStmt, deleteMultiRoomVisibility, arg.UserID, arg.Type, arg.RoomID)
return err
}
const deleteMultiRoomVisibilityByExpireTS = `-- name: DeleteMultiRoomVisibilityByExpireTS :execrows
DELETE FROM syncapi_multiroom_visibility
WHERE expire_ts <= $1
`
func (q *Queries) DeleteMultiRoomVisibilityByExpireTS(ctx context.Context, expireTs int64) (int64, error) {
result, err := q.exec(ctx, q.deleteMultiRoomVisibilityByExpireTSStmt, deleteMultiRoomVisibilityByExpireTS, expireTs)
if err != nil {
return 0, err
}
return result.RowsAffected()
}
const insertMultiRoomData = `-- name: InsertMultiRoomData :one
INSERT INTO syncapi_multiroom_data (
user_id,
type,
data
) VALUES (
$1,
$2,
$3
) ON CONFLICT (user_id, type) DO UPDATE SET id = nextval('syncapi_multiroom_id'), data = $3, ts = current_timestamp
RETURNING id
`
type InsertMultiRoomDataParams struct {
UserID string `json:"user_id"`
Type string `json:"type"`
Data []byte `json:"data"`
}
func (q *Queries) InsertMultiRoomData(ctx context.Context, arg InsertMultiRoomDataParams) (int64, error) {
row := q.queryRow(ctx, q.insertMultiRoomDataStmt, insertMultiRoomData, arg.UserID, arg.Type, arg.Data)
var id int64
err := row.Scan(&id)
return id, err
}
const insertMultiRoomVisibility = `-- name: InsertMultiRoomVisibility :exec
INSERT INTO syncapi_multiroom_visibility (
user_id,
type,
room_id,
expire_ts
) VALUES (
$1,
$2,
$3,
$4
) ON CONFLICT (user_id, type, room_id) DO UPDATE SET expire_ts = $4
`
type InsertMultiRoomVisibilityParams struct {
UserID string `json:"user_id"`
Type string `json:"type"`
RoomID string `json:"room_id"`
ExpireTs int64 `json:"expire_ts"`
}
func (q *Queries) InsertMultiRoomVisibility(ctx context.Context, arg InsertMultiRoomVisibilityParams) error {
_, err := q.exec(ctx, q.insertMultiRoomVisibilityStmt, insertMultiRoomVisibility,
arg.UserID,
arg.Type,
arg.RoomID,
arg.ExpireTs,
)
return err
}
const selectMaxId = `-- name: SelectMaxId :one
SELECT MAX(id) FROM syncapi_multiroom_data
`
func (q *Queries) SelectMaxId(ctx context.Context) (interface{}, error) {
row := q.queryRow(ctx, q.selectMaxIdStmt, selectMaxId)
var max interface{}
err := row.Scan(&max)
return max, err
}
const selectMultiRoomVisibilityRooms = `-- name: SelectMultiRoomVisibilityRooms :many
SELECT room_id FROM syncapi_multiroom_visibility
WHERE user_id = $1
AND expire_ts > $2
`
type SelectMultiRoomVisibilityRoomsParams struct {
UserID string `json:"user_id"`
ExpireTs int64 `json:"expire_ts"`
}
func (q *Queries) SelectMultiRoomVisibilityRooms(ctx context.Context, arg SelectMultiRoomVisibilityRoomsParams) ([]string, error) {
rows, err := q.query(ctx, q.selectMultiRoomVisibilityRoomsStmt, selectMultiRoomVisibilityRooms, arg.UserID, arg.ExpireTs)
if err != nil {
return nil, err
}
defer rows.Close()
var items []string
for rows.Next() {
var room_id string
if err := rows.Scan(&room_id); err != nil {
return nil, err
}
items = append(items, room_id)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}

View file

@ -0,0 +1,8 @@
version: 1
packages:
- path: ../mrd
engine: postgresql
schema: ../postgres/schema.sql
queries: queries.sql
emit_json_tags: true
emit_prepared_queries: true

View file

@ -0,0 +1,6 @@
package mrd
type StateEvent struct {
Hidden bool `json:"hidden"`
ExpireTs int `json:"expire_ts"`
}

View file

@ -0,0 +1,61 @@
package postgres
import (
"context"
"database/sql"
_ "embed"
"fmt"
"time"
"github.com/lib/pq"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/syncapi/storage/tables"
"github.com/matrix-org/dendrite/syncapi/types"
)
//go:embed schema.sql
var schema string
var selectMultiRoomCastSQL = `SELECT d.user_id, d.type, d.data, d.ts FROM syncapi_multiroom_data AS d
JOIN syncapi_multiroom_visibility AS v
ON d.user_id = v.user_id
AND d.type = v.type
WHERE v.room_id = ANY($1)
AND id > $2
AND id <= $3`
type multiRoomStatements struct {
selectMultiRoomCast *sql.Stmt
}
func NewPostgresMultiRoomCastTable(db *sql.DB) (tables.MultiRoom, error) {
r := &multiRoomStatements{}
_, err := db.Exec(schema)
if err != nil {
return nil, err
}
return r, sqlutil.StatementList{
{&r.selectMultiRoomCast, selectMultiRoomCastSQL},
}.Prepare(db)
}
func (s *multiRoomStatements) SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string, txn *sql.Tx) ([]*types.MultiRoomDataRow, error) {
rows, err := sqlutil.TxStmt(txn, s.selectMultiRoomCast).QueryContext(ctx, pq.StringArray(joinedRooms), r.Low(), r.High())
if err != nil {
return nil, err
}
data := make([]*types.MultiRoomDataRow, 0)
defer internal.CloseAndLogIfError(ctx, rows, "SelectMultiRoomData: rows.close() failed")
var t time.Time
for rows.Next() {
r := types.MultiRoomDataRow{}
err = rows.Scan(&r.UserId, &r.Type, &r.Data, &t)
r.Timestamp = t.Unix()
if err != nil {
return nil, fmt.Errorf("rows scan: %w", err)
}
data = append(data, &r)
}
return data, rows.Err()
}

View file

@ -62,6 +62,10 @@ const upsertPresenceFromSyncSQL = "" +
" presence = $2, last_active_ts = $3" +
" RETURNING id"
const updateLastActiveSQL = `UPDATE syncapi_presence
SET last_active_ts = $1
WHERE user_id = $2`
const selectPresenceForUserSQL = "" +
"SELECT presence, status_msg, last_active_ts" +
" FROM syncapi_presence" +
@ -76,12 +80,24 @@ const selectPresenceAfter = "" +
" WHERE id > $1 AND last_active_ts >= $2" +
" ORDER BY id ASC LIMIT $3"
const expirePresenceSQL = `UPDATE syncapi_presence SET
id = nextval('syncapi_presence_id'),
presence = 3
WHERE
to_timestamp(last_active_ts / 1000) < NOW() - INTERVAL` + types.PresenceExpire + `
AND
presence != 3
RETURNING id, user_id
`
type presenceStatements struct {
upsertPresenceStmt *sql.Stmt
upsertPresenceFromSyncStmt *sql.Stmt
selectPresenceForUsersStmt *sql.Stmt
selectMaxPresenceStmt *sql.Stmt
selectPresenceAfterStmt *sql.Stmt
expirePresenceStmt *sql.Stmt
updateLastActiveStmt *sql.Stmt
}
func NewPostgresPresenceTable(db *sql.DB) (*presenceStatements, error) {
@ -96,6 +112,8 @@ func NewPostgresPresenceTable(db *sql.DB) (*presenceStatements, error) {
{&s.selectPresenceForUsersStmt, selectPresenceForUserSQL},
{&s.selectMaxPresenceStmt, selectMaxPresenceSQL},
{&s.selectPresenceAfterStmt, selectPresenceAfter},
{&s.expirePresenceStmt, expirePresenceSQL},
{&s.updateLastActiveStmt, updateLastActiveSQL},
}.Prepare(db)
}
@ -166,3 +184,28 @@ func (p *presenceStatements) GetPresenceAfter(
}
return presences, rows.Err()
}
func (p *presenceStatements) ExpirePresence(
ctx context.Context,
) ([]types.PresenceNotify, error) {
rows, err := p.expirePresenceStmt.QueryContext(ctx)
if err != nil {
return nil, err
}
presences := make([]types.PresenceNotify, 0)
i := 0
for rows.Next() {
presences = append(presences, types.PresenceNotify{})
err = rows.Scan(&presences[i].StreamPos, &presences[i].UserID)
if err != nil {
return nil, err
}
i++
}
return presences, err
}
func (p *presenceStatements) UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error {
_, err := p.updateLastActiveStmt.Exec(&lastActiveTs, &userId)
return err
}

View file

@ -0,0 +1,19 @@
CREATE SEQUENCE IF NOT EXISTS syncapi_multiroom_id;
CREATE TABLE IF NOT EXISTS syncapi_multiroom_data (
id BIGINT PRIMARY KEY DEFAULT nextval('syncapi_multiroom_id'),
user_id TEXT NOT NULL,
type TEXT NOT NULL,
data BYTEA NOT NULL,
ts TIMESTAMP NOT NULL DEFAULT current_timestamp
);
CREATE UNIQUE INDEX IF NOT EXISTS syncapi_multiroom_data_user_id_type_idx ON syncapi_multiroom_data(user_id, type);
CREATE TABLE IF NOT EXISTS syncapi_multiroom_visibility (
user_id TEXT NOT NULL,
type TEXT NOT NULL,
room_id TEXT NOT NULL,
expire_ts BIGINT NOT NULL DEFAULT 0,
PRIMARY KEY(user_id, type, room_id)
)

View file

@ -23,6 +23,7 @@ import (
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/syncapi/storage/mrd"
"github.com/matrix-org/dendrite/syncapi/storage/postgres/deltas"
"github.com/matrix-org/dendrite/syncapi/storage/shared"
)
@ -102,6 +103,11 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
if err != nil {
return nil, err
}
mr, err := NewPostgresMultiRoomCastTable(d.db)
if err != nil {
return nil, err
}
mrq := mrd.New(d.db)
// apply migrations which need multiple tables
m := sqlutil.NewMigrator(d.db)
@ -134,6 +140,8 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions)
Ignores: ignores,
Presence: presence,
Relations: relations,
MultiRoom: mr,
MultiRoomQ: mrq,
}
return &d, nil
}

View file

@ -19,6 +19,7 @@ import (
"database/sql"
"encoding/json"
"fmt"
"strings"
"github.com/tidwall/gjson"
@ -30,6 +31,7 @@ import (
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/syncapi/storage/mrd"
"github.com/matrix-org/dendrite/syncapi/storage/tables"
"github.com/matrix-org/dendrite/syncapi/types"
)
@ -54,6 +56,8 @@ type Database struct {
Ignores tables.Ignores
Presence tables.Presence
Relations tables.Relations
MultiRoomQ *mrd.Queries
MultiRoom tables.MultiRoom
}
func (d *Database) NewDatabaseSnapshot(ctx context.Context) (*DatabaseTransaction, error) {
@ -336,6 +340,13 @@ func (d *Database) updateRoomState(
}
}
if strings.HasPrefix(event.Type(), "connect.mrd") {
err := d.UpdateMultiRoomVisibility(ctx, event)
if err != nil {
logrus.WithError(err).WithField("event_id", event.EventID()).Error("failed to update multi room visibility")
}
}
if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membership, pduPosition); err != nil {
return fmt.Errorf("d.CurrentRoomState.UpsertRoomState: %w", err)
}
@ -625,3 +636,49 @@ func (d *Database) SelectMemberships(
) (eventIDs []string, err error) {
return d.Memberships.SelectMemberships(ctx, nil, roomID, pos, membership, notMembership)
}
func (s *Database) ExpirePresence(ctx context.Context) ([]types.PresenceNotify, error) {
return s.Presence.ExpirePresence(ctx)
}
func (d *Database) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) {
return d.Presence.GetMaxPresenceID(ctx, nil)
}
func (d *Database) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) {
return d.Presence.GetPresenceAfter(ctx, nil, after, filter)
}
func (s *Database) UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error {
return s.Presence.UpdateLastActive(ctx, userId, lastActiveTs)
}
func (d *Database) UpdateMultiRoomVisibility(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error {
var mrdEv mrd.StateEvent
err := json.Unmarshal(event.Content(), &mrdEv)
if err != nil {
return fmt.Errorf("unmarshal multiroom visibility failed: %w", err)
}
if mrdEv.Hidden {
err = d.MultiRoomQ.DeleteMultiRoomVisibility(ctx, mrd.DeleteMultiRoomVisibilityParams{
UserID: event.Sender(),
Type: event.Type(),
RoomID: event.RoomID(),
})
if err != nil {
return fmt.Errorf("delete multiroom visibility failed: %w", err)
}
}
if mrdEv.ExpireTs > 0 {
err = d.MultiRoomQ.InsertMultiRoomVisibility(ctx, mrd.InsertMultiRoomVisibilityParams{
UserID: event.Sender(),
Type: event.Type(),
RoomID: event.RoomID(),
ExpireTs: int64(mrdEv.ExpireTs),
})
if err != nil {
return fmt.Errorf("insert multiroom visibility failed: %w", err)
}
}
return nil
}

View file

@ -688,3 +688,22 @@ func (d *DatabaseTransaction) RelationsFor(ctx context.Context, roomID, eventID,
return events, prevBatch, nextBatch, nil
}
func (d *DatabaseTransaction) SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string) (types.MultiRoom, error) {
rows, err := d.MultiRoom.SelectMultiRoomData(ctx, r, joinedRooms, d.txn)
if err != nil {
return nil, fmt.Errorf("select multi room data: %w", err)
}
mr := make(types.MultiRoom, 3)
for _, row := range rows {
if mr[row.UserId] == nil {
mr[row.UserId] = make(map[string]types.MultiRoomData)
}
mr[row.UserId][row.Type] = types.MultiRoomData{
Content: row.Data,
Timestamp: row.Timestamp,
}
}
return mr, nil
}

View file

@ -180,3 +180,15 @@ func (p *presenceStatements) GetPresenceAfter(
}
return presences, rows.Err()
}
func (p *presenceStatements) ExpirePresence(
ctx context.Context,
) ([]types.PresenceNotify, error) {
// TODO implement
return nil, nil
}
func (p *presenceStatements) UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error {
// TODO implement
return nil
}

View file

@ -22,18 +22,26 @@ import (
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/syncapi/storage/mrd"
"github.com/matrix-org/dendrite/syncapi/storage/postgres"
"github.com/matrix-org/dendrite/syncapi/storage/sqlite3"
)
// NewSyncServerDatasource opens a database connection.
func NewSyncServerDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) {
func NewSyncServerDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, *mrd.Queries, error) {
switch {
case dbProperties.ConnectionString.IsSQLite():
return sqlite3.NewDatabase(base, dbProperties)
ds, err := sqlite3.NewDatabase(base, dbProperties)
return ds, nil, err
case dbProperties.ConnectionString.IsPostgres():
return postgres.NewDatabase(base, dbProperties)
ds, err := postgres.NewDatabase(base, dbProperties)
if err != nil {
return nil, nil, err
}
mrq := mrd.New(ds.DB)
return ds, mrq, nil
default:
return nil, fmt.Errorf("unexpected database type")
return nil, nil, fmt.Errorf("unexpected database type")
}
}

View file

@ -21,7 +21,7 @@ var ctx = context.Background()
func MustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func(), func()) {
connStr, close := test.PrepareDBConnectionString(t, dbType)
base, closeBase := testrig.CreateBaseDendrite(t, dbType)
db, err := storage.NewSyncServerDatasource(base, &config.DatabaseOptions{
db, _, err := storage.NewSyncServerDatasource(base, &config.DatabaseOptions{
ConnectionString: config.DataSource(connStr),
})
if err != nil {

View file

@ -210,6 +210,8 @@ type Presence interface {
GetPresenceForUser(ctx context.Context, txn *sql.Tx, userID string) (presence *types.PresenceInternal, err error)
GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error)
GetPresenceAfter(ctx context.Context, txn *sql.Tx, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (presences map[string]*types.PresenceInternal, err error)
ExpirePresence(ctx context.Context) ([]types.PresenceNotify, error)
UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error
}
type Relations interface {
@ -230,3 +232,7 @@ type Relations interface {
// "from" or want to work forwards and don't have a "to").
SelectMaxRelationID(ctx context.Context, txn *sql.Tx) (id int64, err error)
}
type MultiRoom interface {
SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string, txn *sql.Tx) ([]*types.MultiRoomDataRow, error)
}

View file

@ -0,0 +1,72 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package streams
import (
"context"
"database/sql"
"github.com/matrix-org/dendrite/syncapi/notifier"
"github.com/matrix-org/dendrite/syncapi/storage"
"github.com/matrix-org/dendrite/syncapi/storage/mrd"
"github.com/matrix-org/dendrite/syncapi/types"
)
type MultiRoomDataStreamProvider struct {
DefaultStreamProvider
notifier *notifier.Notifier
mrdDb *mrd.Queries
}
func (p *MultiRoomDataStreamProvider) Setup(ctx context.Context, snapshot storage.DatabaseTransaction) {
p.DefaultStreamProvider.Setup(ctx, snapshot)
id, err := p.mrdDb.SelectMaxId(context.Background())
if err != nil && err != sql.ErrNoRows {
panic(err)
}
p.latestMutex.Lock()
defer p.latestMutex.Unlock()
if id == nil {
p.latest = types.StreamPosition(0)
} else {
p.latest = types.StreamPosition(id.(int64))
}
}
func (p *MultiRoomDataStreamProvider) CompleteSync(
ctx context.Context,
snapshot storage.DatabaseTransaction,
req *types.SyncRequest,
) types.StreamPosition {
return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx))
}
func (p *MultiRoomDataStreamProvider) IncrementalSync(
ctx context.Context,
snapshot storage.DatabaseTransaction,
req *types.SyncRequest,
from, to types.StreamPosition,
) types.StreamPosition {
mr, err := snapshot.SelectMultiRoomData(ctx, &types.Range{From: from, To: to}, req.JoinedRooms)
if err != nil {
req.Log.WithError(err).Error("GetUserUnreadNotificationCountsForRooms failed")
return from
}
req.Log.Tracef("MultiRoomDataStreamProvider IncrementalSync: %+v", mr)
req.Response.MultiRoom = mr
return to
}

View file

@ -77,6 +77,7 @@ func (p *PDUStreamProvider) CompleteSync(
req.Log.WithError(err).Error("p.DB.RoomIDsWithMembership failed")
return from
}
req.JoinedRooms = joinedRoomIDs
stateFilter := req.Filter.Room.State
eventFilter := req.Filter.Room.Timeline
@ -170,6 +171,7 @@ func (p *PDUStreamProvider) IncrementalSync(
for _, roomID := range syncJoinedRooms {
req.Rooms[roomID] = gomatrixserverlib.Join
}
req.JoinedRooms = syncJoinedRooms
if len(stateDeltas) == 0 {
return to

View file

@ -17,7 +17,6 @@ package streams
import (
"context"
"encoding/json"
"sync"
"github.com/matrix-org/gomatrixserverlib"
"github.com/tidwall/gjson"
@ -29,8 +28,6 @@ import (
type PresenceStreamProvider struct {
DefaultStreamProvider
// cache contains previously sent presence updates to avoid unneeded updates
cache sync.Map
notifier *notifier.Notifier
}
@ -114,19 +111,6 @@ func (p *PresenceStreamProvider) IncrementalSync(
if req.Device.UserID != presence.UserID && !p.notifier.IsSharedUser(req.Device.UserID, presence.UserID) {
continue
}
cacheKey := req.Device.UserID + req.Device.ID + presence.UserID
pres, ok := p.cache.Load(cacheKey)
if ok {
// skip already sent presence
prevPresence := pres.(*types.PresenceInternal)
currentlyActive := prevPresence.CurrentlyActive()
skip := prevPresence.Equals(presence) && currentlyActive && req.Device.UserID != presence.UserID
_, membershipChange := req.MembershipChanges[presence.UserID]
if skip && !membershipChange {
req.Log.Tracef("Skipping presence, no change (%s)", presence.UserID)
continue
}
}
if _, known := types.PresenceFromString(presence.ClientFields.Presence); known {
presence.ClientFields.LastActiveAgo = presence.LastActiveAgo()
@ -154,7 +138,6 @@ func (p *PresenceStreamProvider) IncrementalSync(
if len(req.Response.Presence.Events) == req.Filter.Presence.Limit {
break
}
p.cache.Store(cacheKey, presence)
}
if len(req.Response.Presence.Events) == 0 {

View file

@ -9,6 +9,7 @@ import (
rsapi "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/syncapi/notifier"
"github.com/matrix-org/dendrite/syncapi/storage"
"github.com/matrix-org/dendrite/syncapi/storage/mrd"
"github.com/matrix-org/dendrite/syncapi/types"
userapi "github.com/matrix-org/dendrite/userapi/api"
)
@ -23,12 +24,14 @@ type Streams struct {
DeviceListStreamProvider StreamProvider
NotificationDataStreamProvider StreamProvider
PresenceStreamProvider StreamProvider
MultiRoomStreamProvider StreamProvider
}
func NewSyncStreamProviders(
d storage.Database, userAPI userapi.SyncUserAPI,
rsAPI rsapi.SyncRoomserverAPI, keyAPI keyapi.SyncKeyAPI,
eduCache *caching.EDUCache, lazyLoadCache caching.LazyLoadCache, notifier *notifier.Notifier,
mrdb *mrd.Queries,
) *Streams {
streams := &Streams{
PDUStreamProvider: &PDUStreamProvider{
@ -66,6 +69,11 @@ func NewSyncStreamProviders(
DefaultStreamProvider: DefaultStreamProvider{DB: d},
notifier: notifier,
},
MultiRoomStreamProvider: &MultiRoomDataStreamProvider{
DefaultStreamProvider: DefaultStreamProvider{DB: d},
notifier: notifier,
mrdDb: mrdb,
},
}
ctx := context.TODO()
@ -85,6 +93,7 @@ func NewSyncStreamProviders(
streams.NotificationDataStreamProvider.Setup(ctx, snapshot)
streams.DeviceListStreamProvider.Setup(ctx, snapshot)
streams.PresenceStreamProvider.Setup(ctx, snapshot)
streams.MultiRoomStreamProvider.Setup(ctx, snapshot)
succeeded = true
return streams

View file

@ -80,6 +80,8 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat
filter.AccountData.Limit = math.MaxInt32
filter.Room.AccountData.Limit = math.MaxInt32
}
// Ignore state limit filter so that e.g. correct room name appears on clients.
filter.Room.State.Limit = math.MaxInt32
logger := util.GetLogger(req.Context()).WithFields(logrus.Fields{
"user_id": device.UserID,

View file

@ -51,7 +51,7 @@ type RequestPool struct {
keyAPI keyapi.SyncKeyAPI
rsAPI roomserverAPI.SyncRoomserverAPI
lastseen *sync.Map
presence *sync.Map
Presence *sync.Map
streams *streams.Streams
Notifier *notifier.Notifier
producer PresencePublisher
@ -86,14 +86,14 @@ func NewRequestPool(
keyAPI: keyAPI,
rsAPI: rsAPI,
lastseen: &sync.Map{},
presence: &sync.Map{},
Presence: &sync.Map{},
streams: streams,
Notifier: notifier,
producer: producer,
consumer: consumer,
}
go rp.cleanLastSeen()
go rp.cleanPresence(db, time.Minute*5)
// go rp.cleanPresence(db, time.Minute*5)
return rp
}
@ -112,11 +112,11 @@ func (rp *RequestPool) cleanPresence(db storage.Presence, cleanupTime time.Durat
return
}
for {
rp.presence.Range(func(key interface{}, v interface{}) bool {
rp.Presence.Range(func(key interface{}, v interface{}) bool {
p := v.(types.PresenceInternal)
if time.Since(p.LastActiveTS.Time()) > cleanupTime {
rp.updatePresence(db, types.PresenceUnavailable.String(), p.UserID)
rp.presence.Delete(key)
rp.Presence.Delete(key)
}
return true
})
@ -154,13 +154,22 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user
}
newPresence.ClientFields.Presence = presenceID.String()
defer rp.presence.Store(userID, newPresence)
defer rp.Presence.Store(userID, newPresence)
// avoid spamming presence updates when syncing
existingPresence, ok := rp.presence.LoadOrStore(userID, newPresence)
existingPresence, ok := rp.Presence.LoadOrStore(userID, newPresence)
if ok {
p := existingPresence.(types.PresenceInternal)
if p.ClientFields.Presence == newPresence.ClientFields.Presence {
return
if dbPresence != nil {
if p.Presence == newPresence.Presence && newPresence.LastActiveTS-dbPresence.LastActiveTS < types.PresenceNoOpMs {
return
}
if dbPresence.Presence == types.PresenceOnline && presenceID == types.PresenceOnline && newPresence.LastActiveTS-dbPresence.LastActiveTS >= types.PresenceNoOpMs {
err := db.UpdateLastActive(context.Background(), userID, uint64(newPresence.LastActiveTS))
if err != nil {
logrus.WithError(err).Error("failed to update last active")
}
return
}
}
}
@ -248,7 +257,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
defer activeSyncRequests.Dec()
rp.updateLastSeen(req, device)
rp.updatePresence(rp.db, req.FormValue("set_presence"), device.UserID)
rp.updatePresence(rp.db, "", device.UserID)
waitingSyncRequests.Inc()
defer waitingSyncRequests.Dec()
@ -398,6 +407,14 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
)
},
),
MultiRoomDataPosition: withTransaction(
syncReq.Since.MultiRoomDataPosition,
func(txn storage.DatabaseTransaction) types.StreamPosition {
return rp.streams.MultiRoomStreamProvider.CompleteSync(
syncReq.Context, txn, syncReq,
)
},
),
}
} else {
// Incremental sync
@ -483,6 +500,15 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.
)
},
),
MultiRoomDataPosition: withTransaction(
syncReq.Since.MultiRoomDataPosition,
func(snapshot storage.DatabaseTransaction) types.StreamPosition {
return rp.streams.MultiRoomStreamProvider.IncrementalSync(
syncReq.Context, snapshot, syncReq,
syncReq.Since.MultiRoomDataPosition, rp.Notifier.CurrentPosition().MultiRoomDataPosition,
)
},
),
}
// it's possible for there to be no updates for this user even though since < current pos,
// e.g busy servers with a quiet user. In this scenario, we don't want to return a no-op

View file

@ -7,6 +7,7 @@ import (
"time"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/syncapi/storage"
"github.com/matrix-org/dendrite/syncapi/types"
"github.com/matrix-org/gomatrixserverlib"
)
@ -23,7 +24,9 @@ func (d *dummyPublisher) SendPresence(userID string, presence types.Presence, st
return nil
}
type dummyDB struct{}
type dummyDB struct {
storage.Database
}
func (d dummyDB) UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) {
return 0, nil
@ -109,7 +112,7 @@ func TestRequestPool_updatePresence(t *testing.T) {
},
}
rp := &RequestPool{
presence: &syncMap,
Presence: &syncMap,
producer: publisher,
consumer: consumer,
cfg: &config.SyncAPI{

View file

@ -16,6 +16,7 @@ package syncapi
import (
"context"
"time"
"github.com/sirupsen/logrus"
@ -34,6 +35,7 @@ import (
"github.com/matrix-org/dendrite/syncapi/storage"
"github.com/matrix-org/dendrite/syncapi/streams"
"github.com/matrix-org/dendrite/syncapi/sync"
"github.com/matrix-org/dendrite/syncapi/types"
)
// AddPublicRoutes sets up and registers HTTP handlers for the SyncAPI
@ -48,14 +50,26 @@ func AddPublicRoutes(
js, natsClient := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
syncDB, err := storage.NewSyncServerDatasource(base, &cfg.Database)
syncDB, mrq, err := storage.NewSyncServerDatasource(base, &cfg.Database)
if err != nil {
logrus.WithError(err).Panicf("failed to connect to sync db")
}
go func() {
var affected int64
for {
affected, err = mrq.DeleteMultiRoomVisibilityByExpireTS(context.Background(), time.Now().Unix())
if err != nil {
logrus.WithError(err).Error("failed to expire multiroom visibility")
}
logrus.WithField("rows", affected).Info("expired multiroom visibility")
time.Sleep(time.Minute)
}
}()
eduCache := caching.NewTypingCache()
notifier := notifier.NewNotifier()
streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache, base.Caches, notifier)
streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache, base.Caches, notifier, mrq)
notifier.SetCurrentPosition(streams.Latest(context.Background()))
if err = notifier.Load(context.Background(), syncDB); err != nil {
logrus.WithError(err).Panicf("failed to load notifier ")
@ -130,8 +144,35 @@ func AddPublicRoutes(
logrus.WithError(err).Panicf("failed to start receipts consumer")
}
multiRoomConsumer := consumers.NewOutputMultiRoomDataConsumer(
base.ProcessContext, cfg, js, mrq, notifier, streams.MultiRoomStreamProvider,
)
if err = multiRoomConsumer.Start(); err != nil {
logrus.WithError(err).Panicf("failed to start multiroom consumer")
}
routing.Setup(
base.PublicClientAPIMux, requestPool, syncDB, userAPI,
rsAPI, cfg, base.Caches, base.Fulltext,
)
go func() {
ctx := context.Background()
for {
notify, err := syncDB.ExpirePresence(ctx)
if err != nil {
logrus.WithError(err).Error("failed to expire presence")
}
for i := range notify {
requestPool.Presence.Store(notify[i].UserID, types.PresenceInternal{
Presence: types.PresenceOffline,
})
notifier.OnNewPresence(types.StreamingToken{
PresencePosition: notify[i].StreamPos,
}, notify[i].UserID)
}
time.Sleep(types.PresenceExpireInterval)
}
}()
}

View file

@ -0,0 +1,21 @@
package types
type MultiRoom map[string]map[string]MultiRoomData
type MultiRoomContent []byte
type MultiRoomData struct {
Content MultiRoomContent `json:"content"`
Timestamp int64 `json:"timestamp"`
}
func (d MultiRoomContent) MarshalJSON() ([]byte, error) {
return d, nil
}
type MultiRoomDataRow struct {
Data []byte
Type string
UserId string
Timestamp int64
}

View file

@ -0,0 +1,21 @@
package types
import (
"encoding/json"
"testing"
"github.com/matryer/is"
)
func TestMarshallMultiRoom(t *testing.T) {
is := is.New(t)
m, err := json.Marshal(
MultiRoom{
"@3:example.com": map[string]MultiRoomData{
"location": {
Content: MultiRoomContent(`{"foo":"bar"}`),
Timestamp: 123,
}}})
is.NoErr(err)
is.Equal(m, []byte(`{"@3:example.com":{"location":{"content":{"foo":"bar"},"timestamp":123}}}`))
}

View file

@ -21,6 +21,12 @@ import (
"github.com/matrix-org/gomatrixserverlib"
)
const (
PresenceNoOpMs = 60_000
PresenceExpire = "'4 minutes'"
PresenceExpireInterval = time.Second * 30
)
type Presence uint8
const (
@ -66,6 +72,11 @@ type PresenceInternal struct {
Presence Presence `json:"-"`
}
type PresenceNotify struct {
StreamPos StreamPosition
UserID string
}
// Equals compares p1 with p2.
func (p1 *PresenceInternal) Equals(p2 *PresenceInternal) bool {
return p1.ClientFields.Presence == p2.ClientFields.Presence &&

View file

@ -21,7 +21,8 @@ type SyncRequest struct {
WantFullState bool
// Updated by the PDU stream.
Rooms map[string]string
Rooms map[string]string
JoinedRooms []string
// Updated by the PDU stream.
MembershipChanges map[string]struct{}
// Updated by the PDU stream.

View file

@ -22,6 +22,7 @@ import (
"strings"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
"github.com/tidwall/gjson"
"github.com/matrix-org/dendrite/roomserver/api"
@ -114,6 +115,7 @@ type StreamingToken struct {
DeviceListPosition StreamPosition
NotificationDataPosition StreamPosition
PresencePosition StreamPosition
MultiRoomDataPosition StreamPosition
}
// This will be used as a fallback by json.Marshal.
@ -129,12 +131,12 @@ func (s *StreamingToken) UnmarshalText(text []byte) (err error) {
func (t StreamingToken) String() string {
posStr := fmt.Sprintf(
"s%d_%d_%d_%d_%d_%d_%d_%d_%d",
"s%d_%d_%d_%d_%d_%d_%d_%d_%d_%d",
t.PDUPosition, t.TypingPosition,
t.ReceiptPosition, t.SendToDevicePosition,
t.InvitePosition, t.AccountDataPosition,
t.DeviceListPosition, t.NotificationDataPosition,
t.PresencePosition,
t.PresencePosition, t.MultiRoomDataPosition,
)
return posStr
}
@ -160,12 +162,14 @@ func (t *StreamingToken) IsAfter(other StreamingToken) bool {
return true
case t.PresencePosition > other.PresencePosition:
return true
case t.MultiRoomDataPosition > other.MultiRoomDataPosition:
return true
}
return false
}
func (t *StreamingToken) IsEmpty() bool {
return t == nil || t.PDUPosition+t.TypingPosition+t.ReceiptPosition+t.SendToDevicePosition+t.InvitePosition+t.AccountDataPosition+t.DeviceListPosition+t.NotificationDataPosition+t.PresencePosition == 0
return t == nil || t.PDUPosition+t.TypingPosition+t.ReceiptPosition+t.SendToDevicePosition+t.InvitePosition+t.AccountDataPosition+t.DeviceListPosition+t.NotificationDataPosition+t.PresencePosition+t.MultiRoomDataPosition == 0
}
// WithUpdates returns a copy of the StreamingToken with updates applied from another StreamingToken.
@ -209,6 +213,9 @@ func (t *StreamingToken) ApplyUpdates(other StreamingToken) {
if other.PresencePosition > t.PresencePosition {
t.PresencePosition = other.PresencePosition
}
if other.MultiRoomDataPosition > t.MultiRoomDataPosition {
t.MultiRoomDataPosition = other.MultiRoomDataPosition
}
}
type TopologyToken struct {
@ -294,9 +301,11 @@ func NewTopologyTokenFromString(tok string) (token TopologyToken, err error) {
func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
if len(tok) < 1 {
err = ErrMalformedSyncToken
logrus.WithField("token", tok).Info("invalid stream token: bad length")
return
}
if tok[0] != SyncTokenTypeStream[0] {
logrus.WithField("token", tok).Info("invalid stream token: not starting from s")
err = ErrMalformedSyncToken
return
}
@ -304,7 +313,7 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
// s478_0_0_0_0_13.dl-0-2 but we have now removed partitioned stream positions
tok = strings.Split(tok, ".")[0]
parts := strings.Split(tok[1:], "_")
var positions [9]StreamPosition
var positions [10]StreamPosition
for i, p := range parts {
if i >= len(positions) {
break
@ -312,6 +321,7 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
var pos int
pos, err = strconv.Atoi(p)
if err != nil {
logrus.WithField("token", tok).Info("invalid stream token: strconv")
err = ErrMalformedSyncToken
return
}
@ -327,6 +337,7 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
DeviceListPosition: positions[6],
NotificationDataPosition: positions[7],
PresencePosition: positions[8],
MultiRoomDataPosition: positions[9],
}
return token, nil
}
@ -363,6 +374,7 @@ type Response struct {
ToDevice *ToDeviceResponse `json:"to_device,omitempty"`
DeviceLists *DeviceLists `json:"device_lists,omitempty"`
DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"`
MultiRoom MultiRoom `json:"multiroom,omitempty"`
}
func (r Response) MarshalJSON() ([]byte, error) {
@ -401,7 +413,8 @@ func (r *Response) HasUpdates() bool {
len(r.Rooms.Peek) > 0 ||
len(r.ToDevice.Events) > 0 ||
len(r.DeviceLists.Changed) > 0 ||
len(r.DeviceLists.Left) > 0)
len(r.DeviceLists.Left) > 0) ||
len(r.MultiRoom) > 0
}
// NewResponse creates an empty response with initialised maps.

View file

@ -9,10 +9,10 @@ import (
func TestSyncTokens(t *testing.T) {
shouldPass := map[string]string{
"s4_0_0_0_0_0_0_0_3": StreamingToken{4, 0, 0, 0, 0, 0, 0, 0, 3}.String(),
"s3_1_0_0_0_0_2_0_5": StreamingToken{3, 1, 0, 0, 0, 0, 2, 0, 5}.String(),
"s3_1_2_3_5_0_0_0_6": StreamingToken{3, 1, 2, 3, 5, 0, 0, 0, 6}.String(),
"t3_1": TopologyToken{3, 1}.String(),
"s4_0_0_0_0_0_0_0_3_0": StreamingToken{4, 0, 0, 0, 0, 0, 0, 0, 3, 0}.String(),
"s3_1_0_0_0_0_2_0_5_1": StreamingToken{3, 1, 0, 0, 0, 0, 2, 0, 5, 1}.String(),
"s3_1_2_3_5_0_0_0_6_2": StreamingToken{3, 1, 2, 3, 5, 0, 0, 0, 6, 2}.String(),
"t3_1": TopologyToken{3, 1}.String(),
}
for a, b := range shouldPass {

View file

@ -48,4 +48,19 @@ If a device list update goes missing, the server resyncs on the next one
Leaves are present in non-gapped incremental syncs
# Below test was passing for the wrong reason, failing correctly since #2858
New federated private chats get full presence information (SYN-115)
New federated private chats get full presence information (SYN-115)
If a device list update goes missing, the server resyncs on the next one
# You'll be shocked to discover this is flakey too
Inbound /v1/send_join rejects joins from other servers
# For notifications extension on iOS
/event/ does not allow access to events before the user joined
# Failing after recent updates with presence
Newly joined room includes presence in incremental sync
User sees their own presence in a sync
User is offline if they set_presence=offline in their sync
User sees updates to presence from other users in the incremental sync.

View file

@ -204,7 +204,6 @@ Deleted tags appear in an incremental v2 /sync
/event/ on non world readable room does not work
Outbound federation can query profile data
/event/ on joined room works
/event/ does not allow access to events before the user joined
Federation key API allows unsigned requests for keys
GET /publicRooms lists rooms
GET /publicRooms includes avatar URLs
@ -758,4 +757,4 @@ Can get rooms/{roomId}/members at a given point
Can filter rooms/{roomId}/members
Current state appears in timeline in private history with many messages after
AS can publish rooms in their own list
AS and main public room lists are separate
AS and main public room lists are separate

View file

@ -171,7 +171,6 @@ func PrepareDBConnectionString(t *testing.T, dbType DBType) (connStr string, clo
func WithAllDatabases(t *testing.T, testFn func(t *testing.T, db DBType)) {
dbs := map[string]DBType{
"postgres": DBTypePostgres,
"sqlite": DBTypeSQLite,
}
for dbName, dbType := range dbs {
dbt := dbType

View file

@ -533,7 +533,7 @@ type PerformPusherSetRequest struct {
type PerformPusherDeletionRequest struct {
Localpart string
SessionID int64
SessionID int64 // Pusher corresponding to this SessionID will not be deleted
}
// Pusher represents a push notification subscriber

View file

@ -0,0 +1,6 @@
package api
type MulticastMetadata struct {
ExpireMs int
ExcludeRoomIds []string
}

View file

@ -613,14 +613,25 @@ func (a *UserInternalAPI) PerformAccountDeactivation(ctx context.Context, req *a
return err
}
threepids, err := a.DB.GetThreePIDsForLocalpart(ctx, req.Localpart)
if err != nil {
return err
}
for i := 0; i < len(threepids); i++ {
err = a.DB.RemoveThreePIDAssociation(ctx, threepids[i].Address, threepids[i].Medium)
if err != nil {
return err
}
}
pusherReq := &api.PerformPusherDeletionRequest{
Localpart: req.Localpart,
}
if err := a.PerformPusherDeletion(ctx, pusherReq, &struct{}{}); err != nil {
if err = a.PerformPusherDeletion(ctx, pusherReq, &struct{}{}); err != nil {
return err
}
err := a.DB.DeactivateAccount(ctx, req.Localpart)
err = a.DB.DeactivateAccount(ctx, req.Localpart)
res.AccountDeactivated = err == nil
return err
}

View file

@ -98,6 +98,11 @@ func NewPostgresAccountsTable(db *sql.DB, serverName gomatrixserverlib.ServerNam
Up: deltas.UpAddAccountType,
Down: deltas.DownAddAccountType,
},
{
Version: "userapi: no guests",
Up: deltas.UpNoGuests,
Down: deltas.DownNoGuests,
},
}...)
err = m.Up(context.Background())
if err != nil {

View file

@ -0,0 +1,20 @@
package deltas
import (
"context"
"database/sql"
"fmt"
)
func UpNoGuests(ctx context.Context, tx *sql.Tx) error {
// AddAccountType introduced a bug where each user that had was registered as a regular user, but without user_id, became a guest.
_, err := tx.ExecContext(ctx, "UPDATE userapi_accounts SET account_type = 1 WHERE account_type = 2;")
if err != nil {
return fmt.Errorf("failed to execute upgrade: %w", err)
}
return nil
}
func DownNoGuests(ctx context.Context, tx *sql.Tx) error {
return nil
}

View file

@ -72,7 +72,7 @@ const selectNotificationSQL = "" +
") AND NOT read ORDER BY localpart, id LIMIT $4"
const selectNotificationCountSQL = "" +
"SELECT COUNT(*) FROM userapi_notifications WHERE localpart = $1 AND (" +
"SELECT COUNT(DISTINCT(room_id)) FROM userapi_notifications WHERE localpart = $1 AND (" +
"(($2 & 1) <> 0 AND highlight) OR (($2 & 2) <> 0 AND NOT highlight)" +
") AND NOT read"

View file

@ -565,7 +565,7 @@ func (d *Database) CreateDevice(
ctx context.Context, localpart string, deviceID *string, accessToken string,
displayName *string, ipAddr, userAgent string,
) (dev *api.Device, returnErr error) {
if deviceID != nil {
if deviceID != nil && *deviceID != "" {
returnErr = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
var err error
// Revoke existing tokens for this device

View file

@ -72,7 +72,7 @@ const selectNotificationSQL = "" +
") AND NOT read ORDER BY localpart, id LIMIT $4"
const selectNotificationCountSQL = "" +
"SELECT COUNT(*) FROM userapi_notifications WHERE localpart = $1 AND (" +
"SELECT COUNT(DISTINCT(room_id)) FROM userapi_notifications WHERE localpart = $1 AND (" +
"(($2 & 1) <> 0 AND highlight) OR (($2 & 2) <> 0 AND NOT highlight)" +
") AND NOT read"

View file

@ -128,6 +128,11 @@ func Test_Accounts(t *testing.T) {
_, err = db.GetAccountByPassword(ctx, aliceLocalpart, "newPassword")
assert.Error(t, err, "expected an error, got none")
// This should return an empty slice, as the account is deactivated and the 3pid is unbound
threepids, err := db.GetThreePIDsForLocalpart(ctx, aliceLocalpart)
assert.NoError(t, err, "failed to get 3pid for account")
assert.Equal(t, len(threepids), 0)
_, err = db.GetAccountByLocalpart(ctx, "unusename")
assert.Error(t, err, "expected an error for non existent localpart")
@ -533,7 +538,7 @@ func Test_Notification(t *testing.T) {
// get notifications
count, err := db.GetNotificationCount(ctx, aliceLocalpart, tables.AllNotifications)
assert.NoError(t, err, "unable to get notification count")
assert.Equal(t, int64(10), count)
assert.Equal(t, int64(2), count)
notifs, count, err := db.GetNotifications(ctx, aliceLocalpart, 0, 15, tables.AllNotifications)
assert.NoError(t, err, "unable to get notifications")
assert.Equal(t, int64(10), count)