diff --git a/.cloudbuild/dev.yaml b/.cloudbuild/dev.yaml new file mode 100644 index 000000000..0a8919477 --- /dev/null +++ b/.cloudbuild/dev.yaml @@ -0,0 +1,12 @@ +steps: + - name: gcr.io/cloud-builders/docker + args: ['build', '-t', 'gcr.io/$PROJECT_ID/dendrite-monolith:$COMMIT_SHA', '-f', 'build/docker/Dockerfile.monolith', '.'] + - name: gcr.io/cloud-builders/kubectl + args: ['-n', 'dendrite', 'set', 'image', 'deployment/dendrite', 'dendrite=gcr.io/$PROJECT_ID/dendrite-monolith:$COMMIT_SHA'] + env: + - CLOUDSDK_CORE_PROJECT=globekeeper-development + - CLOUDSDK_COMPUTE_ZONE=europe-west2-a + - CLOUDSDK_CONTAINER_CLUSTER=synapse +images: + - gcr.io/$PROJECT_ID/dendrite-monolith:$COMMIT_SHA +timeout: 480s diff --git a/.cloudbuild/prod.yaml b/.cloudbuild/prod.yaml new file mode 100644 index 000000000..145d86752 --- /dev/null +++ b/.cloudbuild/prod.yaml @@ -0,0 +1,12 @@ +steps: + - name: gcr.io/cloud-builders/docker + args: ['build', '-t', 'gcr.io/$PROJECT_ID/dendrite-monolith:$TAG_NAME', '-f', 'build/docker/Dockerfile.monolith', '.'] + - name: gcr.io/cloud-builders/kubectl + args: ['set', 'image', 'deployment/dendrite', 'dendrite=gcr.io/$PROJECT_ID/dendrite-monolith:$TAG_NAME'] + env: + - CLOUDSDK_CORE_PROJECT=globekeeper-production + - CLOUDSDK_COMPUTE_ZONE=europe-west2-a + - CLOUDSDK_CONTAINER_CLUSTER=synapse-production +images: + - gcr.io/$PROJECT_ID/dendrite-monolith:$TAG_NAME +timeout: 480s diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index a37e45e46..ddf3c4943 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -14,51 +14,6 @@ concurrency: cancel-in-progress: true jobs: - wasm: - name: WASM build test - timeout-minutes: 5 - runs-on: ubuntu-latest - if: ${{ false }} # disable for now - steps: - - uses: actions/checkout@v3 - - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: 1.18 - - - uses: actions/cache@v2 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-wasm-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-wasm - - - name: Install Node - uses: actions/setup-node@v2 - with: - node-version: 14 - - - uses: actions/cache@v2 - with: - path: ~/.npm - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- - - - name: Reconfigure Git to use HTTPS auth for repo packages - run: > - git config --global url."https://github.com/".insteadOf - ssh://git@github.com/ - - - name: Install test dependencies - working-directory: ./test/wasm - run: npm ci - - - name: Test - run: ./test-dendritejs.sh # Run golangci-lint lint: @@ -74,7 +29,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v3 - # run go test with different go versions + # run go test with go 1.19 test: timeout-minutes: 5 name: Unit tests (Go ${{ matrix.go }}) @@ -102,7 +57,7 @@ jobs: strategy: fail-fast: false matrix: - go: ["1.18", "1.19"] + go: ["1.19"] steps: - uses: actions/checkout@v3 - name: Setup go @@ -129,7 +84,7 @@ jobs: POSTGRES_PASSWORD: postgres POSTGRES_DB: dendrite - # build Dendrite for linux with different architectures and go versions + # build Dendrite for linux amd64 with go 1.18 build: name: Build for Linux timeout-minutes: 10 @@ -137,9 +92,9 @@ jobs: strategy: fail-fast: false matrix: - go: ["1.18", "1.19"] + go: ["1.19"] goos: ["linux"] - goarch: ["amd64", "386"] + goarch: ["amd64"] steps: - uses: actions/checkout@v3 - name: Setup go @@ -164,43 +119,10 @@ jobs: CGO_CFLAGS: -fno-stack-protector run: go build -trimpath -v -o "bin/" ./cmd/... - # build for Windows 64-bit - build_windows: - name: Build for Windows - timeout-minutes: 10 - runs-on: ubuntu-latest - strategy: - matrix: - go: ["1.18", "1.19"] - goos: ["windows"] - goarch: ["amd64"] - steps: - - uses: actions/checkout@v3 - - name: Setup Go ${{ matrix.go }} - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go }} - - name: Install dependencies - run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc - - uses: actions/cache@v3 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }} - - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - CGO_ENABLED: 1 - CC: "/usr/bin/x86_64-w64-mingw32-gcc" - run: go build -trimpath -v -o "bin/" ./cmd/... - # Dummy step to gate other tests on without repeating the whole list initial-tests-done: name: Initial tests passed - needs: [lint, test, build, build_windows] + needs: [lint, test, build] runs-on: ubuntu-latest if: ${{ !cancelled() }} # Run this even if prior jobs were skipped steps: @@ -209,56 +131,6 @@ jobs: with: jobs: ${{ toJSON(needs) }} - # run database upgrade tests - upgrade_test: - name: Upgrade tests - timeout-minutes: 20 - needs: initial-tests-done - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Setup go - uses: actions/setup-go@v3 - with: - go-version: "1.18" - - uses: actions/cache@v3 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-upgrade - - name: Build upgrade-tests - run: go build ./cmd/dendrite-upgrade-tests - - name: Test upgrade - run: ./dendrite-upgrade-tests --head . - - # run database upgrade tests, skipping over one version - upgrade_test_direct: - name: Upgrade tests from HEAD-2 - timeout-minutes: 20 - needs: initial-tests-done - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Setup go - uses: actions/setup-go@v3 - with: - go-version: "1.18" - - uses: actions/cache@v3 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-upgrade - - name: Build upgrade-tests - run: go build ./cmd/dendrite-upgrade-tests - - name: Test upgrade - run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head . - # run Sytest in different variations sytest: timeout-minutes: 20 @@ -269,18 +141,6 @@ jobs: fail-fast: false matrix: include: - - label: SQLite native - - - label: SQLite Cgo - cgo: 1 - - - label: SQLite native, full HTTP APIs - api: full-http - - - label: SQLite Cgo, full HTTP APIs - api: full-http - cgo: 1 - - label: PostgreSQL postgres: postgres @@ -313,13 +173,12 @@ jobs: run: /src/are-we-synapse-yet.py /logs/results.tap -v continue-on-error: true # not fatal - name: Upload Sytest logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }}) path: | - /logs/results.tap - /logs/**/*.log* + /logs # run Complement complement: @@ -331,18 +190,6 @@ jobs: fail-fast: false matrix: include: - - label: SQLite native - - - label: SQLite Cgo - cgo: 1 - - - label: SQLite native, full HTTP APIs - api: full-http - - - label: SQLite Cgo, full HTTP APIs - api: full-http - cgo: 1 - - label: PostgreSQL postgres: Postgres @@ -390,7 +237,7 @@ jobs: continue fi - (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break + (wget -O - "https://github.com/globekeeper/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break done # Build initial Dendrite image @@ -416,8 +263,6 @@ jobs: needs: [ initial-tests-done, - upgrade_test, - upgrade_test_direct, sytest, complement, ] @@ -428,15 +273,3 @@ jobs: uses: re-actors/alls-green@release/v1 with: jobs: ${{ toJSON(needs) }} - - update-docker-images: - name: Update Docker images - permissions: - packages: write - contents: read - security-events: write # To upload Trivy sarif files - if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main' - needs: [integration-tests-done] - uses: matrix-org/dendrite/.github/workflows/docker.yml@main - secrets: - DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index f0500ccc5..000000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,313 +0,0 @@ -# Based on https://github.com/docker/build-push-action - -name: "Docker" - -on: - release: # A GitHub release was published - types: [published] - workflow_dispatch: # A build was manually requested - workflow_call: # Another pipeline called us - secrets: - DOCKER_TOKEN: - required: true - -env: - DOCKER_NAMESPACE: matrixdotorg - DOCKER_HUB_USER: dendritegithub - GHCR_NAMESPACE: matrix-org - PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7 - -jobs: - monolith: - name: Monolith image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - security-events: write # To upload Trivy sarif files - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Get release tag & build flags - if: github.event_name == 'release' # Only for GitHub releases - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ env.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to GitHub Containers - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build main monolith image - if: github.ref_name == 'main' - id: docker_build_monolith - uses: docker/build-push-action@v3 - with: - cache-from: type=gha - cache-to: type=gha,mode=max - context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} - target: monolith - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} - - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} - format: "sarif" - output: "trivy-results.sarif" - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" - - - name: Build release monolith image - if: github.event_name == 'release' # Only for GitHub releases - id: docker_build_monolith_release - uses: docker/build-push-action@v3 - with: - cache-from: type=gha - cache-to: type=gha,mode=max - context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} - target: monolith - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:latest - ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:latest - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }} - - polylith: - name: Polylith image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - security-events: write # To upload Trivy sarif files - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Get release tag & build flags - if: github.event_name == 'release' # Only for GitHub releases - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ env.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to GitHub Containers - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build main polylith image - if: github.ref_name == 'main' - id: docker_build_polylith - uses: docker/build-push-action@v3 - with: - cache-from: type=gha - cache-to: type=gha,mode=max - context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} - target: polylith - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} - - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} - format: "sarif" - output: "trivy-results.sarif" - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" - - - name: Build release polylith image - if: github.event_name == 'release' # Only for GitHub releases - id: docker_build_polylith_release - uses: docker/build-push-action@v3 - with: - cache-from: type=gha - cache-to: type=gha,mode=max - context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} - target: polylith - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest - ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }} - - demo-pinecone: - name: Pinecone demo image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Get release tag & build flags - if: github.event_name == 'release' # Only for GitHub releases - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ env.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to GitHub Containers - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build main Pinecone demo image - if: github.ref_name == 'main' - id: docker_build_demo_pinecone - uses: docker/build-push-action@v3 - with: - cache-from: type=gha - cache-to: type=gha,mode=max - context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} - file: ./build/docker/Dockerfile.demo-pinecone - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }} - - - name: Build release Pinecone demo image - if: github.event_name == 'release' # Only for GitHub releases - id: docker_build_demo_pinecone_release - uses: docker/build-push-action@v3 - with: - cache-from: type=gha - cache-to: type=gha,mode=max - context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} - file: ./build/docker/Dockerfile.demo-pinecone - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} - - demo-yggdrasil: - name: Yggdrasil demo image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Get release tag & build flags - if: github.event_name == 'release' # Only for GitHub releases - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ env.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Login to GitHub Containers - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build main Yggdrasil demo image - if: github.ref_name == 'main' - id: docker_build_demo_yggdrasil - uses: docker/build-push-action@v3 - with: - cache-from: type=gha - cache-to: type=gha,mode=max - context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} - file: ./build/docker/Dockerfile.demo-yggdrasil - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }} - - - name: Build release Yggdrasil demo image - if: github.event_name == 'release' # Only for GitHub releases - id: docker_build_demo_yggdrasil_release - uses: docker/build-push-action@v3 - with: - cache-from: type=gha - cache-to: type=gha,mode=max - context: . - build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }} - file: ./build/docker/Dockerfile.demo-yggdrasil - platforms: ${{ env.PLATFORMS }} - push: true - tags: | - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest - ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest - ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }} diff --git a/.gitignore b/.gitignore index e4f0112c4..662d3ae97 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ # Hidden files .* +!.vscode +!.cloudbuild # Allow GitHub config !.github @@ -73,3 +75,7 @@ complement/ docs/_site media_store/ + +__debug_bin + +cmd/dendrite-monolith-server/dendrite-monolith-server \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index a327370e1..bb8d38a8b 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -179,7 +179,6 @@ linters-settings: linters: enable: - - deadcode - errcheck - goconst - gocyclo @@ -191,10 +190,8 @@ linters: - misspell # Check code comments, whereas misspell in CI checks *.md files - nakedret - staticcheck - - structcheck - unparam - unused - - varcheck enable-all: false disable: - bodyclose diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..6142a8df0 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + "configurations": [ + { + "name": "Launch Package", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/dendrite-monolith-server", + "args": [ + "-really-enable-open-registration", + "-config", + "../../../adminas/.ci/config/dendrite-local/dendrite.yaml" + ], + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..f9731b7f8 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,9 @@ +{ + "go.lintTool": "golangci-lint", + "go.testEnvVars": { + "POSTGRES_HOST": "localhost", + "POSTGRES_USER": "postgres", + "POSTGRES_PASSWORD": "foobar", + "POSTGRES_DB": "postgres" + } +} \ No newline at end of file diff --git a/build/dendritejs-pinecone/jsServer.go b/build/dendritejs-pinecone/jsServer.go index 4298c2ae9..a2fc39d42 100644 --- a/build/dendritejs-pinecone/jsServer.go +++ b/build/dendritejs-pinecone/jsServer.go @@ -34,13 +34,16 @@ type JSServer struct { // OnRequestFromJS is the function that JS will invoke when there is a new request. // The JS function signature is: -// function(reqString: string): Promise<{result: string, error: string}> +// +// function(reqString: string): Promise<{result: string, error: string}> +// // Usage is like: -// const res = await global._go_js_server.fetch(reqString); -// if (res.error) { -// // handle error: this is a 'network' error, not a non-2xx error. -// } -// const rawHttpResponse = res.result; +// +// const res = await global._go_js_server.fetch(reqString); +// if (res.error) { +// // handle error: this is a 'network' error, not a non-2xx error. +// } +// const rawHttpResponse = res.result; func (h *JSServer) OnRequestFromJS(this js.Value, args []js.Value) interface{} { // we HAVE to spawn a new goroutine and return immediately or else Go will deadlock // if this request blocks at all e.g for /sync calls diff --git a/clientapi/auth/authtypes/logintypes.go b/clientapi/auth/authtypes/logintypes.go index f01e48f80..00253fede 100644 --- a/clientapi/auth/authtypes/logintypes.go +++ b/clientapi/auth/authtypes/logintypes.go @@ -11,4 +11,6 @@ const ( LoginTypeRecaptcha = "m.login.recaptcha" LoginTypeApplicationService = "m.login.application_service" LoginTypeToken = "m.login.token" + LoginTypeJwt = "org.matrix.login.jwt" + LoginTypeEmail = "m.login.email.identity" ) diff --git a/clientapi/auth/login.go b/clientapi/auth/login.go index 5467e814d..5a88295a3 100644 --- a/clientapi/auth/login.go +++ b/clientapi/auth/login.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/clientapi/ratelimit" "github.com/matrix-org/dendrite/setup/config" uapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" @@ -32,7 +33,7 @@ import ( // called after authorization has completed, with the result of the authorization. // If the final return value is non-nil, an error occurred and the cleanup function // is nil. -func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserLoginAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) { +func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.ClientUserAPI, cfg *config.ClientAPI, rt *ratelimit.RtFailedLogin) (*Login, LoginCleanupFunc, *util.JSONResponse) { reqBytes, err := io.ReadAll(r) if err != nil { err := &util.JSONResponse{ @@ -43,7 +44,8 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U } var header struct { - Type string `json:"type"` + Type string `json:"type"` + InhibitDevice bool `json:"inhibit_device"` } if err := json.Unmarshal(reqBytes, &header); err != nil { err := &util.JSONResponse{ @@ -57,14 +59,20 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U switch header.Type { case authtypes.LoginTypePassword: typ = &LoginTypePassword{ - GetAccountByPassword: useraccountAPI.QueryAccountByPassword, - Config: cfg, + UserApi: useraccountAPI, + Config: cfg, + Rt: rt, + InhibitDevice: header.InhibitDevice, } case authtypes.LoginTypeToken: typ = &LoginTypeToken{ - UserAPI: userAPI, + UserAPI: useraccountAPI, Config: cfg, } + case authtypes.LoginTypeJwt: + typ = &LoginTypeTokenJwt{ + Config: cfg, + } default: err := util.JSONResponse{ Code: http.StatusBadRequest, diff --git a/clientapi/auth/login_jwt.go b/clientapi/auth/login_jwt.go new file mode 100644 index 000000000..35c7d1948 --- /dev/null +++ b/clientapi/auth/login_jwt.go @@ -0,0 +1,74 @@ +package auth + +import ( + "context" + "fmt" + "net/http" + + "github.com/golang-jwt/jwt/v4" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/util" +) + +// LoginTypeToken describes how to authenticate with a login token. +type LoginTypeTokenJwt struct { + // UserAPI uapi.LoginTokenInternalAPI + Config *config.ClientAPI +} + +// Name implements Type. +func (t *LoginTypeTokenJwt) Name() string { + return authtypes.LoginTypeJwt +} + +type Claims struct { + jwt.StandardClaims +} + +const mIdUser = "m.id.user" + +// LoginFromJSON implements Type. The cleanup function deletes the token from +// the database on success. +func (t *LoginTypeTokenJwt) LoginFromJSON(ctx context.Context, reqBytes []byte) (*Login, LoginCleanupFunc, *util.JSONResponse) { + var r loginTokenRequest + if err := httputil.UnmarshalJSON(reqBytes, &r); err != nil { + return nil, nil, err + } + + if r.Token == "" { + return nil, nil, &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("Token field for JWT is missing"), + } + } + c := &Claims{} + token, err := jwt.ParseWithClaims(r.Token, c, func(token *jwt.Token) (interface{}, error) { + if _, ok := token.Method.(*jwt.SigningMethodEd25519); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", token.Method.Alg()) + } + return t.Config.JwtConfig.SecretKey, nil + }) + + if err != nil { + util.GetLogger(ctx).WithError(err).Error("jwt.ParseWithClaims failed") + return nil, nil, &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("Couldn't parse JWT"), + } + } + + if !token.Valid { + return nil, nil, &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("Invalid JWT"), + } + } + + r.Login.Identifier.User = c.Subject + r.Login.Identifier.Type = mIdUser + + return &r.Login, func(context.Context, *util.JSONResponse) {}, nil +} diff --git a/clientapi/auth/login_test.go b/clientapi/auth/login_test.go index 5085f0170..4017c26d5 100644 --- a/clientapi/auth/login_test.go +++ b/clientapi/auth/login_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/clientapi/ratelimit" "github.com/matrix-org/dendrite/setup/config" uapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" @@ -68,8 +69,11 @@ func TestLoginFromJSONReader(t *testing.T) { Matrix: &config.Global{ ServerName: serverName, }, + RtFailedLogin: ratelimit.RtFailedLoginConfig{ + Enabled: false, + }, } - login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg) + login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, cfg, nil) if err != nil { t.Fatalf("LoginFromJSONReader failed: %+v", err) } @@ -147,7 +151,7 @@ func TestBadLoginFromJSONReader(t *testing.T) { ServerName: serverName, }, } - _, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg) + _, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, cfg, nil) if errRes == nil { cleanup(ctx, nil) t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode) @@ -159,6 +163,7 @@ func TestBadLoginFromJSONReader(t *testing.T) { } type fakeUserInternalAPI struct { + uapi.ClientUserAPI UserInternalAPIForLogin DeletedTokens []string } diff --git a/clientapi/auth/login_token.go b/clientapi/auth/login_token.go index 845eb5de9..293b9a460 100644 --- a/clientapi/auth/login_token.go +++ b/clientapi/auth/login_token.go @@ -58,7 +58,7 @@ func (t *LoginTypeToken) LoginFromJSON(ctx context.Context, reqBytes []byte) (*L } } - r.Login.Identifier.Type = "m.id.user" + r.Login.Identifier.Type = mIdUser r.Login.Identifier.User = res.Data.UserID cleanup := func(ctx context.Context, authRes *util.JSONResponse) { diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index 700a72f5d..a13520899 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/clientapi/ratelimit" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/api" @@ -33,12 +34,18 @@ type GetAccountByPassword func(ctx context.Context, req *api.QueryAccountByPassw type PasswordRequest struct { Login Password string `json:"password"` + Address string `json:"address"` + Medium string `json:"medium"` } +const email = "email" + // LoginTypePassword implements https://matrix.org/docs/spec/client_server/r0.6.1#password-based type LoginTypePassword struct { - GetAccountByPassword GetAccountByPassword - Config *config.ClientAPI + UserApi api.ClientUserAPI + Config *config.ClientAPI + Rt *ratelimit.RtFailedLogin + InhibitDevice bool } func (t *LoginTypePassword) Name() string { @@ -55,13 +62,42 @@ func (t *LoginTypePassword) LoginFromJSON(ctx context.Context, reqBytes []byte) if err != nil { return nil, nil, err } + login.InhibitDevice = t.InhibitDevice return login, func(context.Context, *util.JSONResponse) {}, nil } func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) { r := req.(*PasswordRequest) - username := strings.ToLower(r.Username()) + if r.Identifier.Address != "" { + r.Address = r.Identifier.Address + } + if r.Identifier.Medium != "" { + r.Medium = r.Identifier.Medium + } + var username string + if r.Medium == email && r.Address != "" { + r.Address = strings.ToLower(r.Address) + res := api.QueryLocalpartForThreePIDResponse{} + err := t.UserApi.QueryLocalpartForThreePID(ctx, &api.QueryLocalpartForThreePIDRequest{ + ThreePID: r.Address, + Medium: email, + }, &res) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("userApi.QueryLocalpartForThreePID failed") + resp := jsonerror.InternalServerError() + return nil, &resp + } + username = res.Localpart + if username == "" { + return nil, &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: jsonerror.Forbidden("Invalid username or password"), + } + } + } else { + username = strings.ToLower(r.Username()) + } if username == "" { return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, @@ -83,7 +119,17 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, } // Squash username to all lowercase letters res := &api.QueryAccountByPasswordResponse{} - err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{Localpart: strings.ToLower(localpart), PlaintextPassword: r.Password}, res) + localpart = strings.ToLower(localpart) + if t.Rt != nil { + ok, retryIn := t.Rt.CanAct(localpart) + if !ok { + return nil, &util.JSONResponse{ + Code: http.StatusTooManyRequests, + JSON: jsonerror.LimitExceeded("Too Many Requests", retryIn.Milliseconds()), + } + } + } + err = t.UserApi.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{Localpart: localpart, PlaintextPassword: r.Password}, res) if err != nil { return nil, &util.JSONResponse{ Code: http.StatusInternalServerError, @@ -92,7 +138,7 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, } if !res.Exists { - err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{ + err = t.UserApi.QueryAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{ Localpart: localpart, PlaintextPassword: r.Password, }, res) @@ -105,11 +151,15 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, // Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows // but that would leak the existence of the user. if !res.Exists { + if t.Rt != nil { + t.Rt.Act(localpart) + } return nil, &util.JSONResponse{ Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("The username or password was incorrect or the account does not exist."), + JSON: jsonerror.Forbidden("Invalid username or password"), } } } + r.Login.User = username return &r.Login, nil } diff --git a/clientapi/auth/user_interactive.go b/clientapi/auth/user_interactive.go index 9971bf8a4..68921e1c7 100644 --- a/clientapi/auth/user_interactive.go +++ b/clientapi/auth/user_interactive.go @@ -66,6 +66,7 @@ type LoginIdentifier struct { type Login struct { LoginIdentifier // Flat fields deprecated in favour of `identifier`. Identifier LoginIdentifier `json:"identifier"` + InhibitDevice bool `json:"inhibit_device,omitempty"` // Both DeviceID and InitialDisplayName can be omitted, or empty strings ("") // Thus a pointer is needed to differentiate between the two @@ -75,7 +76,7 @@ type Login struct { // Username returns the user localpart/user_id in this request, if it exists. func (r *Login) Username() string { - if r.Identifier.Type == "m.id.user" { + if r.Identifier.Type == mIdUser { return r.Identifier.User } // deprecated but without it Element iOS won't log in @@ -88,8 +89,8 @@ func (r *Login) ThirdPartyID() (medium, address string) { return r.Identifier.Medium, r.Identifier.Address } // deprecated - if r.Medium == "email" { - return "email", r.Address + if r.Medium == email { + return email, r.Address } return "", "" } @@ -111,10 +112,10 @@ type UserInteractive struct { Sessions map[string][]string } -func NewUserInteractive(userAccountAPI api.UserLoginAPI, cfg *config.ClientAPI) *UserInteractive { +func NewUserInteractive(userAccountAPI api.ClientUserAPI, cfg *config.ClientAPI) *UserInteractive { typePassword := &LoginTypePassword{ - GetAccountByPassword: userAccountAPI.QueryAccountByPassword, - Config: cfg, + UserApi: userAccountAPI, + Config: cfg, } return &UserInteractive{ Flows: []userInteractiveFlow{ diff --git a/clientapi/auth/user_interactive_test.go b/clientapi/auth/user_interactive_test.go index 001b1a6d4..8267d2222 100644 --- a/clientapi/auth/user_interactive_test.go +++ b/clientapi/auth/user_interactive_test.go @@ -24,7 +24,9 @@ var ( } ) -type fakeAccountDatabase struct{} +type fakeAccountDatabase struct { + api.ClientUserAPI +} func (d *fakeAccountDatabase) PerformPasswordUpdate(ctx context.Context, req *api.PerformPasswordUpdateRequest, res *api.PerformPasswordUpdateResponse) error { return nil diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 080d4d9fa..bcaae0c3e 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -52,6 +52,7 @@ func AddPublicRoutes( TopicSendToDeviceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), TopicTypingEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), TopicPresenceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent), + TopicMultiRoomCast: cfg.Matrix.JetStream.Prefixed(jetstream.OutputMultiRoomCast), UserAPI: userAPI, ServerName: cfg.Matrix.ServerName, } diff --git a/clientapi/producers/syncapi.go b/clientapi/producers/syncapi.go index 2dc0c4843..d0ea08418 100644 --- a/clientapi/producers/syncapi.go +++ b/clientapi/producers/syncapi.go @@ -36,6 +36,7 @@ type SyncAPIProducer struct { TopicSendToDeviceEvent string TopicTypingEvent string TopicPresenceEvent string + TopicMultiRoomCast string JetStream nats.JetStreamContext ServerName gomatrixserverlib.ServerName UserAPI userapi.ClientUserAPI @@ -159,3 +160,14 @@ func (p *SyncAPIProducer) SendPresence( _, err := p.JetStream.PublishMsg(m, nats.Context(ctx)) return err } + +func (p *SyncAPIProducer) SendMultiroom( + ctx context.Context, userID string, dataType string, message []byte, +) error { + m := nats.NewMsg(p.TopicMultiRoomCast) + m.Header.Set(jetstream.UserID, userID) + m.Header.Set("type", dataType) + m.Data = message + _, err := p.JetStream.PublishMsg(m, nats.Context(ctx)) + return err +} diff --git a/clientapi/ratelimit/rt_failed_login.go b/clientapi/ratelimit/rt_failed_login.go new file mode 100644 index 000000000..291af581d --- /dev/null +++ b/clientapi/ratelimit/rt_failed_login.go @@ -0,0 +1,117 @@ +package ratelimit + +import ( + "container/list" + "sync" + "time" +) + +type rateLimit struct { + cfg *RtFailedLoginConfig + times *list.List +} + +type RtFailedLogin struct { + cfg *RtFailedLoginConfig + mtx sync.RWMutex + rts map[string]*rateLimit +} + +type RtFailedLoginConfig struct { + Enabled bool `yaml:"enabled"` + Limit int `yaml:"burst"` + Interval time.Duration `yaml:"interval"` +} + +// New creates a new rate limiter for the limit and interval. +func NewRtFailedLogin(cfg *RtFailedLoginConfig) *RtFailedLogin { + if !cfg.Enabled { + return nil + } + rt := &RtFailedLogin{ + cfg: cfg, + mtx: sync.RWMutex{}, + rts: make(map[string]*rateLimit), + } + go rt.clean() + return rt +} + +// CanAct is expected to be called before Act +func (r *RtFailedLogin) CanAct(key string) (ok bool, remaining time.Duration) { + r.mtx.RLock() + rt, ok := r.rts[key] + if !ok { + r.mtx.RUnlock() + return true, 0 + } + ok, remaining = rt.canAct() + r.mtx.RUnlock() + return +} + +// Act can be called after CanAct returns true. +func (r *RtFailedLogin) Act(key string) { + r.mtx.Lock() + rt, ok := r.rts[key] + if !ok { + rt = &rateLimit{ + cfg: r.cfg, + times: list.New(), + } + r.rts[key] = rt + } + rt.act() + r.mtx.Unlock() +} + +func (r *RtFailedLogin) clean() { + for { + r.mtx.Lock() + for k, v := range r.rts { + if v.empty() { + delete(r.rts, k) + } + } + r.mtx.Unlock() + time.Sleep(time.Hour) + } +} + +func (r *rateLimit) empty() bool { + back := r.times.Back() + if back == nil { + return true + } + v := back.Value + b := v.(time.Time) + now := time.Now() + return now.Sub(b) > r.cfg.Interval +} + +func (r *rateLimit) canAct() (ok bool, remaining time.Duration) { + now := time.Now() + l := r.times.Len() + if l < r.cfg.Limit { + return true, 0 + } + frnt := r.times.Front() + t := frnt.Value.(time.Time) + diff := now.Sub(t) + if diff < r.cfg.Interval { + return false, r.cfg.Interval - diff + } + return true, 0 +} + +func (r *rateLimit) act() { + now := time.Now() + l := r.times.Len() + if l < r.cfg.Limit { + r.times.PushBack(now) + return + } + frnt := r.times.Front() + frnt.Value = now + r.times.MoveToBack(frnt) +} diff --git a/clientapi/ratelimit/rt_failed_login_test.go b/clientapi/ratelimit/rt_failed_login_test.go new file mode 100644 index 000000000..5281bc765 --- /dev/null +++ b/clientapi/ratelimit/rt_failed_login_test.go @@ -0,0 +1,40 @@ +package ratelimit + +import ( + "testing" + "time" + + "github.com/matryer/is" +) + +func TestRtFailedLogin(t *testing.T) { + is := is.New(t) + rtfl := NewRtFailedLogin(&RtFailedLoginConfig{ + Enabled: true, + Limit: 3, + Interval: 10 * time.Millisecond, + }) + var ( + can bool + remaining time.Duration + remainingB time.Duration + ) + for i := 0; i < 3; i++ { + can, remaining = rtfl.CanAct("foo") + is.True(can) + is.Equal(remaining, time.Duration(0)) + rtfl.Act("foo") + } + can, remaining = rtfl.CanAct("foo") + is.True(!can) + is.True(remaining > time.Millisecond*9) + can, remainingB = rtfl.CanAct("bar") + is.True(can) + is.Equal(remainingB, time.Duration(0)) + rtfl.Act("bar") + rtfl.Act("bar") + time.Sleep(remaining + time.Millisecond) + can, remaining = rtfl.CanAct("foo") + is.True(can) + is.Equal(remaining, time.Duration(0)) +} diff --git a/clientapi/routing/deactivate.go b/clientapi/routing/deactivate.go index f213db7f3..9640b7f59 100644 --- a/clientapi/routing/deactivate.go +++ b/clientapi/routing/deactivate.go @@ -27,13 +27,17 @@ func Deactivate( JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()), } } - - login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, deviceAPI) - if errRes != nil { - return *errRes + var userId string + if deviceAPI.AccountType != api.AccountTypeAppService { + login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, deviceAPI) + if errRes != nil { + return *errRes + } + userId = login.Username() + } else { + userId = deviceAPI.UserID } - - localpart, _, err := gomatrixserverlib.SplitID('@', login.Username()) + localpart, _, err := gomatrixserverlib.SplitID('@', userId) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") return jsonerror.InternalServerError() diff --git a/clientapi/routing/key_crosssigning.go b/clientapi/routing/key_crosssigning.go index 2570db09c..ca6ecefd2 100644 --- a/clientapi/routing/key_crosssigning.go +++ b/clientapi/routing/key_crosssigning.go @@ -63,8 +63,8 @@ func UploadCrossSigningDeviceKeys( } } typePassword := auth.LoginTypePassword{ - GetAccountByPassword: accountAPI.QueryAccountByPassword, - Config: cfg, + UserApi: accountAPI, + Config: cfg, } if _, authErr := typePassword.Login(req.Context(), &uploadReq.Auth.PasswordRequest); authErr != nil { return *authErr diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index 7f5a8c4f8..e33bb3f96 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -20,6 +20,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/clientapi/ratelimit" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -55,6 +56,7 @@ func passwordLogin() flows { func Login( req *http.Request, userAPI userapi.ClientUserAPI, cfg *config.ClientAPI, + rt *ratelimit.RtFailedLogin, ) util.JSONResponse { if req.Method == http.MethodGet { // TODO: support other forms of login other than password, depending on config options @@ -63,10 +65,21 @@ func Login( JSON: passwordLogin(), } } else if req.Method == http.MethodPost { - login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, userAPI, userAPI, cfg) + login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, userAPI, cfg, rt) if authErr != nil { return *authErr } + if login.InhibitDevice { + return util.JSONResponse{ + Code: http.StatusOK, + JSON: loginResponse{ + UserID: userutil.MakeUserID(login.Username(), cfg.Matrix.ServerName), + AccessToken: "", + HomeServer: cfg.Matrix.ServerName, + DeviceID: "", + }, + } + } // make a device/access token authErr2 := completeAuth(req.Context(), cfg.Matrix, userAPI, login, req.RemoteAddr, req.UserAgent()) cleanup(req.Context(), &authErr2) diff --git a/clientapi/routing/multiroom.go b/clientapi/routing/multiroom.go new file mode 100644 index 000000000..14d3c29b6 --- /dev/null +++ b/clientapi/routing/multiroom.go @@ -0,0 +1,48 @@ +package routing + +import ( + "io" + "net/http" + + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + log "github.com/sirupsen/logrus" +) + +func PostMultiroom( + req *http.Request, + device *api.Device, + producer *producers.SyncAPIProducer, + dataType string, +) util.JSONResponse { + b, err := io.ReadAll(req.Body) + if err != nil { + log.WithError(err).Errorf("failed to read request body") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.InternalServerError(), + } + } + canonicalB, err := gomatrixserverlib.CanonicalJSON(b) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("The request body is not valid canonical JSON." + err.Error()), + } + } + err = producer.SendMultiroom(req.Context(), device.UserID, dataType, canonicalB) + if err != nil { + log.WithError(err).Errorf("failed to send multiroomcast") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.InternalServerError(), + } + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} diff --git a/clientapi/routing/password.go b/clientapi/routing/password.go index 6dc9af508..44ca153f2 100644 --- a/clientapi/routing/password.go +++ b/clientapi/routing/password.go @@ -1,12 +1,14 @@ package routing import ( + "fmt" "net/http" "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/clientapi/threepid" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -24,6 +26,7 @@ type newPasswordAuth struct { Type string `json:"type"` Session string `json:"session"` auth.PasswordRequest + ThreePidCreds threepid.Credentials `json:"threepid_creds"` } func Password( @@ -33,13 +36,17 @@ func Password( cfg *config.ClientAPI, ) util.JSONResponse { // Check that the existing password is right. + var fields logrus.Fields + if device != nil { + fields = logrus.Fields{ + "sessionId": device.SessionID, + "userId": device.UserID, + } + } var r newPasswordRequest r.LogoutDevices = true - logrus.WithFields(logrus.Fields{ - "sessionId": device.SessionID, - "userId": device.UserID, - }).Debug("Changing password") + logrus.WithFields(fields).Debug("Changing password") // Unmarshal the request. resErr := httputil.UnmarshalJSONRequest(req, &r) @@ -53,45 +60,95 @@ func Password( // Generate a new, random session ID sessionID = util.RandomString(sessionIDLength) } - - // Require password auth to change the password. - if r.Auth.Type != authtypes.LoginTypePassword { - return util.JSONResponse{ - Code: http.StatusUnauthorized, - JSON: newUserInteractiveResponse( - sessionID, - []authtypes.Flow{ - { - Stages: []authtypes.LoginType{authtypes.LoginTypePassword}, - }, + var localpart string + switch r.Auth.Type { + case authtypes.LoginTypePassword: + // Check if the existing password is correct. + typePassword := auth.LoginTypePassword{ + UserApi: userAPI, + Config: cfg, + } + if _, authErr := typePassword.Login(req.Context(), &r.Auth.PasswordRequest); authErr != nil { + return *authErr + } + // Get the local part. + var err error + localpart, _, err = gomatrixserverlib.SplitID('@', device.UserID) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() + } + sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypePassword) + case authtypes.LoginTypeEmail: + threePid := &authtypes.ThreePID{} + r.Auth.ThreePidCreds.IDServer = cfg.ThreePidDelegate + var ( + bound bool + err error + ) + bound, threePid.Address, threePid.Medium, err = threepid.CheckAssociation(req.Context(), r.Auth.ThreePidCreds, cfg) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAssociation failed") + return jsonerror.InternalServerError() + } + if !bound { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MatrixError{ + ErrCode: "M_THREEPID_AUTH_FAILED", + Err: "Failed to auth 3pid", }, - nil, - ), + } + } + var res api.QueryLocalpartForThreePIDResponse + err = userAPI.QueryLocalpartForThreePID(req.Context(), &api.QueryLocalpartForThreePIDRequest{ + Medium: threePid.Medium, + ThreePID: threePid.Address, + }, &res) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryLocalpartForThreePID failed") + return jsonerror.InternalServerError() + } + if res.Localpart == "" { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MatrixError{ + ErrCode: "M_THREEPID_NOT_FOUND", + Err: "3pid is not bound to any account", + }, + } + } + localpart = res.Localpart + sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeEmail) + default: + flows := []authtypes.Flow{ + { + Stages: []authtypes.LoginType{authtypes.LoginTypePassword}, + }, + } + if cfg.ThreePidDelegate != "" { + flows = append(flows, authtypes.Flow{ + Stages: []authtypes.LoginType{authtypes.LoginTypeEmail}, + }) + } + // Require password auth to change the password. + if r.Auth.Type == authtypes.LoginTypePassword { + return util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: newUserInteractiveResponse( + sessionID, + flows, + nil, + ), + } } } - // Check if the existing password is correct. - typePassword := auth.LoginTypePassword{ - GetAccountByPassword: userAPI.QueryAccountByPassword, - Config: cfg, - } - if _, authErr := typePassword.Login(req.Context(), &r.Auth.PasswordRequest); authErr != nil { - return *authErr - } - sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypePassword) - // Check the new password strength. if resErr = validatePassword(r.NewPassword); resErr != nil { return *resErr } - // Get the local part. - localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") - return jsonerror.InternalServerError() - } - // Ask the user API to perform the password change. passwordReq := &api.PerformPasswordUpdateRequest{ Localpart: localpart, @@ -109,11 +166,23 @@ func Password( // If the request asks us to log out all other devices then // ask the user API to do that. + if r.LogoutDevices { - logoutReq := &api.PerformDeviceDeletionRequest{ - UserID: device.UserID, - DeviceIDs: nil, - ExceptDeviceID: device.ID, + var logoutReq *api.PerformDeviceDeletionRequest + var sessionId int64 + if device == nil { + logoutReq = &api.PerformDeviceDeletionRequest{ + UserID: fmt.Sprintf("@%s:%s", localpart, cfg.Matrix.ServerName), + DeviceIDs: []string{}, + } + sessionId = 0 + } else { + logoutReq = &api.PerformDeviceDeletionRequest{ + UserID: device.UserID, + DeviceIDs: nil, + ExceptDeviceID: device.ID, + } + sessionId = device.SessionID } logoutRes := &api.PerformDeviceDeletionResponse{} if err := userAPI.PerformDeviceDeletion(req.Context(), logoutReq, logoutRes); err != nil { @@ -123,7 +192,7 @@ func Password( pushersReq := &api.PerformPusherDeletionRequest{ Localpart: localpart, - SessionID: device.SessionID, + SessionID: sessionId, } if err := userAPI.PerformPusherDeletion(req.Context(), pushersReq, &struct{}{}); err != nil { util.GetLogger(req.Context()).WithError(err).Error("PerformPusherDeletion failed") diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 4d9e1f8a5..20ed5ede0 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -106,12 +106,6 @@ func SetAvatarURL( if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil { return *resErr } - if r.AvatarURL == "" { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("'avatar_url' must be supplied."), - } - } localpart, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { diff --git a/clientapi/routing/pusher.go b/clientapi/routing/pusher.go index d6a6eb936..48d319ebd 100644 --- a/clientapi/routing/pusher.go +++ b/clientapi/routing/pusher.go @@ -86,8 +86,8 @@ func SetPusher( if err != nil { return invalidParam("malformed url passed") } - if pushUrl.Scheme != "https" { - return invalidParam("only https scheme is allowed") + if pushUrl.Scheme != "https" && pushUrl.Scheme != "http" { + return invalidParam("only https and http schemes are allowed") } } diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index b9ebb0518..a96942999 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -45,6 +45,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/clientapi/threepid" "github.com/matrix-org/dendrite/clientapi/userutil" userapi "github.com/matrix-org/dendrite/userapi/api" ) @@ -238,6 +239,7 @@ type authDict struct { // Recaptcha Response string `json:"response"` // TODO: Lots of custom keys depending on the type + ThreePidCreds threepid.Credentials `json:"threepid_creds"` } // http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#user-interactive-authentication-api @@ -747,6 +749,7 @@ func handleRegistrationFlow( } } + var threePid *authtypes.ThreePID switch r.Auth.Type { case authtypes.LoginTypeRecaptcha: // Check given captcha response @@ -763,6 +766,29 @@ func handleRegistrationFlow( // Add Dummy to the list of completed registration stages sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeDummy) + case authtypes.LoginTypeEmail: + threePid = &authtypes.ThreePID{} + r.Auth.ThreePidCreds.IDServer = cfg.ThreePidDelegate + var ( + bound bool + err error + ) + bound, threePid.Address, threePid.Medium, err = threepid.CheckAssociation(req.Context(), r.Auth.ThreePidCreds, cfg) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAssociation failed") + return jsonerror.InternalServerError() + } + if !bound { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MatrixError{ + ErrCode: "M_THREEPID_AUTH_FAILED", + Err: "Failed to auth 3pid", + }, + } + } + sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeEmail) + case "": // An empty auth type means that we want to fetch the available // flows. It can also mean that we want to register as an appservice @@ -778,7 +804,7 @@ func handleRegistrationFlow( // A response with current registration flow and remaining available methods // will be returned if a flow has not been successfully completed yet return checkAndCompleteFlow(sessions.getCompletedStages(sessionID), - req, r, sessionID, cfg, userAPI) + req, r, sessionID, cfg, userAPI, threePid) } // handleApplicationServiceRegistration handles the registration of an @@ -820,7 +846,7 @@ func handleApplicationServiceRegistration( // application service registration is entirely separate. return completeRegistration( req.Context(), userAPI, r.Username, "", appserviceID, req.RemoteAddr, req.UserAgent(), r.Auth.Session, - r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeAppService, + r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeAppService, nil, ) } @@ -834,12 +860,13 @@ func checkAndCompleteFlow( sessionID string, cfg *config.ClientAPI, userAPI userapi.ClientUserAPI, + threePid *authtypes.ThreePID, ) util.JSONResponse { if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) { // This flow was completed, registration can continue return completeRegistration( req.Context(), userAPI, r.Username, r.Password, "", req.RemoteAddr, req.UserAgent(), sessionID, - r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeUser, + r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeUser, threePid, ) } sessions.addParams(sessionID, r) @@ -865,6 +892,7 @@ func completeRegistration( inhibitLogin eventutil.WeakBoolean, displayName, deviceID *string, accType userapi.AccountType, + threePid *authtypes.ThreePID, ) util.JSONResponse { if username == "" { return util.JSONResponse{ @@ -903,6 +931,21 @@ func completeRegistration( // Increment prometheus counter for created users amtRegUsers.Inc() + // TODO-entry refuse register if threepid is already bound to account. + if threePid != nil { + err = userAPI.PerformSaveThreePIDAssociation(ctx, &userapi.PerformSaveThreePIDAssociationRequest{ + Medium: threePid.Medium, + ThreePID: threePid.Address, + Localpart: accRes.Account.Localpart, + }, &struct{}{}) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.Unknown("Failed to save 3PID association: " + err.Error()), + } + } + } + // Check whether inhibit_login option is set. If so, don't create an access // token or a device for this user if inhibitLogin { @@ -1094,5 +1137,5 @@ func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.Clien if ssrr.Admin { accType = userapi.AccountTypeAdmin } - return completeRegistration(req.Context(), userAPI, ssrr.User, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType) + return completeRegistration(req.Context(), userAPI, ssrr.User, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType, nil) } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index f35aa7e12..12622f293 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -32,6 +32,7 @@ import ( clientutil "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/dendrite/clientapi/ratelimit" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/transactions" @@ -66,6 +67,7 @@ func Setup( prometheus.MustRegister(amtRegUsers, sendEventDuration) rateLimits := httputil.NewRateLimits(&cfg.RateLimiting) + rateLimitsFailedLogin := ratelimit.NewRtFailedLogin(&cfg.RtFailedLogin) userInteractiveAuth := auth.NewUserInteractive(userAPI, cfg) unstableFeatures := map[string]bool{ @@ -434,6 +436,17 @@ func Setup( }), ).Methods(http.MethodPut, http.MethodOptions) + v3mux.Handle("/multiroom/{dataType}", + httputil.MakeAuthAPI("send_multiroom", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + dataType := vars["dataType"] + return PostMultiroom(req, device, syncProducer, dataType) + }), + ).Methods(http.MethodPost, http.MethodOptions) + v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { if r := rateLimits.Limit(req, nil); r != nil { return *r @@ -602,7 +615,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) v3mux.Handle("/account/password", - httputil.MakeAuthAPI("password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + httputil.MakeConditionalAuthAPI("password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { if r := rateLimits.Limit(req, device); r != nil { return *r } @@ -626,7 +639,7 @@ func Setup( if r := rateLimits.Limit(req, nil); r != nil { return *r } - return Login(req, userAPI, cfg) + return Login(req, userAPI, cfg, rateLimitsFailedLogin) }), ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go index 12984c39a..a687211b6 100644 --- a/clientapi/routing/state.go +++ b/clientapi/routing/state.go @@ -101,7 +101,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a } // If the user has never been in the room then stop at this point. // We won't tell the user about a room they have never joined. - if !membershipRes.HasBeenInRoom { + if !membershipRes.HasBeenInRoom && membershipRes.Membership != gomatrixserverlib.Invite { return util.JSONResponse{ Code: http.StatusForbidden, JSON: jsonerror.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)), @@ -241,7 +241,7 @@ func OnIncomingStateTypeRequest( } // If the user has never been in the room then stop at this point. // We won't tell the user about a room they have never joined. - if !membershipRes.HasBeenInRoom || membershipRes.Membership == gomatrixserverlib.Ban { + if !membershipRes.HasBeenInRoom && membershipRes.Membership != gomatrixserverlib.Invite || membershipRes.Membership == gomatrixserverlib.Ban { return util.JSONResponse{ Code: http.StatusForbidden, JSON: jsonerror.Forbidden(fmt.Sprintf("Unknown room %q or user %q has never joined this room", roomID, device.UserID)), diff --git a/clientapi/threepid/threepid.go b/clientapi/threepid/threepid.go index 1e64e3034..a6a469670 100644 --- a/clientapi/threepid/threepid.go +++ b/clientapi/threepid/threepid.go @@ -103,11 +103,8 @@ func CreateSession( func CheckAssociation( ctx context.Context, creds Credentials, cfg *config.ClientAPI, ) (bool, string, string, error) { - if err := isTrusted(creds.IDServer, cfg); err != nil { - return false, "", "", err - } - requestURL := fmt.Sprintf("https://%s/_matrix/identity/api/v1/3pid/getValidated3pid?sid=%s&client_secret=%s", creds.IDServer, creds.SID, creds.Secret) + requestURL := fmt.Sprintf("%s/_matrix/identity/api/v1/3pid/getValidated3pid?sid=%s&client_secret=%s", cfg.ThreePidDelegate, creds.SID, creds.Secret) req, err := http.NewRequest(http.MethodGet, requestURL, nil) if err != nil { return false, "", "", err diff --git a/cmd/dendrite-monolith-server/Dockerfile.dev b/cmd/dendrite-monolith-server/Dockerfile.dev new file mode 100644 index 000000000..7fbf6c667 --- /dev/null +++ b/cmd/dendrite-monolith-server/Dockerfile.dev @@ -0,0 +1,8 @@ +FROM alpine:latest + +COPY dendrite-monolith-server /usr/bin/ + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] diff --git a/cmd/dendrite-monolith-server/build_dev.sh b/cmd/dendrite-monolith-server/build_dev.sh new file mode 100755 index 000000000..5d121890a --- /dev/null +++ b/cmd/dendrite-monolith-server/build_dev.sh @@ -0,0 +1,12 @@ +set -xe +if [ -z "$(git status --porcelain)" ]; then + CGO_ENABLED=0 go build . + TAG=$(git rev-parse --short HEAD) + docker build -f Dockerfile.dev -t gcr.io/globekeeper-development/dendrite-monolith:$TAG -t gcr.io/globekeeper-development/dendrite-monolith -t gcr.io/globekeeper-production/dendrite-monolith:$TAG . + docker push gcr.io/globekeeper-development/dendrite-monolith:$TAG + docker push gcr.io/globekeeper-production/dendrite-monolith:$TAG + docker push gcr.io/globekeeper-development/dendrite-monolith +else + echo "Please commit changes" + exit 0 +fi \ No newline at end of file diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index ff980dc1c..62e004474 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -16,6 +16,7 @@ package main import ( "flag" + "log" "os" "github.com/matrix-org/dendrite/appservice" @@ -45,6 +46,16 @@ var ( func main() { cfg := setup.ParseFlags(true) httpAddr := config.HTTPAddress("http://" + *httpBindAddr) + for _, logging := range cfg.Logging { + if logging.Type == "std" { + level, err := logrus.ParseLevel(logging.Level) + if err != nil { + log.Fatal(err) + } + logrus.SetLevel(level) + logrus.SetFormatter(&logrus.JSONFormatter{}) + } + } httpsAddr := config.HTTPAddress("https://" + *httpsBindAddr) httpAPIAddr := httpAddr options := []basepkg.BaseDendriteOptions{} diff --git a/docs/installation/10_optimisation.md b/docs/installation/10_optimisation.md new file mode 100644 index 000000000..c19b7a75e --- /dev/null +++ b/docs/installation/10_optimisation.md @@ -0,0 +1,71 @@ +--- +title: Optimise your installation +parent: Installation +has_toc: true +nav_order: 10 +permalink: /installation/start/optimisation +--- + +# Optimise your installation + +Now that you have Dendrite running, the following tweaks will improve the reliability +and performance of your installation. + +## File descriptor limit + +Most platforms have a limit on how many file descriptors a single process can open. All +connections made by Dendrite consume file descriptors — this includes database connections +and network requests to remote homeservers. When participating in large federated rooms +where Dendrite must talk to many remote servers, it is often very easy to exhaust default +limits which are quite low. + +We currently recommend setting the file descriptor limit to 65535 to avoid such +issues. Dendrite will log immediately after startup if the file descriptor limit is too low: + +``` +level=warning msg="IMPORTANT: Process file descriptor limit is currently 1024, it is recommended to raise the limit for Dendrite to at least 65535 to avoid issues" +``` + +UNIX systems have two limits: a hard limit and a soft limit. You can view the soft limit +by running `ulimit -Sn` and the hard limit with `ulimit -Hn`: + +```bash +$ ulimit -Hn +1048576 + +$ ulimit -Sn +1024 +``` + +Increase the soft limit before starting Dendrite: + +```bash +ulimit -Sn 65535 +``` + +The log line at startup should no longer appear if the limit is sufficient. + +If you are running under a systemd service, you can instead add `LimitNOFILE=65535` option +to the `[Service]` section of your service unit file. + +## DNS caching + +Dendrite has a built-in DNS cache which significantly reduces the load that Dendrite will +place on your DNS resolver. This may also speed up outbound federation. + +Consider enabling the DNS cache by modifying the `global` section of your configuration file: + +```yaml + dns_cache: + enabled: true + cache_size: 4096 + cache_lifetime: 600s +``` + +## Time synchronisation + +Matrix relies heavily on TLS which requires the system time to be correct. If the clock +drifts then you may find that federation no works reliably (or at all) and clients may +struggle to connect to your Dendrite server. + +Ensure that the time is synchronised on your system by enabling NTP sync. diff --git a/go.mod b/go.mod index 7dd9e0b2c..e4d2cbd60 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ require ( github.com/Arceliar/ironwood v0.0.0-20221025225125-45b4281814c2 github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/MFAshby/stdemuxerhook v1.0.0 github.com/Masterminds/semver/v3 v3.1.1 github.com/blevesearch/bleve/v2 v2.3.4 github.com/codeclysm/extract v2.2.0+incompatible @@ -12,6 +11,7 @@ require ( github.com/docker/docker v20.10.19+incompatible github.com/docker/go-connections v0.4.0 github.com/getsentry/sentry-go v0.14.0 + github.com/golang-jwt/jwt/v4 v4.4.1 github.com/gologme/log v1.3.0 github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.0 @@ -55,6 +55,8 @@ require ( nhooyr.io/websocket v1.8.7 ) +require github.com/matryer/is v1.4.0 + require ( github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect @@ -82,7 +84,7 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect + github.com/gogo/protobuf v1.1.1 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/mock v1.6.0 // indirect diff --git a/go.sum b/go.sum index 5d172e86b..1a61ff544 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,6 @@ github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20O github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= -github.com/MFAshby/stdemuxerhook v1.0.0 h1:1XFGzakrsHMv76AeanPDL26NOgwjPl/OUxbGhJthwMc= -github.com/MFAshby/stdemuxerhook v1.0.0/go.mod h1:nLMI9FUf9Hz98n+yAXsTMUR4RZQy28uCTLG1Fzvj/uY= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= @@ -202,9 +200,10 @@ github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= +github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ= +github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= @@ -316,7 +315,6 @@ github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5l github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= @@ -354,6 +352,8 @@ github.com/matrix-org/pinecone v0.0.0-20221103125849-37f2e9b9ba37 h1:CQWFrgH9TJO github.com/matrix-org/pinecone v0.0.0-20221103125849-37f2e9b9ba37/go.mod h1:F3GHppRuHCTDeoOmmgjZMeJdbql91+RSGGsATWfC7oc= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= +github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= +github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -757,11 +757,9 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go index 36dcaf453..039f5aef3 100644 --- a/internal/httputil/httpapi.go +++ b/internal/httputil/httpapi.go @@ -87,6 +87,57 @@ func MakeAuthAPI( return MakeExternalAPI(metricsName, h) } +// MakeConditionalAuthAPI turns a util.JSONRequestHandler function into an http.Handler which authenticates the request. +// It passes nil device if header is not provided. +func MakeConditionalAuthAPI( + metricsName string, userAPI userapi.QueryAcccessTokenAPI, + f func(*http.Request, *userapi.Device) util.JSONResponse, +) http.Handler { + h := func(req *http.Request) util.JSONResponse { + var ( + jsonRes util.JSONResponse + dev *userapi.Device + ) + if _, err := auth.ExtractAccessToken(req); err != nil { + dev = nil + } else { + logger := util.GetLogger(req.Context()) + var err *util.JSONResponse + dev, err = auth.VerifyUserFromRequest(req, userAPI) + if err != nil { + logger.Debugf("VerifyUserFromRequest %s -> HTTP %d", req.RemoteAddr, err.Code) + return *err + } + // add the user ID to the logger + logger = logger.WithField("user_id", dev.UserID) + req = req.WithContext(util.ContextWithLogger(req.Context(), logger)) + } + // add the user to Sentry, if enabled + hub := sentry.GetHubFromContext(req.Context()) + if hub != nil { + hub.Scope().SetTag("user_id", dev.UserID) + hub.Scope().SetTag("device_id", dev.ID) + } + defer func() { + if r := recover(); r != nil { + if hub != nil { + hub.CaptureException(fmt.Errorf("%s panicked", req.URL.Path)) + } + // re-panic to return the 500 + panic(r) + } + }() + jsonRes = f(req, dev) + // do not log 4xx as errors as they are client fails, not server fails + if hub != nil && jsonRes.Code >= 500 { + hub.Scope().SetExtra("response", jsonRes) + hub.CaptureException(fmt.Errorf("%s returned HTTP %d", req.URL.Path, jsonRes.Code)) + } + return jsonRes + } + return MakeExternalAPI(metricsName, h) +} + // MakeAdminAPI is a wrapper around MakeAuthAPI which enforces that the request can only be // completed by a user that is a server administrator. func MakeAdminAPI( diff --git a/internal/log_unix.go b/internal/log_unix.go index 75332af73..5e8dcaad6 100644 --- a/internal/log_unix.go +++ b/internal/log_unix.go @@ -18,10 +18,8 @@ package internal import ( - "io" "log/syslog" - "github.com/MFAshby/stdemuxerhook" "github.com/matrix-org/dendrite/setup/config" "github.com/sirupsen/logrus" lSyslog "github.com/sirupsen/logrus/hooks/syslog" @@ -31,7 +29,6 @@ import ( // If something fails here it means that the logging was improperly configured, // so we just exit with the error func SetupHookLogging(hooks []config.LogrusHook, componentName string) { - stdLogAdded := false for _, hook := range hooks { // Check we received a proper logging level level, err := logrus.ParseLevel(hook.Level) @@ -39,12 +36,6 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) { logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err) } - // Perform a first filter on the logs according to the lowest level of all - // (Eg: If we have hook for info and above, prevent logrus from processing debug logs) - if logrus.GetLevel() < level { - logrus.SetLevel(level) - } - switch hook.Type { case "file": checkFileHookParams(hook.Params) @@ -53,17 +44,10 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) { checkSyslogHookParams(hook.Params) setupSyslogHook(hook, level, componentName) case "std": - setupStdLogHook(level) - stdLogAdded = true default: logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type) } } - if !stdLogAdded { - setupStdLogHook(logrus.InfoLevel) - } - // Hooks are now configured for stdout/err, so throw away the default logger output - logrus.SetOutput(io.Discard) } func checkSyslogHookParams(params map[string]interface{}) { @@ -87,10 +71,6 @@ func checkSyslogHookParams(params map[string]interface{}) { } -func setupStdLogHook(level logrus.Level) { - logrus.AddHook(&logLevelHook{level, stdemuxerhook.New(logrus.StandardLogger())}) -} - func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) { syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, componentName) if err == nil { diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go index f60247cd7..d593170d4 100644 --- a/roomserver/internal/perform/perform_invite.go +++ b/roomserver/internal/perform/perform_invite.go @@ -140,7 +140,7 @@ func (r *Inviter) PerformInvite( var isAlreadyJoined bool if info != nil { - _, isAlreadyJoined, _, err = r.DB.GetMembership(ctx, info.RoomNID, *event.StateKey()) + _, _, isAlreadyJoined, _, err = r.DB.GetMembership(ctx, info.RoomNID, *event.StateKey()) if err != nil { return nil, fmt.Errorf("r.DB.GetMembership: %w", err) } diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 8850e5c46..89f9fcbda 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -32,6 +32,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/internal/helpers" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/roomserver/version" ) @@ -180,11 +181,16 @@ func (r *Queryer) QueryMembershipForUser( } response.RoomExists = true - membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.UserID) + membershipEventNID, membershipState, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.UserID) if err != nil { return err } + if membershipState == tables.MembershipStateInvite { + response.Membership = gomatrixserverlib.Invite + response.IsInRoom = true + } + response.IsRoomForgotten = isRoomforgotten if membershipEventNID == 0 { @@ -327,7 +333,7 @@ func (r *Queryer) QueryMembershipsForRoom( return nil } - membershipEventNID, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.Sender) + membershipEventNID, _, stillInRoom, isRoomforgotten, err := r.DB.GetMembership(ctx, info.RoomNID, request.Sender) if err != nil { return err } @@ -942,7 +948,7 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.Query } // At this point we're happy that we are in the room, so now let's // see if the target user is in the room. - _, isIn, _, err = r.DB.GetMembership(ctx, targetRoomInfo.RoomNID, req.UserID) + _, _, isIn, _, err = r.DB.GetMembership(ctx, targetRoomInfo.RoomNID, req.UserID) if err != nil { continue } diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index c39a8cbba..0a2a13254 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -129,7 +129,7 @@ type Database interface { // in this room, along a boolean set to true if the user is still in this room, // false if not. // Returns an error if there was a problem talking to the database. - GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom, isRoomForgotten bool, err error) + GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, membershipNID tables.MembershipState, stillInRoom, isRoomForgotten bool, err error) // Lookup the membership event numeric IDs for all user that are or have // been members of a given room. Only lookup events of "join" membership if // joinOnly is set to true. diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 4455ec3bf..b6692333f 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -400,14 +400,14 @@ func (d *Database) RemoveRoomAlias(ctx context.Context, alias string) error { }) } -func (d *Database) GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom, isRoomforgotten bool, err error) { +func (d *Database) GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, membershipState tables.MembershipState, stillInRoom, isRoomforgotten bool, err error) { var requestSenderUserNID types.EventStateKeyNID err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { requestSenderUserNID, err = d.assignStateKeyNID(ctx, txn, requestSenderUserID) return err }) if err != nil { - return 0, false, false, fmt.Errorf("d.assignStateKeyNID: %w", err) + return 0, 0, false, false, fmt.Errorf("d.assignStateKeyNID: %w", err) } senderMembershipEventNID, senderMembership, isRoomforgotten, err := @@ -416,12 +416,12 @@ func (d *Database) GetMembership(ctx context.Context, roomNID types.RoomNID, req ) if err == sql.ErrNoRows { // The user has never been a member of that room - return 0, false, false, nil + return 0, 0, false, false, nil } else if err != nil { return } - return senderMembershipEventNID, senderMembership == tables.MembershipStateJoin, isRoomforgotten, nil + return senderMembershipEventNID, senderMembership, senderMembership == tables.MembershipStateJoin, isRoomforgotten, nil } func (d *Database) GetMembershipEventNIDsForRoom( diff --git a/setup/base/base.go b/setup/base/base.go index 2e3a3a195..6bcabef4b 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -137,7 +137,6 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, options ...Base logrus.Fatalf("Failed to start due to configuration errors") } - internal.SetupStdLogging() internal.SetupHookLogging(cfg.Logging, componentName) internal.SetupPprof() diff --git a/setup/config/config.go b/setup/config/config.go index e99852ec9..47fb5e812 100644 --- a/setup/config/config.go +++ b/setup/config/config.go @@ -16,6 +16,7 @@ package config import ( "bytes" + "crypto/x509" "encoding/pem" "fmt" "io" @@ -266,6 +267,15 @@ func loadConfig( return nil, fmt.Errorf("either specify a 'private_key' path or supply both 'public_key' and 'key_id'") } } + if c.ClientAPI.JwtConfig.Enabled { + pubPki, _ := pem.Decode([]byte(c.ClientAPI.JwtConfig.Secret)) + var pub interface{} + pub, err = x509.ParsePKIXPublicKey(pubPki.Bytes) + if err != nil { + return nil, err + } + c.ClientAPI.JwtConfig.SecretKey = pub.(ed25519.PublicKey) + } c.MediaAPI.AbsBasePath = Path(absPath(basePath, c.MediaAPI.BasePath)) @@ -305,7 +315,10 @@ func (config *Dendrite) Derive() error { config.Derived.Registration.Flows = append(config.Derived.Registration.Flows, authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeDummy}}) } - + if config.ClientAPI.ThreePidDelegate != "" { + config.Derived.Registration.Flows = append(config.Derived.Registration.Flows, + authtypes.Flow{Stages: []authtypes.LoginType{authtypes.LoginTypeEmail}}) + } // Load application service configuration files if err := loadAppServices(&config.AppServiceAPI, &config.Derived); err != nil { return err diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go index bd21826fe..706d2dfd2 100644 --- a/setup/config/config_appservice.go +++ b/setup/config/config_appservice.go @@ -21,7 +21,6 @@ import ( "regexp" "strings" - log "github.com/sirupsen/logrus" yaml "gopkg.in/yaml.v2" ) @@ -346,11 +345,11 @@ func checkErrors(config *AppServiceAPI, derived *Derived) (err error) { // TODO: Remove once rate_limited is implemented if appservice.RateLimited { - log.Warn("WARNING: Application service option rate_limited is currently unimplemented") + // log.Warn("WARNING: Application service option rate_limited is currently unimplemented") } // TODO: Remove once protocols is implemented if len(appservice.Protocols) > 0 { - log.Warn("WARNING: Application service option protocols is currently unimplemented") + // log.Warn("WARNING: Application service option protocols is currently unimplemented") } } @@ -376,7 +375,7 @@ func validateNamespace( // Check if GroupID for the users namespace is in the correct format if key == "users" && namespace.GroupID != "" { // TODO: Remove once group_id is implemented - log.Warn("WARNING: Application service option group_id is currently unimplemented") + // log.Warn("WARNING: Application service option group_id is currently unimplemented") correctFormat := groupIDRegexp.MatchString(namespace.GroupID) if !correctFormat { diff --git a/setup/config/config_clientapi.go b/setup/config/config_clientapi.go index 0a871da18..7a24c7e44 100644 --- a/setup/config/config_clientapi.go +++ b/setup/config/config_clientapi.go @@ -3,6 +3,9 @@ package config import ( "fmt" "time" + + "github.com/matrix-org/dendrite/clientapi/ratelimit" + "golang.org/x/crypto/ed25519" ) type ClientAPI struct { @@ -52,9 +55,23 @@ type ClientAPI struct { TURN TURN `yaml:"turn"` // Rate-limiting options - RateLimiting RateLimiting `yaml:"rate_limiting"` + RateLimiting RateLimiting `yaml:"rate_limiting"` + RtFailedLogin ratelimit.RtFailedLoginConfig `yaml:"rate_limiting_failed_login"` MSCs *MSCs `yaml:"-"` + + ThreePidDelegate string `yaml:"three_pid_delegate"` + + JwtConfig JwtConfig `yaml:"jwt_config"` +} + +type JwtConfig struct { + Enabled bool `yaml:"enabled"` + Algorithm string `yaml:"algorithm"` + Issuer string `yaml:"issuer"` + Secret string `yaml:"secret"` + SecretKey ed25519.PublicKey + Audiences []string `yaml:"audiences"` } func (c *ClientAPI) Defaults(opts DefaultOpts) { diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go index af4eb2949..adaeb873d 100644 --- a/setup/jetstream/nats.go +++ b/setup/jetstream/nats.go @@ -8,13 +8,13 @@ import ( "sync" "time" - "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" natsclient "github.com/nats-io/nats.go" ) @@ -36,7 +36,7 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS defer natsLock.Unlock() // check if we need an in-process NATS Server if len(cfg.Addresses) != 0 { - return setupNATS(process, cfg, nil) + return setupNATS(cfg, nil) } if s.Server == nil { var err error @@ -72,13 +72,34 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS if err != nil { logrus.Fatalln("Failed to create NATS client") } - return setupNATS(process, cfg, nc) + return setupNATS(cfg, nc) } -func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) { +func setupNATS(cfg *config.JetStream, nc *natsclient.Conn) (natsclient.JetStreamContext, *natsclient.Conn) { + var s nats.JetStreamContext + var err error if nc == nil { - var err error - opts := []natsclient.Option{} + opts := []natsclient.Option{ + natsclient.DisconnectErrHandler(func(c *natsclient.Conn, err error) { + logrus.WithError(err).Error("nats connection: disconnected") + }), + natsclient.ReconnectHandler(func(_ *natsclient.Conn) { + logrus.Info("nats connection: client reconnected") + for _, stream := range []*nats.StreamConfig{ + streams[6], + streams[10], + } { + err = configureStream(stream, cfg, s) + if err != nil { + logrus.WithError(err).WithField("stream", stream.Name).Error("unable to configure a stream") + } + + } + }), + natsclient.ClosedHandler(func(_ *natsclient.Conn) { + logrus.Info("nats connection: client closed") + }), + } if cfg.DisableTLSValidation { opts = append(opts, natsclient.Secure(&tls.Config{ InsecureSkipVerify: true, @@ -91,89 +112,16 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc } } - s, err := nc.JetStream() + s, err = nc.JetStream() if err != nil { logrus.WithError(err).Panic("Unable to get JetStream context") return nil, nil } for _, stream := range streams { // streams are defined in streams.go - name := cfg.Prefixed(stream.Name) - info, err := s.StreamInfo(name) - if err != nil && err != natsclient.ErrStreamNotFound { - logrus.WithError(err).Fatal("Unable to get stream info") - } - subjects := stream.Subjects - if len(subjects) == 0 { - // By default we want each stream to listen for the subjects - // that are either an exact match for the stream name, or where - // the first part of the subject is the stream name. ">" is a - // wildcard in NATS for one or more subject tokens. In the case - // that the stream is called "Foo", this will match any message - // with the subject "Foo", "Foo.Bar" or "Foo.Bar.Baz" etc. - subjects = []string{name, name + ".>"} - } - if info != nil { - switch { - case !reflect.DeepEqual(info.Config.Subjects, subjects): - fallthrough - case info.Config.Retention != stream.Retention: - fallthrough - case info.Config.Storage != stream.Storage: - if err = s.DeleteStream(name); err != nil { - logrus.WithError(err).Fatal("Unable to delete stream") - } - info = nil - } - } - if info == nil { - // If we're trying to keep everything in memory (e.g. unit tests) - // then overwrite the storage policy. - if cfg.InMemory { - stream.Storage = natsclient.MemoryStorage - } - - // Namespace the streams without modifying the original streams - // array, otherwise we end up with namespaces on namespaces. - namespaced := *stream - namespaced.Name = name - namespaced.Subjects = subjects - if _, err = s.AddStream(&namespaced); err != nil { - logger := logrus.WithError(err).WithFields(logrus.Fields{ - "stream": namespaced.Name, - "subjects": namespaced.Subjects, - }) - - // If the stream was supposed to be in-memory to begin with - // then an error here is fatal so we'll give up. - if namespaced.Storage == natsclient.MemoryStorage { - logger.WithError(err).Fatal("Unable to add in-memory stream") - } - - // The stream was supposed to be on disk. Let's try starting - // Dendrite with the stream in-memory instead. That'll mean that - // we can't recover anything that was queued on the disk but we - // will still be able to start and run hopefully in the meantime. - logger.WithError(err).Error("Unable to add stream") - sentry.CaptureException(fmt.Errorf("Unable to add stream %q: %w", namespaced.Name, err)) - - namespaced.Storage = natsclient.MemoryStorage - if _, err = s.AddStream(&namespaced); err != nil { - // We tried to add the stream in-memory instead but something - // went wrong. That's an unrecoverable situation so we will - // give up at this point. - logger.WithError(err).Fatal("Unable to add in-memory stream") - } - - if stream.Storage != namespaced.Storage { - // We've managed to add the stream in memory. What's on the - // disk will be left alone, but our ability to recover from a - // future crash will be limited. Yell about it. - err := fmt.Errorf("Stream %q is running in-memory; this may be due to data corruption in the JetStream storage directory", namespaced.Name) - sentry.CaptureException(err) - process.Degraded(err) - } - } + err = configureStream(stream, cfg, s) + if err != nil { + logrus.WithError(err).WithField("stream", stream.Name).Fatal("unable to configure a stream") } } @@ -203,3 +151,52 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc return s, nc } + +func configureStream(stream *nats.StreamConfig, cfg *config.JetStream, s nats.JetStreamContext) error { + name := cfg.Prefixed(stream.Name) + info, err := s.StreamInfo(name) + if err != nil && err != natsclient.ErrStreamNotFound { + return fmt.Errorf("get stream info: %w", err) + } + subjects := stream.Subjects + if len(subjects) == 0 { + // By default we want each stream to listen for the subjects + // that are either an exact match for the stream name, or where + // the first part of the subject is the stream name. ">" is a + // wildcard in NATS for one or more subject tokens. In the case + // that the stream is called "Foo", this will match any message + // with the subject "Foo", "Foo.Bar" or "Foo.Bar.Baz" etc. + subjects = []string{name, name + ".>"} + } + if info != nil { + switch { + case !reflect.DeepEqual(info.Config.Subjects, subjects): + fallthrough + case info.Config.Retention != stream.Retention: + fallthrough + case info.Config.Storage != stream.Storage: + if err = s.DeleteStream(name); err != nil { + return fmt.Errorf("delete stream: %w", err) + } + info = nil + } + } + if info == nil { + // If we're trying to keep everything in memory (e.g. unit tests) + // then overwrite the storage policy. + if cfg.InMemory { + stream.Storage = natsclient.MemoryStorage + } + + // Namespace the streams without modifying the original streams + // array, otherwise we end up with namespaces on namespaces. + namespaced := *stream + namespaced.Name = name + namespaced.Subjects = subjects + if _, err = s.AddStream(&namespaced); err != nil { + return fmt.Errorf("add stream: %w", err) + } + logrus.Infof("stream created: %s", stream.Name) + } + return nil +} diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go index 590f0cbd9..b68ca498d 100644 --- a/setup/jetstream/streams.go +++ b/setup/jetstream/streams.go @@ -31,6 +31,7 @@ var ( RequestPresence = "GetPresence" OutputPresenceEvent = "OutputPresenceEvent" InputFulltextReindex = "InputFulltextReindex" + OutputMultiRoomCast = "OutputMultiRoomCast" ) var safeCharacters = regexp.MustCompile("[^A-Za-z0-9$]+") @@ -101,4 +102,9 @@ var streams = []*nats.StreamConfig{ Storage: nats.MemoryStorage, MaxAge: time.Minute * 5, }, + { + Name: OutputMultiRoomCast, + Retention: nats.InterestPolicy, + Storage: nats.FileStorage, + }, } diff --git a/syncapi/consumers/multiroomdata.go b/syncapi/consumers/multiroomdata.go new file mode 100644 index 000000000..d58f2b185 --- /dev/null +++ b/syncapi/consumers/multiroomdata.go @@ -0,0 +1,113 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "context" + "time" + + "github.com/getsentry/sentry-go" + "github.com/nats-io/nats.go" + log "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/dendrite/syncapi/notifier" + "github.com/matrix-org/dendrite/syncapi/storage/mrd" + "github.com/matrix-org/dendrite/syncapi/streams" + "github.com/matrix-org/dendrite/syncapi/types" +) + +// OutputMultiRoomDataConsumer consumes events that originated in the client API server. +type OutputMultiRoomDataConsumer struct { + ctx context.Context + jetstream nats.JetStreamContext + durable string + topic string + db *mrd.Queries + stream streams.StreamProvider + notifier *notifier.Notifier +} + +// NewOutputMultiRoomDataConsumer creates a new OutputMultiRoomDataConsumer consumer. Call Start() to begin consuming from room servers. +func NewOutputMultiRoomDataConsumer( + process *process.ProcessContext, + cfg *config.SyncAPI, + js nats.JetStreamContext, + q *mrd.Queries, + notifier *notifier.Notifier, + stream streams.StreamProvider, +) *OutputMultiRoomDataConsumer { + return &OutputMultiRoomDataConsumer{ + ctx: process.Context(), + jetstream: js, + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputMultiRoomCast), + durable: cfg.Matrix.JetStream.Durable("SyncAPIMultiRoomDataConsumer"), + db: q, + notifier: notifier, + stream: stream, + } +} + +func (s *OutputMultiRoomDataConsumer) Start() error { + return jetstream.JetStreamConsumer( + s.ctx, s.jetstream, s.topic, s.durable, 1, + s.onMessage, nats.DeliverAll(), nats.ManualAck(), + ) +} + +func (s *OutputMultiRoomDataConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool { + msg := msgs[0] + userID := msg.Header.Get(jetstream.UserID) + dataType := msg.Header.Get("type") + + log.WithFields(log.Fields{ + "type": dataType, + "user_id": userID, + }).Debug("Received multiroom data from client API server") + + pos, err := s.db.InsertMultiRoomData(ctx, mrd.InsertMultiRoomDataParams{ + UserID: userID, + Type: dataType, + Data: msg.Data, + }) + if err != nil { + sentry.CaptureException(err) + log.WithFields(log.Fields{ + "type": dataType, + "user_id": userID, + }).WithError(err).Errorf("could not insert multi room data") + return false + } + + rooms, err := s.db.SelectMultiRoomVisibilityRooms(ctx, mrd.SelectMultiRoomVisibilityRoomsParams{ + UserID: userID, + ExpireTs: time.Now().Unix(), + }) + if err != nil { + sentry.CaptureException(err) + log.WithFields(log.Fields{ + "type": dataType, + "user_id": userID, + }).WithError(err).Errorf("failed to select multi room visibility") + return false + } + + s.stream.Advance(types.StreamPosition(pos)) + s.notifier.OnNewMultiRoomData(types.StreamingToken{MultiRoomDataPosition: types.StreamPosition(pos)}, rooms) + + return true +} diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go index 27f7c37ba..32c6f04e0 100644 --- a/syncapi/notifier/notifier.go +++ b/syncapi/notifier/notifier.go @@ -280,6 +280,32 @@ func (n *Notifier) _sharedUsers(userID string) []string { return sharedUsers } +func (n *Notifier) OnNewMultiRoomData( + posUpdate types.StreamingToken, roomIds []string, +) { + n.lock.Lock() + defer n.lock.Unlock() + + n.currPos.ApplyUpdates(posUpdate) + usersInRoom := n._usersInRooms(roomIds) + + n._wakeupUsers(usersInRoom, nil, n.currPos) +} + +func (n *Notifier) _usersInRooms(roomIds []string) []string { + for i := range roomIds { + for _, userID := range n._joinedUsers(roomIds[i]) { + n._sharedUserMap[userID] = struct{}{} + } + } + usersInRooms := make([]string, 0, len(n._sharedUserMap)+1) + for userID := range n._sharedUserMap { + usersInRooms = append(usersInRooms, userID) + delete(n._sharedUserMap, userID) + } + return usersInRooms +} + func (n *Notifier) IsSharedUser(userA, userB string) bool { n.lock.RLock() defer n.lock.RUnlock() diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 97c2ced49..12c43cba8 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -109,6 +109,7 @@ type DatabaseTransaction interface { GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) RelationsFor(ctx context.Context, roomID, eventID, relType, eventType string, from, to types.StreamPosition, backwards bool, limit int) (events []types.StreamEvent, prevBatch, nextBatch string, err error) + SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string) (types.MultiRoom, error) } type Database interface { @@ -188,6 +189,10 @@ type Database interface { type Presence interface { GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) + PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) + MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) + ExpirePresence(ctx context.Context) ([]types.PresenceNotify, error) + UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error } type SharedUsers interface { diff --git a/syncapi/storage/mrd/README.md b/syncapi/storage/mrd/README.md new file mode 100644 index 000000000..f2269169b --- /dev/null +++ b/syncapi/storage/mrd/README.md @@ -0,0 +1,3 @@ +## Multiroom storage + +please install `sqlc`: `go install github.com/kyleconroy/sqlc/cmd/sqlc@latest`. Then run `sqlc -f sqlc.yaml generate` in this directory after changing `queries.sql` or `../postgres/schema.sql` files. \ No newline at end of file diff --git a/syncapi/storage/mrd/db.go b/syncapi/storage/mrd/db.go new file mode 100644 index 000000000..8dc9794be --- /dev/null +++ b/syncapi/storage/mrd/db.go @@ -0,0 +1,138 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.15.0 + +package mrd + +import ( + "context" + "database/sql" + "fmt" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +func Prepare(ctx context.Context, db DBTX) (*Queries, error) { + q := Queries{db: db} + var err error + if q.deleteMultiRoomVisibilityStmt, err = db.PrepareContext(ctx, deleteMultiRoomVisibility); err != nil { + return nil, fmt.Errorf("error preparing query DeleteMultiRoomVisibility: %w", err) + } + if q.deleteMultiRoomVisibilityByExpireTSStmt, err = db.PrepareContext(ctx, deleteMultiRoomVisibilityByExpireTS); err != nil { + return nil, fmt.Errorf("error preparing query DeleteMultiRoomVisibilityByExpireTS: %w", err) + } + if q.insertMultiRoomDataStmt, err = db.PrepareContext(ctx, insertMultiRoomData); err != nil { + return nil, fmt.Errorf("error preparing query InsertMultiRoomData: %w", err) + } + if q.insertMultiRoomVisibilityStmt, err = db.PrepareContext(ctx, insertMultiRoomVisibility); err != nil { + return nil, fmt.Errorf("error preparing query InsertMultiRoomVisibility: %w", err) + } + if q.selectMaxIdStmt, err = db.PrepareContext(ctx, selectMaxId); err != nil { + return nil, fmt.Errorf("error preparing query SelectMaxId: %w", err) + } + if q.selectMultiRoomVisibilityRoomsStmt, err = db.PrepareContext(ctx, selectMultiRoomVisibilityRooms); err != nil { + return nil, fmt.Errorf("error preparing query SelectMultiRoomVisibilityRooms: %w", err) + } + return &q, nil +} + +func (q *Queries) Close() error { + var err error + if q.deleteMultiRoomVisibilityStmt != nil { + if cerr := q.deleteMultiRoomVisibilityStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing deleteMultiRoomVisibilityStmt: %w", cerr) + } + } + if q.deleteMultiRoomVisibilityByExpireTSStmt != nil { + if cerr := q.deleteMultiRoomVisibilityByExpireTSStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing deleteMultiRoomVisibilityByExpireTSStmt: %w", cerr) + } + } + if q.insertMultiRoomDataStmt != nil { + if cerr := q.insertMultiRoomDataStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing insertMultiRoomDataStmt: %w", cerr) + } + } + if q.insertMultiRoomVisibilityStmt != nil { + if cerr := q.insertMultiRoomVisibilityStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing insertMultiRoomVisibilityStmt: %w", cerr) + } + } + if q.selectMaxIdStmt != nil { + if cerr := q.selectMaxIdStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing selectMaxIdStmt: %w", cerr) + } + } + if q.selectMultiRoomVisibilityRoomsStmt != nil { + if cerr := q.selectMultiRoomVisibilityRoomsStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing selectMultiRoomVisibilityRoomsStmt: %w", cerr) + } + } + return err +} + +func (q *Queries) exec(ctx context.Context, stmt *sql.Stmt, query string, args ...interface{}) (sql.Result, error) { + switch { + case stmt != nil && q.tx != nil: + return q.tx.StmtContext(ctx, stmt).ExecContext(ctx, args...) + case stmt != nil: + return stmt.ExecContext(ctx, args...) + default: + return q.db.ExecContext(ctx, query, args...) + } +} + +func (q *Queries) query(ctx context.Context, stmt *sql.Stmt, query string, args ...interface{}) (*sql.Rows, error) { + switch { + case stmt != nil && q.tx != nil: + return q.tx.StmtContext(ctx, stmt).QueryContext(ctx, args...) + case stmt != nil: + return stmt.QueryContext(ctx, args...) + default: + return q.db.QueryContext(ctx, query, args...) + } +} + +func (q *Queries) queryRow(ctx context.Context, stmt *sql.Stmt, query string, args ...interface{}) *sql.Row { + switch { + case stmt != nil && q.tx != nil: + return q.tx.StmtContext(ctx, stmt).QueryRowContext(ctx, args...) + case stmt != nil: + return stmt.QueryRowContext(ctx, args...) + default: + return q.db.QueryRowContext(ctx, query, args...) + } +} + +type Queries struct { + db DBTX + tx *sql.Tx + deleteMultiRoomVisibilityStmt *sql.Stmt + deleteMultiRoomVisibilityByExpireTSStmt *sql.Stmt + insertMultiRoomDataStmt *sql.Stmt + insertMultiRoomVisibilityStmt *sql.Stmt + selectMaxIdStmt *sql.Stmt + selectMultiRoomVisibilityRoomsStmt *sql.Stmt +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + tx: tx, + deleteMultiRoomVisibilityStmt: q.deleteMultiRoomVisibilityStmt, + deleteMultiRoomVisibilityByExpireTSStmt: q.deleteMultiRoomVisibilityByExpireTSStmt, + insertMultiRoomDataStmt: q.insertMultiRoomDataStmt, + insertMultiRoomVisibilityStmt: q.insertMultiRoomVisibilityStmt, + selectMaxIdStmt: q.selectMaxIdStmt, + selectMultiRoomVisibilityRoomsStmt: q.selectMultiRoomVisibilityRoomsStmt, + } +} diff --git a/syncapi/storage/mrd/models.go b/syncapi/storage/mrd/models.go new file mode 100644 index 000000000..5f61c2edf --- /dev/null +++ b/syncapi/storage/mrd/models.go @@ -0,0 +1,24 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.15.0 + +package mrd + +import ( + "time" +) + +type SyncapiMultiroomDatum struct { + ID int64 `json:"id"` + UserID string `json:"user_id"` + Type string `json:"type"` + Data []byte `json:"data"` + Ts time.Time `json:"ts"` +} + +type SyncapiMultiroomVisibility struct { + UserID string `json:"user_id"` + Type string `json:"type"` + RoomID string `json:"room_id"` + ExpireTs int64 `json:"expire_ts"` +} diff --git a/syncapi/storage/mrd/queries.sql b/syncapi/storage/mrd/queries.sql new file mode 100644 index 000000000..76d6e6578 --- /dev/null +++ b/syncapi/storage/mrd/queries.sql @@ -0,0 +1,44 @@ +-- name: InsertMultiRoomData :one +INSERT INTO syncapi_multiroom_data ( + user_id, + type, + data +) VALUES ( + $1, + $2, + $3 +) ON CONFLICT (user_id, type) DO UPDATE SET id = nextval('syncapi_multiroom_id'), data = $3, ts = current_timestamp +RETURNING id; + + +-- name: InsertMultiRoomVisibility :exec +INSERT INTO syncapi_multiroom_visibility ( + user_id, + type, + room_id, + expire_ts +) VALUES ( + $1, + $2, + $3, + $4 +) ON CONFLICT (user_id, type, room_id) DO UPDATE SET expire_ts = $4; + +-- name: SelectMultiRoomVisibilityRooms :many +SELECT room_id FROM syncapi_multiroom_visibility +WHERE user_id = $1 +AND expire_ts > $2; + + +-- name: SelectMaxId :one +SELECT MAX(id) FROM syncapi_multiroom_data; + +-- name: DeleteMultiRoomVisibility :exec +DELETE FROM syncapi_multiroom_visibility +WHERE user_id = $1 +AND type = $2 +AND room_id = $3; + +-- name: DeleteMultiRoomVisibilityByExpireTS :execrows +DELETE FROM syncapi_multiroom_visibility +WHERE expire_ts <= $1; \ No newline at end of file diff --git a/syncapi/storage/mrd/queries.sql.go b/syncapi/storage/mrd/queries.sql.go new file mode 100644 index 000000000..fb20b0096 --- /dev/null +++ b/syncapi/storage/mrd/queries.sql.go @@ -0,0 +1,143 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.15.0 +// source: queries.sql + +package mrd + +import ( + "context" +) + +const deleteMultiRoomVisibility = `-- name: DeleteMultiRoomVisibility :exec +DELETE FROM syncapi_multiroom_visibility +WHERE user_id = $1 +AND type = $2 +AND room_id = $3 +` + +type DeleteMultiRoomVisibilityParams struct { + UserID string `json:"user_id"` + Type string `json:"type"` + RoomID string `json:"room_id"` +} + +func (q *Queries) DeleteMultiRoomVisibility(ctx context.Context, arg DeleteMultiRoomVisibilityParams) error { + _, err := q.exec(ctx, q.deleteMultiRoomVisibilityStmt, deleteMultiRoomVisibility, arg.UserID, arg.Type, arg.RoomID) + return err +} + +const deleteMultiRoomVisibilityByExpireTS = `-- name: DeleteMultiRoomVisibilityByExpireTS :execrows +DELETE FROM syncapi_multiroom_visibility +WHERE expire_ts <= $1 +` + +func (q *Queries) DeleteMultiRoomVisibilityByExpireTS(ctx context.Context, expireTs int64) (int64, error) { + result, err := q.exec(ctx, q.deleteMultiRoomVisibilityByExpireTSStmt, deleteMultiRoomVisibilityByExpireTS, expireTs) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +const insertMultiRoomData = `-- name: InsertMultiRoomData :one +INSERT INTO syncapi_multiroom_data ( + user_id, + type, + data +) VALUES ( + $1, + $2, + $3 +) ON CONFLICT (user_id, type) DO UPDATE SET id = nextval('syncapi_multiroom_id'), data = $3, ts = current_timestamp +RETURNING id +` + +type InsertMultiRoomDataParams struct { + UserID string `json:"user_id"` + Type string `json:"type"` + Data []byte `json:"data"` +} + +func (q *Queries) InsertMultiRoomData(ctx context.Context, arg InsertMultiRoomDataParams) (int64, error) { + row := q.queryRow(ctx, q.insertMultiRoomDataStmt, insertMultiRoomData, arg.UserID, arg.Type, arg.Data) + var id int64 + err := row.Scan(&id) + return id, err +} + +const insertMultiRoomVisibility = `-- name: InsertMultiRoomVisibility :exec +INSERT INTO syncapi_multiroom_visibility ( + user_id, + type, + room_id, + expire_ts +) VALUES ( + $1, + $2, + $3, + $4 +) ON CONFLICT (user_id, type, room_id) DO UPDATE SET expire_ts = $4 +` + +type InsertMultiRoomVisibilityParams struct { + UserID string `json:"user_id"` + Type string `json:"type"` + RoomID string `json:"room_id"` + ExpireTs int64 `json:"expire_ts"` +} + +func (q *Queries) InsertMultiRoomVisibility(ctx context.Context, arg InsertMultiRoomVisibilityParams) error { + _, err := q.exec(ctx, q.insertMultiRoomVisibilityStmt, insertMultiRoomVisibility, + arg.UserID, + arg.Type, + arg.RoomID, + arg.ExpireTs, + ) + return err +} + +const selectMaxId = `-- name: SelectMaxId :one +SELECT MAX(id) FROM syncapi_multiroom_data +` + +func (q *Queries) SelectMaxId(ctx context.Context) (interface{}, error) { + row := q.queryRow(ctx, q.selectMaxIdStmt, selectMaxId) + var max interface{} + err := row.Scan(&max) + return max, err +} + +const selectMultiRoomVisibilityRooms = `-- name: SelectMultiRoomVisibilityRooms :many +SELECT room_id FROM syncapi_multiroom_visibility +WHERE user_id = $1 +AND expire_ts > $2 +` + +type SelectMultiRoomVisibilityRoomsParams struct { + UserID string `json:"user_id"` + ExpireTs int64 `json:"expire_ts"` +} + +func (q *Queries) SelectMultiRoomVisibilityRooms(ctx context.Context, arg SelectMultiRoomVisibilityRoomsParams) ([]string, error) { + rows, err := q.query(ctx, q.selectMultiRoomVisibilityRoomsStmt, selectMultiRoomVisibilityRooms, arg.UserID, arg.ExpireTs) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var room_id string + if err := rows.Scan(&room_id); err != nil { + return nil, err + } + items = append(items, room_id) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/syncapi/storage/mrd/sqlc.yaml b/syncapi/storage/mrd/sqlc.yaml new file mode 100644 index 000000000..080407e19 --- /dev/null +++ b/syncapi/storage/mrd/sqlc.yaml @@ -0,0 +1,8 @@ +version: 1 +packages: + - path: ../mrd + engine: postgresql + schema: ../postgres/schema.sql + queries: queries.sql + emit_json_tags: true + emit_prepared_queries: true \ No newline at end of file diff --git a/syncapi/storage/mrd/types.go b/syncapi/storage/mrd/types.go new file mode 100644 index 000000000..a1bf76f81 --- /dev/null +++ b/syncapi/storage/mrd/types.go @@ -0,0 +1,6 @@ +package mrd + +type StateEvent struct { + Hidden bool `json:"hidden"` + ExpireTs int `json:"expire_ts"` +} diff --git a/syncapi/storage/postgres/multiroomcast_table.go b/syncapi/storage/postgres/multiroomcast_table.go new file mode 100644 index 000000000..7c819433d --- /dev/null +++ b/syncapi/storage/postgres/multiroomcast_table.go @@ -0,0 +1,61 @@ +package postgres + +import ( + "context" + "database/sql" + _ "embed" + "fmt" + "time" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" +) + +//go:embed schema.sql +var schema string + +var selectMultiRoomCastSQL = `SELECT d.user_id, d.type, d.data, d.ts FROM syncapi_multiroom_data AS d +JOIN syncapi_multiroom_visibility AS v +ON d.user_id = v.user_id +AND d.type = v.type +WHERE v.room_id = ANY($1) +AND id > $2 +AND id <= $3` + +type multiRoomStatements struct { + selectMultiRoomCast *sql.Stmt +} + +func NewPostgresMultiRoomCastTable(db *sql.DB) (tables.MultiRoom, error) { + r := &multiRoomStatements{} + _, err := db.Exec(schema) + if err != nil { + return nil, err + } + return r, sqlutil.StatementList{ + {&r.selectMultiRoomCast, selectMultiRoomCastSQL}, + }.Prepare(db) +} + +func (s *multiRoomStatements) SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string, txn *sql.Tx) ([]*types.MultiRoomDataRow, error) { + rows, err := sqlutil.TxStmt(txn, s.selectMultiRoomCast).QueryContext(ctx, pq.StringArray(joinedRooms), r.Low(), r.High()) + if err != nil { + return nil, err + } + data := make([]*types.MultiRoomDataRow, 0) + defer internal.CloseAndLogIfError(ctx, rows, "SelectMultiRoomData: rows.close() failed") + var t time.Time + for rows.Next() { + r := types.MultiRoomDataRow{} + err = rows.Scan(&r.UserId, &r.Type, &r.Data, &t) + r.Timestamp = t.Unix() + if err != nil { + return nil, fmt.Errorf("rows scan: %w", err) + } + data = append(data, &r) + } + return data, rows.Err() +} diff --git a/syncapi/storage/postgres/presence_table.go b/syncapi/storage/postgres/presence_table.go index 7194afea6..6f0aa8991 100644 --- a/syncapi/storage/postgres/presence_table.go +++ b/syncapi/storage/postgres/presence_table.go @@ -62,6 +62,10 @@ const upsertPresenceFromSyncSQL = "" + " presence = $2, last_active_ts = $3" + " RETURNING id" +const updateLastActiveSQL = `UPDATE syncapi_presence +SET last_active_ts = $1 +WHERE user_id = $2` + const selectPresenceForUserSQL = "" + "SELECT presence, status_msg, last_active_ts" + " FROM syncapi_presence" + @@ -76,12 +80,24 @@ const selectPresenceAfter = "" + " WHERE id > $1 AND last_active_ts >= $2" + " ORDER BY id ASC LIMIT $3" +const expirePresenceSQL = `UPDATE syncapi_presence SET + id = nextval('syncapi_presence_id'), + presence = 3 +WHERE + to_timestamp(last_active_ts / 1000) < NOW() - INTERVAL` + types.PresenceExpire + ` +AND + presence != 3 +RETURNING id, user_id +` + type presenceStatements struct { upsertPresenceStmt *sql.Stmt upsertPresenceFromSyncStmt *sql.Stmt selectPresenceForUsersStmt *sql.Stmt selectMaxPresenceStmt *sql.Stmt selectPresenceAfterStmt *sql.Stmt + expirePresenceStmt *sql.Stmt + updateLastActiveStmt *sql.Stmt } func NewPostgresPresenceTable(db *sql.DB) (*presenceStatements, error) { @@ -96,6 +112,8 @@ func NewPostgresPresenceTable(db *sql.DB) (*presenceStatements, error) { {&s.selectPresenceForUsersStmt, selectPresenceForUserSQL}, {&s.selectMaxPresenceStmt, selectMaxPresenceSQL}, {&s.selectPresenceAfterStmt, selectPresenceAfter}, + {&s.expirePresenceStmt, expirePresenceSQL}, + {&s.updateLastActiveStmt, updateLastActiveSQL}, }.Prepare(db) } @@ -166,3 +184,28 @@ func (p *presenceStatements) GetPresenceAfter( } return presences, rows.Err() } + +func (p *presenceStatements) ExpirePresence( + ctx context.Context, +) ([]types.PresenceNotify, error) { + rows, err := p.expirePresenceStmt.QueryContext(ctx) + if err != nil { + return nil, err + } + presences := make([]types.PresenceNotify, 0) + i := 0 + for rows.Next() { + presences = append(presences, types.PresenceNotify{}) + err = rows.Scan(&presences[i].StreamPos, &presences[i].UserID) + if err != nil { + return nil, err + } + i++ + } + return presences, err +} + +func (p *presenceStatements) UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error { + _, err := p.updateLastActiveStmt.Exec(&lastActiveTs, &userId) + return err +} diff --git a/syncapi/storage/postgres/schema.sql b/syncapi/storage/postgres/schema.sql new file mode 100644 index 000000000..5702fba9d --- /dev/null +++ b/syncapi/storage/postgres/schema.sql @@ -0,0 +1,19 @@ +CREATE SEQUENCE IF NOT EXISTS syncapi_multiroom_id; + +CREATE TABLE IF NOT EXISTS syncapi_multiroom_data ( + id BIGINT PRIMARY KEY DEFAULT nextval('syncapi_multiroom_id'), + user_id TEXT NOT NULL, + type TEXT NOT NULL, + data BYTEA NOT NULL, + ts TIMESTAMP NOT NULL DEFAULT current_timestamp +); + +CREATE UNIQUE INDEX IF NOT EXISTS syncapi_multiroom_data_user_id_type_idx ON syncapi_multiroom_data(user_id, type); + +CREATE TABLE IF NOT EXISTS syncapi_multiroom_visibility ( + user_id TEXT NOT NULL, + type TEXT NOT NULL, + room_id TEXT NOT NULL, + expire_ts BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY(user_id, type, room_id) +) diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 850d24a07..2151b0032 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/storage/mrd" "github.com/matrix-org/dendrite/syncapi/storage/postgres/deltas" "github.com/matrix-org/dendrite/syncapi/storage/shared" ) @@ -102,6 +103,11 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) if err != nil { return nil, err } + mr, err := NewPostgresMultiRoomCastTable(d.db) + if err != nil { + return nil, err + } + mrq := mrd.New(d.db) // apply migrations which need multiple tables m := sqlutil.NewMigrator(d.db) @@ -134,6 +140,8 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) Ignores: ignores, Presence: presence, Relations: relations, + MultiRoom: mr, + MultiRoomQ: mrq, } return &d, nil } diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go index 23f53d11f..47917eebf 100644 --- a/syncapi/storage/shared/storage_consumer.go +++ b/syncapi/storage/shared/storage_consumer.go @@ -19,6 +19,7 @@ import ( "database/sql" "encoding/json" "fmt" + "strings" "github.com/tidwall/gjson" @@ -30,6 +31,7 @@ import ( "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/storage/mrd" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -54,6 +56,8 @@ type Database struct { Ignores tables.Ignores Presence tables.Presence Relations tables.Relations + MultiRoomQ *mrd.Queries + MultiRoom tables.MultiRoom } func (d *Database) NewDatabaseSnapshot(ctx context.Context) (*DatabaseTransaction, error) { @@ -336,6 +340,13 @@ func (d *Database) updateRoomState( } } + if strings.HasPrefix(event.Type(), "connect.mrd") { + err := d.UpdateMultiRoomVisibility(ctx, event) + if err != nil { + logrus.WithError(err).WithField("event_id", event.EventID()).Error("failed to update multi room visibility") + } + } + if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membership, pduPosition); err != nil { return fmt.Errorf("d.CurrentRoomState.UpsertRoomState: %w", err) } @@ -625,3 +636,49 @@ func (d *Database) SelectMemberships( ) (eventIDs []string, err error) { return d.Memberships.SelectMemberships(ctx, nil, roomID, pos, membership, notMembership) } + +func (s *Database) ExpirePresence(ctx context.Context) ([]types.PresenceNotify, error) { + return s.Presence.ExpirePresence(ctx) +} + +func (d *Database) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) { + return d.Presence.GetMaxPresenceID(ctx, nil) +} + +func (d *Database) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) { + return d.Presence.GetPresenceAfter(ctx, nil, after, filter) +} + +func (s *Database) UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error { + return s.Presence.UpdateLastActive(ctx, userId, lastActiveTs) +} + +func (d *Database) UpdateMultiRoomVisibility(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error { + var mrdEv mrd.StateEvent + err := json.Unmarshal(event.Content(), &mrdEv) + if err != nil { + return fmt.Errorf("unmarshal multiroom visibility failed: %w", err) + } + if mrdEv.Hidden { + err = d.MultiRoomQ.DeleteMultiRoomVisibility(ctx, mrd.DeleteMultiRoomVisibilityParams{ + UserID: event.Sender(), + Type: event.Type(), + RoomID: event.RoomID(), + }) + if err != nil { + return fmt.Errorf("delete multiroom visibility failed: %w", err) + } + } + if mrdEv.ExpireTs > 0 { + err = d.MultiRoomQ.InsertMultiRoomVisibility(ctx, mrd.InsertMultiRoomVisibilityParams{ + UserID: event.Sender(), + Type: event.Type(), + RoomID: event.RoomID(), + ExpireTs: int64(mrdEv.ExpireTs), + }) + if err != nil { + return fmt.Errorf("insert multiroom visibility failed: %w", err) + } + } + return nil +} diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index c3763521c..1faafa3da 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -688,3 +688,22 @@ func (d *DatabaseTransaction) RelationsFor(ctx context.Context, roomID, eventID, return events, prevBatch, nextBatch, nil } + +func (d *DatabaseTransaction) SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string) (types.MultiRoom, error) { + rows, err := d.MultiRoom.SelectMultiRoomData(ctx, r, joinedRooms, d.txn) + if err != nil { + return nil, fmt.Errorf("select multi room data: %w", err) + } + mr := make(types.MultiRoom, 3) + for _, row := range rows { + if mr[row.UserId] == nil { + mr[row.UserId] = make(map[string]types.MultiRoomData) + } + mr[row.UserId][row.Type] = types.MultiRoomData{ + Content: row.Data, + Timestamp: row.Timestamp, + } + } + return mr, nil + +} diff --git a/syncapi/storage/sqlite3/presence_table.go b/syncapi/storage/sqlite3/presence_table.go index b61a825df..fe6b3ce84 100644 --- a/syncapi/storage/sqlite3/presence_table.go +++ b/syncapi/storage/sqlite3/presence_table.go @@ -180,3 +180,15 @@ func (p *presenceStatements) GetPresenceAfter( } return presences, rows.Err() } + +func (p *presenceStatements) ExpirePresence( + ctx context.Context, +) ([]types.PresenceNotify, error) { + // TODO implement + return nil, nil +} + +func (p *presenceStatements) UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error { + // TODO implement + return nil +} diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index 5b20c6cc2..a47efa38e 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -22,18 +22,26 @@ import ( "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/storage/mrd" "github.com/matrix-org/dendrite/syncapi/storage/postgres" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) // NewSyncServerDatasource opens a database connection. -func NewSyncServerDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, error) { +func NewSyncServerDatasource(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) (Database, *mrd.Queries, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(base, dbProperties) + ds, err := sqlite3.NewDatabase(base, dbProperties) + return ds, nil, err + case dbProperties.ConnectionString.IsPostgres(): - return postgres.NewDatabase(base, dbProperties) + ds, err := postgres.NewDatabase(base, dbProperties) + if err != nil { + return nil, nil, err + } + mrq := mrd.New(ds.DB) + return ds, mrq, nil default: - return nil, fmt.Errorf("unexpected database type") + return nil, nil, fmt.Errorf("unexpected database type") } } diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 5ff185a32..74f4c830f 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -21,7 +21,7 @@ var ctx = context.Background() func MustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func(), func()) { connStr, close := test.PrepareDBConnectionString(t, dbType) base, closeBase := testrig.CreateBaseDendrite(t, dbType) - db, err := storage.NewSyncServerDatasource(base, &config.DatabaseOptions{ + db, _, err := storage.NewSyncServerDatasource(base, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }) if err != nil { diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 2c4f04ec2..e027cf59e 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -210,6 +210,8 @@ type Presence interface { GetPresenceForUser(ctx context.Context, txn *sql.Tx, userID string) (presence *types.PresenceInternal, err error) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) GetPresenceAfter(ctx context.Context, txn *sql.Tx, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (presences map[string]*types.PresenceInternal, err error) + ExpirePresence(ctx context.Context) ([]types.PresenceNotify, error) + UpdateLastActive(ctx context.Context, userId string, lastActiveTs uint64) error } type Relations interface { @@ -230,3 +232,7 @@ type Relations interface { // "from" or want to work forwards and don't have a "to"). SelectMaxRelationID(ctx context.Context, txn *sql.Tx) (id int64, err error) } + +type MultiRoom interface { + SelectMultiRoomData(ctx context.Context, r *types.Range, joinedRooms []string, txn *sql.Tx) ([]*types.MultiRoomDataRow, error) +} diff --git a/syncapi/streams/stream_multiroomdata.go b/syncapi/streams/stream_multiroomdata.go new file mode 100644 index 000000000..2ba35191f --- /dev/null +++ b/syncapi/streams/stream_multiroomdata.go @@ -0,0 +1,72 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package streams + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/syncapi/notifier" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/storage/mrd" + "github.com/matrix-org/dendrite/syncapi/types" +) + +type MultiRoomDataStreamProvider struct { + DefaultStreamProvider + notifier *notifier.Notifier + mrdDb *mrd.Queries +} + +func (p *MultiRoomDataStreamProvider) Setup(ctx context.Context, snapshot storage.DatabaseTransaction) { + p.DefaultStreamProvider.Setup(ctx, snapshot) + + id, err := p.mrdDb.SelectMaxId(context.Background()) + if err != nil && err != sql.ErrNoRows { + panic(err) + } + p.latestMutex.Lock() + defer p.latestMutex.Unlock() + if id == nil { + p.latest = types.StreamPosition(0) + } else { + p.latest = types.StreamPosition(id.(int64)) + } +} + +func (p *MultiRoomDataStreamProvider) CompleteSync( + ctx context.Context, + snapshot storage.DatabaseTransaction, + req *types.SyncRequest, +) types.StreamPosition { + return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx)) +} + +func (p *MultiRoomDataStreamProvider) IncrementalSync( + ctx context.Context, + snapshot storage.DatabaseTransaction, + req *types.SyncRequest, + from, to types.StreamPosition, +) types.StreamPosition { + mr, err := snapshot.SelectMultiRoomData(ctx, &types.Range{From: from, To: to}, req.JoinedRooms) + if err != nil { + req.Log.WithError(err).Error("GetUserUnreadNotificationCountsForRooms failed") + return from + } + req.Log.Tracef("MultiRoomDataStreamProvider IncrementalSync: %+v", mr) + req.Response.MultiRoom = mr + return to + +} diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 65ca8e2a3..0d15ba726 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -77,6 +77,7 @@ func (p *PDUStreamProvider) CompleteSync( req.Log.WithError(err).Error("p.DB.RoomIDsWithMembership failed") return from } + req.JoinedRooms = joinedRoomIDs stateFilter := req.Filter.Room.State eventFilter := req.Filter.Room.Timeline @@ -170,6 +171,7 @@ func (p *PDUStreamProvider) IncrementalSync( for _, roomID := range syncJoinedRooms { req.Rooms[roomID] = gomatrixserverlib.Join } + req.JoinedRooms = syncJoinedRooms if len(stateDeltas) == 0 { return to diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go index 030b7c5d5..bbd12be0d 100644 --- a/syncapi/streams/stream_presence.go +++ b/syncapi/streams/stream_presence.go @@ -17,7 +17,6 @@ package streams import ( "context" "encoding/json" - "sync" "github.com/matrix-org/gomatrixserverlib" "github.com/tidwall/gjson" @@ -29,8 +28,6 @@ import ( type PresenceStreamProvider struct { DefaultStreamProvider - // cache contains previously sent presence updates to avoid unneeded updates - cache sync.Map notifier *notifier.Notifier } @@ -114,19 +111,6 @@ func (p *PresenceStreamProvider) IncrementalSync( if req.Device.UserID != presence.UserID && !p.notifier.IsSharedUser(req.Device.UserID, presence.UserID) { continue } - cacheKey := req.Device.UserID + req.Device.ID + presence.UserID - pres, ok := p.cache.Load(cacheKey) - if ok { - // skip already sent presence - prevPresence := pres.(*types.PresenceInternal) - currentlyActive := prevPresence.CurrentlyActive() - skip := prevPresence.Equals(presence) && currentlyActive && req.Device.UserID != presence.UserID - _, membershipChange := req.MembershipChanges[presence.UserID] - if skip && !membershipChange { - req.Log.Tracef("Skipping presence, no change (%s)", presence.UserID) - continue - } - } if _, known := types.PresenceFromString(presence.ClientFields.Presence); known { presence.ClientFields.LastActiveAgo = presence.LastActiveAgo() @@ -154,7 +138,6 @@ func (p *PresenceStreamProvider) IncrementalSync( if len(req.Response.Presence.Events) == req.Filter.Presence.Limit { break } - p.cache.Store(cacheKey, presence) } if len(req.Response.Presence.Events) == 0 { diff --git a/syncapi/streams/streams.go b/syncapi/streams/streams.go index dc8547621..8cc028bdf 100644 --- a/syncapi/streams/streams.go +++ b/syncapi/streams/streams.go @@ -9,6 +9,7 @@ import ( rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/notifier" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/storage/mrd" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" ) @@ -23,12 +24,14 @@ type Streams struct { DeviceListStreamProvider StreamProvider NotificationDataStreamProvider StreamProvider PresenceStreamProvider StreamProvider + MultiRoomStreamProvider StreamProvider } func NewSyncStreamProviders( d storage.Database, userAPI userapi.SyncUserAPI, rsAPI rsapi.SyncRoomserverAPI, keyAPI keyapi.SyncKeyAPI, eduCache *caching.EDUCache, lazyLoadCache caching.LazyLoadCache, notifier *notifier.Notifier, + mrdb *mrd.Queries, ) *Streams { streams := &Streams{ PDUStreamProvider: &PDUStreamProvider{ @@ -66,6 +69,11 @@ func NewSyncStreamProviders( DefaultStreamProvider: DefaultStreamProvider{DB: d}, notifier: notifier, }, + MultiRoomStreamProvider: &MultiRoomDataStreamProvider{ + DefaultStreamProvider: DefaultStreamProvider{DB: d}, + notifier: notifier, + mrdDb: mrdb, + }, } ctx := context.TODO() @@ -85,6 +93,7 @@ func NewSyncStreamProviders( streams.NotificationDataStreamProvider.Setup(ctx, snapshot) streams.DeviceListStreamProvider.Setup(ctx, snapshot) streams.PresenceStreamProvider.Setup(ctx, snapshot) + streams.MultiRoomStreamProvider.Setup(ctx, snapshot) succeeded = true return streams diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index e5e5fdb5b..6a97e25bc 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -80,6 +80,8 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat filter.AccountData.Limit = math.MaxInt32 filter.Room.AccountData.Limit = math.MaxInt32 } + // Ignore state limit filter so that e.g. correct room name appears on clients. + filter.Room.State.Limit = math.MaxInt32 logger := util.GetLogger(req.Context()).WithFields(logrus.Fields{ "user_id": device.UserID, diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 29d92b293..1733a15b0 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -51,7 +51,7 @@ type RequestPool struct { keyAPI keyapi.SyncKeyAPI rsAPI roomserverAPI.SyncRoomserverAPI lastseen *sync.Map - presence *sync.Map + Presence *sync.Map streams *streams.Streams Notifier *notifier.Notifier producer PresencePublisher @@ -86,14 +86,14 @@ func NewRequestPool( keyAPI: keyAPI, rsAPI: rsAPI, lastseen: &sync.Map{}, - presence: &sync.Map{}, + Presence: &sync.Map{}, streams: streams, Notifier: notifier, producer: producer, consumer: consumer, } go rp.cleanLastSeen() - go rp.cleanPresence(db, time.Minute*5) + // go rp.cleanPresence(db, time.Minute*5) return rp } @@ -112,11 +112,11 @@ func (rp *RequestPool) cleanPresence(db storage.Presence, cleanupTime time.Durat return } for { - rp.presence.Range(func(key interface{}, v interface{}) bool { + rp.Presence.Range(func(key interface{}, v interface{}) bool { p := v.(types.PresenceInternal) if time.Since(p.LastActiveTS.Time()) > cleanupTime { rp.updatePresence(db, types.PresenceUnavailable.String(), p.UserID) - rp.presence.Delete(key) + rp.Presence.Delete(key) } return true }) @@ -154,13 +154,22 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user } newPresence.ClientFields.Presence = presenceID.String() - defer rp.presence.Store(userID, newPresence) + defer rp.Presence.Store(userID, newPresence) // avoid spamming presence updates when syncing - existingPresence, ok := rp.presence.LoadOrStore(userID, newPresence) + existingPresence, ok := rp.Presence.LoadOrStore(userID, newPresence) if ok { p := existingPresence.(types.PresenceInternal) - if p.ClientFields.Presence == newPresence.ClientFields.Presence { - return + if dbPresence != nil { + if p.Presence == newPresence.Presence && newPresence.LastActiveTS-dbPresence.LastActiveTS < types.PresenceNoOpMs { + return + } + if dbPresence.Presence == types.PresenceOnline && presenceID == types.PresenceOnline && newPresence.LastActiveTS-dbPresence.LastActiveTS >= types.PresenceNoOpMs { + err := db.UpdateLastActive(context.Background(), userID, uint64(newPresence.LastActiveTS)) + if err != nil { + logrus.WithError(err).Error("failed to update last active") + } + return + } } } @@ -248,7 +257,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. defer activeSyncRequests.Dec() rp.updateLastSeen(req, device) - rp.updatePresence(rp.db, req.FormValue("set_presence"), device.UserID) + rp.updatePresence(rp.db, "", device.UserID) waitingSyncRequests.Inc() defer waitingSyncRequests.Dec() @@ -398,6 +407,14 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. ) }, ), + MultiRoomDataPosition: withTransaction( + syncReq.Since.MultiRoomDataPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.MultiRoomStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, + ), } } else { // Incremental sync @@ -483,6 +500,15 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. ) }, ), + MultiRoomDataPosition: withTransaction( + syncReq.Since.MultiRoomDataPosition, + func(snapshot storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.MultiRoomStreamProvider.IncrementalSync( + syncReq.Context, snapshot, syncReq, + syncReq.Since.MultiRoomDataPosition, rp.Notifier.CurrentPosition().MultiRoomDataPosition, + ) + }, + ), } // it's possible for there to be no updates for this user even though since < current pos, // e.g busy servers with a quiet user. In this scenario, we don't want to return a no-op diff --git a/syncapi/sync/requestpool_test.go b/syncapi/sync/requestpool_test.go index 3e5769d8c..cdc658331 100644 --- a/syncapi/sync/requestpool_test.go +++ b/syncapi/sync/requestpool_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -23,7 +24,9 @@ func (d *dummyPublisher) SendPresence(userID string, presence types.Presence, st return nil } -type dummyDB struct{} +type dummyDB struct { + storage.Database +} func (d dummyDB) UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) { return 0, nil @@ -109,7 +112,7 @@ func TestRequestPool_updatePresence(t *testing.T) { }, } rp := &RequestPool{ - presence: &syncMap, + Presence: &syncMap, producer: publisher, consumer: consumer, cfg: &config.SyncAPI{ diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index be19310f2..8a7216228 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -16,6 +16,7 @@ package syncapi import ( "context" + "time" "github.com/sirupsen/logrus" @@ -34,6 +35,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/streams" "github.com/matrix-org/dendrite/syncapi/sync" + "github.com/matrix-org/dendrite/syncapi/types" ) // AddPublicRoutes sets up and registers HTTP handlers for the SyncAPI @@ -48,14 +50,26 @@ func AddPublicRoutes( js, natsClient := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream) - syncDB, err := storage.NewSyncServerDatasource(base, &cfg.Database) + syncDB, mrq, err := storage.NewSyncServerDatasource(base, &cfg.Database) if err != nil { logrus.WithError(err).Panicf("failed to connect to sync db") } + go func() { + var affected int64 + for { + affected, err = mrq.DeleteMultiRoomVisibilityByExpireTS(context.Background(), time.Now().Unix()) + if err != nil { + logrus.WithError(err).Error("failed to expire multiroom visibility") + } + logrus.WithField("rows", affected).Info("expired multiroom visibility") + time.Sleep(time.Minute) + } + }() + eduCache := caching.NewTypingCache() notifier := notifier.NewNotifier() - streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache, base.Caches, notifier) + streams := streams.NewSyncStreamProviders(syncDB, userAPI, rsAPI, keyAPI, eduCache, base.Caches, notifier, mrq) notifier.SetCurrentPosition(streams.Latest(context.Background())) if err = notifier.Load(context.Background(), syncDB); err != nil { logrus.WithError(err).Panicf("failed to load notifier ") @@ -130,8 +144,35 @@ func AddPublicRoutes( logrus.WithError(err).Panicf("failed to start receipts consumer") } + multiRoomConsumer := consumers.NewOutputMultiRoomDataConsumer( + base.ProcessContext, cfg, js, mrq, notifier, streams.MultiRoomStreamProvider, + ) + if err = multiRoomConsumer.Start(); err != nil { + logrus.WithError(err).Panicf("failed to start multiroom consumer") + } + routing.Setup( base.PublicClientAPIMux, requestPool, syncDB, userAPI, rsAPI, cfg, base.Caches, base.Fulltext, ) + + go func() { + ctx := context.Background() + for { + notify, err := syncDB.ExpirePresence(ctx) + if err != nil { + logrus.WithError(err).Error("failed to expire presence") + } + for i := range notify { + requestPool.Presence.Store(notify[i].UserID, types.PresenceInternal{ + Presence: types.PresenceOffline, + }) + notifier.OnNewPresence(types.StreamingToken{ + PresencePosition: notify[i].StreamPos, + }, notify[i].UserID) + + } + time.Sleep(types.PresenceExpireInterval) + } + }() } diff --git a/syncapi/types/multiroom.go b/syncapi/types/multiroom.go new file mode 100644 index 000000000..f1b5d7e49 --- /dev/null +++ b/syncapi/types/multiroom.go @@ -0,0 +1,21 @@ +package types + +type MultiRoom map[string]map[string]MultiRoomData + +type MultiRoomContent []byte + +type MultiRoomData struct { + Content MultiRoomContent `json:"content"` + Timestamp int64 `json:"timestamp"` +} + +func (d MultiRoomContent) MarshalJSON() ([]byte, error) { + return d, nil +} + +type MultiRoomDataRow struct { + Data []byte + Type string + UserId string + Timestamp int64 +} diff --git a/syncapi/types/multiroom_test.go b/syncapi/types/multiroom_test.go new file mode 100644 index 000000000..7fa906974 --- /dev/null +++ b/syncapi/types/multiroom_test.go @@ -0,0 +1,21 @@ +package types + +import ( + "encoding/json" + "testing" + + "github.com/matryer/is" +) + +func TestMarshallMultiRoom(t *testing.T) { + is := is.New(t) + m, err := json.Marshal( + MultiRoom{ + "@3:example.com": map[string]MultiRoomData{ + "location": { + Content: MultiRoomContent(`{"foo":"bar"}`), + Timestamp: 123, + }}}) + is.NoErr(err) + is.Equal(m, []byte(`{"@3:example.com":{"location":{"content":{"foo":"bar"},"timestamp":123}}}`)) +} diff --git a/syncapi/types/presence.go b/syncapi/types/presence.go index 30e025b9f..760225de8 100644 --- a/syncapi/types/presence.go +++ b/syncapi/types/presence.go @@ -21,6 +21,12 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) +const ( + PresenceNoOpMs = 60_000 + PresenceExpire = "'4 minutes'" + PresenceExpireInterval = time.Second * 30 +) + type Presence uint8 const ( @@ -66,6 +72,11 @@ type PresenceInternal struct { Presence Presence `json:"-"` } +type PresenceNotify struct { + StreamPos StreamPosition + UserID string +} + // Equals compares p1 with p2. func (p1 *PresenceInternal) Equals(p2 *PresenceInternal) bool { return p1.ClientFields.Presence == p2.ClientFields.Presence && diff --git a/syncapi/types/provider.go b/syncapi/types/provider.go index 9a533002b..f8d52b479 100644 --- a/syncapi/types/provider.go +++ b/syncapi/types/provider.go @@ -21,7 +21,8 @@ type SyncRequest struct { WantFullState bool // Updated by the PDU stream. - Rooms map[string]string + Rooms map[string]string + JoinedRooms []string // Updated by the PDU stream. MembershipChanges map[string]struct{} // Updated by the PDU stream. diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 295187acc..bba9be4ba 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" "github.com/tidwall/gjson" "github.com/matrix-org/dendrite/roomserver/api" @@ -114,6 +115,7 @@ type StreamingToken struct { DeviceListPosition StreamPosition NotificationDataPosition StreamPosition PresencePosition StreamPosition + MultiRoomDataPosition StreamPosition } // This will be used as a fallback by json.Marshal. @@ -129,12 +131,12 @@ func (s *StreamingToken) UnmarshalText(text []byte) (err error) { func (t StreamingToken) String() string { posStr := fmt.Sprintf( - "s%d_%d_%d_%d_%d_%d_%d_%d_%d", + "s%d_%d_%d_%d_%d_%d_%d_%d_%d_%d", t.PDUPosition, t.TypingPosition, t.ReceiptPosition, t.SendToDevicePosition, t.InvitePosition, t.AccountDataPosition, t.DeviceListPosition, t.NotificationDataPosition, - t.PresencePosition, + t.PresencePosition, t.MultiRoomDataPosition, ) return posStr } @@ -160,12 +162,14 @@ func (t *StreamingToken) IsAfter(other StreamingToken) bool { return true case t.PresencePosition > other.PresencePosition: return true + case t.MultiRoomDataPosition > other.MultiRoomDataPosition: + return true } return false } func (t *StreamingToken) IsEmpty() bool { - return t == nil || t.PDUPosition+t.TypingPosition+t.ReceiptPosition+t.SendToDevicePosition+t.InvitePosition+t.AccountDataPosition+t.DeviceListPosition+t.NotificationDataPosition+t.PresencePosition == 0 + return t == nil || t.PDUPosition+t.TypingPosition+t.ReceiptPosition+t.SendToDevicePosition+t.InvitePosition+t.AccountDataPosition+t.DeviceListPosition+t.NotificationDataPosition+t.PresencePosition+t.MultiRoomDataPosition == 0 } // WithUpdates returns a copy of the StreamingToken with updates applied from another StreamingToken. @@ -209,6 +213,9 @@ func (t *StreamingToken) ApplyUpdates(other StreamingToken) { if other.PresencePosition > t.PresencePosition { t.PresencePosition = other.PresencePosition } + if other.MultiRoomDataPosition > t.MultiRoomDataPosition { + t.MultiRoomDataPosition = other.MultiRoomDataPosition + } } type TopologyToken struct { @@ -294,9 +301,11 @@ func NewTopologyTokenFromString(tok string) (token TopologyToken, err error) { func NewStreamTokenFromString(tok string) (token StreamingToken, err error) { if len(tok) < 1 { err = ErrMalformedSyncToken + logrus.WithField("token", tok).Info("invalid stream token: bad length") return } if tok[0] != SyncTokenTypeStream[0] { + logrus.WithField("token", tok).Info("invalid stream token: not starting from s") err = ErrMalformedSyncToken return } @@ -304,7 +313,7 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) { // s478_0_0_0_0_13.dl-0-2 but we have now removed partitioned stream positions tok = strings.Split(tok, ".")[0] parts := strings.Split(tok[1:], "_") - var positions [9]StreamPosition + var positions [10]StreamPosition for i, p := range parts { if i >= len(positions) { break @@ -312,6 +321,7 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) { var pos int pos, err = strconv.Atoi(p) if err != nil { + logrus.WithField("token", tok).Info("invalid stream token: strconv") err = ErrMalformedSyncToken return } @@ -327,6 +337,7 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) { DeviceListPosition: positions[6], NotificationDataPosition: positions[7], PresencePosition: positions[8], + MultiRoomDataPosition: positions[9], } return token, nil } @@ -363,6 +374,7 @@ type Response struct { ToDevice *ToDeviceResponse `json:"to_device,omitempty"` DeviceLists *DeviceLists `json:"device_lists,omitempty"` DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"` + MultiRoom MultiRoom `json:"multiroom,omitempty"` } func (r Response) MarshalJSON() ([]byte, error) { @@ -401,7 +413,8 @@ func (r *Response) HasUpdates() bool { len(r.Rooms.Peek) > 0 || len(r.ToDevice.Events) > 0 || len(r.DeviceLists.Changed) > 0 || - len(r.DeviceLists.Left) > 0) + len(r.DeviceLists.Left) > 0) || + len(r.MultiRoom) > 0 } // NewResponse creates an empty response with initialised maps. diff --git a/syncapi/types/types_test.go b/syncapi/types/types_test.go index 19fcfc150..08614ebb7 100644 --- a/syncapi/types/types_test.go +++ b/syncapi/types/types_test.go @@ -9,10 +9,10 @@ import ( func TestSyncTokens(t *testing.T) { shouldPass := map[string]string{ - "s4_0_0_0_0_0_0_0_3": StreamingToken{4, 0, 0, 0, 0, 0, 0, 0, 3}.String(), - "s3_1_0_0_0_0_2_0_5": StreamingToken{3, 1, 0, 0, 0, 0, 2, 0, 5}.String(), - "s3_1_2_3_5_0_0_0_6": StreamingToken{3, 1, 2, 3, 5, 0, 0, 0, 6}.String(), - "t3_1": TopologyToken{3, 1}.String(), + "s4_0_0_0_0_0_0_0_3_0": StreamingToken{4, 0, 0, 0, 0, 0, 0, 0, 3, 0}.String(), + "s3_1_0_0_0_0_2_0_5_1": StreamingToken{3, 1, 0, 0, 0, 0, 2, 0, 5, 1}.String(), + "s3_1_2_3_5_0_0_0_6_2": StreamingToken{3, 1, 2, 3, 5, 0, 0, 0, 6, 2}.String(), + "t3_1": TopologyToken{3, 1}.String(), } for a, b := range shouldPass { diff --git a/sytest-blacklist b/sytest-blacklist index c35b03bd7..74e670d0c 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -48,4 +48,19 @@ If a device list update goes missing, the server resyncs on the next one Leaves are present in non-gapped incremental syncs # Below test was passing for the wrong reason, failing correctly since #2858 -New federated private chats get full presence information (SYN-115) \ No newline at end of file +New federated private chats get full presence information (SYN-115) +If a device list update goes missing, the server resyncs on the next one +# You'll be shocked to discover this is flakey too + +Inbound /v1/send_join rejects joins from other servers + +# For notifications extension on iOS + +/event/ does not allow access to events before the user joined + +# Failing after recent updates with presence + +Newly joined room includes presence in incremental sync +User sees their own presence in a sync +User is offline if they set_presence=offline in their sync +User sees updates to presence from other users in the incremental sync. diff --git a/sytest-whitelist b/sytest-whitelist index f4311d339..4002ce719 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -204,7 +204,6 @@ Deleted tags appear in an incremental v2 /sync /event/ on non world readable room does not work Outbound federation can query profile data /event/ on joined room works -/event/ does not allow access to events before the user joined Federation key API allows unsigned requests for keys GET /publicRooms lists rooms GET /publicRooms includes avatar URLs @@ -758,4 +757,4 @@ Can get rooms/{roomId}/members at a given point Can filter rooms/{roomId}/members Current state appears in timeline in private history with many messages after AS can publish rooms in their own list -AS and main public room lists are separate \ No newline at end of file +AS and main public room lists are separate diff --git a/test/db.go b/test/db.go index 17f637e18..3de3d267f 100644 --- a/test/db.go +++ b/test/db.go @@ -171,7 +171,6 @@ func PrepareDBConnectionString(t *testing.T, dbType DBType) (connStr string, clo func WithAllDatabases(t *testing.T, testFn func(t *testing.T, db DBType)) { dbs := map[string]DBType{ "postgres": DBTypePostgres, - "sqlite": DBTypeSQLite, } for dbName, dbType := range dbs { dbt := dbType diff --git a/userapi/api/api.go b/userapi/api/api.go index 8d7f783de..c5ba2bddf 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -533,7 +533,7 @@ type PerformPusherSetRequest struct { type PerformPusherDeletionRequest struct { Localpart string - SessionID int64 + SessionID int64 // Pusher corresponding to this SessionID will not be deleted } // Pusher represents a push notification subscriber diff --git a/userapi/api/api_multicast.go b/userapi/api/api_multicast.go new file mode 100644 index 000000000..e98a39a58 --- /dev/null +++ b/userapi/api/api_multicast.go @@ -0,0 +1,6 @@ +package api + +type MulticastMetadata struct { + ExpireMs int + ExcludeRoomIds []string +} diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 9ca76965d..a814a89ab 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -613,14 +613,25 @@ func (a *UserInternalAPI) PerformAccountDeactivation(ctx context.Context, req *a return err } + threepids, err := a.DB.GetThreePIDsForLocalpart(ctx, req.Localpart) + if err != nil { + return err + } + for i := 0; i < len(threepids); i++ { + err = a.DB.RemoveThreePIDAssociation(ctx, threepids[i].Address, threepids[i].Medium) + if err != nil { + return err + } + } + pusherReq := &api.PerformPusherDeletionRequest{ Localpart: req.Localpart, } - if err := a.PerformPusherDeletion(ctx, pusherReq, &struct{}{}); err != nil { + if err = a.PerformPusherDeletion(ctx, pusherReq, &struct{}{}); err != nil { return err } - err := a.DB.DeactivateAccount(ctx, req.Localpart) + err = a.DB.DeactivateAccount(ctx, req.Localpart) res.AccountDeactivated = err == nil return err } diff --git a/userapi/storage/postgres/accounts_table.go b/userapi/storage/postgres/accounts_table.go index 7c309eb4f..d3d0bbcd5 100644 --- a/userapi/storage/postgres/accounts_table.go +++ b/userapi/storage/postgres/accounts_table.go @@ -98,6 +98,11 @@ func NewPostgresAccountsTable(db *sql.DB, serverName gomatrixserverlib.ServerNam Up: deltas.UpAddAccountType, Down: deltas.DownAddAccountType, }, + { + Version: "userapi: no guests", + Up: deltas.UpNoGuests, + Down: deltas.DownNoGuests, + }, }...) err = m.Up(context.Background()) if err != nil { diff --git a/userapi/storage/postgres/deltas/2022080800000000_no_guests.go b/userapi/storage/postgres/deltas/2022080800000000_no_guests.go new file mode 100644 index 000000000..9985fd822 --- /dev/null +++ b/userapi/storage/postgres/deltas/2022080800000000_no_guests.go @@ -0,0 +1,20 @@ +package deltas + +import ( + "context" + "database/sql" + "fmt" +) + +func UpNoGuests(ctx context.Context, tx *sql.Tx) error { + // AddAccountType introduced a bug where each user that had was registered as a regular user, but without user_id, became a guest. + _, err := tx.ExecContext(ctx, "UPDATE userapi_accounts SET account_type = 1 WHERE account_type = 2;") + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} + +func DownNoGuests(ctx context.Context, tx *sql.Tx) error { + return nil +} diff --git a/userapi/storage/postgres/notifications_table.go b/userapi/storage/postgres/notifications_table.go index 24a30b2f5..420b1babc 100644 --- a/userapi/storage/postgres/notifications_table.go +++ b/userapi/storage/postgres/notifications_table.go @@ -72,7 +72,7 @@ const selectNotificationSQL = "" + ") AND NOT read ORDER BY localpart, id LIMIT $4" const selectNotificationCountSQL = "" + - "SELECT COUNT(*) FROM userapi_notifications WHERE localpart = $1 AND (" + + "SELECT COUNT(DISTINCT(room_id)) FROM userapi_notifications WHERE localpart = $1 AND (" + "(($2 & 1) <> 0 AND highlight) OR (($2 & 2) <> 0 AND NOT highlight)" + ") AND NOT read" diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index f8b8d02c9..e4242913d 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -565,7 +565,7 @@ func (d *Database) CreateDevice( ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string, ipAddr, userAgent string, ) (dev *api.Device, returnErr error) { - if deviceID != nil { + if deviceID != nil && *deviceID != "" { returnErr = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { var err error // Revoke existing tokens for this device diff --git a/userapi/storage/sqlite3/notifications_table.go b/userapi/storage/sqlite3/notifications_table.go index a35ec7be5..d124bfbc5 100644 --- a/userapi/storage/sqlite3/notifications_table.go +++ b/userapi/storage/sqlite3/notifications_table.go @@ -72,7 +72,7 @@ const selectNotificationSQL = "" + ") AND NOT read ORDER BY localpart, id LIMIT $4" const selectNotificationCountSQL = "" + - "SELECT COUNT(*) FROM userapi_notifications WHERE localpart = $1 AND (" + + "SELECT COUNT(DISTINCT(room_id)) FROM userapi_notifications WHERE localpart = $1 AND (" + "(($2 & 1) <> 0 AND highlight) OR (($2 & 2) <> 0 AND NOT highlight)" + ") AND NOT read" diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index 354f085fc..53c51416b 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -128,6 +128,11 @@ func Test_Accounts(t *testing.T) { _, err = db.GetAccountByPassword(ctx, aliceLocalpart, "newPassword") assert.Error(t, err, "expected an error, got none") + // This should return an empty slice, as the account is deactivated and the 3pid is unbound + threepids, err := db.GetThreePIDsForLocalpart(ctx, aliceLocalpart) + assert.NoError(t, err, "failed to get 3pid for account") + assert.Equal(t, len(threepids), 0) + _, err = db.GetAccountByLocalpart(ctx, "unusename") assert.Error(t, err, "expected an error for non existent localpart") @@ -533,7 +538,7 @@ func Test_Notification(t *testing.T) { // get notifications count, err := db.GetNotificationCount(ctx, aliceLocalpart, tables.AllNotifications) assert.NoError(t, err, "unable to get notification count") - assert.Equal(t, int64(10), count) + assert.Equal(t, int64(2), count) notifs, count, err := db.GetNotifications(ctx, aliceLocalpart, 0, 15, tables.AllNotifications) assert.NoError(t, err, "unable to get notifications") assert.Equal(t, int64(10), count)