diff --git a/.github/codecov.yaml b/.github/codecov.yaml index 78122c990..3462e91ee 100644 --- a/.github/codecov.yaml +++ b/.github/codecov.yaml @@ -7,7 +7,7 @@ coverage: project: default: target: auto - threshold: 0% + threshold: 0.1% base: auto flags: - unittests diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index ac40f06b0..d9c883dab 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -28,12 +28,12 @@ jobs: runs-on: ubuntu-latest if: ${{ false }} # disable for now steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: - go-version: "stable" + go-version-file: 'go.mod' cache: true - name: Install Node @@ -41,7 +41,7 @@ jobs: with: node-version: 14 - - uses: actions/cache@v3 + - uses: actions/cache@v4 with: path: ~/.npm key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} @@ -66,13 +66,13 @@ jobs: name: Linting runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install libolm run: sudo apt-get install libolm-dev libolm3 - name: Install Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: - go-version: "stable" + go-version-file: 'go.mod' - name: golangci-lint uses: golangci/golangci-lint-action@v3 @@ -102,14 +102,14 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install libolm run: sudo apt-get install libolm-dev libolm3 - name: Setup go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: - go-version: "stable" - - uses: actions/cache@v3 + go-version-file: 'go.mod' + - uses: actions/cache@v4 # manually set up caches, as they otherwise clash with different steps using setup-go with cache=true with: path: | @@ -141,12 +141,12 @@ jobs: goos: ["linux"] goarch: ["amd64", "386"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: - go-version: "stable" - - uses: actions/cache@v3 + go-version-file: 'go.mod' + - uses: actions/cache@v4 with: path: | ~/.cache/go-build @@ -174,12 +174,12 @@ jobs: goos: ["windows"] goarch: ["amd64"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: - go-version: "stable" - - uses: actions/cache@v3 + go-version-file: 'go.mod' + - uses: actions/cache@v4 with: path: | ~/.cache/go-build @@ -235,19 +235,19 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install libolm run: sudo apt-get install libolm-dev libolm3 - name: Setup go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: - go-version: "stable" + go-version-file: 'go.mod' - name: Set up gotestfmt uses: gotesttools/gotestfmt-action@v2 with: # Optional: pass GITHUB_TOKEN to avoid rate limiting. token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/cache@v3 + - uses: actions/cache@v4 with: path: | ~/.cache/go-build @@ -262,10 +262,11 @@ jobs: POSTGRES_PASSWORD: postgres POSTGRES_DB: dendrite - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: flags: unittests fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} # run database upgrade tests upgrade_test: @@ -274,12 +275,20 @@ jobs: needs: initial-tests-done runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: - go-version: "stable" + go-version-file: 'go.mod' cache: true + - uses: actions/cache@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-upgrade-test-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-upgrade-test- - name: Docker version run: docker version - name: Build upgrade-tests @@ -296,12 +305,20 @@ jobs: needs: initial-tests-done runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: - go-version: "stable" + go-version-file: 'go.mod' cache: true + - uses: actions/cache@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-upgrade-direct-test-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-upgrade-direct-test- - name: Docker version run: docker version - name: Build upgrade-tests @@ -340,8 +357,8 @@ jobs: SYTEST_BRANCH: ${{ github.head_ref }} CGO_ENABLED: ${{ matrix.cgo && 1 }} steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 + - uses: actions/checkout@v4 + - uses: actions/cache@v4 with: path: | ~/.cache/go-build @@ -364,7 +381,7 @@ jobs: run: /src/are-we-synapse-yet.py /logs/results.tap -v continue-on-error: true # not fatal - name: Upload Sytest logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }}) @@ -404,8 +421,8 @@ jobs: run: | sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest - - name: Run actions/checkout@v3 for dendrite - uses: actions/checkout@v3 + - name: Run actions/checkout@v4 for dendrite + uses: actions/checkout@v4 with: path: dendrite diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8d3a8d674..c795cd366 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -27,22 +27,22 @@ jobs: security-events: write # To upload Trivy sarif files steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases run: | echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -98,22 +98,22 @@ jobs: packages: write steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases run: | echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -159,22 +159,22 @@ jobs: packages: write steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases run: | echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 9df3cceae..30f55b7c8 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Pages uses: actions/configure-pages@v2 - name: Build with Jekyll diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 9a5eb2b62..10eb7c020 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -32,7 +32,7 @@ jobs: version: v3.10.0 - name: Run chart-releaser - uses: helm/chart-releaser-action@ed43eb303604cbc0eeec8390544f7748dc6c790d # specific commit, since `mark_as_latest` is not yet in a release + uses: helm/chart-releaser-action@v1.6.0 env: CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" with: diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml index af2750356..a49042bf2 100644 --- a/.github/workflows/k8s.yml +++ b/.github/workflows/k8s.yml @@ -17,7 +17,7 @@ jobs: outputs: changed: ${{ steps.list-changed.outputs.changed }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: azure/setup-helm@v3 @@ -48,7 +48,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ inputs.checkoutCommit }} @@ -66,7 +66,7 @@ jobs: - name: Create k3d cluster uses: nolar/setup-k3d-k3s@v1 with: - version: v1.21 + version: v1.28 - name: Remove node taints run: | kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml index 509861860..e339c14d3 100644 --- a/.github/workflows/schedules.yaml +++ b/.github/workflows/schedules.yaml @@ -10,8 +10,26 @@ concurrency: cancel-in-progress: true jobs: + check_date: # https://stackoverflow.com/questions/63014786/how-to-schedule-a-github-actions-nightly-build-but-run-it-only-when-there-where + runs-on: ubuntu-latest + name: Check latest commit + outputs: + should_run: ${{ steps.should_run.outputs.should_run }} + steps: + - uses: actions/checkout@v4 + - name: print latest_commit + run: echo ${{ github.sha }} + + - id: should_run + continue-on-error: true + name: check latest commit is less than a day + if: ${{ github.event_name == 'schedule' }} + run: test -z $(git rev-list --after="24 hours" ${{ github.sha }}) && echo "::set-output name=should_run::false" + # run Sytest in different variations sytest: + needs: check_date + if: ${{ needs.check_date.outputs.should_run != 'false' }} timeout-minutes: 60 name: "Sytest (${{ matrix.label }})" runs-on: ubuntu-latest @@ -38,8 +56,8 @@ jobs: RACE_DETECTION: 1 COVER: 1 steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 + - uses: actions/checkout@v4 + - uses: actions/cache@v4 with: path: | ~/.cache/go-build @@ -62,7 +80,7 @@ jobs: run: /src/are-we-synapse-yet.py /logs/results.tap -v continue-on-error: true # not fatal - name: Upload Sytest logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: name: Sytest Logs - ${{ job.status }} - (Dendrite ${{ join(matrix.*, ' ') }}) @@ -75,31 +93,34 @@ jobs: timeout-minutes: 5 name: "Sytest Coverage" runs-on: ubuntu-latest - needs: sytest # only run once Sytest is done - if: ${{ always() }} + needs: [ sytest, check_date ] # only run once Sytest is done and there was a commit + if: ${{ always() && needs.check_date.outputs.should_run != 'false' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: 'stable' cache: true - name: Download all artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Collect coverage run: | go tool covdata textfmt -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov grep -Ev 'relayapi|setup/mscs|api_trace' sytest.cov > final.cov go tool covdata func -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: files: ./final.cov flags: sytest fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} # run Complement complement: + needs: check_date + if: ${{ needs.check_date.outputs.should_run != 'false' }} name: "Complement (${{ matrix.label }})" timeout-minutes: 60 runs-on: ubuntu-latest @@ -129,8 +150,8 @@ jobs: run: | sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest - - name: Run actions/checkout@v3 for dendrite - uses: actions/checkout@v3 + - name: Run actions/checkout@v4 for dendrite + uses: actions/checkout@v4 with: path: dendrite @@ -174,7 +195,7 @@ jobs: # Run Complement - run: | set -o pipefail && - go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt + go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all shell: bash name: Run Complement Tests env: @@ -185,7 +206,7 @@ jobs: working-directory: complement - name: Upload Complement logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: name: Complement Logs - (Dendrite ${{ join(matrix.*, ' ') }}) @@ -196,30 +217,32 @@ jobs: timeout-minutes: 5 name: "Complement Coverage" runs-on: ubuntu-latest - needs: complement # only run once Complement is done - if: ${{ always() }} + needs: [ complement, check_date ] # only run once Complements is done and there was a commit + if: ${{ always() && needs.check_date.outputs.should_run != 'false' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: 'stable' cache: true - name: Download all artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Collect coverage run: | go tool covdata textfmt -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o complement.cov grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov go tool covdata func -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: files: ./final.cov flags: complement fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} # required element-web: + if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright timeout-minutes: 120 runs-on: ubuntu-latest steps: @@ -228,7 +251,7 @@ jobs: # Our test suite includes some screenshot tests with unusual diacritics, which are # supposed to be covered by STIXGeneral. tools: fonts-stix - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: repository: matrix-org/matrix-react-sdk - uses: actions/setup-node@v3 @@ -259,6 +282,7 @@ jobs: TMPDIR: ${{ runner.temp }} element-web-pinecone: + if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright timeout-minutes: 120 runs-on: ubuntu-latest steps: @@ -267,7 +291,7 @@ jobs: # Our test suite includes some screenshot tests with unusual diacritics, which are # supposed to be covered by STIXGeneral. tools: fonts-stix - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: repository: matrix-org/matrix-react-sdk - uses: actions/setup-node@v3 diff --git a/.gitignore b/.gitignore index 043956ee4..ce1c9461d 100644 --- a/.gitignore +++ b/.gitignore @@ -77,4 +77,7 @@ media_store/ build # golang workspaces -go.work* \ No newline at end of file +go.work* + +# helm chart +helm/dendrite/charts/ diff --git a/.golangci.yml b/.golangci.yml index 5bee0a885..6f3fd3627 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -6,7 +6,7 @@ run: concurrency: 4 # timeout for analysis, e.g. 30s, 5m, default is 1m - deadline: 30m + timeout: 5m # exit code when at least one issue was found, default is 1 issues-exit-code: 1 @@ -18,24 +18,6 @@ run: #build-tags: # - mytag - # which dirs to skip: they won't be analyzed; - # can use regexp here: generated.*, regexp is applied on full path; - # default value is empty list, but next dirs are always skipped independently - # from this option's value: - # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ - skip-dirs: - - bin - - docs - - # which files to skip: they will be analyzed, but issues from them - # won't be reported. Default value is empty list, but there is - # no need to include all autogenerated files, we confidently recognize - # autogenerated files. If it's not please let us know. - skip-files: - - ".*\\.md$" - - ".*\\.sh$" - - "^cmd/syncserver-integration-tests/testdata.go$" - # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules": # If invoked with -mod=readonly, the go command is disallowed from the implicit # automatic updating of go.mod described above. Instead, it fails when any changes @@ -50,7 +32,8 @@ run: # output configuration options output: # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" - format: colored-line-number + formats: + - format: colored-line-number # print lines of code with issue, default is true print-issued-lines: true @@ -79,9 +62,8 @@ linters-settings: # see https://github.com/kisielk/errcheck#excluding-functions for details #exclude: /path/to/file.txt govet: - # report about shadowed variables - check-shadowing: true - + enable: + - shadow # settings per analyzer settings: printf: # analyzer name, run `go tool vet help` to see all analyzers @@ -217,6 +199,24 @@ linters: issues: + # which files to skip: they will be analyzed, but issues from them + # won't be reported. Default value is empty list, but there is + # no need to include all autogenerated files, we confidently recognize + # autogenerated files. If it's not please let us know. + exclude-files: + - ".*\\.md$" + - ".*\\.sh$" + - "^cmd/syncserver-integration-tests/testdata.go$" + + # which dirs to skip: they won't be analyzed; + # can use regexp here: generated.*, regexp is applied on full path; + # default value is empty list, but next dirs are always skipped independently + # from this option's value: + # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + exclude-dirs: + - bin + - docs + # List of regexps of issue texts to exclude, empty list by default. # But independently from this option we use default exclude patterns, # it can be disabled by `exclude-use-default: false`. To list all diff --git a/CHANGES.md b/CHANGES.md index 57e3a3d4f..c2c1a73f7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,91 @@ # Changelog +## Dendrite 0.13.8 (2024-09-13) + +### Features + + - The required Go version to build Dendrite is now 1.21 + - Support for authenticated media ([MSC3916](https://github.com/matrix-org/matrix-spec-proposals/pull/3916)) has been added + - NATS can now connect to servers requiring authentication (contributed by [paigeadelethompson](https://github.com/paigeadelethompson)) + - Updated dependencies + - Internal NATS Server has been updated from v2.10.7 to v2.10.20 (contributed by [neilalexander](https://github.com/neilalexander)) + +### Fixes + + - Fix parsing `?ts` query param (contributed by [tulir](https://github.com/tulir)) + - Don't query the database if we could fetch all keys from cache + - Fix media DB potentially leaking connections + - Fixed a bug where we would return that an account exists if we encountered an unhandled error case + - Fixed an issues where edited message could appear twice in search results (contributed by [adnull](https://github.com/adnull)) + - Outgoing threepid HTTP requests now correctly close the returned body (contributed by [ testwill](https://github.com/testwill)) + - Presence conflicts are handled better, reducing the amount of outgoing federation requests (contributed by [jjj333-p](https://github.com/jjj333-p)) + - Internal NATS now uses `SyncAlways` which should improve resilience against crashes (contributed by [neilalexander](https://github.com/neilalexander)) + - Whitespaces in the `X-Matrix` header are now handled correctly + - `/.well-known/matrix/server` lookups now timeout after 30 seconds + - Purging rooms has seen a huge speed-up + +## Dendrite 0.13.7 (2024-04-09) + +### Fixes + +- Fixed an issue where the displayname/avatar of an invited user was replaced with the inviter's details +- Improved server startup performance by avoiding unnecessary room ACL queries + - This change reduces memory footprint as it caches ACL regex patterns once instead of for each room + - Unnecessary Relay related queries have been removed. **Note**: To use relays, you now need to explicitly enable them using the `federation_api.enable_relays` config +- Fixed space summaries over federation +- Improved usage of external NATS JetStream by reusing existing connections instead of opening new ones unnecessarily + +### Features + +- Modernized Appservices (contributed by [tulir](https://github.com/tulir)) +- Added event reporting with Synapse Admin endpoints for querying them +- Updated dependencies + +## Dendrite 0.13.6 (2024-01-26) + +Upgrading to this version is **highly** recommended, as it contains several QoL improvements. + +### Fixes + +- Use `AckExplicitPolicy` for JetStream consumers, so messages don't pile up in NATS +- A rare panic when assigning a state key NID has been fixed +- A rare panic when checking powerlevels has been fixed +- Notary keys requests for all keys now work correctly +- Spec compliance: + - Return `M_INVALID_PARAM` when querying room aliases + - Handle empty `from` parameter when requesting `/messages` + - Add CORP headers on media endpoints + - Remove `aliases` from `/publicRooms` responses + - Allow `+` in MXIDs (Contributed by [RosstheRoss](https://github.com/RosstheRoss)) +- Fixes membership transitions from `knock` to `join` in `knock_restricted` rooms +- Incremental syncs now batch querying events (Contributed by [recht](https://github.com/recht)) +- Move `/joined_members` back to the clientAPI/roomserver, which should make bridges happier again +- Backfilling from other servers now only uses at max 100 events instead of potentially thousands + +## Dendrite 0.13.5 (2023-12-12) + +Upgrading to this version is **highly** recommended, as it fixes several long-standing bugs in +our CanonicalJSON implementation. + +### Fixes + +- Convert unicode escapes to lowercase (gomatrixserverlib) +- Fix canonical json utf-16 surrogate pair detection logic (gomatrixserverlib) +- Handle negative zero and exponential numbers in Canonical JSON verification (gomatrixserverlib) +- Avoid logging unnecessary messages when unable to fetch server keys if multiple fetchers are used (gomatrixserverlib) +- Issues around the device list updater have been fixed, which should ensure that there are always + workers available to process incoming device list updates. +- A panic in the `/hierarchy` endpoints used for spaces has been fixed (client-server and server-server API) +- Fixes around the way we handle database transactions (including a potential connection leak) +- ACLs are now updated when received as outliers +- A race condition, which could lead to bridges instantly leaving a room after joining it, between the SyncAPI and + Appservices has been fixed + +### Features + +- **Appservice login is now supported!** +- Users can now kick themselves (used by some bridges) + ## Dendrite 0.13.4 (2023-10-25) Upgrading to this version is **highly** recommended, as it fixes a long-standing bug in the state resolution diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..028424406 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,59 @@ +# Contributing to Dendrite + +Thank you for taking the time to contribute to Matrix! + +This is the repository for Dendrite, a second-generation Matrix homeserver written in Go. + +## Sign off + +We ask that everybody who contributes to this project signs off their contributions, as explained below. + +We follow a simple 'inbound=outbound' model for contributions: the act of submitting an 'inbound' contribution means that the contributor agrees to license their contribution under the same terms as the project's overall 'outbound' license - in our case, this is Apache Software License v2 (see [LICENSE](./LICENSE)). + +In order to have a concrete record that your contribution is intentional and you agree to license it under the same terms as the project's license, we've adopted the same lightweight approach used by the [Linux Kernel](https://www.kernel.org/doc/html/latest/process/submitting-patches.html), [Docker](https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other projects: the [Developer Certificate of Origin](https://developercertificate.org/) (DCO). This is a simple declaration that you wrote the contribution or otherwise have the right to contribute it to Matrix: + +``` +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +660 York Street, Suite 102, +San Francisco, CA 94110 USA + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. +``` + +If you agree to this for your contribution, then all that's needed is to include the line in your commit or pull request comment: + +``` +Signed-off-by: Your Name +``` + +Git allows you to add this signoff automatically when using the `-s` flag to `git commit`, which uses the name and email set in your `user.name` and `user.email` git configs. \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 8c8f1588f..bf8069213 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # # base installs required dependencies and runs go mod download to cache dependencies # -FROM --platform=${BUILDPLATFORM} docker.io/golang:1.21-alpine AS base +FROM --platform=${BUILDPLATFORM} docker.io/golang:1.22-alpine AS base RUN apk --update --no-cache add bash build-base curl git # diff --git a/README.md b/README.md index bde19b07e..1f3545874 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,13 @@ # Dendrite +## Dendrite is now maintained at [element-hq/dendrite](https://github.com/element-hq/dendrite) + +Dendrite is an open-source [Matrix](https://matrix.org/) homeserver developed from 2019 through 2023 as part of the Matrix.org Foundation. +The Matrix.org Foundation is not able to resource maintenance of Dendrite and it [continues to be developed by Element](https://github.com/element-hq/dendrite) +additionally you have the choice of [other Matrix homeservers](https://matrix.org/ecosystem/servers/>) + +See [The future of Synapse and Dendrite](https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/) blog post for more information. + [![Build status](https://github.com/matrix-org/dendrite/actions/workflows/dendrite.yml/badge.svg?event=push)](https://github.com/matrix-org/dendrite/actions/workflows/dendrite.yml) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) Dendrite is a second-generation Matrix homeserver written in Go. @@ -36,7 +44,7 @@ If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or j See the [Planning your Installation](https://matrix-org.github.io/dendrite/installation/planning) page for more information on requirements. -To build Dendrite, you will need Go 1.20 or later. +To build Dendrite, you will need Go 1.21 or later. For a usable federating Dendrite deployment, you will also need: diff --git a/appservice/api/query.go b/appservice/api/query.go index 472266d9e..8e159152e 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -82,9 +82,17 @@ type UserIDExistsResponse struct { } const ( - ASProtocolPath = "/_matrix/app/unstable/thirdparty/protocol/" - ASUserPath = "/_matrix/app/unstable/thirdparty/user" - ASLocationPath = "/_matrix/app/unstable/thirdparty/location" + ASProtocolLegacyPath = "/_matrix/app/unstable/thirdparty/protocol/" + ASUserLegacyPath = "/_matrix/app/unstable/thirdparty/user" + ASLocationLegacyPath = "/_matrix/app/unstable/thirdparty/location" + ASRoomAliasExistsLegacyPath = "/rooms/" + ASUserExistsLegacyPath = "/users/" + + ASProtocolPath = "/_matrix/app/v1/thirdparty/protocol/" + ASUserPath = "/_matrix/app/v1/thirdparty/user" + ASLocationPath = "/_matrix/app/v1/thirdparty/location" + ASRoomAliasExistsPath = "/_matrix/app/v1/rooms/" + ASUserExistsPath = "/_matrix/app/v1/users/" ) type ProtocolRequest struct { diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go index ddc24477b..aa2af9f24 100644 --- a/appservice/appservice_test.go +++ b/appservice/appservice_test.go @@ -14,7 +14,18 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/clientapi" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/federationapi/statistics" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/syncapi" + uapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/nats-io/nats.go" "github.com/stretchr/testify/assert" + "github.com/tidwall/gjson" "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/appservice/api" @@ -32,6 +43,10 @@ import ( "github.com/matrix-org/dendrite/test/testrig" ) +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + func TestAppserviceInternalAPI(t *testing.T) { // Set expected results @@ -144,7 +159,7 @@ func TestAppserviceInternalAPI(t *testing.T) { cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) runCases(t, asAPI) @@ -239,7 +254,7 @@ func TestAppserviceInternalAPI_UnixSocket_Simple(t *testing.T) { cm := sqlutil.NewConnectionManager(ctx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) t.Run("UserIDExists", func(t *testing.T) { @@ -378,7 +393,7 @@ func TestRoomserverConsumerOneInvite(t *testing.T) { // Create required internal APIs rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // start the consumer appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI) @@ -402,3 +417,190 @@ func TestRoomserverConsumerOneInvite(t *testing.T) { close(evChan) }) } + +// Note: If this test panics, it is because we timed out waiting for the +// join event to come through to the appservice and we close the DB/shutdown Dendrite. This makes the +// syncAPI unhappy, as it is unable to write to the database. +func TestOutputAppserviceEvent(t *testing.T) { + alice := test.NewUser(t) + bob := test.NewUser(t) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType) + defer closeDB() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + natsInstance := &jetstream.NATSInstance{} + + evChan := make(chan struct{}) + + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + // Create required internal APIs + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + + // Create the router, so we can hit `/joined_members` + routers := httputil.NewRouters() + + accessTokens := map[*test.User]userDevice{ + bob: {}, + } + + usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) + clientapi.AddPublicRoutes(processCtx, routers, cfg, natsInstance, nil, rsAPI, nil, nil, nil, usrAPI, nil, nil, caching.DisableMetrics) + createAccessTokens(t, accessTokens, usrAPI, processCtx.Context(), routers) + + room := test.NewRoom(t, alice) + + // Invite Bob + room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{ + "membership": "invite", + }, test.WithStateKey(bob.ID)) + + // create a dummy AS url, handling the events + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var txn consumers.ApplicationServiceTransaction + err := json.NewDecoder(r.Body).Decode(&txn) + if err != nil { + t.Fatal(err) + } + for _, ev := range txn.Events { + if ev.Type != spec.MRoomMember { + continue + } + if ev.StateKey != nil && *ev.StateKey == bob.ID { + membership := gjson.GetBytes(ev.Content, "membership").Str + t.Logf("Processing membership: %s", membership) + switch membership { + case spec.Invite: + // Accept the invite + joinEv := room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID)) + + if err := rsapi.SendEvents(context.Background(), rsAPI, rsapi.KindNew, []*types.HeaderedEvent{joinEv}, "test", "test", "test", nil, false); err != nil { + t.Fatalf("failed to send events: %v", err) + } + case spec.Join: // the AS has received the join event, now hit `/joined_members` to validate that + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/rooms/"+room.ID+"/joined_members", nil) + req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken) + routers.Client.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String()) + } + + // Both Alice and Bob should be joined. If not, we have a race condition + if !gjson.GetBytes(rec.Body.Bytes(), "joined."+alice.ID).Exists() { + t.Errorf("Alice is not joined to the room") // in theory should not happen + } + if !gjson.GetBytes(rec.Body.Bytes(), "joined."+bob.ID).Exists() { + t.Errorf("Bob is not joined to the room") + } + evChan <- struct{}{} + default: + t.Fatalf("Unexpected membership: %s", membership) + } + } + } + })) + defer srv.Close() + + as := &config.ApplicationService{ + ID: "someID", + URL: srv.URL, + ASToken: "", + HSToken: "", + SenderLocalpart: "senderLocalPart", + NamespaceMap: map[string][]config.ApplicationServiceNamespace{ + "users": {{RegexpObject: regexp.MustCompile(bob.ID)}}, + "aliases": {{RegexpObject: regexp.MustCompile(room.ID)}}, + }, + } + as.CreateHTTPClient(cfg.AppServiceAPI.DisableTLSValidation) + + // Create a dummy application service + cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as} + + // Prepare AS Streams on the old topic to validate that they get deleted + jsCtx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + + token := jetstream.Tokenise(as.ID) + if err := jetstream.JetStreamConsumer( + processCtx.Context(), jsCtx, cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent), + cfg.Global.JetStream.Durable("Appservice_"+token), + 50, // maximum number of events to send in a single transaction + func(ctx context.Context, msgs []*nats.Msg) bool { + return true + }, + ); err != nil { + t.Fatal(err) + } + + // Start the syncAPI to have `/joined_members` available + syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, natsInstance, usrAPI, rsAPI, caches, caching.DisableMetrics) + + // start the consumer + appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI) + + // At this point, the old JetStream consumers should be deleted + for consumer := range jsCtx.Consumers(cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent)) { + if consumer.Name == cfg.Global.JetStream.Durable("Appservice_"+token)+"Pull" { + t.Fatalf("Consumer still exists") + } + } + + // Create the room, this triggers the AS to receive an invite for Bob. + if err := rsapi.SendEvents(context.Background(), rsAPI, rsapi.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { + t.Fatalf("failed to send events: %v", err) + } + + select { + // Pretty generous timeout duration... + case <-time.After(time.Millisecond * 1000): // wait for the AS to process the events + t.Errorf("Timed out waiting for join event") + case <-evChan: + } + close(evChan) + }) +} + +type userDevice struct { + accessToken string + deviceID string + password string +} + +func createAccessTokens(t *testing.T, accessTokens map[*test.User]userDevice, userAPI uapi.UserInternalAPI, ctx context.Context, routers httputil.Routers) { + t.Helper() + for u := range accessTokens { + localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID) + userRes := &uapi.PerformAccountCreationResponse{} + password := util.RandomString(8) + if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{ + AccountType: u.AccountType, + Localpart: localpart, + ServerName: serverName, + Password: password, + }, userRes); err != nil { + t.Errorf("failed to create account: %s", err) + } + req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{ + "type": authtypes.LoginTypePassword, + "identifier": map[string]interface{}{ + "type": "m.id.user", + "user": u.ID, + }, + "password": password, + })) + rec := httptest.NewRecorder() + routers.Client.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("failed to login: %s", rec.Body.String()) + } + accessTokens[u] = userDevice{ + accessToken: gjson.GetBytes(rec.Body.Bytes(), "access_token").String(), + deviceID: gjson.GetBytes(rec.Body.Bytes(), "device_id").String(), + password: password, + } + } +} diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index e8b9211c4..b07b24fcc 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -71,13 +71,14 @@ func NewOutputRoomEventConsumer( ctx: process.Context(), cfg: cfg, jetstream: js, - topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputAppserviceEvent), rsAPI: rsAPI, } } // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { + durableNames := make([]string, 0, len(s.cfg.Derived.ApplicationServices)) for _, as := range s.cfg.Derived.ApplicationServices { appsvc := as state := &appserviceState{ @@ -95,6 +96,15 @@ func (s *OutputRoomEventConsumer) Start() error { ); err != nil { return fmt.Errorf("failed to create %q consumer: %w", token, err) } + durableNames = append(durableNames, s.cfg.Matrix.JetStream.Durable("Appservice_"+token)) + } + // Cleanup any consumers still existing on the OutputRoomEvent stream + // to avoid messages not being deleted + for _, consumerName := range durableNames { + err := s.jetstream.DeleteConsumer(s.cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent), consumerName+"Pull") + if err != nil && err != nats.ErrConsumerNotFound { + return err + } } return nil } @@ -196,13 +206,21 @@ func (s *OutputRoomEventConsumer) sendEvents( } // Send the transaction to the appservice. - // https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid - address := fmt.Sprintf("%s/transactions/%s?access_token=%s", state.RequestUrl(), txnID, url.QueryEscape(state.HSToken)) + // https://spec.matrix.org/v1.9/application-service-api/#pushing-events + path := "_matrix/app/v1/transactions" + if s.cfg.LegacyPaths { + path = "transactions" + } + address := fmt.Sprintf("%s/%s/%s", state.RequestUrl(), path, txnID) + if s.cfg.LegacyAuth { + address += "?access_token=" + url.QueryEscape(state.HSToken) + } req, err := http.NewRequestWithContext(ctx, "PUT", address, bytes.NewBuffer(transaction)) if err != nil { return err } req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", state.HSToken)) resp, err := state.HTTPClient.Do(req) if err != nil { return state.backoffAndPause(err) diff --git a/appservice/query/query.go b/appservice/query/query.go index 5c736f379..7f33e17f8 100644 --- a/appservice/query/query.go +++ b/appservice/query/query.go @@ -19,10 +19,10 @@ package query import ( "context" "encoding/json" + "fmt" "io" "net/http" "net/url" - "strings" "sync" log "github.com/sirupsen/logrus" @@ -32,9 +32,6 @@ import ( "github.com/matrix-org/dendrite/setup/config" ) -const roomAliasExistsPath = "/rooms/" -const userIDExistsPath = "/users/" - // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI type AppServiceQueryAPI struct { Cfg *config.AppServiceAPI @@ -55,14 +52,23 @@ func (a *AppServiceQueryAPI) RoomAliasExists( // Determine which application service should handle this request for _, appservice := range a.Cfg.Derived.ApplicationServices { if appservice.URL != "" && appservice.IsInterestedInRoomAlias(request.Alias) { + path := api.ASRoomAliasExistsPath + if a.Cfg.LegacyPaths { + path = api.ASRoomAliasExistsLegacyPath + } // The full path to the rooms API, includes hs token - URL, err := url.Parse(appservice.RequestUrl() + roomAliasExistsPath) + URL, err := url.Parse(appservice.RequestUrl() + path) if err != nil { return err } URL.Path += request.Alias - apiURL := URL.String() + "?access_token=" + appservice.HSToken + if a.Cfg.LegacyAuth { + q := URL.Query() + q.Set("access_token", appservice.HSToken) + URL.RawQuery = q.Encode() + } + apiURL := URL.String() // Send a request to each application service. If one responds that it has // created the room, immediately return. @@ -70,6 +76,7 @@ func (a *AppServiceQueryAPI) RoomAliasExists( if err != nil { return err } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", appservice.HSToken)) req = req.WithContext(ctx) resp, err := appservice.HTTPClient.Do(req) @@ -123,12 +130,21 @@ func (a *AppServiceQueryAPI) UserIDExists( for _, appservice := range a.Cfg.Derived.ApplicationServices { if appservice.URL != "" && appservice.IsInterestedInUserID(request.UserID) { // The full path to the rooms API, includes hs token - URL, err := url.Parse(appservice.RequestUrl() + userIDExistsPath) + path := api.ASUserExistsPath + if a.Cfg.LegacyPaths { + path = api.ASUserExistsLegacyPath + } + URL, err := url.Parse(appservice.RequestUrl() + path) if err != nil { return err } URL.Path += request.UserID - apiURL := URL.String() + "?access_token=" + appservice.HSToken + if a.Cfg.LegacyAuth { + q := URL.Query() + q.Set("access_token", appservice.HSToken) + URL.RawQuery = q.Encode() + } + apiURL := URL.String() // Send a request to each application service. If one responds that it has // created the user, immediately return. @@ -136,6 +152,7 @@ func (a *AppServiceQueryAPI) UserIDExists( if err != nil { return err } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", appservice.HSToken)) resp, err := appservice.HTTPClient.Do(req.WithContext(ctx)) if resp != nil { defer func() { @@ -176,25 +193,22 @@ type thirdpartyResponses interface { api.ASProtocolResponse | []api.ASUserResponse | []api.ASLocationResponse } -func requestDo[T thirdpartyResponses](client *http.Client, url string, response *T) (err error) { - origURL := url - // try v1 and unstable appservice endpoints - for _, version := range []string{"v1", "unstable"} { - var resp *http.Response - var body []byte - asURL := strings.Replace(origURL, "unstable", version, 1) - resp, err = client.Get(asURL) - if err != nil { - continue - } - defer resp.Body.Close() // nolint: errcheck - body, err = io.ReadAll(resp.Body) - if err != nil { - continue - } - return json.Unmarshal(body, &response) +func requestDo[T thirdpartyResponses](as *config.ApplicationService, url string, response *T) error { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return err } - return err + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", as.HSToken)) + resp, err := as.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() // nolint: errcheck + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + return json.Unmarshal(body, &response) } func (a *AppServiceQueryAPI) Locations( @@ -207,16 +221,22 @@ func (a *AppServiceQueryAPI) Locations( return err } + path := api.ASLocationPath + if a.Cfg.LegacyPaths { + path = api.ASLocationLegacyPath + } for _, as := range a.Cfg.Derived.ApplicationServices { var asLocations []api.ASLocationResponse - params.Set("access_token", as.HSToken) + if a.Cfg.LegacyAuth { + params.Set("access_token", as.HSToken) + } - url := as.RequestUrl() + api.ASLocationPath + url := as.RequestUrl() + path if req.Protocol != "" { url += "/" + req.Protocol } - if err := requestDo[[]api.ASLocationResponse](as.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil { + if err := requestDo[[]api.ASLocationResponse](&as, url+"?"+params.Encode(), &asLocations); err != nil { log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'locations' from application service") continue } @@ -242,16 +262,22 @@ func (a *AppServiceQueryAPI) User( return err } + path := api.ASUserPath + if a.Cfg.LegacyPaths { + path = api.ASUserLegacyPath + } for _, as := range a.Cfg.Derived.ApplicationServices { var asUsers []api.ASUserResponse - params.Set("access_token", as.HSToken) + if a.Cfg.LegacyAuth { + params.Set("access_token", as.HSToken) + } - url := as.RequestUrl() + api.ASUserPath + url := as.RequestUrl() + path if req.Protocol != "" { url += "/" + req.Protocol } - if err := requestDo[[]api.ASUserResponse](as.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil { + if err := requestDo[[]api.ASUserResponse](&as, url+"?"+params.Encode(), &asUsers); err != nil { log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'user' from application service") continue } @@ -272,6 +298,10 @@ func (a *AppServiceQueryAPI) Protocols( req *api.ProtocolRequest, resp *api.ProtocolResponse, ) error { + protocolPath := api.ASProtocolPath + if a.Cfg.LegacyPaths { + protocolPath = api.ASProtocolLegacyPath + } // get a single protocol response if req.Protocol != "" { @@ -289,7 +319,7 @@ func (a *AppServiceQueryAPI) Protocols( response := api.ASProtocolResponse{} for _, as := range a.Cfg.Derived.ApplicationServices { var proto api.ASProtocolResponse - if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+req.Protocol, &proto); err != nil { + if err := requestDo[api.ASProtocolResponse](&as, as.RequestUrl()+protocolPath+req.Protocol, &proto); err != nil { log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service") continue } @@ -319,7 +349,7 @@ func (a *AppServiceQueryAPI) Protocols( for _, as := range a.Cfg.Derived.ApplicationServices { for _, p := range as.Protocols { var proto api.ASProtocolResponse - if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+p, &proto); err != nil { + if err := requestDo[api.ASProtocolResponse](&as, as.RequestUrl()+protocolPath+p, &proto); err != nil { log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service") continue } diff --git a/build/dendritejs-pinecone/main.go b/build/dendritejs-pinecone/main.go index 61baed902..6acc93c7b 100644 --- a/build/dendritejs-pinecone/main.go +++ b/build/dendritejs-pinecone/main.go @@ -38,6 +38,7 @@ import ( "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/userapi" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/gomatrixserverlib" @@ -190,13 +191,13 @@ func startup() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) + fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fedSenderAPI.IsBlacklistedOrBackingOff) asQuery := appservice.NewInternalAPI( processCtx, cfg, &natsInstance, userAPI, rsAPI, ) rsAPI.SetAppserviceAPI(asQuery) - fedSenderAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true) rsAPI.SetFederationAPI(fedSenderAPI, keyRing) monolith := setup.Monolith{ diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone index ab50cf318..dc6b22d7d 100644 --- a/build/docker/Dockerfile.demo-pinecone +++ b/build/docker/Dockerfile.demo-pinecone @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.21-alpine AS base +FROM docker.io/golang:1.22-alpine AS base # # Needs to be separate from the main Dockerfile for OpenShift, diff --git a/build/docker/Dockerfile.demo-yggdrasil b/build/docker/Dockerfile.demo-yggdrasil index b9e387666..6d6b37ea3 100644 --- a/build/docker/Dockerfile.demo-yggdrasil +++ b/build/docker/Dockerfile.demo-yggdrasil @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.21-alpine AS base +FROM docker.io/golang:1.22 AS base # # Needs to be separate from the main Dockerfile for OpenShift, diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 720ce37eb..2b227d373 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -216,7 +216,7 @@ func (m *DendriteMonolith) Start() { processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true, ) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) diff --git a/build/scripts/Complement.Dockerfile b/build/scripts/Complement.Dockerfile index 453d89765..660b84a46 100644 --- a/build/scripts/Complement.Dockerfile +++ b/build/scripts/Complement.Dockerfile @@ -1,6 +1,6 @@ #syntax=docker/dockerfile:1.2 -FROM golang:1.20-bullseye as build +FROM golang:1.22-bookworm as build RUN apt-get update && apt-get install -y sqlite3 WORKDIR /build diff --git a/build/scripts/ComplementLocal.Dockerfile b/build/scripts/ComplementLocal.Dockerfile index 0b80cfc40..8fc847650 100644 --- a/build/scripts/ComplementLocal.Dockerfile +++ b/build/scripts/ComplementLocal.Dockerfile @@ -8,7 +8,7 @@ # # Use these mounts to make use of this dockerfile: # COMPLEMENT_HOST_MOUNTS='/your/local/dendrite:/dendrite:ro;/your/go/path:/go:ro' -FROM golang:1.18-stretch +FROM golang:1.22-bookworm RUN apt-get update && apt-get install -y sqlite3 ENV SERVER_NAME=localhost diff --git a/build/scripts/ComplementPostgres.Dockerfile b/build/scripts/ComplementPostgres.Dockerfile index 77071b450..0026842d8 100644 --- a/build/scripts/ComplementPostgres.Dockerfile +++ b/build/scripts/ComplementPostgres.Dockerfile @@ -1,19 +1,19 @@ #syntax=docker/dockerfile:1.2 -FROM golang:1.20-bullseye as build +FROM golang:1.22-bookworm as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build -# No password when connecting over localhost -RUN sed -i "s%127.0.0.1/32 md5%127.0.0.1/32 trust%g" /etc/postgresql/13/main/pg_hba.conf && \ +# No password when connecting to Postgres +RUN sed -i "s%peer%trust%g" /etc/postgresql/15/main/pg_hba.conf && \ # Bump up max conns for moar concurrency - sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/13/main/postgresql.conf + sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/15/main/postgresql.conf # This entry script starts postgres, waits for it to be up then starts dendrite RUN echo '\ #!/bin/bash -eu \n\ pg_lsclusters \n\ - pg_ctlcluster 13 main start \n\ + pg_ctlcluster 15 main start \n\ \n\ until pg_isready \n\ do \n\ @@ -50,7 +50,7 @@ EXPOSE 8008 8448 # At runtime, generate TLS cert based on the CA now mounted at /ca # At runtime, replace the SERVER_NAME with what we are told CMD /build/run_postgres.sh && ./generate-keys --keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \ - ./generate-config -server $SERVER_NAME --ci --db postgresql://postgres@localhost/postgres?sslmode=disable > dendrite.yaml && \ + ./generate-config -server $SERVER_NAME --ci --db "user=postgres database=postgres host=/var/run/postgresql/" > dendrite.yaml && \ # Bump max_open_conns up here in the global database config sed -i 's/max_open_conns:.*$/max_open_conns: 1990/g' dendrite.yaml && \ cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index 66667b03c..b2adeb757 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -2,10 +2,12 @@ package clientapi import ( "context" + "encoding/json" "fmt" "net/http" "net/http/httptest" "reflect" + "strings" "testing" "time" @@ -45,7 +47,7 @@ func TestAdminCreateToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -196,7 +198,7 @@ func TestAdminListRegistrationTokens(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -314,7 +316,7 @@ func TestAdminGetRegistrationToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -415,7 +417,7 @@ func TestAdminDeleteRegistrationToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -509,7 +511,7 @@ func TestAdminUpdateRegistrationToken(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) accessTokens := map[*test.User]userDevice{ aliceAdmin: {}, @@ -693,7 +695,7 @@ func TestAdminResetPassword(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed for changing the password/login - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -791,7 +793,7 @@ func TestPurgeRoom(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) // Create the room @@ -863,7 +865,7 @@ func TestAdminEvacuateRoom(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { @@ -964,7 +966,7 @@ func TestAdminEvacuateUser(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // Create the room if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil { @@ -1055,7 +1057,7 @@ func TestAdminMarkAsStale(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1092,3 +1094,382 @@ func TestAdminMarkAsStale(t *testing.T) { } }) } + +func TestAdminQueryEventReports(t *testing.T) { + alice := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin)) + bob := test.NewUser(t) + room := test.NewRoom(t, alice) + room2 := test.NewRoom(t, alice) + + // room2 has a name and canonical alias + room2.CreateAndInsert(t, alice, spec.MRoomName, map[string]string{"name": "Testing"}, test.WithStateKey("")) + room2.CreateAndInsert(t, alice, spec.MRoomCanonicalAlias, map[string]string{"alias": "#testing"}, test.WithStateKey("")) + + // Join the rooms with Bob + room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID)) + room2.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID)) + + // Create a few events to report + eventsToReportPerRoom := make(map[string][]string) + for i := 0; i < 10; i++ { + ev1 := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"}) + ev2 := room2.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"}) + eventsToReportPerRoom[room.ID] = append(eventsToReportPerRoom[room.ID], ev1.EventID()) + eventsToReportPerRoom[room2.ID] = append(eventsToReportPerRoom[room2.ID], ev2.EventID()) + } + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + /*if dbType == test.DBTypeSQLite { + t.Skip() + }*/ + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + defer close() + natsInstance := jetstream.NATSInstance{} + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) + + // Use an actual roomserver for this + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) + + if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { + t.Fatalf("failed to send events: %v", err) + } + if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room2.Events(), "test", "test", "test", nil, false); err != nil { + t.Fatalf("failed to send events: %v", err) + } + + // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + accessTokens := map[*test.User]userDevice{ + alice: {}, + bob: {}, + } + createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers) + + reqBody := map[string]any{ + "reason": "baaad", + "score": -100, + } + body, err := json.Marshal(reqBody) + if err != nil { + t.Fatal(err) + } + + w := httptest.NewRecorder() + + var req *http.Request + // Report all events + for roomID, eventIDs := range eventsToReportPerRoom { + for _, eventID := range eventIDs { + req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", roomID, eventID), strings.NewReader(string(body))) + req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken) + + routers.Client.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected report to succeed, got HTTP %d instead: %s", w.Code, w.Body.String()) + } + } + } + + type response struct { + EventReports []api.QueryAdminEventReportsResponse `json:"event_reports"` + Total int64 `json:"total"` + NextToken *int64 `json:"next_token,omitempty"` + } + + t.Run("Can query all reports", func(t *testing.T) { + w = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/_synapse/admin/v1/event_reports", strings.NewReader(string(body))) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + + routers.SynapseAdmin.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String()) + } + var resp response + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + wantCount := 20 + // Only validating the count + if len(resp.EventReports) != wantCount { + t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports)) + } + if resp.Total != int64(wantCount) { + t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total) + } + }) + + t.Run("Can filter on room", func(t *testing.T) { + w = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_synapse/admin/v1/event_reports?room_id=%s", room.ID), strings.NewReader(string(body))) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + + routers.SynapseAdmin.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String()) + } + var resp response + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + wantCount := 10 + // Only validating the count + if len(resp.EventReports) != wantCount { + t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports)) + } + if resp.Total != int64(wantCount) { + t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total) + } + }) + + t.Run("Can filter on user_id", func(t *testing.T) { + w = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_synapse/admin/v1/event_reports?user_id=%s", "@doesnotexist:test"), strings.NewReader(string(body))) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + + routers.SynapseAdmin.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String()) + } + var resp response + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + + // The user does not exist, so we expect no results + wantCount := 0 + // Only validating the count + if len(resp.EventReports) != wantCount { + t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports)) + } + if resp.Total != int64(wantCount) { + t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total) + } + }) + + t.Run("Can set direction=f", func(t *testing.T) { + w = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_synapse/admin/v1/event_reports?room_id=%s&dir=f", room.ID), strings.NewReader(string(body))) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + + routers.SynapseAdmin.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String()) + } + var resp response + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + wantCount := 10 + // Only validating the count + if len(resp.EventReports) != wantCount { + t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports)) + } + if resp.Total != int64(wantCount) { + t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total) + } + // we now should have the first reported event + wantEventID := eventsToReportPerRoom[room.ID][0] + gotEventID := resp.EventReports[0].EventID + if gotEventID != wantEventID { + t.Fatalf("expected eventID to be %v, got %v", wantEventID, gotEventID) + } + }) + + t.Run("Can limit and paginate", func(t *testing.T) { + var from int64 = 0 + var limit int64 = 5 + var wantTotal int64 = 10 // We expect there to be 10 events in total + var resp response + for from+limit <= wantTotal { + resp = response{} + t.Logf("Getting reports starting from %d", from) + w = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/_synapse/admin/v1/event_reports?room_id=%s&limit=%d&from=%d", room2.ID, limit, from), strings.NewReader(string(body))) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + + routers.SynapseAdmin.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected getting reports to succeed, got HTTP %d instead: %s", w.Code, w.Body.String()) + } + + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + + wantCount := 5 // we are limited to 5 + if len(resp.EventReports) != wantCount { + t.Fatalf("expected %d events, got %d", wantCount, len(resp.EventReports)) + } + if resp.Total != int64(wantTotal) { + t.Fatalf("expected total to be %d, got %d", wantCount, resp.Total) + } + + // We've reached the end + if (from + int64(len(resp.EventReports))) == wantTotal { + return + } + + // The next_token should be set + if resp.NextToken == nil { + t.Fatal("expected nextToken to be set") + } + from = *resp.NextToken + } + }) + }) +} + +func TestEventReportsGetDelete(t *testing.T) { + alice := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin)) + bob := test.NewUser(t) + room := test.NewRoom(t, alice) + + // Add a name and alias + roomName := "Testing" + alias := "#testing" + room.CreateAndInsert(t, alice, spec.MRoomName, map[string]string{"name": roomName}, test.WithStateKey("")) + room.CreateAndInsert(t, alice, spec.MRoomCanonicalAlias, map[string]string{"alias": alias}, test.WithStateKey("")) + + // Join the rooms with Bob + room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID)) + + // Create a few events to report + + eventIDToReport := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"}) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + defer close() + natsInstance := jetstream.NATSInstance{} + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) + + // Use an actual roomserver for this + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) + + if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { + t.Fatalf("failed to send events: %v", err) + } + + // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + accessTokens := map[*test.User]userDevice{ + alice: {}, + bob: {}, + } + createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers) + + reqBody := map[string]any{ + "reason": "baaad", + "score": -100, + } + body, err := json.Marshal(reqBody) + if err != nil { + t.Fatal(err) + } + + w := httptest.NewRecorder() + + var req *http.Request + // Report the event + req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", room.ID, eventIDToReport.EventID()), strings.NewReader(string(body))) + req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken) + + routers.Client.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected report to succeed, got HTTP %d instead: %s", w.Code, w.Body.String()) + } + + t.Run("Can not query with invalid ID", func(t *testing.T) { + w = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/_synapse/admin/v1/event_reports/abc", strings.NewReader(string(body))) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + + routers.SynapseAdmin.ServeHTTP(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("expected getting report to fail, got HTTP %d instead: %s", w.Code, w.Body.String()) + } + }) + + t.Run("Can query with valid ID", func(t *testing.T) { + w = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/_synapse/admin/v1/event_reports/1", strings.NewReader(string(body))) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + + routers.SynapseAdmin.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected getting report to fail, got HTTP %d instead: %s", w.Code, w.Body.String()) + } + resp := api.QueryAdminEventReportResponse{} + if err = json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + // test a few things + if resp.EventID != eventIDToReport.EventID() { + t.Fatalf("expected eventID to be %s, got %s instead", eventIDToReport.EventID(), resp.EventID) + } + if resp.RoomName != roomName { + t.Fatalf("expected roomName to be %s, got %s instead", roomName, resp.RoomName) + } + if resp.CanonicalAlias != alias { + t.Fatalf("expected alias to be %s, got %s instead", alias, resp.CanonicalAlias) + } + if reflect.DeepEqual(resp.EventJSON, eventIDToReport.JSON()) { + t.Fatal("mismatching eventJSON") + } + }) + + t.Run("Can delete with a valid ID", func(t *testing.T) { + w = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodDelete, "/_synapse/admin/v1/event_reports/1", strings.NewReader(string(body))) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + + routers.SynapseAdmin.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected getting report to fail, got HTTP %d instead: %s", w.Code, w.Body.String()) + } + }) + + t.Run("Can not query deleted report", func(t *testing.T) { + w = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/_synapse/admin/v1/event_reports/1", strings.NewReader(string(body))) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + + routers.SynapseAdmin.ServeHTTP(w, req) + + if w.Code == http.StatusOK { + t.Fatalf("expected getting report to fail, got HTTP %d instead: %s", w.Code, w.Body.String()) + } + }) + }) +} diff --git a/clientapi/auth/login.go b/clientapi/auth/login.go index 77835614e..58a27e593 100644 --- a/clientapi/auth/login.go +++ b/clientapi/auth/login.go @@ -15,7 +15,6 @@ package auth import ( - "context" "encoding/json" "io" "net/http" @@ -32,8 +31,13 @@ import ( // called after authorization has completed, with the result of the authorization. // If the final return value is non-nil, an error occurred and the cleanup function // is nil. -func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.UserLoginAPI, userAPI UserInternalAPIForLogin, cfg *config.ClientAPI) (*Login, LoginCleanupFunc, *util.JSONResponse) { - reqBytes, err := io.ReadAll(r) +func LoginFromJSONReader( + req *http.Request, + useraccountAPI uapi.UserLoginAPI, + userAPI UserInternalAPIForLogin, + cfg *config.ClientAPI, +) (*Login, LoginCleanupFunc, *util.JSONResponse) { + reqBytes, err := io.ReadAll(req.Body) if err != nil { err := &util.JSONResponse{ Code: http.StatusBadRequest, @@ -65,6 +69,20 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U UserAPI: userAPI, Config: cfg, } + case authtypes.LoginTypeApplicationService: + token, err := ExtractAccessToken(req) + if err != nil { + err := &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.MissingToken(err.Error()), + } + return nil, nil, err + } + + typ = &LoginTypeApplicationService{ + Config: cfg, + Token: token, + } default: err := util.JSONResponse{ Code: http.StatusBadRequest, @@ -73,7 +91,7 @@ func LoginFromJSONReader(ctx context.Context, r io.Reader, useraccountAPI uapi.U return nil, nil, &err } - return typ.LoginFromJSON(ctx, reqBytes) + return typ.LoginFromJSON(req.Context(), reqBytes) } // UserInternalAPIForLogin contains the aspects of UserAPI required for logging in. diff --git a/clientapi/auth/login_application_service.go b/clientapi/auth/login_application_service.go new file mode 100644 index 000000000..dd4a9cbb4 --- /dev/null +++ b/clientapi/auth/login_application_service.go @@ -0,0 +1,55 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "context" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/util" +) + +// LoginTypeApplicationService describes how to authenticate as an +// application service +type LoginTypeApplicationService struct { + Config *config.ClientAPI + Token string +} + +// Name implements Type +func (t *LoginTypeApplicationService) Name() string { + return authtypes.LoginTypeApplicationService +} + +// LoginFromJSON implements Type +func (t *LoginTypeApplicationService) LoginFromJSON( + ctx context.Context, reqBytes []byte, +) (*Login, LoginCleanupFunc, *util.JSONResponse) { + var r Login + if err := httputil.UnmarshalJSON(reqBytes, &r); err != nil { + return nil, nil, err + } + + _, err := internal.ValidateApplicationServiceRequest(t.Config, r.Identifier.User, t.Token) + if err != nil { + return nil, nil, err + } + + cleanup := func(ctx context.Context, j *util.JSONResponse) {} + return &r, cleanup, nil +} diff --git a/clientapi/auth/login_test.go b/clientapi/auth/login_test.go index 93d3e2713..a2c2a719c 100644 --- a/clientapi/auth/login_test.go +++ b/clientapi/auth/login_test.go @@ -17,7 +17,9 @@ package auth import ( "context" "net/http" + "net/http/httptest" "reflect" + "regexp" "strings" "testing" @@ -33,8 +35,9 @@ func TestLoginFromJSONReader(t *testing.T) { ctx := context.Background() tsts := []struct { - Name string - Body string + Name string + Body string + Token string WantUsername string WantDeviceID string @@ -62,6 +65,30 @@ func TestLoginFromJSONReader(t *testing.T) { WantDeviceID: "adevice", WantDeletedTokens: []string{"atoken"}, }, + { + Name: "appServiceWorksUserID", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "@alice:example.com" }, + "device_id": "adevice" + }`, + Token: "astoken", + + WantUsername: "@alice:example.com", + WantDeviceID: "adevice", + }, + { + Name: "appServiceWorksLocalpart", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "alice" }, + "device_id": "adevice" + }`, + Token: "astoken", + + WantUsername: "alice", + WantDeviceID: "adevice", + }, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { @@ -72,11 +99,35 @@ func TestLoginFromJSONReader(t *testing.T) { ServerName: serverName, }, }, + Derived: &config.Derived{ + ApplicationServices: []config.ApplicationService{ + { + ID: "anapplicationservice", + ASToken: "astoken", + NamespaceMap: map[string][]config.ApplicationServiceNamespace{ + "users": { + { + Exclusive: true, + Regex: "@alice:example.com", + RegexpObject: regexp.MustCompile("@alice:example.com"), + }, + }, + }, + }, + }, + }, } - login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg) - if err != nil { - t.Fatalf("LoginFromJSONReader failed: %+v", err) + + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tst.Body)) + if tst.Token != "" { + req.Header.Add("Authorization", "Bearer "+tst.Token) } + + login, cleanup, jsonErr := LoginFromJSONReader(req, &userAPI, &userAPI, cfg) + if jsonErr != nil { + t.Fatalf("LoginFromJSONReader failed: %+v", jsonErr) + } + cleanup(ctx, &util.JSONResponse{Code: http.StatusOK}) if login.Username() != tst.WantUsername { @@ -104,8 +155,9 @@ func TestBadLoginFromJSONReader(t *testing.T) { ctx := context.Background() tsts := []struct { - Name string - Body string + Name string + Body string + Token string WantErrCode spec.MatrixErrorCode }{ @@ -142,6 +194,45 @@ func TestBadLoginFromJSONReader(t *testing.T) { }`, WantErrCode: spec.ErrorInvalidParam, }, + { + Name: "noASToken", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "@alice:example.com" }, + "device_id": "adevice" + }`, + WantErrCode: "M_MISSING_TOKEN", + }, + { + Name: "badASToken", + Token: "badastoken", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "@alice:example.com" }, + "device_id": "adevice" + }`, + WantErrCode: "M_UNKNOWN_TOKEN", + }, + { + Name: "badASNamespace", + Token: "astoken", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "@bob:example.com" }, + "device_id": "adevice" + }`, + WantErrCode: "M_EXCLUSIVE", + }, + { + Name: "badASUserID", + Token: "astoken", + Body: `{ + "type": "m.login.application_service", + "identifier": { "type": "m.id.user", "user": "@alice:wrong.example.com" }, + "device_id": "adevice" + }`, + WantErrCode: "M_INVALID_USERNAME", + }, } for _, tst := range tsts { t.Run(tst.Name, func(t *testing.T) { @@ -152,8 +243,30 @@ func TestBadLoginFromJSONReader(t *testing.T) { ServerName: serverName, }, }, + Derived: &config.Derived{ + ApplicationServices: []config.ApplicationService{ + { + ID: "anapplicationservice", + ASToken: "astoken", + NamespaceMap: map[string][]config.ApplicationServiceNamespace{ + "users": { + { + Exclusive: true, + Regex: "@alice:example.com", + RegexpObject: regexp.MustCompile("@alice:example.com"), + }, + }, + }, + }, + }, + }, } - _, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg) + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tst.Body)) + if tst.Token != "" { + req.Header.Add("Authorization", "Bearer "+tst.Token) + } + + _, cleanup, errRes := LoginFromJSONReader(req, &userAPI, &userAPI, cfg) if errRes == nil { cleanup(ctx, nil) t.Fatalf("LoginFromJSONReader err: got %+v, want code %q", errRes, tst.WantErrCode) diff --git a/clientapi/auth/user_interactive.go b/clientapi/auth/user_interactive.go index 92d83ad29..9831450cc 100644 --- a/clientapi/auth/user_interactive.go +++ b/clientapi/auth/user_interactive.go @@ -55,7 +55,7 @@ type LoginCleanupFunc func(context.Context, *util.JSONResponse) // https://matrix.org/docs/spec/client_server/r0.6.1#identifier-types type LoginIdentifier struct { Type string `json:"type"` - // when type = m.id.user + // when type = m.id.user or m.id.application_service User string `json:"user"` // when type = m.id.thirdparty Medium string `json:"medium"` diff --git a/clientapi/clientapi_test.go b/clientapi/clientapi_test.go index f2d617cb9..c550b2083 100644 --- a/clientapi/clientapi_test.go +++ b/clientapi/clientapi_test.go @@ -17,6 +17,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/routing" "github.com/matrix-org/dendrite/clientapi/threepid" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/pushrules" @@ -49,6 +50,10 @@ type userDevice struct { password string } +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + func TestGetPutDevices(t *testing.T) { alice := test.NewUser(t) bob := test.NewUser(t) @@ -121,7 +126,7 @@ func TestGetPutDevices(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -170,7 +175,7 @@ func TestDeleteDevice(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -275,7 +280,7 @@ func TestDeleteDevices(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -442,7 +447,7 @@ func TestSetDisplayname(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI) AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -554,7 +559,7 @@ func TestSetAvatarURL(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asPI := appservice.NewInternalAPI(processCtx, cfg, natsInstance, userAPI, rsAPI) AddPublicRoutes(processCtx, routers, cfg, natsInstance, base.CreateFederationClient(cfg, nil), rsAPI, asPI, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -632,7 +637,7 @@ func TestTyping(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed to create accounts - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -716,7 +721,7 @@ func TestMembership(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed to create accounts - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) rsAPI.SetUserAPI(userAPI) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -953,9 +958,10 @@ func TestCapabilities(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) // Needed to create accounts - rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1000,9 +1006,10 @@ func TestTurnserver(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) // Needed to create accounts - rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) //rsAPI.SetUserAPI(userAPI) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1098,9 +1105,10 @@ func Test3PID(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) // Needed to create accounts - rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1276,7 +1284,7 @@ func TestPushRules(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -1663,7 +1671,7 @@ func TestKeys(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -2125,7 +2133,7 @@ func TestKeyBackup(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) @@ -2146,3 +2154,284 @@ func TestKeyBackup(t *testing.T) { } }) } + +func TestGetMembership(t *testing.T) { + alice := test.NewUser(t) + bob := test.NewUser(t) + + testCases := []struct { + name string + roomID string + user *test.User + additionalEvents func(t *testing.T, room *test.Room) + request func(t *testing.T, room *test.Room, accessToken string) *http.Request + wantOK bool + wantMemberCount int + }{ + + { + name: "/joined_members - Bob never joined", + user: bob, + request: func(t *testing.T, room *test.Room, accessToken string) *http.Request { + return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{ + "access_token": accessToken, + })) + }, + wantOK: false, + }, + { + name: "/joined_members - Alice joined", + user: alice, + request: func(t *testing.T, room *test.Room, accessToken string) *http.Request { + return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{ + "access_token": accessToken, + })) + }, + wantOK: true, + wantMemberCount: 1, + }, + { + name: "/joined_members - Alice leaves, shouldn't be able to see members ", + user: alice, + request: func(t *testing.T, room *test.Room, accessToken string) *http.Request { + return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{ + "access_token": accessToken, + })) + }, + additionalEvents: func(t *testing.T, room *test.Room) { + room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{ + "membership": "leave", + }, test.WithStateKey(alice.ID)) + }, + wantOK: false, + }, + { + name: "/joined_members - Bob joins, Alice sees two members", + user: alice, + request: func(t *testing.T, room *test.Room, accessToken string) *http.Request { + return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{ + "access_token": accessToken, + })) + }, + additionalEvents: func(t *testing.T, room *test.Room) { + room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID)) + }, + wantOK: true, + wantMemberCount: 2, + }, + } + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + defer close() + natsInstance := jetstream.NATSInstance{} + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) + + // Use an actual roomserver for this + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) + + // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + accessTokens := map[*test.User]userDevice{ + alice: {}, + bob: {}, + } + createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + room := test.NewRoom(t, alice) + t.Cleanup(func() { + t.Logf("running cleanup for %s", tc.name) + }) + // inject additional events + if tc.additionalEvents != nil { + tc.additionalEvents(t, room) + } + if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { + t.Fatalf("failed to send events: %v", err) + } + + w := httptest.NewRecorder() + routers.Client.ServeHTTP(w, tc.request(t, room, accessTokens[tc.user].accessToken)) + if w.Code != 200 && tc.wantOK { + t.Logf("%s", w.Body.String()) + t.Fatalf("got HTTP %d want %d", w.Code, 200) + } + t.Logf("[%s] Resp: %s", tc.name, w.Body.String()) + + // check we got the expected events + if tc.wantOK { + memberCount := len(gjson.GetBytes(w.Body.Bytes(), "joined").Map()) + if memberCount != tc.wantMemberCount { + t.Fatalf("expected %d members, got %d", tc.wantMemberCount, memberCount) + } + } + }) + } + }) +} + +func TestCreateRoomInvite(t *testing.T) { + alice := test.NewUser(t) + bob := test.NewUser(t) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + defer close() + natsInstance := jetstream.NATSInstance{} + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) + + // Use an actual roomserver for this + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) + + // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + accessTokens := map[*test.User]userDevice{ + alice: {}, + } + createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers) + + reqBody := map[string]any{ + "invite": []string{bob.ID}, + } + body, err := json.Marshal(reqBody) + if err != nil { + t.Fatal(err) + } + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/createRoom", strings.NewReader(string(body))) + req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken) + + routers.Client.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected room creation to be successful, got HTTP %d instead: %s", w.Code, w.Body.String()) + } + + roomID := gjson.GetBytes(w.Body.Bytes(), "room_id").Str + validRoomID, _ := spec.NewRoomID(roomID) + // Now ask the roomserver about the membership event of Bob + ev, err := rsAPI.CurrentStateEvent(context.Background(), *validRoomID, spec.MRoomMember, bob.ID) + if err != nil { + t.Fatal(err) + } + + if ev == nil { + t.Fatal("Membership event for Bob does not exist") + } + + // Validate that there is NO displayname in content + if gjson.GetBytes(ev.Content(), "displayname").Exists() { + t.Fatal("Found displayname in invite") + } + }) +} + +func TestReportEvent(t *testing.T) { + alice := test.NewUser(t) + bob := test.NewUser(t) + charlie := test.NewUser(t) + room := test.NewRoom(t, alice) + + room.CreateAndInsert(t, charlie, spec.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(charlie.ID)) + eventToReport := room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world"}) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + defer close() + natsInstance := jetstream.NATSInstance{} + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) + + // Use an actual roomserver for this + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) + + if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { + t.Fatalf("failed to send events: %v", err) + } + + // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) + + accessTokens := map[*test.User]userDevice{ + alice: {}, + bob: {}, + charlie: {}, + } + createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers) + + reqBody := map[string]any{ + "reason": "baaad", + "score": -100, + } + body, err := json.Marshal(reqBody) + if err != nil { + t.Fatal(err) + } + + w := httptest.NewRecorder() + + var req *http.Request + t.Run("Bob is not joined and should not be able to report the event", func(t *testing.T) { + req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", room.ID, eventToReport.EventID()), strings.NewReader(string(body))) + req.Header.Set("Authorization", "Bearer "+accessTokens[bob].accessToken) + + routers.Client.ServeHTTP(w, req) + + if w.Code != http.StatusNotFound { + t.Fatalf("expected report to fail, got HTTP %d instead: %s", w.Code, w.Body.String()) + } + }) + + t.Run("Charlie is joined but the event does not exist", func(t *testing.T) { + w = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/$doesNotExist", room.ID), strings.NewReader(string(body))) + req.Header.Set("Authorization", "Bearer "+accessTokens[charlie].accessToken) + + routers.Client.ServeHTTP(w, req) + + if w.Code != http.StatusNotFound { + t.Fatalf("expected report to fail, got HTTP %d instead: %s", w.Code, w.Body.String()) + } + }) + + t.Run("Charlie is joined and allowed to report the event", func(t *testing.T) { + w = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/_matrix/client/v3/rooms/%s/report/%s", room.ID, eventToReport.EventID()), strings.NewReader(string(body))) + req.Header.Set("Authorization", "Bearer "+accessTokens[charlie].accessToken) + + routers.Client.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected report to be successful, got HTTP %d instead: %s", w.Code, w.Body.String()) + } + }) + }) +} diff --git a/clientapi/httputil/parse.go b/clientapi/httputil/parse.go index c83583345..a952d1778 100644 --- a/clientapi/httputil/parse.go +++ b/clientapi/httputil/parse.go @@ -35,5 +35,5 @@ func ParseTSParam(req *http.Request) (time.Time, error) { return time.Time{}, fmt.Errorf("param 'ts' is no valid int (%s)", err.Error()) } - return time.Unix(ts/1000, 0), nil + return time.UnixMilli(ts), nil } diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go index 519666076..73a0afc32 100644 --- a/clientapi/routing/admin.go +++ b/clientapi/routing/admin.go @@ -328,7 +328,7 @@ func AdminPurgeRoom(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI) } } -func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.Device, userAPI api.ClientUserAPI) util.JSONResponse { +func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *api.Device, userAPI userapi.ClientUserAPI) util.JSONResponse { if req.Body == nil { return util.JSONResponse{ Code: http.StatusBadRequest, @@ -423,7 +423,7 @@ func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *api.Device, } } -func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.ClientKeyAPI) util.JSONResponse { +func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI userapi.ClientKeyAPI) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -495,3 +495,93 @@ func AdminDownloadState(req *http.Request, device *api.Device, rsAPI roomserverA JSON: struct{}{}, } } + +// GetEventReports returns reported events for a given user/room. +func GetEventReports( + req *http.Request, + rsAPI roomserverAPI.ClientRoomserverAPI, + from, limit uint64, + backwards bool, + userID, roomID string, +) util.JSONResponse { + + eventReports, count, err := rsAPI.QueryAdminEventReports(req.Context(), from, limit, backwards, userID, roomID) + if err != nil { + logrus.WithError(err).Error("failed to query event reports") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + resp := map[string]any{ + "event_reports": eventReports, + "total": count, + } + + // Add a next_token if there are still reports + if int64(from+limit) < count { + resp["next_token"] = int(from) + len(eventReports) + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: resp, + } +} + +func GetEventReport(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, reportID string) util.JSONResponse { + parsedReportID, err := strconv.ParseUint(reportID, 10, 64) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + // Given this is an admin endpoint, let them know what didn't work. + JSON: spec.InvalidParam(err.Error()), + } + } + + report, err := rsAPI.QueryAdminEventReport(req.Context(), parsedReportID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown(err.Error()), + } + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: report, + } +} + +func DeleteEventReport(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, reportID string) util.JSONResponse { + parsedReportID, err := strconv.ParseUint(reportID, 10, 64) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + // Given this is an admin endpoint, let them know what didn't work. + JSON: spec.InvalidParam(err.Error()), + } + } + + err = rsAPI.PerformAdminDeleteEventReport(req.Context(), parsedReportID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.Unknown(err.Error()), + } + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} + +func parseUint64OrDefault(input string, defaultValue uint64) uint64 { + v, err := strconv.ParseUint(input, 10, 64) + if err != nil { + return defaultValue + } + return v +} diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 907727662..9466f583f 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -55,7 +55,7 @@ func DirectoryRoom( if err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: spec.BadJSON("Room alias must be in the form '#localpart:domain'"), + JSON: spec.InvalidParam("Room alias must be in the form '#localpart:domain'"), } } @@ -134,7 +134,7 @@ func SetLocalAlias( if err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: spec.BadJSON("Room alias must be in the form '#localpart:domain'"), + JSON: spec.InvalidParam("Room alias must be in the form '#localpart:domain'"), } } diff --git a/clientapi/routing/joinroom_test.go b/clientapi/routing/joinroom_test.go index 933ea8d3a..bd854efa8 100644 --- a/clientapi/routing/joinroom_test.go +++ b/clientapi/routing/joinroom_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/jetstream" @@ -21,6 +22,10 @@ import ( uapi "github.com/matrix-org/dendrite/userapi/api" ) +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + func TestJoinRoomByIDOrAlias(t *testing.T) { alice := test.NewUser(t) bob := test.NewUser(t) @@ -36,7 +41,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { natsInstance := jetstream.NATSInstance{} rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) // Create the users in the userapi diff --git a/clientapi/routing/keys.go b/clientapi/routing/keys.go index 72785cda8..871b8b08e 100644 --- a/clientapi/routing/keys.go +++ b/clientapi/routing/keys.go @@ -93,7 +93,6 @@ func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device) type queryKeysRequest struct { Timeout int `json:"timeout"` - Token string `json:"token"` DeviceKeys map[string][]string `json:"device_keys"` } @@ -119,7 +118,6 @@ func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device) u UserID: device.UserID, UserToDevices: r.DeviceKeys, Timeout: r.GetTimeout(), - // TODO: Token? }, &queryRes) return util.JSONResponse{ Code: 200, diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index bc38b8340..0f55c8816 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -19,6 +19,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/auth" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -40,28 +41,25 @@ type flow struct { Type string `json:"type"` } -func passwordLogin() flows { - f := flows{} - s := flow{ - Type: "m.login.password", - } - f.Flows = append(f.Flows, s) - return f -} - // Login implements GET and POST /login func Login( req *http.Request, userAPI userapi.ClientUserAPI, cfg *config.ClientAPI, ) util.JSONResponse { if req.Method == http.MethodGet { - // TODO: support other forms of login other than password, depending on config options + loginFlows := []flow{{Type: authtypes.LoginTypePassword}} + if len(cfg.Derived.ApplicationServices) > 0 { + loginFlows = append(loginFlows, flow{Type: authtypes.LoginTypeApplicationService}) + } + // TODO: support other forms of login, depending on config options return util.JSONResponse{ Code: http.StatusOK, - JSON: passwordLogin(), + JSON: flows{ + Flows: loginFlows, + }, } } else if req.Method == http.MethodPost { - login, cleanup, authErr := auth.LoginFromJSONReader(req.Context(), req.Body, userAPI, userAPI, cfg) + login, cleanup, authErr := auth.LoginFromJSONReader(req, userAPI, userAPI, cfg) if authErr != nil { return *authErr } diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go index 252017db2..3f8934490 100644 --- a/clientapi/routing/login_test.go +++ b/clientapi/routing/login_test.go @@ -49,7 +49,7 @@ func TestLogin(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) // Needed for /login - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. Setup(routers, cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, caching.DisableMetrics) @@ -114,6 +114,44 @@ func TestLogin(t *testing.T) { ctx := context.Background() + // Inject a dummy application service, so we have a "m.login.application_service" + // in the login flows + as := &config.ApplicationService{} + cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{*as} + + t.Run("Supported log-in flows are returned", func(t *testing.T) { + req := test.NewRequest(t, http.MethodGet, "/_matrix/client/v3/login") + rec := httptest.NewRecorder() + routers.Client.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("failed to get log-in flows: %s", rec.Body.String()) + } + + t.Logf("response: %s", rec.Body.String()) + resp := flows{} + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + + appServiceFound := false + passwordFound := false + for _, flow := range resp.Flows { + if flow.Type == "m.login.password" { + passwordFound = true + } else if flow.Type == "m.login.application_service" { + appServiceFound = true + } else { + t.Fatalf("got unknown login flow: %s", flow.Type) + } + } + if !appServiceFound { + t.Fatal("m.login.application_service missing from login flows") + } + if !passwordFound { + t.Fatal("m.login.password missing from login flows") + } + }) + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{ diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 8b8cc47bc..9e41a3794 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -181,18 +181,6 @@ func SendKick( return *errRes } - pl, errRes := getPowerlevels(req, rsAPI, roomID) - if errRes != nil { - return *errRes - } - allowedToKick := pl.UserLevel(*senderID) >= pl.Kick - if !allowedToKick { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: spec.Forbidden("You don't have permission to kick this user, power level too low."), - } - } - bodyUserID, err := spec.NewUserID(body.UserID, true) if err != nil { return util.JSONResponse{ @@ -200,6 +188,19 @@ func SendKick( JSON: spec.BadJSON("body userID is invalid"), } } + + pl, errRes := getPowerlevels(req, rsAPI, roomID) + if errRes != nil { + return *errRes + } + allowedToKick := pl.UserLevel(*senderID) >= pl.Kick || bodyUserID.String() == deviceUserID.String() + if !allowedToKick { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("You don't have permission to kick this user, power level too low."), + } + } + var queryRes roomserverAPI.QueryMembershipForUserResponse err = rsAPI.QueryMembershipForUser(req.Context(), &roomserverAPI.QueryMembershipForUserRequest{ RoomID: roomID, @@ -323,19 +324,18 @@ func SendInvite( } // We already received the return value, so no need to check for an error here. - response, _ := sendInvite(req.Context(), profileAPI, device, roomID, body.UserID, body.Reason, cfg, rsAPI, asAPI, evTime) + response, _ := sendInvite(req.Context(), device, roomID, body.UserID, body.Reason, cfg, rsAPI, evTime) return response } // sendInvite sends an invitation to a user. Returns a JSONResponse and an error func sendInvite( ctx context.Context, - profileAPI userapi.ClientUserAPI, device *userapi.Device, roomID, userID, reason string, cfg *config.ClientAPI, rsAPI roomserverAPI.ClientRoomserverAPI, - asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time, + evTime time.Time, ) (util.JSONResponse, error) { validRoomID, err := spec.NewRoomID(roomID) if err != nil { @@ -358,13 +358,7 @@ func sendInvite( JSON: spec.InvalidParam("UserID is invalid"), }, err } - profile, err := loadProfile(ctx, userID, cfg, profileAPI, asAPI) - if err != nil { - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - }, err - } + identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain()) if err != nil { return util.JSONResponse{ @@ -374,16 +368,14 @@ func sendInvite( } err = rsAPI.PerformInvite(ctx, &api.PerformInviteRequest{ InviteInput: roomserverAPI.InviteInput{ - RoomID: *validRoomID, - Inviter: *inviter, - Invitee: *invitee, - DisplayName: profile.DisplayName, - AvatarURL: profile.AvatarURL, - Reason: reason, - IsDirect: false, - KeyID: identity.KeyID, - PrivateKey: identity.PrivateKey, - EventTime: evTime, + RoomID: *validRoomID, + Inviter: *inviter, + Invitee: *invitee, + Reason: reason, + IsDirect: false, + KeyID: identity.KeyID, + PrivateKey: identity.PrivateKey, + EventTime: evTime, }, InviteRoomState: nil, // ask the roomserver to draw up invite room state for us SendAsServer: string(device.UserDomain()), diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go new file mode 100644 index 000000000..84be498d6 --- /dev/null +++ b/clientapi/routing/memberships.go @@ -0,0 +1,139 @@ +// Copyright 2024 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "encoding/json" + "net/http" + + "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" +) + +// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members +type getJoinedMembersResponse struct { + Joined map[string]joinedMember `json:"joined"` +} + +type joinedMember struct { + DisplayName string `json:"display_name"` + AvatarURL string `json:"avatar_url"` +} + +// The database stores 'displayname' without an underscore. +// Deserialize into this and then change to the actual API response +type databaseJoinedMember struct { + DisplayName string `json:"displayname"` + AvatarURL string `json:"avatar_url"` +} + +// GetJoinedMembers implements +// +// GET /rooms/{roomId}/joined_members +func GetJoinedMembers( + req *http.Request, device *userapi.Device, roomID string, + rsAPI api.ClientRoomserverAPI, +) util.JSONResponse { + // Validate the userID + userID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("Device UserID is invalid"), + } + } + + // Validate the roomID + validRoomID, err := spec.NewRoomID(roomID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.InvalidParam("RoomID is invalid"), + } + } + + // Get the current memberships for the requesting user to determine + // if they are allowed to query this endpoint. + queryReq := api.QueryMembershipForUserRequest{ + RoomID: validRoomID.String(), + UserID: *userID, + } + + var queryRes api.QueryMembershipForUserResponse + if queryErr := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); queryErr != nil { + util.GetLogger(req.Context()).WithError(queryErr).Error("rsAPI.QueryMembershipsForRoom failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + if !queryRes.HasBeenInRoom { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."), + } + } + + if !queryRes.IsInRoom { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."), + } + } + + // Get the current membership events + var membershipsForRoomResp api.QueryMembershipsForRoomResponse + if err = rsAPI.QueryMembershipsForRoom(req.Context(), &api.QueryMembershipsForRoomRequest{ + JoinedOnly: true, + RoomID: validRoomID.String(), + }, &membershipsForRoomResp); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryEventsByID failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + var res getJoinedMembersResponse + res.Joined = make(map[string]joinedMember) + for _, ev := range membershipsForRoomResp.JoinEvents { + var content databaseJoinedMember + if err := json.Unmarshal(ev.Content, &content); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + userID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, spec.SenderID(ev.Sender)) + if err != nil || userID == nil { + util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryUserIDForSender failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{}, + } + } + + res.Joined[userID.String()] = joinedMember(content) + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: res, + } +} diff --git a/clientapi/routing/pushrules.go b/clientapi/routing/pushrules.go index 74873d5c9..43c034f9d 100644 --- a/clientapi/routing/pushrules.go +++ b/clientapi/routing/pushrules.go @@ -70,7 +70,7 @@ func GetPushRulesByKind(ctx context.Context, scope, kind string, device *userapi } rulesPtr := pushRuleSetKindPointer(ruleSet, pushrules.Kind(kind)) // Even if rulesPtr is not nil, there may not be any rules for this kind - if rulesPtr == nil || (rulesPtr != nil && len(*rulesPtr) == 0) { + if rulesPtr == nil || len(*rulesPtr) == 0 { return errorResponse(ctx, spec.InvalidParam("invalid push rules kind"), "pushRuleSetKindPointer failed") } return util.JSONResponse{ diff --git a/clientapi/routing/receipt.go b/clientapi/routing/receipt.go index be6542979..1d7e35562 100644 --- a/clientapi/routing/receipt.go +++ b/clientapi/routing/receipt.go @@ -23,13 +23,12 @@ import ( "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/gomatrixserverlib/spec" - "github.com/matrix-org/dendrite/userapi/api" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) -func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse { +func SetReceipt(req *http.Request, userAPI userapi.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse { timestamp := spec.AsTimestamp(time.Now()) logrus.WithFields(logrus.Fields{ "roomID": roomID, @@ -54,13 +53,13 @@ func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *prod } } - dataReq := api.InputAccountDataRequest{ + dataReq := userapi.InputAccountDataRequest{ UserID: device.UserID, DataType: "m.fully_read", RoomID: roomID, AccountData: data, } - dataRes := api.InputAccountDataResponse{} + dataRes := userapi.InputAccountDataResponse{} if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") return util.ErrorResponse(err) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 090a2fc20..5235e9092 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -630,6 +630,7 @@ func handleGuestRegistration( AccessToken: token, IPAddr: req.RemoteAddr, UserAgent: req.UserAgent(), + FromRegistration: true, }, &devRes) if err != nil { return util.JSONResponse{ @@ -647,6 +648,16 @@ func handleGuestRegistration( } } +// localpartMatchesExclusiveNamespaces will check if a given username matches any +// application service's exclusive users namespace +func localpartMatchesExclusiveNamespaces( + cfg *config.ClientAPI, + localpart string, +) bool { + userID := userutil.MakeUserID(localpart, cfg.Matrix.ServerName) + return cfg.Derived.ExclusiveApplicationServicesUsernameRegexp.MatchString(userID) +} + // handleRegistrationFlow will direct and complete registration flow stages // that the client has requested. // nolint: gocyclo @@ -695,7 +706,7 @@ func handleRegistrationFlow( // If an access token is provided, ignore this check this is an appservice // request and we will validate in validateApplicationService if len(cfg.Derived.ApplicationServices) != 0 && - UsernameMatchesExclusiveNamespaces(cfg, r.Username) { + localpartMatchesExclusiveNamespaces(cfg, r.Username) { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.ASExclusive("This username is reserved by an application service."), @@ -772,7 +783,7 @@ func handleApplicationServiceRegistration( // Check application service register user request is valid. // The application service's ID is returned if so. - appserviceID, err := validateApplicationService( + appserviceID, err := internal.ValidateApplicationServiceRequest( cfg, r.Username, accessToken, ) if err != nil { @@ -909,6 +920,7 @@ func completeRegistration( DeviceID: deviceID, IPAddr: ipAddr, UserAgent: userAgent, + FromRegistration: true, }, &devRes) if err != nil { return util.JSONResponse{ diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index 0a1986cf7..7fa740e7f 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -298,25 +298,29 @@ func Test_register(t *testing.T) { guestsDisabled bool enableRecaptcha bool captchaBody string - wantResponse util.JSONResponse + // in case of an error, the expected response + wantErrorResponse util.JSONResponse + // in case of success, the expected username assigned + wantUsername string }{ { name: "disallow guests", kind: "guest", guestsDisabled: true, - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Forbidden(`Guest registration is disabled on "test"`), }, }, { - name: "allow guests", - kind: "guest", + name: "allow guests", + kind: "guest", + wantUsername: "1", }, { name: "unknown login type", loginType: "im.not.known", - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusNotImplemented, JSON: spec.Unknown("unknown/unimplemented auth type"), }, @@ -324,25 +328,33 @@ func Test_register(t *testing.T) { { name: "disabled registration", registrationDisabled: true, - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Forbidden(`Registration is disabled on "test"`), }, }, { - name: "successful registration, numeric ID", - username: "", - password: "someRandomPassword", - forceEmpty: true, + name: "successful registration, numeric ID", + username: "", + password: "someRandomPassword", + forceEmpty: true, + wantUsername: "2", }, { name: "successful registration", username: "success", }, + { + name: "successful registration, sequential numeric ID", + username: "", + password: "someRandomPassword", + forceEmpty: true, + wantUsername: "3", + }, { name: "failing registration - user already exists", username: "success", - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.UserInUse("Desired user ID is already taken."), }, @@ -352,14 +364,14 @@ func Test_register(t *testing.T) { username: "LOWERCASED", // this is going to be lower-cased }, { - name: "invalid username", - username: "#totalyNotValid", - wantResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid), + name: "invalid username", + username: "#totalyNotValid", + wantErrorResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid), }, { name: "numeric username is forbidden", username: "1337", - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.InvalidUsername("Numeric user IDs are reserved"), }, @@ -367,7 +379,7 @@ func Test_register(t *testing.T) { { name: "disabled recaptcha login", loginType: authtypes.LoginTypeRecaptcha, - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusForbidden, JSON: spec.Unknown(ErrCaptchaDisabled.Error()), }, @@ -376,7 +388,7 @@ func Test_register(t *testing.T) { name: "enabled recaptcha, no response defined", enableRecaptcha: true, loginType: authtypes.LoginTypeRecaptcha, - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusBadRequest, JSON: spec.BadJSON(ErrMissingResponse.Error()), }, @@ -386,7 +398,7 @@ func Test_register(t *testing.T) { enableRecaptcha: true, loginType: authtypes.LoginTypeRecaptcha, captchaBody: `notvalid`, - wantResponse: util.JSONResponse{ + wantErrorResponse: util.JSONResponse{ Code: http.StatusUnauthorized, JSON: spec.BadJSON(ErrInvalidCaptcha.Error()), }, @@ -398,11 +410,11 @@ func Test_register(t *testing.T) { captchaBody: `success`, }, { - name: "captcha invalid from remote", - enableRecaptcha: true, - loginType: authtypes.LoginTypeRecaptcha, - captchaBody: `i should fail for other reasons`, - wantResponse: util.JSONResponse{Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}}, + name: "captcha invalid from remote", + enableRecaptcha: true, + loginType: authtypes.LoginTypeRecaptcha, + captchaBody: `i should fail for other reasons`, + wantErrorResponse: util.JSONResponse{Code: http.StatusInternalServerError, JSON: spec.InternalServerError{}}, }, } @@ -416,7 +428,7 @@ func Test_register(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -486,8 +498,8 @@ func Test_register(t *testing.T) { t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, cfg.Derived.Registration.Flows) } case spec.MatrixError: - if !reflect.DeepEqual(tc.wantResponse, resp) { - t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantResponse) + if !reflect.DeepEqual(tc.wantErrorResponse, resp) { + t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantErrorResponse) } return case registerResponse: @@ -505,6 +517,13 @@ func Test_register(t *testing.T) { if r.DeviceID == "" { t.Fatalf("missing deviceID in response") } + // if an expected username is provided, assert that it is a match + if tc.wantUsername != "" { + wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", tc.wantUsername, "test")) + if wantUserID != r.UserID { + t.Fatalf("unexpected userID: %s, want %s", r.UserID, wantUserID) + } + } return default: t.Logf("Got response: %T", resp.JSON) @@ -541,44 +560,29 @@ func Test_register(t *testing.T) { resp = Register(req, userAPI, &cfg.ClientAPI) - switch resp.JSON.(type) { - case spec.InternalServerError: - if !reflect.DeepEqual(tc.wantResponse, resp) { - t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse) + switch rr := resp.JSON.(type) { + case spec.InternalServerError, spec.MatrixError, util.JSONResponse: + if !reflect.DeepEqual(tc.wantErrorResponse, resp) { + t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantErrorResponse) } return - case spec.MatrixError: - if !reflect.DeepEqual(tc.wantResponse, resp) { - t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse) + case registerResponse: + // validate the response + if tc.wantUsername != "" { + // if an expected username is provided, assert that it is a match + wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", tc.wantUsername, "test")) + if wantUserID != rr.UserID { + t.Fatalf("unexpected userID: %s, want %s", rr.UserID, wantUserID) + } } - return - case util.JSONResponse: - if !reflect.DeepEqual(tc.wantResponse, resp) { - t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse) + if rr.DeviceID != *reg.DeviceID { + t.Fatalf("unexpected deviceID: %s, want %s", rr.DeviceID, *reg.DeviceID) } - return - } - - rr, ok := resp.JSON.(registerResponse) - if !ok { - t.Fatalf("expected a registerresponse, got %T", resp.JSON) - } - - // validate the response - if tc.forceEmpty { - // when not supplying a username, one will be generated. Given this _SHOULD_ be - // the second user, set the username accordingly - reg.Username = "2" - } - wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", reg.Username, "test")) - if wantUserID != rr.UserID { - t.Fatalf("unexpected userID: %s, want %s", rr.UserID, wantUserID) - } - if rr.DeviceID != *reg.DeviceID { - t.Fatalf("unexpected deviceID: %s, want %s", rr.DeviceID, *reg.DeviceID) - } - if rr.AccessToken == "" { - t.Fatalf("missing accessToken in response") + if rr.AccessToken == "" { + t.Fatalf("missing accessToken in response") + } + default: + t.Fatalf("expected one of internalservererror, matrixerror, jsonresponse, registerresponse, got %T", resp.JSON) } }) } @@ -596,7 +600,7 @@ func TestRegisterUserWithDisplayName(t *testing.T) { cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) deviceName, deviceID := "deviceName", "deviceID" expectedDisplayName := "DisplayName" response := completeRegistration( @@ -637,7 +641,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) { caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) expectedDisplayName := "rabbit" jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`) diff --git a/clientapi/routing/report_event.go b/clientapi/routing/report_event.go new file mode 100644 index 000000000..4dc6498d8 --- /dev/null +++ b/clientapi/routing/report_event.go @@ -0,0 +1,93 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/roomserver/api" + userAPI "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" +) + +type reportEventRequest struct { + Reason string `json:"reason"` + Score int64 `json:"score"` +} + +func ReportEvent( + req *http.Request, + device *userAPI.Device, + roomID, eventID string, + rsAPI api.ClientRoomserverAPI, +) util.JSONResponse { + defer req.Body.Close() // nolint: errcheck + + deviceUserID, err := spec.NewUserID(device.UserID, true) + if err != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: spec.NotFound("You don't have permission to report this event, bad userID"), + } + } + // The requesting user must be a member of the room + errRes := checkMemberInRoom(req.Context(), rsAPI, *deviceUserID, roomID) + if errRes != nil { + return util.JSONResponse{ + Code: http.StatusNotFound, // Spec demands this... + JSON: spec.NotFound("The event was not found or you are not joined to the room."), + } + } + + // Parse the request + report := reportEventRequest{} + if resErr := httputil.UnmarshalJSONRequest(req, &report); resErr != nil { + return *resErr + } + + queryRes := &api.QueryEventsByIDResponse{} + if err = rsAPI.QueryEventsByID(req.Context(), &api.QueryEventsByIDRequest{ + RoomID: roomID, + EventIDs: []string{eventID}, + }, queryRes); err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{Err: err.Error()}, + } + } + + // No event was found or it was already redacted + if len(queryRes.Events) == 0 || queryRes.Events[0].Redacted() { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: spec.NotFound("The event was not found or you are not joined to the room."), + } + } + + _, err = rsAPI.InsertReportedEvent(req.Context(), roomID, eventID, device.UserID, report.Reason, report.Score) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: spec.InternalServerError{Err: err.Error()}, + } + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} diff --git a/clientapi/routing/room_hierarchy.go b/clientapi/routing/room_hierarchy.go index 2884d2c32..cf9d43dd1 100644 --- a/clientapi/routing/room_hierarchy.go +++ b/clientapi/routing/room_hierarchy.go @@ -138,7 +138,7 @@ func QueryRoomHierarchy(req *http.Request, device *userapi.Device, roomIDStr str walker = *cachedWalker } - discoveredRooms, nextWalker, err := rsAPI.QueryNextRoomHierarchyPage(req.Context(), walker, limit) + discoveredRooms, _, nextWalker, err := rsAPI.QueryNextRoomHierarchyPage(req.Context(), walker, limit) if err != nil { switch err.(type) { diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index d4aa1d08d..e82c8861c 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -94,6 +94,7 @@ func Setup( unstableFeatures := map[string]bool{ "org.matrix.e2e_cross_signing": true, "org.matrix.msc2285.stable": true, + "org.matrix.msc3916.stable": true, } for _, msc := range cfg.MSCs.MSCs { unstableFeatures["org.matrix."+msc] = true @@ -255,7 +256,7 @@ func Setup( logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice") serverNotificationSender, err := getSenderDevice(context.Background(), rsAPI, userAPI, cfg) if err != nil { - logrus.WithError(err).Fatal("unable to get account for sending sending server notices") + logrus.WithError(err).Fatal("unable to get account for sending server notices") } synapseAdminRouter.Handle("/admin/v1/send_server_notice/{txnID}", @@ -732,7 +733,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) v3mux.Handle("/auth/{authType}/fallback/web", - httputil.MakeHTMLAPI("auth_fallback", enableMetrics, func(w http.ResponseWriter, req *http.Request) { + httputil.MakeHTTPAPI("auth_fallback", userAPI, enableMetrics, func(w http.ResponseWriter, req *http.Request) { vars := mux.Vars(req) AuthFallback(w, req, vars["authType"], cfg) }), @@ -1513,4 +1514,58 @@ func Setup( return GetPresence(req, device, natsClient, cfg.Matrix.JetStream.Prefixed(jetstream.RequestPresence), vars["userId"]) }), ).Methods(http.MethodGet, http.MethodOptions) + + v3mux.Handle("/rooms/{roomID}/joined_members", + httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetJoinedMembers(req, device, vars["roomID"], rsAPI) + }), + ).Methods(http.MethodGet, http.MethodOptions) + + v3mux.Handle("/rooms/{roomID}/report/{eventID}", + httputil.MakeAuthAPI("report_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return ReportEvent(req, device, vars["roomID"], vars["eventID"], rsAPI) + }), + ).Methods(http.MethodPost, http.MethodOptions) + + synapseAdminRouter.Handle("/admin/v1/event_reports", + httputil.MakeAdminAPI("admin_report_events", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + from := parseUint64OrDefault(req.URL.Query().Get("from"), 0) + limit := parseUint64OrDefault(req.URL.Query().Get("limit"), 100) + dir := req.URL.Query().Get("dir") + userID := req.URL.Query().Get("user_id") + roomID := req.URL.Query().Get("room_id") + + // Go backwards if direction is empty or "b" + backwards := dir == "" || dir == "b" + return GetEventReports(req, rsAPI, from, limit, backwards, userID, roomID) + }), + ).Methods(http.MethodGet, http.MethodOptions) + + synapseAdminRouter.Handle("/admin/v1/event_reports/{reportID}", + httputil.MakeAdminAPI("admin_report_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetEventReport(req, rsAPI, vars["reportID"]) + }), + ).Methods(http.MethodGet, http.MethodOptions) + + synapseAdminRouter.Handle("/admin/v1/event_reports/{reportID}", + httputil.MakeAdminAPI("admin_report_event_delete", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return DeleteEventReport(req, rsAPI, vars["reportID"]) + }), + ).Methods(http.MethodDelete, http.MethodOptions) } diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 69131966b..44e82aed0 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -224,7 +224,7 @@ func SendEvent( req.Context(), rsAPI, api.KindNew, []*types.HeaderedEvent{ - &types.HeaderedEvent{PDU: e}, + {PDU: e}, }, device.UserDomain(), domain, diff --git a/clientapi/routing/sendevent_test.go b/clientapi/routing/sendevent_test.go index 9cdd75358..00d19154a 100644 --- a/clientapi/routing/sendevent_test.go +++ b/clientapi/routing/sendevent_test.go @@ -265,7 +265,7 @@ func createEvents(eventsJSON []string, roomVer gomatrixserverlib.RoomVersion) ([ for i, eventJSON := range eventsJSON { pdu, evErr := roomVerImpl.NewEventFromTrustedJSON([]byte(eventJSON), false) if evErr != nil { - return nil, fmt.Errorf("failed to make event: %s", err.Error()) + return nil, fmt.Errorf("failed to make event: %s", evErr.Error()) } ev := types.HeaderedEvent{PDU: pdu} events[i] = &ev diff --git a/clientapi/routing/server_notices.go b/clientapi/routing/server_notices.go index 5deb559df..d4644b3e5 100644 --- a/clientapi/routing/server_notices.go +++ b/clientapi/routing/server_notices.go @@ -215,7 +215,7 @@ func SendServerNotice( } if !membershipRes.IsInRoom { // re-invite the user - res, err := sendInvite(ctx, userAPI, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, asAPI, time.Now()) + res, err := sendInvite(ctx, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, time.Now()) if err != nil { return res } diff --git a/clientapi/threepid/threepid.go b/clientapi/threepid/threepid.go index d61052cc0..ad94a49c6 100644 --- a/clientapi/threepid/threepid.go +++ b/clientapi/threepid/threepid.go @@ -83,6 +83,7 @@ func CreateSession( if err != nil { return "", err } + defer resp.Body.Close() // nolint: errcheck // Error if the status isn't OK if resp.StatusCode != http.StatusOK { diff --git a/cmd/dendrite-demo-pinecone/monolith/monolith.go b/cmd/dendrite-demo-pinecone/monolith/monolith.go index 41af568a6..d9f44b5cc 100644 --- a/cmd/dendrite-demo-pinecone/monolith/monolith.go +++ b/cmd/dendrite-demo-pinecone/monolith/monolith.go @@ -145,7 +145,7 @@ func (p *P2PMonolith) SetupDendrite( ) rsAPI.SetFederationAPI(fsAPI, keyRing) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, enableMetrics, fsAPI.IsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) diff --git a/cmd/dendrite-demo-pinecone/relay/retriever.go b/cmd/dendrite-demo-pinecone/relay/retriever.go index 3c76ad600..9c918fb67 100644 --- a/cmd/dendrite-demo-pinecone/relay/retriever.go +++ b/cmd/dendrite-demo-pinecone/relay/retriever.go @@ -17,13 +17,13 @@ package relay import ( "context" "sync" + "sync/atomic" "time" federationAPI "github.com/matrix-org/dendrite/federationapi/api" relayServerAPI "github.com/matrix-org/dendrite/relayapi/api" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/sirupsen/logrus" - "go.uber.org/atomic" ) const ( @@ -54,7 +54,7 @@ func NewRelayServerRetriever( federationAPI: federationAPI, relayAPI: relayAPI, relayServersQueried: make(map[spec.ServerName]bool), - running: *atomic.NewBool(false), + running: atomic.Bool{}, quit: quit, } } diff --git a/cmd/dendrite-demo-yggdrasil/README.md b/cmd/dendrite-demo-yggdrasil/README.md index 23304c214..a950720dd 100644 --- a/cmd/dendrite-demo-yggdrasil/README.md +++ b/cmd/dendrite-demo-yggdrasil/README.md @@ -1,6 +1,6 @@ # Yggdrasil Demo -This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.20 or later. +This is the Dendrite Yggdrasil demo! It's easy to get started - all you need is Go 1.21 or later. To run the homeserver, start at the root of the Dendrite repository and run: diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 3ec550113..b02f131ae 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -59,7 +59,7 @@ var ( instanceName = flag.String("name", "dendrite-p2p-ygg", "the name of this P2P demo instance") instancePort = flag.Int("port", 8008, "the port that the client API will listen on") instancePeer = flag.String("peer", "", "the static Yggdrasil peers to connect to, comma separated-list") - instanceListen = flag.String("listen", "tcp://:0", "the port Yggdrasil peers can connect to") + instanceListen = flag.String("listen", "tls://:0", "the port Yggdrasil peers can connect to") instanceDir = flag.String("dir", ".", "the directory to store the databases in (if --config not specified)") ) @@ -134,6 +134,7 @@ func main() { cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(*instanceDir, *instanceName))) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(*instanceDir, *instanceName))) cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", filepath.Join(*instanceDir, *instanceName))) + cfg.RelayAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-relayapi.db", filepath.Join(*instanceDir, *instanceName))) cfg.MSCs.MSCs = []string{"msc2836"} cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName))) cfg.ClientAPI.RegistrationDisabled = false @@ -213,16 +214,16 @@ func main() { natsInstance := jetstream.NATSInstance{} rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation) - - asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) - rsAPI.SetAppserviceAPI(asAPI) fsAPI := federationapi.NewInternalAPI( processCtx, cfg, cm, &natsInstance, federation, rsAPI, caches, keyRing, true, ) - rsAPI.SetFederationAPI(fsAPI, keyRing) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federation, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff) + + asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) + rsAPI.SetAppserviceAPI(asAPI) + monolith := setup.Monolith{ Config: cfg, Client: ygg.CreateClient(), diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go index 26c30e489..3c230f623 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go @@ -18,16 +18,15 @@ import ( "context" "crypto/ed25519" "encoding/hex" - "fmt" "net" "regexp" "strings" "github.com/matrix-org/gomatrixserverlib/spec" - "github.com/neilalexander/utp" "github.com/sirupsen/logrus" + "github.com/yggdrasil-network/yggquic" - ironwoodtypes "github.com/Arceliar/ironwood/types" + "github.com/yggdrasil-network/yggdrasil-go/src/config" "github.com/yggdrasil-network/yggdrasil-go/src/core" yggdrasilcore "github.com/yggdrasil-network/yggdrasil-go/src/core" "github.com/yggdrasil-network/yggdrasil-go/src/multicast" @@ -40,25 +39,23 @@ type Node struct { core *yggdrasilcore.Core multicast *yggdrasilmulticast.Multicast log *gologme.Logger - utpSocket *utp.Socket - incoming chan net.Conn + *yggquic.YggdrasilTransport } func (n *Node) DialerContext(ctx context.Context, _, address string) (net.Conn, error) { tokens := strings.Split(address, ":") - raw, err := hex.DecodeString(tokens[0]) - if err != nil { - return nil, fmt.Errorf("hex.DecodeString: %w", err) - } - pk := make(ironwoodtypes.Addr, ed25519.PublicKeySize) - copy(pk, raw[:]) - return n.utpSocket.DialAddrContext(ctx, pk) + return n.DialContext(ctx, "yggdrasil", tokens[0]) } func Setup(sk ed25519.PrivateKey, instanceName, storageDirectory, peerURI, listenURI string) (*Node, error) { n := &Node{ - log: gologme.New(logrus.StandardLogger().Writer(), "", 0), - incoming: make(chan net.Conn), + log: gologme.New(logrus.StandardLogger().Writer(), "", 0), + } + + cfg := config.GenerateConfig() + cfg.PrivateKey = config.KeyBytes(sk) + if err := cfg.GenerateSelfSignedCertificate(); err != nil { + panic(err) } n.log.EnableLevel("error") @@ -78,12 +75,12 @@ func Setup(sk ed25519.PrivateKey, instanceName, storageDirectory, peerURI, liste }) } } - if n.core, err = core.New(sk[:], n.log, options...); err != nil { + if n.core, err = core.New(cfg.Certificate, n.log, options...); err != nil { panic(err) } n.core.SetLogger(n.log) - if n.utpSocket, err = utp.NewSocketFromPacketConnNoClose(n.core); err != nil { + if n.YggdrasilTransport, err = yggquic.New(n.core, *cfg.Certificate, nil); err != nil { panic(err) } } @@ -106,8 +103,6 @@ func Setup(sk ed25519.PrivateKey, instanceName, storageDirectory, peerURI, liste } n.log.Printf("Public key: %x", n.core.PublicKey()) - go n.listenFromYgg() - return n, nil } diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/session.go b/cmd/dendrite-demo-yggdrasil/yggconn/session.go deleted file mode 100644 index b35b40435..000000000 --- a/cmd/dendrite-demo-yggdrasil/yggconn/session.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package yggconn - -import ( - "context" - "net" -) - -func (n *Node) listenFromYgg() { - for { - conn, err := n.utpSocket.Accept() - if err != nil { - n.log.Println("n.utpSocket.Accept:", err) - return - } - n.incoming <- conn - } -} - -// Implements net.Listener -func (n *Node) Accept() (net.Conn, error) { - return <-n.incoming, nil -} - -// Implements net.Listener -func (n *Node) Close() error { - return n.utpSocket.Close() -} - -// Implements net.Listener -func (n *Node) Addr() net.Addr { - return n.utpSocket.Addr() -} - -// Implements http.Transport.Dial -func (n *Node) Dial(network, address string) (net.Conn, error) { - return n.DialContext(context.TODO(), network, address) -} - -// Implements http.Transport.DialContext -func (n *Node) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - return n.utpSocket.DialContext(ctx, network, address) -} diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go index b78c5f605..871a605d4 100644 --- a/cmd/dendrite-upgrade-tests/main.go +++ b/cmd/dendrite-upgrade-tests/main.go @@ -54,7 +54,7 @@ var latest, _ = semver.NewVersion("v6.6.6") // Dummy version, used as "HEAD" // due to the error: // When using COPY with more than one source file, the destination must be a directory and end with a / // We need to run a postgres anyway, so use the dockerfile associated with Complement instead. -const DockerfilePostgreSQL = `FROM golang:1.20-bookworm as build +const DockerfilePostgreSQL = `FROM golang:1.22-bookworm as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build ARG BINARY @@ -67,13 +67,11 @@ RUN go build ./cmd/${BINARY} RUN go build ./cmd/generate-keys RUN go build ./cmd/generate-config RUN go build ./cmd/create-account -RUN ./generate-config --ci > dendrite.yaml +RUN ./generate-config --ci --db "user=postgres database=postgres host=/var/run/postgresql/" > dendrite.yaml RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key -# Replace the connection string with a single postgres DB, using user/db = 'postgres' and no password -RUN sed -i "s%connection_string:.*$%connection_string: postgresql://postgres@localhost/postgres?sslmode=disable%g" dendrite.yaml -# No password when connecting over localhost -RUN sed -i "s%127.0.0.1/32 scram-sha-256%127.0.0.1/32 trust%g" /etc/postgresql/15/main/pg_hba.conf +# No password when connecting to Postgres +RUN sed -i "s%peer%trust%g" /etc/postgresql/15/main/pg_hba.conf # Bump up max conns for moar concurrency RUN sed -i 's/max_connections = 100/max_connections = 2000/g' /etc/postgresql/15/main/postgresql.conf RUN sed -i 's/max_open_conns:.*$/max_open_conns: 100/g' dendrite.yaml @@ -100,7 +98,7 @@ ENV BINARY=dendrite EXPOSE 8008 8448 CMD /build/run_dendrite.sh` -const DockerfileSQLite = `FROM golang:1.20-bookworm as build +const DockerfileSQLite = `FROM golang:1.22-bookworm as build RUN apt-get update && apt-get install -y postgresql WORKDIR /build ARG BINARY @@ -410,7 +408,7 @@ func runImage(dockerClient *client.Client, volumeName string, branchNameToImageI } containerID = body.ID - err = dockerClient.ContainerStart(ctx, containerID, types.ContainerStartOptions{}) + err = dockerClient.ContainerStart(ctx, containerID, container.StartOptions{}) if err != nil { return "", "", fmt.Errorf("failed to ContainerStart: %s", err) } @@ -442,7 +440,7 @@ func runImage(dockerClient *client.Client, volumeName string, branchNameToImageI lastErr = nil break } - logs, err := dockerClient.ContainerLogs(context.Background(), containerID, types.ContainerLogsOptions{ + logs, err := dockerClient.ContainerLogs(context.Background(), containerID, container.LogsOptions{ ShowStdout: true, ShowStderr: true, Follow: true, @@ -463,7 +461,7 @@ func runImage(dockerClient *client.Client, volumeName string, branchNameToImageI } func destroyContainer(dockerClient *client.Client, containerID string) { - err := dockerClient.ContainerRemove(context.TODO(), containerID, types.ContainerRemoveOptions{ + err := dockerClient.ContainerRemove(context.TODO(), containerID, container.RemoveOptions{ Force: true, }) if err != nil { @@ -550,7 +548,7 @@ func verifyTests(dockerClient *client.Client, volumeName string, versions []*sem // cleanup old containers/volumes from a previous run func cleanup(dockerClient *client.Client) { // ignore all errors, we are just cleaning up and don't want to fail just because we fail to cleanup - containers, _ := dockerClient.ContainerList(context.Background(), types.ContainerListOptions{ + containers, _ := dockerClient.ContainerList(context.Background(), container.ListOptions{ Filters: label(dendriteUpgradeTestLabel), All: true, }) @@ -558,7 +556,7 @@ func cleanup(dockerClient *client.Client) { log.Printf("Removing container: %v %v\n", c.ID, c.Names) timeout := 1 _ = dockerClient.ContainerStop(context.Background(), c.ID, container.StopOptions{Timeout: &timeout}) - _ = dockerClient.ContainerRemove(context.Background(), c.ID, types.ContainerRemoveOptions{ + _ = dockerClient.ContainerRemove(context.Background(), c.ID, container.RemoveOptions{ Force: true, }) } diff --git a/cmd/dendrite/main.go b/cmd/dendrite/main.go index f3140b4e2..5234b7504 100644 --- a/cmd/dendrite/main.go +++ b/cmd/dendrite/main.go @@ -162,7 +162,7 @@ func main() { // dependency. Other components also need updating after their dependencies are up. rsAPI.SetFederationAPI(fsAPI, keyRing) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff) asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) diff --git a/contrib/dendrite-demo-i2p/main.go b/contrib/dendrite-demo-i2p/main.go new file mode 100644 index 000000000..92cfab49a --- /dev/null +++ b/contrib/dendrite-demo-i2p/main.go @@ -0,0 +1,185 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "os" + "time" + + "github.com/getsentry/sentry-go" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/gomatrixserverlib/fclient" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/setup" + basepkg "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/mscs" + "github.com/matrix-org/dendrite/userapi" +) + +var ( + samAddr = flag.String("samaddr", "127.0.0.1:7656", "Address to connect to the I2P SAMv3 API") + _, skip = os.LookupEnv("CI") +) + +func main() { + cfg := setup.ParseFlags(true) + if skip { + return + } + + configErrors := &config.ConfigErrors{} + cfg.Verify(configErrors) + if len(*configErrors) > 0 { + for _, err := range *configErrors { + logrus.Errorf("Configuration error: %s", err) + } + logrus.Fatalf("Failed to start due to configuration errors") + } + processCtx := process.NewProcessContext() + + internal.SetupStdLogging() + internal.SetupHookLogging(cfg.Logging) + internal.SetupPprof() + + basepkg.PlatformSanityChecks() + + logrus.Infof("Dendrite version %s", internal.VersionString()) + if !cfg.ClientAPI.RegistrationDisabled && cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled { + logrus.Warn("Open registration is enabled") + } + + // create DNS cache + var dnsCache *fclient.DNSCache + if cfg.Global.DNSCache.Enabled { + dnsCache = fclient.NewDNSCache( + cfg.Global.DNSCache.CacheSize, + cfg.Global.DNSCache.CacheLifetime, + ) + logrus.Infof( + "DNS cache enabled (size %d, lifetime %s)", + cfg.Global.DNSCache.CacheSize, + cfg.Global.DNSCache.CacheLifetime, + ) + } + + // setup tracing + closer, err := cfg.SetupTracing() + if err != nil { + logrus.WithError(err).Panicf("failed to start opentracing") + } + defer closer.Close() // nolint: errcheck + + // setup sentry + if cfg.Global.Sentry.Enabled { + logrus.Info("Setting up Sentry for debugging...") + err = sentry.Init(sentry.ClientOptions{ + Dsn: cfg.Global.Sentry.DSN, + Environment: cfg.Global.Sentry.Environment, + Debug: true, + ServerName: string(cfg.Global.ServerName), + Release: "dendrite@" + internal.VersionString(), + AttachStacktrace: true, + }) + if err != nil { + logrus.WithError(err).Panic("failed to start Sentry") + } + go func() { + processCtx.ComponentStarted() + <-processCtx.WaitForShutdown() + if !sentry.Flush(time.Second * 5) { + logrus.Warnf("failed to flush all Sentry events!") + } + processCtx.ComponentFinished() + }() + } + + federationClient := basepkg.CreateFederationClient(cfg, dnsCache) + httpClient := basepkg.CreateClient(cfg, dnsCache) + + // prepare required dependencies + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + routers := httputil.NewRouters() + + caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.EnableMetrics) + natsInstance := jetstream.NATSInstance{} + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics) + fsAPI := federationapi.NewInternalAPI( + processCtx, cfg, cm, &natsInstance, federationClient, rsAPI, caches, nil, false, + ) + + keyRing := fsAPI.KeyRing() + + // The underlying roomserver implementation needs to be able to call the fedsender. + // This is different to rsAPI which can be the http client which doesn't need this + // dependency. Other components also need updating after their dependencies are up. + rsAPI.SetFederationAPI(fsAPI, keyRing) + + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff) + asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) + + rsAPI.SetAppserviceAPI(asAPI) + rsAPI.SetUserAPI(userAPI) + + monolith := setup.Monolith{ + Config: cfg, + Client: httpClient, + FedClient: federationClient, + KeyRing: keyRing, + + AppserviceAPI: asAPI, + // always use the concrete impl here even in -http mode because adding public routes + // must be done on the concrete impl not an HTTP client else fedapi will call itself + FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + } + monolith.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, caching.EnableMetrics) + + if len(cfg.MSCs.MSCs) > 0 { + if err := mscs.Enable(cfg, cm, routers, &monolith, caches); err != nil { + logrus.WithError(err).Fatalf("Failed to enable MSCs") + } + } + + upCounter := prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "dendrite", + Name: "up", + ConstLabels: map[string]string{ + "version": internal.VersionString(), + }, + }) + upCounter.Add(1) + prometheus.MustRegister(upCounter) + + // Expose the matrix APIs directly rather than putting them under a /api path. + go func() { + SetupAndServeHTTPS(processCtx, cfg, routers) //, httpsAddr, nil, nil) + }() + + // We want to block forever to let the HTTP and HTTPS handler serve the APIs + basepkg.WaitForShutdown(processCtx) +} diff --git a/contrib/dendrite-demo-i2p/main_i2p.go b/contrib/dendrite-demo-i2p/main_i2p.go new file mode 100644 index 000000000..72fed656e --- /dev/null +++ b/contrib/dendrite-demo-i2p/main_i2p.go @@ -0,0 +1,233 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "crypto/tls" + "embed" + "net" + "net/http" + "net/url" + "strings" + "sync/atomic" + "text/template" + + "github.com/cretz/bine/tor" + "github.com/eyedeekay/goSam" + "github.com/eyedeekay/onramp" + sentryhttp "github.com/getsentry/sentry-go/http" + "github.com/gorilla/mux" + "github.com/kardianos/minwinsvc" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/setup/process" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" + + basepkg "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/config" +) + +func client() (*goSam.Client, error) { + if skip { + return nil, nil + } + return goSam.NewClient(*samAddr) +} + +var sam, samError = client() + +func start() (*tor.Tor, error) { + if skip { + return nil, nil + } + return tor.Start(context.Background(), nil) +} + +func dialer() (*tor.Dialer, error) { + if skip { + return nil, nil + } + return t.Dialer(context.TODO(), nil) +} + +var ( + t, terr = start() + tdialer, tderr = dialer() +) + +// Dial a network connection to an I2P server or a unix socket. Use Tor, or Fail for clearnet addresses. +func DialContext(ctx context.Context, network, addr string) (net.Conn, error) { + if samError != nil { + return nil, samError + } + if network == "unix" { + return net.Dial(network, addr) + } + + // convert the addr to a full URL + url, err := url.Parse(addr) + if err != nil { + return nil, err + } + if strings.HasSuffix(url.Host, ".i2p") { + return sam.DialContext(ctx, network, addr) + } + if terr != nil { + return nil, terr + } + if (tderr != nil) || (tdialer == nil) { + return nil, tderr + } + return tdialer.DialContext(ctx, network, addr) +} + +//go:embed static/*.gotmpl +var staticContent embed.FS + +// SetupAndServeHTTPS sets up the HTTPS server to serve client & federation APIs +// and adds a prometheus handler under /_dendrite/metrics. +func SetupAndServeHTTPS( + processContext *process.ProcessContext, + cfg *config.Dendrite, + routers httputil.Routers, +) { + // create a transport that uses SAM to dial TCP Connections + httpClient := &http.Client{ + Transport: &http.Transport{ + DialContext: DialContext, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + } + + http.DefaultClient = httpClient + + garlic, err := onramp.NewGarlic("dendrite", *samAddr, onramp.OPT_HUGE) + if err != nil { + logrus.WithError(err).Fatal("failed to create garlic") + } + defer garlic.Close() // nolint: errcheck + listener, err := garlic.ListenTLS() + if err != nil { + logrus.WithError(err).Fatal("failed to serve HTTPS") + } + defer listener.Close() // nolint: errcheck + + externalHTTPSAddr := config.ServerAddress{} + https, err := config.HTTPAddress("https://" + listener.Addr().String()) + if err != nil { + logrus.WithError(err).Fatalf("Failed to parse http address") + } + externalHTTPSAddr = https + + externalRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() + + externalServ := &http.Server{ + Addr: externalHTTPSAddr.Address, + WriteTimeout: basepkg.HTTPServerTimeout, + Handler: externalRouter, + BaseContext: func(_ net.Listener) context.Context { + return processContext.Context() + }, + } + + // Redirect for Landing Page + externalRouter.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, httputil.PublicStaticPath, http.StatusFound) + }) + + if cfg.Global.Metrics.Enabled { + externalRouter.Handle("/metrics", httputil.WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Global.Metrics.BasicAuth)) + } + + basepkg.ConfigureAdminEndpoints(processContext, routers) + + // Parse and execute the landing page template + tmpl := template.Must(template.ParseFS(staticContent, "static/*.gotmpl")) + landingPage := &bytes.Buffer{} + if err := tmpl.ExecuteTemplate(landingPage, "index.gotmpl", map[string]string{ + "Version": internal.VersionString(), + }); err != nil { + logrus.WithError(err).Fatal("failed to execute landing page template") + } + + routers.Static.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(landingPage.Bytes()) + }) + + var clientHandler http.Handler + clientHandler = routers.Client + if cfg.Global.Sentry.Enabled { + sentryHandler := sentryhttp.New(sentryhttp.Options{ + Repanic: true, + }) + clientHandler = sentryHandler.Handle(routers.Client) + } + var federationHandler http.Handler + federationHandler = routers.Federation + if cfg.Global.Sentry.Enabled { + sentryHandler := sentryhttp.New(sentryhttp.Options{ + Repanic: true, + }) + federationHandler = sentryHandler.Handle(routers.Federation) + } + externalRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(routers.DendriteAdmin) + externalRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(clientHandler) + if !cfg.Global.DisableFederation { + externalRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(routers.Keys) + externalRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(federationHandler) + } + externalRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(routers.SynapseAdmin) + externalRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media) + externalRouter.PathPrefix(httputil.PublicWellKnownPrefix).Handler(routers.WellKnown) + externalRouter.PathPrefix(httputil.PublicStaticPath).Handler(routers.Static) + + externalRouter.NotFoundHandler = httputil.NotFoundCORSHandler + externalRouter.MethodNotAllowedHandler = httputil.NotAllowedHandler + + if externalHTTPSAddr.Enabled() { + go func() { + var externalShutdown atomic.Bool // RegisterOnShutdown can be called more than once + logrus.Infof("Starting external listener on https://%s", externalServ.Addr) + processContext.ComponentStarted() + externalServ.RegisterOnShutdown(func() { + if externalShutdown.CompareAndSwap(false, true) { + processContext.ComponentFinished() + logrus.Infof("Stopped external HTTPS listener") + } + }) + addr := listener.Addr() + externalServ.Addr = addr.String() + if err := externalServ.Serve(listener); err != nil { + if err != http.ErrServerClosed { + logrus.WithError(err).Fatal("failed to serve HTTPS") + } + } + + logrus.Infof("Stopped external listener on %s", externalServ.Addr) + }() + } + + minwinsvc.SetOnExit(processContext.ShutdownDendrite) + <-processContext.WaitForShutdown() + + logrus.Infof("Stopping HTTPS listeners") + _ = externalServ.Shutdown(context.Background()) + logrus.Infof("Stopped HTTPS listeners") +} diff --git a/contrib/dendrite-demo-i2p/main_test.go b/contrib/dendrite-demo-i2p/main_test.go new file mode 100644 index 000000000..cc70ddba4 --- /dev/null +++ b/contrib/dendrite-demo-i2p/main_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "os" + "os/signal" + "strings" + "syscall" + "testing" +) + +// This is an instrumented main, used when running integration tests (sytest) with code coverage. +// Compile: go test -c -race -cover -covermode=atomic -o monolith.debug -coverpkg "github.com/matrix-org/..." ./cmd/dendrite +// Run the monolith: ./monolith.debug -test.coverprofile=/somewhere/to/dump/integrationcover.out DEVEL --config dendrite.yaml +// Generate HTML with coverage: go tool cover -html=/somewhere/where/there/is/integrationcover.out -o cover.html +// Source: https://dzone.com/articles/measuring-integration-test-coverage-rate-in-pouchc +func TestMain(t *testing.T) { + var args []string + + for _, arg := range os.Args { + switch { + case strings.HasPrefix(arg, "DEVEL"): + case strings.HasPrefix(arg, "-test"): + default: + args = append(args, arg) + } + } + + // only run the tests if there are args to be passed + if len(args) <= 1 { + return + } + t.Log(args) + + waitCh := make(chan int, 1) + os.Args = args + go func() { + main() + close(waitCh) + }() + + signalCh := make(chan os.Signal, 1) + signal.Notify(signalCh, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGHUP) + + select { + case <-signalCh: + return + case <-waitCh: + return + } +} diff --git a/contrib/dendrite-demo-i2p/static/index.gotmpl b/contrib/dendrite-demo-i2p/static/index.gotmpl new file mode 100644 index 000000000..b3c5576eb --- /dev/null +++ b/contrib/dendrite-demo-i2p/static/index.gotmpl @@ -0,0 +1,63 @@ + + + + Dendrite is running + + + + +

It works! Dendrite {{ .Version }} is running

+

Your Dendrite server is listening on this port and is ready for messages.

+

To use this server you'll need a Matrix client. +

+

Welcome to the Matrix universe :)

+
+

+ + + matrix.org + + +

+ + diff --git a/contrib/dendrite-demo-tor/main.go b/contrib/dendrite-demo-tor/main.go new file mode 100644 index 000000000..f82d6d538 --- /dev/null +++ b/contrib/dendrite-demo-tor/main.go @@ -0,0 +1,180 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "time" + + "github.com/getsentry/sentry-go" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/gomatrixserverlib/fclient" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/setup" + basepkg "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/mscs" + "github.com/matrix-org/dendrite/userapi" +) + +var _, skip = os.LookupEnv("CI") + +func main() { + cfg := setup.ParseFlags(true) + if skip { + return + } + configErrors := &config.ConfigErrors{} + cfg.Verify(configErrors) + if len(*configErrors) > 0 { + for _, err := range *configErrors { + logrus.Errorf("Configuration error: %s", err) + } + logrus.Fatalf("Failed to start due to configuration errors") + } + processCtx := process.NewProcessContext() + + internal.SetupStdLogging() + internal.SetupHookLogging(cfg.Logging) + internal.SetupPprof() + + basepkg.PlatformSanityChecks() + + logrus.Infof("Dendrite version %s", internal.VersionString()) + if !cfg.ClientAPI.RegistrationDisabled && cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled { + logrus.Warn("Open registration is enabled") + } + + // create DNS cache + var dnsCache *fclient.DNSCache + if cfg.Global.DNSCache.Enabled { + dnsCache = fclient.NewDNSCache( + cfg.Global.DNSCache.CacheSize, + cfg.Global.DNSCache.CacheLifetime, + ) + logrus.Infof( + "DNS cache enabled (size %d, lifetime %s)", + cfg.Global.DNSCache.CacheSize, + cfg.Global.DNSCache.CacheLifetime, + ) + } + + // setup tracing + closer, err := cfg.SetupTracing() + if err != nil { + logrus.WithError(err).Panicf("failed to start opentracing") + } + defer closer.Close() // nolint: errcheck + + // setup sentry + if cfg.Global.Sentry.Enabled { + logrus.Info("Setting up Sentry for debugging...") + err = sentry.Init(sentry.ClientOptions{ + Dsn: cfg.Global.Sentry.DSN, + Environment: cfg.Global.Sentry.Environment, + Debug: true, + ServerName: string(cfg.Global.ServerName), + Release: "dendrite@" + internal.VersionString(), + AttachStacktrace: true, + }) + if err != nil { + logrus.WithError(err).Panic("failed to start Sentry") + } + go func() { + processCtx.ComponentStarted() + <-processCtx.WaitForShutdown() + if !sentry.Flush(time.Second * 5) { + logrus.Warnf("failed to flush all Sentry events!") + } + processCtx.ComponentFinished() + }() + } + + federationClient := basepkg.CreateFederationClient(cfg, dnsCache) + httpClient := basepkg.CreateClient(cfg, dnsCache) + + // prepare required dependencies + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + routers := httputil.NewRouters() + + caches := caching.NewRistrettoCache(cfg.Global.Cache.EstimatedMaxSize, cfg.Global.Cache.MaxAge, caching.EnableMetrics) + natsInstance := jetstream.NATSInstance{} + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.EnableMetrics) + fsAPI := federationapi.NewInternalAPI( + processCtx, cfg, cm, &natsInstance, federationClient, rsAPI, caches, nil, false, + ) + + keyRing := fsAPI.KeyRing() + + // The underlying roomserver implementation needs to be able to call the fedsender. + // This is different to rsAPI which can be the http client which doesn't need this + // dependency. Other components also need updating after their dependencies are up. + rsAPI.SetFederationAPI(fsAPI, keyRing) + + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, federationClient, caching.EnableMetrics, fsAPI.IsBlacklistedOrBackingOff) + asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) + + rsAPI.SetAppserviceAPI(asAPI) + rsAPI.SetUserAPI(userAPI) + + monolith := setup.Monolith{ + Config: cfg, + Client: httpClient, + FedClient: federationClient, + KeyRing: keyRing, + + AppserviceAPI: asAPI, + // always use the concrete impl here even in -http mode because adding public routes + // must be done on the concrete impl not an HTTP client else fedapi will call itself + FederationAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + } + monolith.AddAllPublicRoutes(processCtx, cfg, routers, cm, &natsInstance, caches, caching.EnableMetrics) + + if len(cfg.MSCs.MSCs) > 0 { + if err := mscs.Enable(cfg, cm, routers, &monolith, caches); err != nil { + logrus.WithError(err).Fatalf("Failed to enable MSCs") + } + } + + upCounter := prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "dendrite", + Name: "up", + ConstLabels: map[string]string{ + "version": internal.VersionString(), + }, + }) + upCounter.Add(1) + prometheus.MustRegister(upCounter) + + // Expose the matrix APIs directly rather than putting them under a /api path. + go func() { + SetupAndServeHTTPS(processCtx, cfg, routers) //, httpsAddr, nil, nil) + }() + + // We want to block forever to let the HTTP and HTTPS handler serve the APIs + basepkg.WaitForShutdown(processCtx) +} diff --git a/contrib/dendrite-demo-tor/main_test.go b/contrib/dendrite-demo-tor/main_test.go new file mode 100644 index 000000000..cc70ddba4 --- /dev/null +++ b/contrib/dendrite-demo-tor/main_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "os" + "os/signal" + "strings" + "syscall" + "testing" +) + +// This is an instrumented main, used when running integration tests (sytest) with code coverage. +// Compile: go test -c -race -cover -covermode=atomic -o monolith.debug -coverpkg "github.com/matrix-org/..." ./cmd/dendrite +// Run the monolith: ./monolith.debug -test.coverprofile=/somewhere/to/dump/integrationcover.out DEVEL --config dendrite.yaml +// Generate HTML with coverage: go tool cover -html=/somewhere/where/there/is/integrationcover.out -o cover.html +// Source: https://dzone.com/articles/measuring-integration-test-coverage-rate-in-pouchc +func TestMain(t *testing.T) { + var args []string + + for _, arg := range os.Args { + switch { + case strings.HasPrefix(arg, "DEVEL"): + case strings.HasPrefix(arg, "-test"): + default: + args = append(args, arg) + } + } + + // only run the tests if there are args to be passed + if len(args) <= 1 { + return + } + t.Log(args) + + waitCh := make(chan int, 1) + os.Args = args + go func() { + main() + close(waitCh) + }() + + signalCh := make(chan os.Signal, 1) + signal.Notify(signalCh, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGHUP) + + select { + case <-signalCh: + return + case <-waitCh: + return + } +} diff --git a/contrib/dendrite-demo-tor/main_tor.go b/contrib/dendrite-demo-tor/main_tor.go new file mode 100644 index 000000000..8f889c41a --- /dev/null +++ b/contrib/dendrite-demo-tor/main_tor.go @@ -0,0 +1,215 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "crypto/tls" + "embed" + "net" + "net/http" + "net/url" + "sync/atomic" + "text/template" + + "github.com/cretz/bine/tor" + "github.com/eyedeekay/onramp" + sentryhttp "github.com/getsentry/sentry-go/http" + "github.com/gorilla/mux" + "github.com/kardianos/minwinsvc" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/setup/process" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" + + basepkg "github.com/matrix-org/dendrite/setup/base" + "github.com/matrix-org/dendrite/setup/config" +) + +func start() (*tor.Tor, error) { + if skip { + return nil, nil + } + return tor.Start(context.Background(), nil) +} + +func dialer() (*tor.Dialer, error) { + if skip { + return nil, nil + } + return t.Dialer(context.TODO(), nil) +} + +var ( + t, terr = start() + tdialer, tderr = dialer() +) + +// Dial either a unix socket address, or connect to a remote address over Tor. Always uses Tor. +func DialContext(ctx context.Context, network, addr string) (net.Conn, error) { + if terr != nil { + return nil, terr + } + if (tderr != nil) || (tdialer == nil) { + return nil, tderr + } + if network == "unix" { + return net.Dial(network, addr) + } + // convert the addr to a full URL + url, err := url.Parse(addr) + if err != nil { + return nil, err + } + return tdialer.DialContext(ctx, network, url.Host) +} + +//go:embed static/*.gotmpl +var staticContent embed.FS + +// SetupAndServeHTTPS sets up the HTTPS server to serve client & federation APIs +// and adds a prometheus handler under /_dendrite/metrics. +func SetupAndServeHTTPS( + processContext *process.ProcessContext, + cfg *config.Dendrite, + routers httputil.Routers, +) { + // create a transport that uses SAM to dial TCP Connections + httpClient := &http.Client{ + Transport: &http.Transport{ + DialContext: DialContext, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + } + + http.DefaultClient = httpClient + + onion, err := onramp.NewOnion("dendrite-onion") + if err != nil { + logrus.WithError(err).Fatal("failed to create onion") + } + defer onion.Close() // nolint: errcheck + listener, err := onion.ListenTLS() + if err != nil { + logrus.WithError(err).Fatal("failed to serve HTTPS") + } + defer listener.Close() // nolint: errcheck + + externalHTTPSAddr := config.ServerAddress{} + https, err := config.HTTPAddress("https://" + listener.Addr().String()) + if err != nil { + logrus.WithError(err).Fatalf("Failed to parse http address") + } + externalHTTPSAddr = https + + externalRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() + + externalServ := &http.Server{ + Addr: externalHTTPSAddr.Address, + WriteTimeout: basepkg.HTTPServerTimeout, + Handler: externalRouter, + BaseContext: func(_ net.Listener) context.Context { + return processContext.Context() + }, + } + + // Redirect for Landing Page + externalRouter.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, httputil.PublicStaticPath, http.StatusFound) + }) + + if cfg.Global.Metrics.Enabled { + externalRouter.Handle("/metrics", httputil.WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Global.Metrics.BasicAuth)) + } + + basepkg.ConfigureAdminEndpoints(processContext, routers) + + // Parse and execute the landing page template + tmpl := template.Must(template.ParseFS(staticContent, "static/*.gotmpl")) + landingPage := &bytes.Buffer{} + if err := tmpl.ExecuteTemplate(landingPage, "index.gotmpl", map[string]string{ + "Version": internal.VersionString(), + }); err != nil { + logrus.WithError(err).Fatal("failed to execute landing page template") + } + + routers.Static.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(landingPage.Bytes()) + }) + + var clientHandler http.Handler + clientHandler = routers.Client + if cfg.Global.Sentry.Enabled { + sentryHandler := sentryhttp.New(sentryhttp.Options{ + Repanic: true, + }) + clientHandler = sentryHandler.Handle(routers.Client) + } + var federationHandler http.Handler + federationHandler = routers.Federation + if cfg.Global.Sentry.Enabled { + sentryHandler := sentryhttp.New(sentryhttp.Options{ + Repanic: true, + }) + federationHandler = sentryHandler.Handle(routers.Federation) + } + externalRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(routers.DendriteAdmin) + externalRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(clientHandler) + if !cfg.Global.DisableFederation { + externalRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(routers.Keys) + externalRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(federationHandler) + } + externalRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(routers.SynapseAdmin) + externalRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(routers.Media) + externalRouter.PathPrefix(httputil.PublicWellKnownPrefix).Handler(routers.WellKnown) + externalRouter.PathPrefix(httputil.PublicStaticPath).Handler(routers.Static) + + externalRouter.NotFoundHandler = httputil.NotFoundCORSHandler + externalRouter.MethodNotAllowedHandler = httputil.NotAllowedHandler + + if externalHTTPSAddr.Enabled() { + go func() { + var externalShutdown atomic.Bool // RegisterOnShutdown can be called more than once + logrus.Infof("Starting external listener on https://%s", externalServ.Addr) + processContext.ComponentStarted() + externalServ.RegisterOnShutdown(func() { + if externalShutdown.CompareAndSwap(false, true) { + processContext.ComponentFinished() + logrus.Infof("Stopped external HTTPS listener") + } + }) + addr := listener.Addr() + externalServ.Addr = addr.String() + if err := externalServ.Serve(listener); err != nil { + if err != http.ErrServerClosed { + logrus.WithError(err).Fatal("failed to serve HTTPS") + } + } + + logrus.Infof("Stopped external listener on %s", externalServ.Addr) + }() + } + + minwinsvc.SetOnExit(processContext.ShutdownDendrite) + <-processContext.WaitForShutdown() + + logrus.Infof("Stopping HTTPS listeners") + _ = externalServ.Shutdown(context.Background()) + logrus.Infof("Stopped HTTPS listeners") +} diff --git a/contrib/dendrite-demo-tor/static/index.gotmpl b/contrib/dendrite-demo-tor/static/index.gotmpl new file mode 100644 index 000000000..b3c5576eb --- /dev/null +++ b/contrib/dendrite-demo-tor/static/index.gotmpl @@ -0,0 +1,63 @@ + + + + Dendrite is running + + + + +

It works! Dendrite {{ .Version }} is running

+

Your Dendrite server is listening on this port and is ready for messages.

+

To use this server you'll need a Matrix client. +

+

Welcome to the Matrix universe :)

+
+

+ + + matrix.org + + +

+ + diff --git a/dendrite-sample.yaml b/dendrite-sample.yaml index 7affc2599..8616e120e 100644 --- a/dendrite-sample.yaml +++ b/dendrite-sample.yaml @@ -154,6 +154,13 @@ app_service_api: # to be sent to an insecure endpoint. disable_tls_validation: false + # Send the access_token query parameter with appservice requests in addition + # to the Authorization header. This can cause hs_tokens to be saved to logs, + # so it should not be enabled unless absolutely necessary. + legacy_auth: false + # Use the legacy unprefixed paths for appservice requests. + legacy_paths: false + # Appservice configuration files to load into this homeserver. config_files: # - /path/to/appservice_registration.yaml @@ -325,6 +332,10 @@ user_api: auto_join_rooms: # - "#main:matrix.org" + # The number of workers to start for the DeviceListUpdater. Defaults to 8. + # This only needs updating if the "InputDeviceListUpdate" stream keeps growing indefinitely. + # worker_count: 8 + # Configuration for Opentracing. # See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on # how this works and how to set it up. diff --git a/docs/FAQ.md b/docs/FAQ.md index 2cfb13f15..136ec9200 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -4,6 +4,8 @@ nav_order: 1 permalink: /faq --- +{% include deprecation.html %} + # FAQ ## Why does Dendrite exist? @@ -24,7 +26,7 @@ No, although a good portion of the Matrix specification has been implemented. Mo Dendrite development is currently supported by a small team of developers and due to those limited resources, the majority of the effort is focused on getting Dendrite to be specification complete. If there are major features you're requesting (e.g. new administration endpoints), we'd like to strongly encourage you to join the community in supporting -the development efforts through [contributing](../development/contributing). +the development efforts through [contributing](./development/CONTRIBUTING.md). ## Is there a migration path from Synapse to Dendrite? @@ -105,7 +107,7 @@ This can be done by performing a room upgrade. Use the command `/upgraderoom 2.1) minitest (5.17.0) multipart-post (2.1.1) - nokogiri (1.14.3-arm64-darwin) + nokogiri (1.16.2-arm64-darwin) racc (~> 1.4) - nokogiri (1.14.3-x86_64-linux) + nokogiri (1.16.2-x86_64-linux) racc (~> 1.4) octokit (4.22.0) faraday (>= 0.9) @@ -241,11 +241,12 @@ GEM pathutil (0.16.2) forwardable-extended (~> 2.6) public_suffix (4.0.7) - racc (1.6.2) + racc (1.7.3) rb-fsevent (0.11.1) rb-inotify (0.10.1) ffi (~> 1.0) - rexml (3.2.5) + rexml (3.3.2) + strscan rouge (3.26.0) ruby2_keywords (0.0.5) rubyzip (2.3.2) @@ -260,6 +261,7 @@ GEM faraday (> 0.8, < 2.0) simpleidn (0.2.1) unf (~> 0.1.4) + strscan (3.1.0) terminal-table (1.8.0) unicode-display_width (~> 1.1, >= 1.1.1) thread_safe (0.3.6) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 8e72da971..786735c5a 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -1,3 +1,6 @@ + +{% include deprecation.html %} + # Installation Please note that new installation instructions can be found diff --git a/docs/_includes/deprecation.html b/docs/_includes/deprecation.html new file mode 100644 index 000000000..cb7073b1a --- /dev/null +++ b/docs/_includes/deprecation.html @@ -0,0 +1,6 @@ +{: .warning-title } +> This documentation is out of date! +> +> This documentation site is for the versions of Dendrite maintained by the Matrix.org Foundation (github.com/matrix-org/dendrite), available under the Apache 2.0 licence. +> +> If you are interested in the documentation for a later version of Dendrite, please refer to https://element-hq.github.io/dendrite/. \ No newline at end of file diff --git a/docs/_sass/custom/custom.scss b/docs/_sass/custom/custom.scss deleted file mode 100644 index 8a5ed3d8d..000000000 --- a/docs/_sass/custom/custom.scss +++ /dev/null @@ -1,3 +0,0 @@ -footer.site-footer { - opacity: 10%; -} \ No newline at end of file diff --git a/docs/administration.md b/docs/administration.md index 08ad7803e..15add5b32 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -5,6 +5,8 @@ nav_order: 4 permalink: /administration --- +{% include deprecation.html %} + # Administration This section contains documentation on managing your existing Dendrite deployment. diff --git a/docs/administration/1_createusers.md b/docs/administration/1_createusers.md index cbdccd18b..7052004d8 100644 --- a/docs/administration/1_createusers.md +++ b/docs/administration/1_createusers.md @@ -5,6 +5,8 @@ permalink: /administration/createusers nav_order: 1 --- +{% include deprecation.html %} + # Creating user accounts User accounts can be created on a Dendrite instance in a number of ways. diff --git a/docs/administration/2_registration.md b/docs/administration/2_registration.md index 66949f2ca..8599e614c 100644 --- a/docs/administration/2_registration.md +++ b/docs/administration/2_registration.md @@ -5,6 +5,8 @@ permalink: /administration/registration nav_order: 2 --- +{% include deprecation.html %} + # Enabling registration Enabling registration allows users to register their own user accounts on your diff --git a/docs/administration/3_presence.md b/docs/administration/3_presence.md index 858025370..22c22c9e2 100644 --- a/docs/administration/3_presence.md +++ b/docs/administration/3_presence.md @@ -5,6 +5,8 @@ permalink: /administration/presence nav_order: 3 --- +{% include deprecation.html %} + # Enabling presence Dendrite supports presence, which allows you to send your online/offline status diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md index 40d02622b..1e840d621 100644 --- a/docs/administration/4_adminapi.md +++ b/docs/administration/4_adminapi.md @@ -5,6 +5,8 @@ nav_order: 4 permalink: /administration/adminapi --- +{% include deprecation.html %} + # Supported admin APIs Dendrite supports, at present, a very small number of endpoints that allow diff --git a/docs/administration/5_optimisation.md b/docs/administration/5_optimisation.md index 57b7924d3..9bcfb658a 100644 --- a/docs/administration/5_optimisation.md +++ b/docs/administration/5_optimisation.md @@ -6,6 +6,8 @@ nav_order: 5 permalink: /administration/optimisation --- +{% include deprecation.html %} + # Optimise your installation Now that you have Dendrite running, the following tweaks will improve the reliability diff --git a/docs/administration/6_troubleshooting.md b/docs/administration/6_troubleshooting.md index 5f11f9931..27916b64c 100644 --- a/docs/administration/6_troubleshooting.md +++ b/docs/administration/6_troubleshooting.md @@ -5,6 +5,8 @@ nav_order: 6 permalink: /administration/troubleshooting --- +{% include deprecation.html %} + # Troubleshooting If your Dendrite installation is acting strangely, there are a few things you should diff --git a/docs/development.md b/docs/development.md index cf296fb53..2ed616c31 100644 --- a/docs/development.md +++ b/docs/development.md @@ -4,6 +4,10 @@ has_children: true permalink: /development --- +{% include deprecation.html %} + +{% include deprecation.html %} + # Development This section contains documentation that may be useful when helping to develop diff --git a/docs/development/CONTRIBUTING.md b/docs/development/CONTRIBUTING.md index caab1e749..0529071d1 100644 --- a/docs/development/CONTRIBUTING.md +++ b/docs/development/CONTRIBUTING.md @@ -5,6 +5,8 @@ nav_order: 1 permalink: /development/contributing --- +{% include deprecation.html %} + # Contributing to Dendrite Everyone is welcome to contribute to Dendrite! We aim to make it as easy as diff --git a/docs/development/PROFILING.md b/docs/development/PROFILING.md index dc4eca7b7..d91416285 100644 --- a/docs/development/PROFILING.md +++ b/docs/development/PROFILING.md @@ -5,6 +5,8 @@ nav_order: 4 permalink: /development/profiling --- +{% include deprecation.html %} + # Profiling Dendrite If you are running into problems with Dendrite using excessive resources (e.g. CPU or RAM) then you can use the profiler to work out what is happening. diff --git a/docs/development/coverage.md b/docs/development/coverage.md index 1b15f71a2..3ae92c9c9 100644 --- a/docs/development/coverage.md +++ b/docs/development/coverage.md @@ -5,6 +5,8 @@ nav_order: 3 permalink: /development/coverage --- +{% include deprecation.html %} + ## Running unit tests with coverage enabled Running unit tests with coverage enabled can be done with the following commands, this will generate a `integrationcover.log` diff --git a/docs/development/sytest.md b/docs/development/sytest.md index 2f681f3e5..274412bac 100644 --- a/docs/development/sytest.md +++ b/docs/development/sytest.md @@ -5,6 +5,8 @@ nav_order: 2 permalink: /development/sytest --- +{% include deprecation.html %} + # SyTest Dendrite uses [SyTest](https://github.com/matrix-org/sytest) for its diff --git a/docs/index.md b/docs/index.md index 64836152c..d60702265 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,6 +3,8 @@ layout: home nav_exclude: true --- +{% include deprecation.html %} + # Dendrite Dendrite is a second-generation Matrix homeserver written in Go! Following the microservice diff --git a/docs/installation.md b/docs/installation.md index c38a6dbb2..142d2a689 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -5,6 +5,8 @@ nav_order: 2 permalink: /installation --- +{% include deprecation.html %} + # Installation This section contains documentation on installing a new Dendrite deployment. diff --git a/docs/installation/1_planning.md b/docs/installation/1_planning.md index 37ca5702a..9174d3726 100644 --- a/docs/installation/1_planning.md +++ b/docs/installation/1_planning.md @@ -5,6 +5,8 @@ nav_order: 1 permalink: /installation/planning --- +{% include deprecation.html %} + # Planning your installation ## Database @@ -59,7 +61,7 @@ In order to install Dendrite, you will need to satisfy the following dependencie ### Go -At this time, Dendrite supports being built with Go 1.20 or later. We do not support building +At this time, Dendrite supports being built with Go 1.21 or later. We do not support building Dendrite with older versions of Go than this. If you are installing Go using a package manager, you should check (by running `go version`) that you are using a suitable version before you start. diff --git a/docs/installation/2_domainname.md b/docs/installation/2_domainname.md index d86a664cb..f658b0f74 100644 --- a/docs/installation/2_domainname.md +++ b/docs/installation/2_domainname.md @@ -5,6 +5,8 @@ nav_order: 2 permalink: /installation/domainname --- +{% include deprecation.html %} + # Setting up the domain Every Matrix server deployment requires a server name which uniquely identifies it. For diff --git a/docs/installation/docker.md b/docs/installation/docker.md index 1ecc7c6ee..2066f83d4 100644 --- a/docs/installation/docker.md +++ b/docs/installation/docker.md @@ -6,6 +6,8 @@ nav_order: 4 permalink: /docker --- +{% include deprecation.html %} + # Installation using Docker This section contains documentation how to install Dendrite using Docker diff --git a/docs/installation/docker/1_docker.md b/docs/installation/docker/1_docker.md index 1fe792636..4d082d9c4 100644 --- a/docs/installation/docker/1_docker.md +++ b/docs/installation/docker/1_docker.md @@ -7,6 +7,8 @@ nav_order: 1 permalink: /installation/docker/install --- +{% include deprecation.html %} + # Installing Dendrite using Docker Compose Dendrite provides an [example](https://github.com/matrix-org/dendrite/blob/main/build/docker/docker-compose.yml) @@ -26,6 +28,8 @@ docker run --rm --entrypoint="/usr/bin/generate-keys" \ -v $(pwd)/config:/mnt \ matrixdotorg/dendrite-monolith:latest \ -private-key /mnt/matrix_key.pem + +# Windows equivalent: docker run --rm --entrypoint="/usr/bin/generate-keys" -v %cd%/config:/mnt matrixdotorg/dendrite-monolith:latest -private-key /mnt/matrix_key.pem ``` (**NOTE**: This only needs to be executed **once**, as you otherwise overwrite the key) @@ -44,6 +48,8 @@ docker run --rm --entrypoint="/bin/sh" \ -dir /var/dendrite/ \ -db postgres://dendrite:itsasecret@postgres/dendrite?sslmode=disable \ -server YourDomainHere > /mnt/dendrite.yaml" + +# Windows equivalent: docker run --rm --entrypoint="/bin/sh" -v %cd%/config:/mnt matrixdotorg/dendrite-monolith:latest -c "/usr/bin/generate-config -dir /var/dendrite/ -db postgres://dendrite:itsasecret@postgres/dendrite?sslmode=disable -server YourDomainHere > /mnt/dendrite.yaml" ``` You can then change `config/dendrite.yaml` to your liking. diff --git a/docs/installation/helm.md b/docs/installation/helm.md index dd20e0261..b9bf3f3c9 100644 --- a/docs/installation/helm.md +++ b/docs/installation/helm.md @@ -6,6 +6,8 @@ nav_order: 3 permalink: /helm --- +{% include deprecation.html %} + # Helm This section contains documentation how to use [Helm](https://helm.sh/) to install Dendrite on a [Kubernetes](https://kubernetes.io/) cluster. diff --git a/docs/installation/helm/1_helm.md b/docs/installation/helm/1_helm.md index 00fe4fdca..992b418fd 100644 --- a/docs/installation/helm/1_helm.md +++ b/docs/installation/helm/1_helm.md @@ -7,6 +7,8 @@ nav_order: 1 permalink: /installation/helm/install --- +{% include deprecation.html %} + # Installing Dendrite using Helm To install Dendrite using the Helm chart, you first have to add the repository using the following commands: diff --git a/docs/installation/manual.md b/docs/installation/manual.md index 3ab1fd627..f59893543 100644 --- a/docs/installation/manual.md +++ b/docs/installation/manual.md @@ -6,6 +6,8 @@ nav_order: 5 permalink: /manual --- +{% include deprecation.html %} + # Manual Installation This section contains documentation how to manually install Dendrite diff --git a/docs/installation/manual/1_build.md b/docs/installation/manual/1_build.md index 73a626882..f54b1ac43 100644 --- a/docs/installation/manual/1_build.md +++ b/docs/installation/manual/1_build.md @@ -7,6 +7,8 @@ nav_order: 1 permalink: /installation/manual/build --- +{% include deprecation.html %} + # Build all Dendrite commands Dendrite has numerous utility commands in addition to the actual server binaries. diff --git a/docs/installation/manual/2_database.md b/docs/installation/manual/2_database.md index 1be602c66..e19912750 100644 --- a/docs/installation/manual/2_database.md +++ b/docs/installation/manual/2_database.md @@ -7,6 +7,8 @@ grand_parent: Installation permalink: /installation/manual/database --- +{% include deprecation.html %} + # Preparing database storage Dendrite uses SQL databases to store data. Depending on the database engine being used, you diff --git a/docs/installation/manual/3_signingkey.md b/docs/installation/manual/3_signingkey.md index 91289fd6a..6a62fa5e1 100644 --- a/docs/installation/manual/3_signingkey.md +++ b/docs/installation/manual/3_signingkey.md @@ -6,6 +6,8 @@ nav_order: 3 permalink: /installation/manual/signingkeys --- +{% include deprecation.html %} + # Generating signing keys All Matrix homeservers require a signing private key, which will be used to authenticate diff --git a/docs/installation/manual/4_configuration.md b/docs/installation/manual/4_configuration.md index 624cc4155..ea00c6658 100644 --- a/docs/installation/manual/4_configuration.md +++ b/docs/installation/manual/4_configuration.md @@ -6,6 +6,8 @@ nav_order: 4 permalink: /installation/manual/configuration --- +{% include deprecation.html %} + # Configuring Dendrite A YAML configuration file is used to configure Dendrite. A sample configuration file is diff --git a/docs/installation/manual/5_starting_dendrite.md b/docs/installation/manual/5_starting_dendrite.md index d13504372..92d598958 100644 --- a/docs/installation/manual/5_starting_dendrite.md +++ b/docs/installation/manual/5_starting_dendrite.md @@ -6,6 +6,8 @@ nav_order: 5 permalink: /installation/manual/start --- +{% include deprecation.html %} + # Starting Dendrite Once you have completed all preparation and installation steps, diff --git a/docs/nginx/dendrite-sample.conf b/docs/nginx/dendrite-sample.conf index 360eb9255..e45fe1685 100644 --- a/docs/nginx/dendrite-sample.conf +++ b/docs/nginx/dendrite-sample.conf @@ -1,5 +1,5 @@ -#change IP to location of monolith server -upstream monolith{ +# change IP to location of monolith server +upstream monolith { server 127.0.0.1:8008; } server { @@ -20,8 +20,9 @@ server { } location /.well-known/matrix/client { - # If your sever_name here doesn't match your matrix homeserver URL + # If your server_name here doesn't match your matrix homeserver URL # (e.g. hostname.com as server_name and matrix.hostname.com as homeserver URL) + # uncomment the following line. # add_header Access-Control-Allow-Origin '*'; return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }'; } diff --git a/docs/other/p2p.md b/docs/other/p2p.md index 9f104f025..558ceb1c6 100644 --- a/docs/other/p2p.md +++ b/docs/other/p2p.md @@ -3,6 +3,8 @@ title: P2P Matrix nav_exclude: true --- +{% include deprecation.html %} + # P2P Matrix These are the instructions for setting up P2P Dendrite, current as of May 2020. There's both Go stuff and JS stuff to do to set this up. diff --git a/docs/other/peeking.md b/docs/other/peeking.md index c4ae89811..940729f9e 100644 --- a/docs/other/peeking.md +++ b/docs/other/peeking.md @@ -2,6 +2,8 @@ nav_exclude: true --- +{% include deprecation.html %} + ## Peeking Local peeking is implemented as per [MSC2753](https://github.com/matrix-org/matrix-doc/pull/2753). diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index e148199fb..d3730c7ca 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -24,7 +24,6 @@ import ( "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/sirupsen/logrus" - "github.com/matrix-org/dendrite/federationapi/api" federationAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/consumers" "github.com/matrix-org/dendrite/federationapi/internal" @@ -102,7 +101,7 @@ func NewInternalAPI( caches *caching.Caches, keyRing *gomatrixserverlib.KeyRing, resetBlacklist bool, -) api.FederationInternalAPI { +) *internal.FederationInternalAPI { cfg := &dendriteCfg.FederationAPI federationDB, err := storage.NewDatabase(processContext.Context(), cm, &cfg.Database, caches, dendriteCfg.Global.IsLocalServerName) @@ -114,10 +113,7 @@ func NewInternalAPI( _ = federationDB.RemoveAllServersFromBlacklist() } - stats := statistics.NewStatistics( - federationDB, - cfg.FederationMaxRetries+1, - cfg.P2PFederationRetriesUntilAssumedOffline+1) + stats := statistics.NewStatistics(federationDB, cfg.FederationMaxRetries+1, cfg.P2PFederationRetriesUntilAssumedOffline+1, cfg.EnableRelays) js, nats := natsInstance.Prepare(processContext, &cfg.Matrix.JetStream) @@ -126,7 +122,7 @@ func NewInternalAPI( queues := queue.NewOutgoingQueues( federationDB, processContext, cfg.Matrix.DisableFederation, - cfg.Matrix.ServerName, federation, rsAPI, &stats, + cfg.Matrix.ServerName, federation, &stats, signingInfo, ) diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index 1ea8c40ea..79f4b3f21 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -5,11 +5,14 @@ import ( "crypto/ed25519" "encoding/json" "fmt" + "net/http" + "net/http/httptest" "strings" "sync" "testing" "time" + "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/sqlutil" @@ -17,7 +20,10 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/matrix-org/util" "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/tidwall/gjson" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationapi/api" @@ -362,3 +368,126 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) { } } } + +func TestNotaryServer(t *testing.T) { + testCases := []struct { + name string + httpBody string + pubKeyRequest *gomatrixserverlib.PublicKeyNotaryLookupRequest + validateFunc func(t *testing.T, response util.JSONResponse) + }{ + { + name: "empty httpBody", + validateFunc: func(t *testing.T, resp util.JSONResponse) { + assert.Equal(t, http.StatusBadRequest, resp.Code) + nk, ok := resp.JSON.(spec.MatrixError) + assert.True(t, ok) + assert.Equal(t, spec.ErrorBadJSON, nk.ErrCode) + }, + }, + { + name: "valid but empty httpBody", + httpBody: "{}", + validateFunc: func(t *testing.T, resp util.JSONResponse) { + want := util.JSONResponse{ + Code: http.StatusOK, + JSON: routing.NotaryKeysResponse{ServerKeys: []json.RawMessage{}}, + } + assert.Equal(t, want, resp) + }, + }, + { + name: "request all keys using an empty criteria", + httpBody: `{"server_keys":{"servera":{}}}`, + validateFunc: func(t *testing.T, resp util.JSONResponse) { + assert.Equal(t, http.StatusOK, resp.Code) + nk, ok := resp.JSON.(routing.NotaryKeysResponse) + assert.True(t, ok) + assert.Equal(t, "servera", gjson.GetBytes(nk.ServerKeys[0], "server_name").Str) + assert.True(t, gjson.GetBytes(nk.ServerKeys[0], "verify_keys.ed25519:someID").Exists()) + }, + }, + { + name: "request all keys using null as the criteria", + httpBody: `{"server_keys":{"servera":null}}`, + validateFunc: func(t *testing.T, resp util.JSONResponse) { + assert.Equal(t, http.StatusOK, resp.Code) + nk, ok := resp.JSON.(routing.NotaryKeysResponse) + assert.True(t, ok) + assert.Equal(t, "servera", gjson.GetBytes(nk.ServerKeys[0], "server_name").Str) + assert.True(t, gjson.GetBytes(nk.ServerKeys[0], "verify_keys.ed25519:someID").Exists()) + }, + }, + { + name: "request specific key", + httpBody: `{"server_keys":{"servera":{"ed25519:someID":{}}}}`, + validateFunc: func(t *testing.T, resp util.JSONResponse) { + assert.Equal(t, http.StatusOK, resp.Code) + nk, ok := resp.JSON.(routing.NotaryKeysResponse) + assert.True(t, ok) + assert.Equal(t, "servera", gjson.GetBytes(nk.ServerKeys[0], "server_name").Str) + assert.True(t, gjson.GetBytes(nk.ServerKeys[0], "verify_keys.ed25519:someID").Exists()) + }, + }, + { + name: "request multiple servers", + httpBody: `{"server_keys":{"servera":{"ed25519:someID":{}},"serverb":{"ed25519:someID":{}}}}`, + validateFunc: func(t *testing.T, resp util.JSONResponse) { + assert.Equal(t, http.StatusOK, resp.Code) + nk, ok := resp.JSON.(routing.NotaryKeysResponse) + assert.True(t, ok) + wantServers := map[string]struct{}{ + "servera": {}, + "serverb": {}, + } + for _, js := range nk.ServerKeys { + serverName := gjson.GetBytes(js, "server_name").Str + _, ok = wantServers[serverName] + assert.True(t, ok, "unexpected servername: %s", serverName) + delete(wantServers, serverName) + assert.True(t, gjson.GetBytes(js, "verify_keys.ed25519:someID").Exists()) + } + if len(wantServers) > 0 { + t.Fatalf("expected response to also contain: %#v", wantServers) + } + }, + }, + } + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, close := testrig.CreateConfig(t, dbType) + defer close() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + natsInstance := jetstream.NATSInstance{} + fc := &fedClient{ + keys: map[spec.ServerName]struct { + key ed25519.PrivateKey + keyID gomatrixserverlib.KeyID + }{ + "servera": { + key: test.PrivateKeyA, + keyID: "ed25519:someID", + }, + "serverb": { + key: test.PrivateKeyB, + keyID: "ed25519:someID", + }, + }, + } + + fedAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, fc, nil, caches, nil, true) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tc.httpBody)) + req.Host = string(cfg.Global.ServerName) + + resp := routing.NotaryKeys(req, &cfg.FederationAPI, fedAPI, tc.pubKeyRequest) + // assert that we received the expected response + tc.validateFunc(t, resp) + }) + } + + }) +} diff --git a/federationapi/internal/api.go b/federationapi/internal/api.go index 3e6f39566..67388a102 100644 --- a/federationapi/internal/api.go +++ b/federationapi/internal/api.go @@ -112,7 +112,7 @@ func NewFederationInternalAPI( } } -func (a *FederationInternalAPI) isBlacklistedOrBackingOff(s spec.ServerName) (*statistics.ServerStatistics, error) { +func (a *FederationInternalAPI) IsBlacklistedOrBackingOff(s spec.ServerName) (*statistics.ServerStatistics, error) { stats := a.statistics.ForServer(s) if stats.Blacklisted() { return stats, &api.FederationClientError{ @@ -151,7 +151,7 @@ func failBlacklistableError(err error, stats *statistics.ServerStatistics) (unti func (a *FederationInternalAPI) doRequestIfNotBackingOffOrBlacklisted( s spec.ServerName, request func() (interface{}, error), ) (interface{}, error) { - stats, err := a.isBlacklistedOrBackingOff(s) + stats, err := a.IsBlacklistedOrBackingOff(s) if err != nil { return nil, err } diff --git a/federationapi/internal/federationclient_test.go b/federationapi/internal/federationclient_test.go index 8c562dd61..47efb11da 100644 --- a/federationapi/internal/federationclient_test.go +++ b/federationapi/internal/federationclient_test.go @@ -61,11 +61,11 @@ func TestFederationClientQueryKeys(t *testing.T) { }, } fedClient := &testFedClient{} - stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline) + stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false) queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedapi := FederationInternalAPI{ @@ -92,11 +92,11 @@ func TestFederationClientQueryKeysBlacklisted(t *testing.T) { }, } fedClient := &testFedClient{} - stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline) + stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false) queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedapi := FederationInternalAPI{ @@ -122,11 +122,11 @@ func TestFederationClientQueryKeysFailure(t *testing.T) { }, } fedClient := &testFedClient{shouldFail: true} - stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline) + stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false) queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedapi := FederationInternalAPI{ @@ -152,11 +152,11 @@ func TestFederationClientClaimKeys(t *testing.T) { }, } fedClient := &testFedClient{} - stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline) + stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false) queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedapi := FederationInternalAPI{ @@ -183,11 +183,11 @@ func TestFederationClientClaimKeysBlacklisted(t *testing.T) { }, } fedClient := &testFedClient{} - stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline) + stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false) queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedapi := FederationInternalAPI{ diff --git a/federationapi/internal/perform_test.go b/federationapi/internal/perform_test.go index 656755f96..82f9b9db1 100644 --- a/federationapi/internal/perform_test.go +++ b/federationapi/internal/perform_test.go @@ -66,11 +66,11 @@ func TestPerformWakeupServers(t *testing.T) { }, } fedClient := &testFedClient{} - stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline) + stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, true) queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedAPI := NewFederationInternalAPI( @@ -112,11 +112,11 @@ func TestQueryRelayServers(t *testing.T) { }, } fedClient := &testFedClient{} - stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline) + stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false) queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedAPI := NewFederationInternalAPI( @@ -153,11 +153,11 @@ func TestRemoveRelayServers(t *testing.T) { }, } fedClient := &testFedClient{} - stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline) + stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false) queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedAPI := NewFederationInternalAPI( @@ -193,11 +193,11 @@ func TestPerformDirectoryLookup(t *testing.T) { }, } fedClient := &testFedClient{} - stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline) + stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false) queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedAPI := NewFederationInternalAPI( @@ -232,11 +232,11 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) { }, } fedClient := &testFedClient{} - stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline) + stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, true) queues := queue.NewOutgoingQueues( testDB, process.NewProcessContext(), false, - cfg.Matrix.ServerName, fedClient, nil, &stats, + cfg.Matrix.ServerName, fedClient, &stats, nil, ) fedAPI := NewFederationInternalAPI( diff --git a/federationapi/internal/query.go b/federationapi/internal/query.go index e53f19ff8..21e77c48d 100644 --- a/federationapi/internal/query.go +++ b/federationapi/internal/query.go @@ -43,6 +43,15 @@ func (a *FederationInternalAPI) fetchServerKeysFromCache( ctx context.Context, req *api.QueryServerKeysRequest, ) ([]gomatrixserverlib.ServerKeys, error) { var results []gomatrixserverlib.ServerKeys + + // We got a request for _all_ server keys, return them. + if len(req.KeyIDToCriteria) == 0 { + serverKeysResponses, _ := a.db.GetNotaryKeys(ctx, req.ServerName, []gomatrixserverlib.KeyID{}) + if len(serverKeysResponses) == 0 { + return nil, fmt.Errorf("failed to find server key response for server %s", req.ServerName) + } + return serverKeysResponses, nil + } for keyID, criteria := range req.KeyIDToCriteria { serverKeysResponses, _ := a.db.GetNotaryKeys(ctx, req.ServerName, []gomatrixserverlib.KeyID{keyID}) if len(serverKeysResponses) == 0 { diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go index 880aee0d3..be43aaf1c 100644 --- a/federationapi/queue/destinationqueue.go +++ b/federationapi/queue/destinationqueue.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" "sync" + "sync/atomic" "time" "github.com/matrix-org/gomatrix" @@ -26,12 +27,10 @@ import ( "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/sirupsen/logrus" - "go.uber.org/atomic" "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/federationapi/storage/shared/receipt" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/process" ) @@ -53,7 +52,6 @@ type destinationQueue struct { db storage.Database process *process.ProcessContext signing map[spec.ServerName]*fclient.SigningIdentity - rsAPI api.FederationRoomserverAPI client fclient.FederationClient // federation client origin spec.ServerName // origin of requests destination spec.ServerName // destination of requests @@ -296,6 +294,10 @@ func (oq *destinationQueue) checkNotificationsOnClose() { // backgroundSend is the worker goroutine for sending events. func (oq *destinationQueue) backgroundSend() { + // Don't try to send transactions if we are shutting down. + if oq.process.Context().Err() != nil { + return + } // Check if a worker is already running, and if it isn't, then // mark it as started. if !oq.running.CompareAndSwap(false, true) { diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 24b3efd2d..892c26a2c 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -27,12 +27,10 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" - "github.com/tidwall/gjson" "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/federationapi/storage/shared/receipt" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/process" ) @@ -43,7 +41,6 @@ type OutgoingQueues struct { db storage.Database process *process.ProcessContext disabled bool - rsAPI api.FederationRoomserverAPI origin spec.ServerName client fclient.FederationClient statistics *statistics.Statistics @@ -90,7 +87,6 @@ func NewOutgoingQueues( disabled bool, origin spec.ServerName, client fclient.FederationClient, - rsAPI api.FederationRoomserverAPI, statistics *statistics.Statistics, signing []*fclient.SigningIdentity, ) *OutgoingQueues { @@ -98,7 +94,6 @@ func NewOutgoingQueues( disabled: disabled, process: process, db: db, - rsAPI: rsAPI, origin: origin, client: client, statistics: statistics, @@ -162,7 +157,6 @@ func (oqs *OutgoingQueues) getQueue(destination spec.ServerName) *destinationQue queues: oqs, db: oqs.db, process: oqs.process, - rsAPI: oqs.rsAPI, origin: oqs.origin, destination: destination, client: oqs.client, @@ -213,18 +207,6 @@ func (oqs *OutgoingQueues) SendEvent( delete(destmap, local) } - // Check if any of the destinations are prohibited by server ACLs. - for destination := range destmap { - if api.IsServerBannedFromRoom( - oqs.process.Context(), - oqs.rsAPI, - ev.RoomID().String(), - destination, - ) { - delete(destmap, destination) - } - } - // If there are no remaining destinations then give up. if len(destmap) == 0 { return nil @@ -303,24 +285,6 @@ func (oqs *OutgoingQueues) SendEDU( delete(destmap, local) } - // There is absolutely no guarantee that the EDU will have a room_id - // field, as it is not required by the spec. However, if it *does* - // (e.g. typing notifications) then we should try to make sure we don't - // bother sending them to servers that are prohibited by the server - // ACLs. - if result := gjson.GetBytes(e.Content, "room_id"); result.Exists() { - for destination := range destmap { - if api.IsServerBannedFromRoom( - oqs.process.Context(), - oqs.rsAPI, - result.Str, - destination, - ) { - delete(destmap, destination) - } - } - } - // If there are no remaining destinations then give up. if len(destmap) == 0 { return nil diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go index e75615e05..7d21a3bb3 100644 --- a/federationapi/queue/queue_test.go +++ b/federationapi/queue/queue_test.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "sync/atomic" "testing" "time" @@ -26,7 +27,6 @@ import ( "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" - "go.uber.org/atomic" "gotest.tools/v3/poll" "github.com/matrix-org/gomatrixserverlib" @@ -34,7 +34,6 @@ import ( "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/federationapi/storage" - rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" @@ -65,15 +64,6 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase } } -type stubFederationRoomServerAPI struct { - rsapi.FederationRoomserverAPI -} - -func (r *stubFederationRoomServerAPI) QueryServerBannedFromRoom(ctx context.Context, req *rsapi.QueryServerBannedFromRoomRequest, res *rsapi.QueryServerBannedFromRoomResponse) error { - res.Banned = false - return nil -} - type stubFederationClient struct { fclient.FederationClient shouldTxSucceed bool @@ -123,12 +113,11 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32 fc := &stubFederationClient{ shouldTxSucceed: shouldTxSucceed, shouldTxRelaySucceed: shouldTxRelaySucceed, - txCount: *atomic.NewUint32(0), - txRelayCount: *atomic.NewUint32(0), + txCount: atomic.Uint32{}, + txRelayCount: atomic.Uint32{}, } - rs := &stubFederationRoomServerAPI{} - stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline) + stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline, false) signingInfo := []*fclient.SigningIdentity{ { KeyID: "ed21019:auto", @@ -136,7 +125,7 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32 ServerName: "localhost", }, } - queues := NewOutgoingQueues(db, processContext, false, "localhost", fc, rs, &stats, signingInfo) + queues := NewOutgoingQueues(db, processContext, false, "localhost", fc, &stats, signingInfo) return db, fc, queues, processContext, close } diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 75a007265..bc4138839 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -95,6 +95,12 @@ func Backfill( } } + // Enforce a limit of 100 events, as not to hit the DB to hard. + // Synapse has a hard limit of 100 events as well. + if req.Limit > 100 { + req.Limit = 100 + } + // Query the Roomserver. if err = rsAPI.PerformBackfill(httpReq.Context(), &req, &res); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("query.PerformBackfill failed") diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go index 3d8ff2dea..38a88e4b1 100644 --- a/federationapi/routing/keys.go +++ b/federationapi/routing/keys.go @@ -197,6 +197,10 @@ func localKeys(cfg *config.FederationAPI, serverName spec.ServerName) (*gomatrix return &keys, err } +type NotaryKeysResponse struct { + ServerKeys []json.RawMessage `json:"server_keys"` +} + func NotaryKeys( httpReq *http.Request, cfg *config.FederationAPI, fsAPI federationAPI.FederationInternalAPI, @@ -217,10 +221,9 @@ func NotaryKeys( } } - var response struct { - ServerKeys []json.RawMessage `json:"server_keys"` + response := NotaryKeysResponse{ + ServerKeys: []json.RawMessage{}, } - response.ServerKeys = []json.RawMessage{} for serverName, kidToCriteria := range req.ServerKeys { var keyList []gomatrixserverlib.ServerKeys diff --git a/federationapi/routing/profile_test.go b/federationapi/routing/profile_test.go index a31b206c1..ba13e07fc 100644 --- a/federationapi/routing/profile_test.go +++ b/federationapi/routing/profile_test.go @@ -26,7 +26,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" fedAPI "github.com/matrix-org/dendrite/federationapi" - fedInternal "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" @@ -67,11 +66,8 @@ func TestHandleQueryProfile(t *testing.T) { keyRing := serverKeyAPI.KeyRing() fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true) userapi := fakeUserAPI{} - r, ok := fedapi.(*fedInternal.FederationInternalAPI) - if !ok { - panic("This is a programming error.") - } - routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics) + + routing.Setup(routers, cfg, nil, fedapi, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics) handler := fedMux.Get(routing.QueryProfileRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 327ba9b08..dac9b1b34 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -146,7 +146,7 @@ func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationReques } walker := roomserverAPI.NewRoomHierarchyWalker(types.NewServerNameNotDevice(request.Origin()), roomID, suggestedOnly, 1) - discoveredRooms, _, err := rsAPI.QueryNextRoomHierarchyPage(httpReq.Context(), walker, -1) + discoveredRooms, inaccessibleRooms, _, err := rsAPI.QueryNextRoomHierarchyPage(httpReq.Context(), walker, -1) if err != nil { switch err.(type) { @@ -175,8 +175,9 @@ func QueryRoomHierarchy(httpReq *http.Request, request *fclient.FederationReques return util.JSONResponse{ Code: 200, JSON: fclient.RoomHierarchyResponse{ - Room: discoveredRooms[0], - Children: discoveredRooms[1:], + Room: discoveredRooms[0], + Children: discoveredRooms[1:], + InaccessibleChildren: inaccessibleRooms, }, } } diff --git a/federationapi/routing/query_test.go b/federationapi/routing/query_test.go index bb14ab031..fd0894d15 100644 --- a/federationapi/routing/query_test.go +++ b/federationapi/routing/query_test.go @@ -25,7 +25,6 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" fedAPI "github.com/matrix-org/dendrite/federationapi" - fedInternal "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" @@ -65,11 +64,8 @@ func TestHandleQueryDirectory(t *testing.T) { keyRing := serverKeyAPI.KeyRing() fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, &fedClient, nil, nil, keyRing, true) userapi := fakeUserAPI{} - r, ok := fedapi.(*fedInternal.FederationInternalAPI) - if !ok { - panic("This is a programming error.") - } - routing.Setup(routers, cfg, nil, r, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics) + + routing.Setup(routers, cfg, nil, fedapi, keyRing, &fedClient, &userapi, &cfg.MSCs, nil, caching.DisableMetrics) handler := fedMux.Get(routing.QueryDirectoryRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index dc7a363e7..91718efdb 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -16,6 +16,7 @@ package routing import ( "context" + "encoding/json" "fmt" "net/http" "sync" @@ -647,6 +648,8 @@ func MakeFedAPI( // add the user to Sentry, if enabled hub := sentry.GetHubFromContext(req.Context()) if hub != nil { + // clone the hub, so we don't send garbage events with e.g. mismatching rooms/event_ids + hub = hub.Clone() hub.Scope().SetTag("origin", string(fedReq.Origin())) hub.Scope().SetTag("uri", fedReq.RequestURI()) } @@ -676,6 +679,53 @@ func MakeFedAPI( return httputil.MakeExternalAPI(metricsName, h) } +// MakeFedHTTPAPI makes an http.Handler that checks matrix federation authentication. +func MakeFedHTTPAPI( + serverName spec.ServerName, + isLocalServerName func(spec.ServerName) bool, + keyRing gomatrixserverlib.JSONVerifier, + f func(http.ResponseWriter, *http.Request), +) http.Handler { + h := func(w http.ResponseWriter, req *http.Request) { + fedReq, errResp := fclient.VerifyHTTPRequest( + req, time.Now(), serverName, isLocalServerName, keyRing, + ) + + enc := json.NewEncoder(w) + logger := util.GetLogger(req.Context()) + if fedReq == nil { + + logger.Debugf("VerifyUserFromRequest %s -> HTTP %d", req.RemoteAddr, errResp.Code) + w.WriteHeader(errResp.Code) + if err := enc.Encode(errResp); err != nil { + logger.WithError(err).Error("failed to encode JSON response") + } + return + } + // add the user to Sentry, if enabled + hub := sentry.GetHubFromContext(req.Context()) + if hub != nil { + // clone the hub, so we don't send garbage events with e.g. mismatching rooms/event_ids + hub = hub.Clone() + hub.Scope().SetTag("origin", string(fedReq.Origin())) + hub.Scope().SetTag("uri", fedReq.RequestURI()) + } + defer func() { + if r := recover(); r != nil { + if hub != nil { + hub.CaptureException(fmt.Errorf("%s panicked", req.URL.Path)) + } + // re-panic to return the 500 + panic(r) + } + }() + + f(w, req) + } + + return http.HandlerFunc(h) +} + type FederationWakeups struct { FsAPI *fedInternal.FederationInternalAPI origins sync.Map diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index f629479da..ff4f7bd06 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -23,7 +23,6 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" fedAPI "github.com/matrix-org/dendrite/federationapi" - fedInternal "github.com/matrix-org/dendrite/federationapi/internal" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" @@ -62,11 +61,8 @@ func TestHandleSend(t *testing.T) { fedapi := fedAPI.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, nil, nil, nil, true) serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() - r, ok := fedapi.(*fedInternal.FederationInternalAPI) - if !ok { - panic("This is a programming error.") - } - routing.Setup(routers, cfg, nil, r, keyRing, nil, nil, &cfg.MSCs, nil, caching.DisableMetrics) + + routing.Setup(routers, cfg, nil, fedapi, keyRing, nil, nil, &cfg.MSCs, nil, caching.DisableMetrics) handler := fedMux.Get(routing.SendRouteName).GetHandler().ServeHTTP _, sk, _ := ed25519.GenerateKey(nil) diff --git a/federationapi/statistics/statistics.go b/federationapi/statistics/statistics.go index e5fc4b940..750c57fd7 100644 --- a/federationapi/statistics/statistics.go +++ b/federationapi/statistics/statistics.go @@ -5,10 +5,10 @@ import ( "math" "math/rand" "sync" + "sync/atomic" "time" "github.com/sirupsen/logrus" - "go.uber.org/atomic" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/gomatrixserverlib/spec" @@ -34,12 +34,15 @@ type Statistics struct { // mark the destination as offline. At this point we should attempt // to send messages to the user's async relay servers if we know them. FailuresUntilAssumedOffline uint32 + + enableRelays bool } func NewStatistics( db storage.Database, failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32, + enableRelays bool, ) Statistics { return Statistics{ DB: db, @@ -47,6 +50,7 @@ func NewStatistics( FailuresUntilAssumedOffline: failuresUntilAssumedOffline, backoffTimers: make(map[spec.ServerName]*time.Timer), servers: make(map[spec.ServerName]*ServerStatistics), + enableRelays: enableRelays, } } @@ -73,6 +77,13 @@ func (s *Statistics) ForServer(serverName spec.ServerName) *ServerStatistics { } else { server.blacklisted.Store(blacklisted) } + + // Don't bother hitting the database 2 additional times + // if we don't want to use relays. + if !s.enableRelays { + return server + } + assumedOffline, err := s.DB.IsServerAssumedOffline(context.Background(), serverName) if err != nil { logrus.WithError(err).Errorf("Failed to get assumed offline entry %q", serverName) @@ -158,7 +169,7 @@ func (s *ServerStatistics) Success(method SendMethod) { // NOTE : Sending to the final destination vs. a relay server has // slightly different semantics. if method == SendDirect { - s.successCounter.Inc() + s.successCounter.Add(1) if s.blacklisted.Load() && s.statistics.DB != nil { if err := s.statistics.DB.RemoveServerFromBlacklist(s.serverName); err != nil { logrus.WithError(err).Errorf("Failed to remove %q from blacklist", s.serverName) @@ -184,7 +195,7 @@ func (s *ServerStatistics) Failure() (time.Time, bool) { // start a goroutine which will wait out the backoff and // unset the backoffStarted flag when done. if s.backoffStarted.CompareAndSwap(false, true) { - backoffCount := s.backoffCount.Inc() + backoffCount := s.backoffCount.Add(1) if backoffCount >= s.statistics.FailuresUntilAssumedOffline { s.assumedOffline.CompareAndSwap(false, true) diff --git a/federationapi/statistics/statistics_test.go b/federationapi/statistics/statistics_test.go index a930bc3b0..4376a9050 100644 --- a/federationapi/statistics/statistics_test.go +++ b/federationapi/statistics/statistics_test.go @@ -16,7 +16,7 @@ const ( ) func TestBackoff(t *testing.T) { - stats := NewStatistics(nil, FailuresUntilBlacklist, FailuresUntilAssumedOffline) + stats := NewStatistics(nil, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false) server := ServerStatistics{ statistics: &stats, serverName: "test.com", @@ -106,7 +106,7 @@ func TestBackoff(t *testing.T) { } func TestRelayServersListing(t *testing.T) { - stats := NewStatistics(test.NewInMemoryFederationDatabase(), FailuresUntilBlacklist, FailuresUntilAssumedOffline) + stats := NewStatistics(test.NewInMemoryFederationDatabase(), FailuresUntilBlacklist, FailuresUntilAssumedOffline, false) server := ServerStatistics{statistics: &stats} server.AddRelayServers([]spec.ServerName{"relay1", "relay1", "relay2"}) relayServers := server.KnownRelayServers() diff --git a/federationapi/storage/cache/keydb.go b/federationapi/storage/cache/keydb.go index b53695ca4..d63c889d5 100644 --- a/federationapi/storage/cache/keydb.go +++ b/federationapi/storage/cache/keydb.go @@ -46,6 +46,10 @@ func (d *KeyDatabase) FetchKeys( delete(requests, req) } } + // Don't bother hitting the DB if we got everything from cache. + if len(requests) == 0 { + return results, nil + } fromDB, err := d.inner.FetchKeys(ctx, requests) if err != nil { return results, err diff --git a/federationapi/storage/postgres/notary_server_keys_metadata_table.go b/federationapi/storage/postgres/notary_server_keys_metadata_table.go index 7a1ec4122..47aa82b48 100644 --- a/federationapi/storage/postgres/notary_server_keys_metadata_table.go +++ b/federationapi/storage/postgres/notary_server_keys_metadata_table.go @@ -151,7 +151,7 @@ func (s *notaryServerKeysMetadataStatements) SelectKeys(ctx context.Context, txn } results = append(results, sk) } - return results, nil + return results, rows.Err() } func (s *notaryServerKeysMetadataStatements) DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error { diff --git a/federationapi/storage/postgres/queue_json_table.go b/federationapi/storage/postgres/queue_json_table.go index 563738dd5..f92e33d5e 100644 --- a/federationapi/storage/postgres/queue_json_table.go +++ b/federationapi/storage/postgres/queue_json_table.go @@ -109,5 +109,5 @@ func (s *queueJSONStatements) SelectQueueJSON( } blobs[nid] = blob } - return blobs, err + return blobs, rows.Err() } diff --git a/federationapi/storage/postgres/relay_servers_table.go b/federationapi/storage/postgres/relay_servers_table.go index 9e1bc5d40..1a47816e2 100644 --- a/federationapi/storage/postgres/relay_servers_table.go +++ b/federationapi/storage/postgres/relay_servers_table.go @@ -110,7 +110,7 @@ func (s *relayServersStatements) SelectRelayServers( } result = append(result, spec.ServerName(relayServer)) } - return result, nil + return result, rows.Err() } func (s *relayServersStatements) DeleteRelayServers( diff --git a/federationapi/storage/postgres/server_key_table.go b/federationapi/storage/postgres/server_key_table.go index c62446da5..fa58f1ea2 100644 --- a/federationapi/storage/postgres/server_key_table.go +++ b/federationapi/storage/postgres/server_key_table.go @@ -94,12 +94,14 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys( } defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectServerKeys: rows.close() failed") results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} + + var serverName string + var keyID string + var key string + var validUntilTS int64 + var expiredTS int64 + var vk gomatrixserverlib.VerifyKey for rows.Next() { - var serverName string - var keyID string - var key string - var validUntilTS int64 - var expiredTS int64 if err = rows.Scan(&serverName, &keyID, &validUntilTS, &expiredTS, &key); err != nil { return nil, err } @@ -107,7 +109,6 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys( ServerName: spec.ServerName(serverName), KeyID: gomatrixserverlib.KeyID(keyID), } - vk := gomatrixserverlib.VerifyKey{} err = vk.Key.Decode(key) if err != nil { return nil, err diff --git a/federationapi/storage/sqlite3/joined_hosts_table.go b/federationapi/storage/sqlite3/joined_hosts_table.go index 2412cacdb..418194312 100644 --- a/federationapi/storage/sqlite3/joined_hosts_table.go +++ b/federationapi/storage/sqlite3/joined_hosts_table.go @@ -216,5 +216,5 @@ func joinedHostsFromStmt( }) } - return result, nil + return result, rows.Err() } diff --git a/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go b/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go index 2fd9ef211..d9b98fc4f 100644 --- a/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go +++ b/federationapi/storage/sqlite3/notary_server_keys_metadata_table.go @@ -154,7 +154,7 @@ func (s *notaryServerKeysMetadataStatements) SelectKeys(ctx context.Context, txn } results = append(results, sk) } - return results, nil + return results, rows.Err() } func (s *notaryServerKeysMetadataStatements) DeleteOldJSONResponses(ctx context.Context, txn *sql.Tx) error { diff --git a/federationapi/storage/sqlite3/queue_json_table.go b/federationapi/storage/sqlite3/queue_json_table.go index 0e2806d56..33ae06131 100644 --- a/federationapi/storage/sqlite3/queue_json_table.go +++ b/federationapi/storage/sqlite3/queue_json_table.go @@ -135,5 +135,5 @@ func (s *queueJSONStatements) SelectQueueJSON( } blobs[nid] = blob } - return blobs, err + return blobs, rows.Err() } diff --git a/federationapi/storage/sqlite3/relay_servers_table.go b/federationapi/storage/sqlite3/relay_servers_table.go index 36cabeb4d..232db32af 100644 --- a/federationapi/storage/sqlite3/relay_servers_table.go +++ b/federationapi/storage/sqlite3/relay_servers_table.go @@ -109,7 +109,7 @@ func (s *relayServersStatements) SelectRelayServers( } result = append(result, spec.ServerName(relayServer)) } - return result, nil + return result, rows.Err() } func (s *relayServersStatements) DeleteRelayServers( diff --git a/federationapi/storage/sqlite3/server_key_table.go b/federationapi/storage/sqlite3/server_key_table.go index f28b89940..65a854ce1 100644 --- a/federationapi/storage/sqlite3/server_key_table.go +++ b/federationapi/storage/sqlite3/server_key_table.go @@ -98,12 +98,13 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys( err := sqlutil.RunLimitedVariablesQuery( ctx, bulkSelectServerSigningKeysSQL, s.db, iKeyIDs, sqlutil.SQLite3MaxVariables, func(rows *sql.Rows) error { + var serverName string + var keyID string + var key string + var validUntilTS int64 + var expiredTS int64 + var vk gomatrixserverlib.VerifyKey for rows.Next() { - var serverName string - var keyID string - var key string - var validUntilTS int64 - var expiredTS int64 if err := rows.Scan(&serverName, &keyID, &validUntilTS, &expiredTS, &key); err != nil { return fmt.Errorf("bulkSelectServerKeys: %v", err) } @@ -111,7 +112,6 @@ func (s *serverSigningKeyStatements) BulkSelectServerKeys( ServerName: spec.ServerName(serverName), KeyID: gomatrixserverlib.KeyID(keyID), } - vk := gomatrixserverlib.VerifyKey{} err := vk.Key.Decode(key) if err != nil { return fmt.Errorf("bulkSelectServerKeys: %v", err) diff --git a/federationapi/storage/tables/server_key_table_test.go b/federationapi/storage/tables/server_key_table_test.go new file mode 100644 index 000000000..e79a086b8 --- /dev/null +++ b/federationapi/storage/tables/server_key_table_test.go @@ -0,0 +1,116 @@ +package tables_test + +import ( + "context" + "testing" + "time" + + "github.com/matrix-org/dendrite/federationapi/storage/postgres" + "github.com/matrix-org/dendrite/federationapi/storage/sqlite3" + "github.com/matrix-org/dendrite/federationapi/storage/tables" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/stretchr/testify/assert" +) + +func mustCreateServerKeyDB(t *testing.T, dbType test.DBType) (tables.FederationServerSigningKeys, func()) { + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + if err != nil { + t.Fatalf("failed to open database: %s", err) + } + var tab tables.FederationServerSigningKeys + switch dbType { + case test.DBTypePostgres: + tab, err = postgres.NewPostgresServerSigningKeysTable(db) + case test.DBTypeSQLite: + tab, err = sqlite3.NewSQLiteServerSigningKeysTable(db) + } + if err != nil { + t.Fatalf("failed to create table: %s", err) + } + return tab, close +} + +func TestServerKeysTable(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + ctx, cancel := context.WithCancel(context.Background()) + tab, close := mustCreateServerKeyDB(t, dbType) + t.Cleanup(func() { + close() + cancel() + }) + + req := gomatrixserverlib.PublicKeyLookupRequest{ + ServerName: "localhost", + KeyID: "ed25519:test", + } + expectedTimestamp := spec.AsTimestamp(time.Now().Add(time.Hour)) + res := gomatrixserverlib.PublicKeyLookupResult{ + VerifyKey: gomatrixserverlib.VerifyKey{Key: make(spec.Base64Bytes, 0)}, + ExpiredTS: 0, + ValidUntilTS: expectedTimestamp, + } + + // Insert the key + err := tab.UpsertServerKeys(ctx, nil, req, res) + assert.NoError(t, err) + + selectKeys := map[gomatrixserverlib.PublicKeyLookupRequest]spec.Timestamp{ + req: spec.AsTimestamp(time.Now()), + } + gotKeys, err := tab.BulkSelectServerKeys(ctx, nil, selectKeys) + assert.NoError(t, err) + + // Now we should have a key for the req above + assert.NotNil(t, gotKeys[req]) + assert.Equal(t, res, gotKeys[req]) + + // "Expire" the key by setting ExpireTS to a non-zero value and ValidUntilTS to 0 + expectedTimestamp = spec.AsTimestamp(time.Now()) + res.ExpiredTS = expectedTimestamp + res.ValidUntilTS = 0 + + // Update the key + err = tab.UpsertServerKeys(ctx, nil, req, res) + assert.NoError(t, err) + + gotKeys, err = tab.BulkSelectServerKeys(ctx, nil, selectKeys) + assert.NoError(t, err) + + // The key should be expired + assert.NotNil(t, gotKeys[req]) + assert.Equal(t, res, gotKeys[req]) + + // Upsert a different key to validate querying multiple keys + req2 := gomatrixserverlib.PublicKeyLookupRequest{ + ServerName: "notlocalhost", + KeyID: "ed25519:test2", + } + expectedTimestamp2 := spec.AsTimestamp(time.Now().Add(time.Hour)) + res2 := gomatrixserverlib.PublicKeyLookupResult{ + VerifyKey: gomatrixserverlib.VerifyKey{Key: make(spec.Base64Bytes, 0)}, + ExpiredTS: 0, + ValidUntilTS: expectedTimestamp2, + } + + err = tab.UpsertServerKeys(ctx, nil, req2, res2) + assert.NoError(t, err) + + // Select multiple keys + selectKeys[req2] = spec.AsTimestamp(time.Now()) + + gotKeys, err = tab.BulkSelectServerKeys(ctx, nil, selectKeys) + assert.NoError(t, err) + + // We now should receive two keys, one of which is expired + assert.Equal(t, 2, len(gotKeys)) + assert.Equal(t, res2, gotKeys[req2]) + assert.Equal(t, res, gotKeys[req]) + }) +} diff --git a/go.mod b/go.mod index de60d9b4e..4add5ae1a 100644 --- a/go.mod +++ b/go.mod @@ -1,20 +1,22 @@ module github.com/matrix-org/dendrite require ( - github.com/Arceliar/ironwood v0.0.0-20221025225125-45b4281814c2 - github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 + github.com/Arceliar/phony v0.0.0-20220903101357-530938a4b13d github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/MFAshby/stdemuxerhook v1.0.0 github.com/Masterminds/semver/v3 v3.1.1 - github.com/blevesearch/bleve/v2 v2.3.8 + github.com/blevesearch/bleve/v2 v2.4.0 github.com/codeclysm/extract v2.2.0+incompatible + github.com/cretz/bine v0.2.0 github.com/dgraph-io/ristretto v0.1.1 - github.com/docker/docker v24.0.5+incompatible + github.com/docker/docker v25.0.6+incompatible github.com/docker/go-connections v0.4.0 + github.com/eyedeekay/goSam v0.32.54 + github.com/eyedeekay/onramp v0.33.8 github.com/getsentry/sentry-go v0.14.0 github.com/gologme/log v1.3.0 - github.com/google/go-cmp v0.5.9 - github.com/google/uuid v1.3.0 + github.com/google/go-cmp v0.6.0 + github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 github.com/kardianos/minwinsvc v1.0.2 @@ -22,125 +24,137 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca + github.com/matrix-org/gomatrixserverlib v0.0.0-20240910190622-2c764912ce93 github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 - github.com/mattn/go-sqlite3 v1.14.17 - github.com/nats-io/nats-server/v2 v2.9.23 - github.com/nats-io/nats.go v1.28.0 - github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 + github.com/mattn/go-sqlite3 v1.14.22 + github.com/nats-io/nats-server/v2 v2.10.20 + github.com/nats-io/nats.go v1.36.0 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/opentracing/opentracing-go v1.2.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.16.0 + github.com/prometheus/client_golang v1.19.1 github.com/sirupsen/logrus v1.9.3 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.9.0 github.com/tidwall/gjson v1.17.0 github.com/tidwall/sjson v1.2.5 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible - github.com/yggdrasil-network/yggdrasil-go v0.4.6 - go.uber.org/atomic v1.10.0 - golang.org/x/crypto v0.14.0 - golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 - golang.org/x/image v0.5.0 - golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e - golang.org/x/sync v0.3.0 - golang.org/x/term v0.13.0 + github.com/yggdrasil-network/yggdrasil-go v0.5.6 + github.com/yggdrasil-network/yggquic v0.0.0-20240802104827-b4e97a928967 + go.uber.org/atomic v1.11.0 + golang.org/x/crypto v0.27.0 + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 + golang.org/x/image v0.18.0 + golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b + golang.org/x/sync v0.8.0 + golang.org/x/term v0.24.0 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 gotest.tools/v3 v3.4.0 maunium.net/go/mautrix v0.15.1 - modernc.org/sqlite v1.23.1 + modernc.org/sqlite v1.29.5 nhooyr.io/websocket v1.8.7 ) require ( + github.com/Arceliar/ironwood v0.0.0-20240529054413-b8e59574e2b2 // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/RoaringBitmap/roaring v1.2.3 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.5.0 // indirect - github.com/blevesearch/bleve_index_api v1.0.5 // indirect - github.com/blevesearch/geo v0.1.17 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect + github.com/bits-and-blooms/bloom/v3 v3.7.0 // indirect + github.com/blevesearch/bleve_index_api v1.1.6 // indirect + github.com/blevesearch/geo v0.1.20 // indirect + github.com/blevesearch/go-faiss v1.0.13 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/gtreap v0.1.1 // indirect github.com/blevesearch/mmap-go v1.0.4 // indirect - github.com/blevesearch/scorch_segment_api/v2 v2.1.4 // indirect + github.com/blevesearch/scorch_segment_api/v2 v2.2.9 // indirect github.com/blevesearch/segment v0.9.1 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect - github.com/blevesearch/vellum v1.0.9 // indirect - github.com/blevesearch/zapx/v11 v11.3.7 // indirect - github.com/blevesearch/zapx/v12 v12.3.7 // indirect - github.com/blevesearch/zapx/v13 v13.3.7 // indirect - github.com/blevesearch/zapx/v14 v14.3.7 // indirect - github.com/blevesearch/zapx/v15 v15.3.10 // indirect + github.com/blevesearch/vellum v1.0.10 // indirect + github.com/blevesearch/zapx/v11 v11.3.10 // indirect + github.com/blevesearch/zapx/v12 v12.3.10 // indirect + github.com/blevesearch/zapx/v13 v13.3.10 // indirect + github.com/blevesearch/zapx/v14 v14.3.10 // indirect + github.com/blevesearch/zapx/v15 v15.3.13 // indirect + github.com/blevesearch/zapx/v16 v16.0.12 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/containerd/log v0.1.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/docker/distribution v2.8.2+incompatible // indirect + github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/eyedeekay/i2pkeys v0.33.8 // indirect + github.com/eyedeekay/sam3 v0.33.8 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/glog v1.0.0 // indirect - github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/pprof v0.0.0-20230808223545-4887780b67fb // indirect github.com/h2non/filetype v1.1.3 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/hjson/hjson-go/v4 v4.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/juju/errors v1.0.0 // indirect - github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/minio/highwayhash v1.0.2 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/minio/highwayhash v1.0.3 // indirect github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mschoch/smat v0.2.0 // indirect - github.com/nats-io/jwt/v2 v2.5.0 // indirect - github.com/nats-io/nkeys v0.4.4 // indirect + github.com/nats-io/jwt/v2 v2.5.8 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect github.com/onsi/ginkgo/v2 v2.11.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect - github.com/quic-go/qtls-go1-20 v0.3.2 // indirect - github.com/quic-go/quic-go v0.37.4 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/quic-go/quic-go v0.45.2 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/rs/zerolog v1.29.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect - go.etcd.io/bbolt v1.3.6 // indirect - golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.12.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + go.etcd.io/bbolt v1.3.7 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.uber.org/mock v0.4.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + golang.org/x/time v0.6.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/macaroon.v2 v2.1.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - lukechampine.com/uint128 v1.2.0 // indirect maunium.net/go/maulogger/v2 v2.4.1 // indirect - modernc.org/cc/v3 v3.40.0 // indirect - modernc.org/ccgo/v3 v3.16.13 // indirect - modernc.org/libc v1.22.5 // indirect - modernc.org/mathutil v1.5.0 // indirect - modernc.org/memory v1.5.0 // indirect - modernc.org/opt v0.1.3 // indirect - modernc.org/strutil v1.1.3 // indirect - modernc.org/token v1.0.1 // indirect + modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect + modernc.org/libc v1.41.0 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.7.2 // indirect + modernc.org/strutil v1.2.0 // indirect + modernc.org/token v1.1.0 // indirect ) -go 1.20 +go 1.21.0 diff --git a/go.sum b/go.sum index 955cdc11c..b64125e3c 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Arceliar/ironwood v0.0.0-20221025225125-45b4281814c2 h1:Usab30pNT2i/vZvpXcN9uOr5IO1RZPcUqoGH0DIAPnU= -github.com/Arceliar/ironwood v0.0.0-20221025225125-45b4281814c2/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk= -github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3ugUxx2EnnWmgba1kCqPkd4Gk1yQ= -github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI= +github.com/Arceliar/ironwood v0.0.0-20240529054413-b8e59574e2b2 h1:SBdYBKeXYUUFef5wi2CMhYmXFVGiYaRpTvbki0Bu+JQ= +github.com/Arceliar/ironwood v0.0.0-20240529054413-b8e59574e2b2/go.mod h1:6WP4799FX0OuWdENGQAh+0RXp9FLh0y7NZ7tM9cJyXk= +github.com/Arceliar/phony v0.0.0-20220903101357-530938a4b13d h1:UK9fsWbWqwIQkMCz1CP+v5pGbsGoWAw6g4AyvMpm1EM= +github.com/Arceliar/phony v0.0.0-20220903101357-530938a4b13d/go.mod h1:BCnxhRf47C/dy/e/D2pmB8NkB3dQVIrkD98b220rx5Q= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -16,70 +16,67 @@ github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030I github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/anacrolix/envpprof v0.0.0-20180404065416-323002cec2fa/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= -github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= -github.com/anacrolix/envpprof v1.1.1 h1:sHQCyj7HtiSfaZAzL2rJrQdyS7odLqlwO6nhk/tG/j8= -github.com/anacrolix/envpprof v1.1.1/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= -github.com/anacrolix/log v0.3.0 h1:Btxh7GkT4JYWvWJ1uKOwgobf+7q/1eFQaDdCUXCtssw= -github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgwU9jwU= -github.com/anacrolix/missinggo v1.1.2-0.20190815015349-b888af804467/go.mod h1:MBJu3Sk/k3ZfGYcS7z18gwfu72Ey/xopPFJJbTi5yIo= -github.com/anacrolix/missinggo v1.2.1 h1:0IE3TqX5y5D0IxeMwTyIgqdDew4QrzcXaaEnJQyjHvw= -github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikzMt+uUjeM21Y= -github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ= -github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= -github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/blevesearch/bleve/v2 v2.3.8 h1:IqFyMJ73n4gY8AmVqM8Sa6EtAZ5beE8yramVqCvs2kQ= -github.com/blevesearch/bleve/v2 v2.3.8/go.mod h1:Lh9aZEHrLKxwPnW4z4lsBEGnflZQ1V/aWP/t+htsiDw= -github.com/blevesearch/bleve_index_api v1.0.5 h1:Lc986kpC4Z0/n1g3gg8ul7H+lxgOQPcXb9SxvQGu+tw= -github.com/blevesearch/bleve_index_api v1.0.5/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= -github.com/blevesearch/geo v0.1.17 h1:AguzI6/5mHXapzB0gE9IKWo+wWPHZmXZoscHcjFgAFA= -github.com/blevesearch/geo v0.1.17/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM= +github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bloom/v3 v3.7.0 h1:VfknkqV4xI+PsaDIsoHueyxVDZrfvMn56jeWUzvzdls= +github.com/bits-and-blooms/bloom/v3 v3.7.0/go.mod h1:VKlUSvp0lFIYqxJjzdnSsZEw4iHb1kOL2tfHTgyJBHg= +github.com/blevesearch/bleve/v2 v2.4.0 h1:2xyg+Wv60CFHYccXc+moGxbL+8QKT/dZK09AewHgKsg= +github.com/blevesearch/bleve/v2 v2.4.0/go.mod h1:IhQHoFAbHgWKYavb9rQgQEJJVMuY99cKdQ0wPpst2aY= +github.com/blevesearch/bleve_index_api v1.1.6 h1:orkqDFCBuNU2oHW9hN2YEJmet+TE9orml3FCGbl1cKk= +github.com/blevesearch/bleve_index_api v1.1.6/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= +github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM= +github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w= +github.com/blevesearch/go-faiss v1.0.13 h1:zfFs7ZYD0NqXVSY37j0JZjZT1BhE9AE4peJfcx/NB4A= +github.com/blevesearch/go-faiss v1.0.13/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= -github.com/blevesearch/scorch_segment_api/v2 v2.1.4 h1:LmGmo5twU3gV+natJbKmOktS9eMhokPGKWuR+jX84vk= -github.com/blevesearch/scorch_segment_api/v2 v2.1.4/go.mod h1:PgVnbbg/t1UkgezPDu8EHLi1BHQ17xUwsFdU6NnOYS0= +github.com/blevesearch/scorch_segment_api/v2 v2.2.9 h1:3nBaSBRFokjE4FtPW3eUDgcAu3KphBg1GP07zy/6Uyk= +github.com/blevesearch/scorch_segment_api/v2 v2.2.9/go.mod h1:ckbeb7knyOOvAdZinn/ASbB7EA3HoagnJkmEV3J7+sg= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A= github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= -github.com/blevesearch/vellum v1.0.9 h1:PL+NWVk3dDGPCV0hoDu9XLLJgqU4E5s/dOeEJByQ2uQ= -github.com/blevesearch/vellum v1.0.9/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= -github.com/blevesearch/zapx/v11 v11.3.7 h1:Y6yIAF/DVPiqZUA/jNgSLXmqewfzwHzuwfKyfdG+Xaw= -github.com/blevesearch/zapx/v11 v11.3.7/go.mod h1:Xk9Z69AoAWIOvWudNDMlxJDqSYGf90LS0EfnaAIvXCA= -github.com/blevesearch/zapx/v12 v12.3.7 h1:DfQ6rsmZfEK4PzzJJRXjiM6AObG02+HWvprlXQ1Y7eI= -github.com/blevesearch/zapx/v12 v12.3.7/go.mod h1:SgEtYIBGvM0mgIBn2/tQE/5SdrPXaJUaT/kVqpAPxm0= -github.com/blevesearch/zapx/v13 v13.3.7 h1:igIQg5eKmjw168I7av0Vtwedf7kHnQro/M+ubM4d2l8= -github.com/blevesearch/zapx/v13 v13.3.7/go.mod h1:yyrB4kJ0OT75UPZwT/zS+Ru0/jYKorCOOSY5dBzAy+s= -github.com/blevesearch/zapx/v14 v14.3.7 h1:gfe+fbWslDWP/evHLtp/GOvmNM3sw1BbqD7LhycBX20= -github.com/blevesearch/zapx/v14 v14.3.7/go.mod h1:9J/RbOkqZ1KSjmkOes03AkETX7hrXT0sFMpWH4ewC4w= -github.com/blevesearch/zapx/v15 v15.3.10 h1:bQ9ZxJCj6rKp873EuVJu2JPxQ+EWQZI1cjJGeroovaQ= -github.com/blevesearch/zapx/v15 v15.3.10/go.mod h1:m7Y6m8soYUvS7MjN9eKlz1xrLCcmqfFadmu7GhWIrLY= -github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= -github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= -github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= -github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= +github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI= +github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= +github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk= +github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ= +github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s= +github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs= +github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8= +github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk= +github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU= +github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= +github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ= +github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= +github.com/blevesearch/zapx/v16 v16.0.12 h1:Uccxvjmn+hQ6ywQP+wIiTpdq9LnAviGoryJOmGwAo/I= +github.com/blevesearch/zapx/v16 v16.0.12/go.mod h1:MYnOshRfSm4C4drxx1LGRI+MVFByykJ2anDY1fxdk9Q= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/codeclysm/extract v2.2.0+incompatible h1:q3wyckoA30bhUSiwdQezMqVhwd8+WGE64/GL//LtUhI= github.com/codeclysm/extract v2.2.0+incompatible/go.mod h1:2nhFMPHiU9At61hz+12bfrlpXSUrOnK+wR+KlGO4Uks= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cretz/bine v0.2.0 h1:8GiDRGlTgz+o8H9DSnsl+5MeBK4HsExxgl6WgzOCuZo= +github.com/cretz/bine v0.2.0/go.mod h1:WU4o9QR9wWp8AVKtTM1XD5vUHkEqnf2vVSo6dBqbetI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -87,40 +84,70 @@ github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWa github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY= -github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v25.0.6+incompatible h1:5cPwbwriIcsua2REJe8HqQV+6WlWc1byg2QSXzBxBGg= +github.com/docker/docker v25.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/eyedeekay/goSam v0.32.54 h1:Uq1F9rePGi5aiHZ8J8ZC0HRpf4hvTUR+PJvmcCBpmWU= +github.com/eyedeekay/goSam v0.32.54/go.mod h1:R+prG/Xans0bG87LhtbbLSx40YiHtJNovhTHL2mEwPE= +github.com/eyedeekay/i2pkeys v0.0.0-20220310055120-b97558c06ac8/go.mod h1:W9KCm9lqZ+Ozwl3dwcgnpPXAML97+I8Jiht7o5A8YBM= +github.com/eyedeekay/i2pkeys v0.33.8 h1:f3llyruchFqs1QwCacBYbShArKPpMSSOqo/DVZXcfVs= +github.com/eyedeekay/i2pkeys v0.33.8/go.mod h1:W9KCm9lqZ+Ozwl3dwcgnpPXAML97+I8Jiht7o5A8YBM= +github.com/eyedeekay/onramp v0.33.8 h1:nc2ZGwWkeLf/GcijOMDdzI53q2mA8hCjepMyGHstcF0= +github.com/eyedeekay/onramp v0.33.8/go.mod h1:YYMgClC/ck/+3lHHAdsYzmDCSmsU8tn5WMkiSy9fcLo= +github.com/eyedeekay/sam3 v0.33.8 h1:emuSZ4qSyoqc1EDjIBFbJ3GXNHOXw6hjbNp2OqdOpgI= +github.com/eyedeekay/sam3 v0.33.8/go.mod h1:ytbwLYLJlW6UA92Ffyc6oioWTKnGeeUMr9CLuJbtqSA= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/getlantern/context v0.0.0-20190109183933-c447772a6520/go.mod h1:L+mq6/vvYHKjCX2oez0CgEAJmbq1fbb/oNJIWQkBybY= +github.com/getlantern/errors v1.0.1/go.mod h1:l+xpFBrCtDLpK9qNjxs+cHU6+BAdlBaxHqikB6Lku3A= +github.com/getlantern/fdcount v0.0.0-20210503151800-5decd65b3731/go.mod h1:XZwE+iIlAgr64OFbXKFNCllBwV4wEipPx8Hlo2gZdbM= +github.com/getlantern/go-socks5 v0.0.0-20171114193258-79d4dd3e2db5/go.mod h1:kGHRXch95rnGLHjER/GhhFiHvfnqNz7KqWD9kGfATHY= +github.com/getlantern/golog v0.0.0-20201105130739-9586b8bde3a9/go.mod h1:ZyIjgH/1wTCl+B+7yH1DqrWp6MPJqESmwmEQ89ZfhvA= +github.com/getlantern/hex v0.0.0-20190417191902-c6586a6fe0b7/go.mod h1:dD3CgOrwlzca8ed61CsZouQS5h5jIzkK9ZWrTcf0s+o= +github.com/getlantern/hidden v0.0.0-20190325191715-f02dbb02be55/go.mod h1:6mmzY2kW1TOOrVy+r41Za2MxXM+hhqTtY3oBKd2AgFA= +github.com/getlantern/mockconn v0.0.0-20200818071412-cb30d065a848/go.mod h1:+F5GJ7qGpQ03DBtcOEyQpM30ix4BLswdaojecFtsdy8= +github.com/getlantern/mtime v0.0.0-20200417132445-23682092d1f7/go.mod h1:GfzwugvtH7YcmNIrHHizeyImsgEdyL88YkdnK28B14c= +github.com/getlantern/netx v0.0.0-20190110220209-9912de6f94fd/go.mod h1:wKdY0ikOgzrWSeB9UyBVKPRhjXQ+vTb+BPeJuypUuNE= +github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f/go.mod h1:D5ao98qkA6pxftxoqzibIBBrLSUli+kYnJqrgBf9cIA= +github.com/getlantern/ops v0.0.0-20200403153110-8476b16edcd6/go.mod h1:D5ao98qkA6pxftxoqzibIBBrLSUli+kYnJqrgBf9cIA= github.com/getsentry/sentry-go v0.14.0 h1:rlOBkuFZRKKdUnKO+0U3JclRDQKlRu5vVQtkWSQvC70= github.com/getsentry/sentry-go v0.14.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2HtRckJk9XVxJ9I= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= -github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= +github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= @@ -130,6 +157,7 @@ github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -139,67 +167,65 @@ github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgR github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gologme/log v1.3.0 h1:l781G4dE+pbigClDSDzSaaYKtiueHCILUa/qSDsmHAo= github.com/gologme/log v1.3.0/go.mod h1:yKT+DvIPdDdDoPtqFrFxheooyVmoqi0BAsw+erN3wA4= -github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20230808223545-4887780b67fb h1:oqpb3Cwpc7EOml5PVGMYbSGmwNui2R7i8IW83gs4W0c= github.com/google/pprof v0.0.0-20230808223545-4887780b67fb/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= -github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= -github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hjson/hjson-go/v4 v4.4.0 h1:D/NPvqOCH6/eisTb5/ztuIS8GUvmpHaLOcNk1Bjr298= +github.com/hjson/hjson-go/v4 v4.4.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v1.0.0 h1:yiq7kjCLll1BiaRuNY53MGI0+EQ3rF6GB+wvboZDefM= github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5Qe8= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5lLdR0= github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e h1:DP5RC0Z3XdyBEW5dKt8YPeN6vZbm6OzVaGVp7f1BQRM= @@ -208,8 +234,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca h1:JCP72vU4Vcmur2071RwYVOSoekR+ZjbC03wZD5lAAK0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20231024124730-58af9a2712ca/go.mod h1:M8m7seOroO5ePlgxA7AFZymnG90Cnh94rYQyngSrZkk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20240910190622-2c764912ce93 h1:FbyZ/xkeBVYHi2xfwAVaNmDhP+4HNbt9e6ucOR+jvBk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20240910190622-2c764912ce93/go.mod h1:HZGsVJ3bUE+DkZtufkH9H0mlsvbhEGK5CpX0Zlavylg= github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4= github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg= github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y= @@ -220,15 +246,14 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= -github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= -github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= -github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= +github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q= +github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -239,84 +264,77 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= -github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak= -github.com/nats-io/jwt/v2 v2.5.0/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= -github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU= -github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0= -github.com/nats-io/nats.go v1.28.0 h1:Th4G6zdsz2d0OqXdfzKLClo6bOfoI/b1kInhRtFIy5c= -github.com/nats-io/nats.go v1.28.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= -github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= -github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= +github.com/nats-io/jwt/v2 v2.5.8 h1:uvdSzwWiEGWGXf+0Q+70qv6AQdvcvxrv9hPM0RiPamE= +github.com/nats-io/jwt/v2 v2.5.8/go.mod h1:ZdWS1nZa6WMZfFwwgpEaqBV8EPGVgOTDHN/wTbz0Y5A= +github.com/nats-io/nats-server/v2 v2.10.20 h1:CXDTYNHeBiAKBTAIP2gjpgbWap2GhATnTLgP8etyvEI= +github.com/nats-io/nats-server/v2 v2.10.20/go.mod h1:hgcPnoUtMfxz1qVOvLZGurVypQ+Cg6GXVXjG53iHk+M= +github.com/nats-io/nats.go v1.36.0 h1:suEUPuWzTSse/XhESwqLxXGuj8vGRuPRoG7MoRN/qyU= +github.com/nats-io/nats.go v1.36.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ= -github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= +github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI= -github.com/quic-go/qtls-go1-20 v0.3.2/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= -github.com/quic-go/quic-go v0.37.4 h1:ke8B73yMCWGq9MfrCCAw0Uzdm7GaViC3i39dsIdDlH4= -github.com/quic-go/quic-go v0.37.4/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/quic-go/quic-go v0.45.2 h1:DfqBmqjb4ExSdxRIb/+qXhPC+7k6+DUNZha4oeiC9fY= +github.com/quic-go/quic-go v0.45.2/go.mod h1:1dLehS7TIR64+vxGR70GDcatWTOtMX2PUtnKsjbTurI= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= -github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -327,7 +345,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= +github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= @@ -336,79 +355,99 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= -github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/yggdrasil-network/yggdrasil-go v0.4.6 h1:GALUDV9QPz/5FVkbazpkTc9EABHufA556JwUJZr41j4= -github.com/yggdrasil-network/yggdrasil-go v0.4.6/go.mod h1:PBMoAOvQjA9geNEeGyMXA9QgCS6Bu+9V+1VkWM84wpw= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/yggdrasil-network/yggdrasil-go v0.5.6 h1:thh5YQYXQgkhkSO6v2D9Ya9fLHXfY38VfsCTZTIbIeI= +github.com/yggdrasil-network/yggdrasil-go v0.5.6/go.mod h1:WAqMZ4e1QSf/EKbzfD77XXTSAIRO/0nWKCVpHsKLg40= +github.com/yggdrasil-network/yggquic v0.0.0-20240802104827-b4e97a928967 h1:IxtZy4a4ZFYc1OiEv1VUc8u4Xl1WF6986wfu1DbY/SI= +github.com/yggdrasil-network/yggquic v0.0.0-20240802104827-b4e97a928967/go.mod h1:RVLAuYojgYebPO/fJwWRSVlzKLXbZzZpWAStnBwiSsk= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= +go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 h1:EDuYyU/MkFXllv9QF9819VlI9a4tzGuCbhG0ExK9o1U= -golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= -golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= +golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e h1:zSgtO19fpg781xknwqiQPmOHaASr6E7ZVlTseLd9Fx4= -golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e/go.mod h1:aAjjkJNdrh3PMckS4B10TGS2nag27cbKR1y2BpUxsiY= +golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b h1:WX7nnnLfCEXg+FmdYZPai2XuP3VqCP1HZVMST0n9DF0= +golang.org/x/mobile v0.0.0-20240520174638-fa72addaaa1b/go.mod h1:EiXZlVfUTaAyySFVJb9rsODuiO+WXu8HrUuySb7nYFw= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -418,22 +457,36 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -443,10 +496,10 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -455,17 +508,24 @@ gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJ gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/h2non/bimg.v1 v1.1.9 h1:wZIUbeOnwr37Ta4aofhIv8OI8v4ujpjXC9mXnAGpQjM= gopkg.in/h2non/bimg.v1 v1.1.9/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= +gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI= gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -478,34 +538,26 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= -lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= -lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8= maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho= maunium.net/go/mautrix v0.15.1 h1:pmCtMjYRpd83+2UL+KTRFYQo5to0373yulimvLK+1k0= maunium.net/go/mautrix v0.15.1/go.mod h1:icQIrvz2NldkRLTuzSGzmaeuMUmw+fzO7UVycPeauN8= -modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= -modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= -modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= -modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= -modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= -modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= -modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= -modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= -modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= -modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= -modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= -modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= -modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= -modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= -modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY= -modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg= -modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= +modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk= +modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= +modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= +modernc.org/sqlite v1.29.5 h1:8l/SQKAjDtZFo9lkJLdk8g9JEOeYRG4/ghStDCCTiTE= +modernc.org/sqlite v1.29.5/go.mod h1:S02dvcmm7TnTRvGhv8IGYyLnIt7AS2KPaB1F/71p75U= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/helm/ct.yaml b/helm/ct.yaml index af706fa3d..8b5aaa875 100644 --- a/helm/ct.yaml +++ b/helm/ct.yaml @@ -4,4 +4,6 @@ chart-repos: - bitnami=https://charts.bitnami.com/bitnami chart-dirs: - helm -validate-maintainers: false \ No newline at end of file +validate-maintainers: false +# this should ensure the tarballs are in the appropriate location for GH pages, rather than repo root +package-path: docs/ \ No newline at end of file diff --git a/helm/dendrite/Chart.yaml b/helm/dendrite/Chart.yaml index 32f479960..9613b5045 100644 --- a/helm/dendrite/Chart.yaml +++ b/helm/dendrite/Chart.yaml @@ -1,9 +1,10 @@ apiVersion: v2 name: dendrite -version: "0.13.5" -appVersion: "0.13.4" +version: "0.14.6" +appVersion: "0.13.8" description: Dendrite Matrix Homeserver type: application +icon: https://avatars.githubusercontent.com/u/8418310?s=48&v=4 keywords: - matrix - chat @@ -13,7 +14,7 @@ home: https://github.com/matrix-org/dendrite sources: - https://github.com/matrix-org/dendrite dependencies: -- name: postgresql - version: 12.1.7 - repository: https://charts.bitnami.com/bitnami - condition: postgresql.enabled + - name: postgresql + version: 14.2.3 + repository: https://charts.bitnami.com/bitnami + condition: postgresql.enabled diff --git a/helm/dendrite/README.md b/helm/dendrite/README.md index 22daa1813..a5b03aa85 100644 --- a/helm/dendrite/README.md +++ b/helm/dendrite/README.md @@ -1,7 +1,7 @@ # dendrite -![Version: 0.13.5](https://img.shields.io/badge/Version-0.13.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.4](https://img.shields.io/badge/AppVersion-0.13.4-informational?style=flat-square) +![Version: 0.14.4](https://img.shields.io/badge/Version-0.14.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.8](https://img.shields.io/badge/AppVersion-0.13.8-informational?style=flat-square) Dendrite Matrix Homeserver Status: **NOT PRODUCTION READY** @@ -37,7 +37,7 @@ Create a folder `appservices` and place your configurations in there. The confi | Repository | Name | Version | |------------|------|---------| -| https://charts.bitnami.com/bitnami | postgresql | 12.1.7 | +| https://charts.bitnami.com/bitnami | postgresql | 14.2.3 | ## Values | Key | Type | Default | Description | @@ -45,19 +45,23 @@ Create a folder `appservices` and place your configurations in there. The confi | image.repository | string | `"ghcr.io/matrix-org/dendrite-monolith"` | Docker repository/image to use | | image.pullPolicy | string | `"IfNotPresent"` | Kubernetes pullPolicy | | image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | +| imagePullSecrets | list | `[]` | Configure image pull secrets to use private container registry https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret | | signing_key.create | bool | `true` | Create a new signing key, if not exists | | signing_key.existingSecret | string | `""` | Use an existing secret | | resources | object | sets some sane default values | Default resource requests/limits. | -| persistence.jetstream | object | `{"capacity":"1Gi","existingClaim":""}` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. # If defined, storageClassName: # If set to "-", storageClassName: "", which disables dynamic provisioning # If undefined (the default) or set to null, no storageClassName spec is # set, choosing the default provisioner. (gp2 on AWS, standard on # GKE, AWS & OpenStack) # storageClass: "" | +| persistence.storageClass | string | `nil` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. If defined, storageClassName: If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. (gp2 on AWS, standard on GKE, AWS & OpenStack) | | persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream | | persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume | +| persistence.jetstream.storageClass | string | `nil` | The storage class to use for volume claims. Defaults to persistence.storageClass If defined, storageClassName: If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. (gp2 on AWS, standard on GKE, AWS & OpenStack) | | persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files | | persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume | +| persistence.media.storageClass | string | `nil` | The storage class to use for volume claims. Defaults to persistence.storageClass If defined, storageClassName: If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. (gp2 on AWS, standard on GKE, AWS & OpenStack) | | persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index | | persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume | +| persistence.search.storageClass | string | `nil` | The storage class to use for volume claims. Defaults to persistence.storageClass If defined, storageClassName: If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. (gp2 on AWS, standard on GKE, AWS & OpenStack) | | extraVolumes | list | `[]` | Add additional volumes to the Dendrite Pod | | extraVolumeMounts | list | `[]` | Configure additional mount points volumes in the Dendrite Pod | -| strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate | +| strategy.type | string | `"Recreate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate | | strategy.rollingUpdate.maxUnavailable | string | `"25%"` | Maximum number of pods that can be unavailable during the update process | | strategy.rollingUpdate.maxSurge | string | `"25%"` | Maximum number of pods that can be scheduled above the desired number of pods | | dendrite_config.version | int | `2` | | @@ -139,7 +143,7 @@ Create a folder `appservices` and place your configurations in there. The confi | dendrite_config.logging | list | `[{"level":"info","type":"std"}]` | Default logging configuration | | postgresql.enabled | bool | See value.yaml | Enable and configure postgres as the database for dendrite. | | postgresql.image.repository | string | `"bitnami/postgresql"` | | -| postgresql.image.tag | string | `"15.1.0"` | | +| postgresql.image.tag | string | `"16.2.0"` | | | postgresql.auth.username | string | `"dendrite"` | | | postgresql.auth.password | string | `"changeme"` | | | postgresql.auth.database | string | `"dendrite"` | | @@ -186,3 +190,5 @@ grafana: ``` PS: The label `release=kube-prometheus-stack` is setup with the helmchart of the Prometheus Operator. For Grafana Dashboards it may be necessary to enable scanning in the correct namespaces (or ALL), enabled by `sidecar.dashboards.searchNamespace` in [Helmchart of grafana](https://artifacthub.io/packages/helm/grafana/grafana) (which is part of PrometheusOperator, so `grafana.sidecar.dashboards.searchNamespace`) +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) \ No newline at end of file diff --git a/helm/dendrite/grafana_dashboards/dendrite-rev2.json b/helm/dendrite/grafana_dashboards/dendrite-rev2.json index 817f950b3..420d8bf1b 100644 --- a/helm/dendrite/grafana_dashboards/dendrite-rev2.json +++ b/helm/dendrite/grafana_dashboards/dendrite-rev2.json @@ -119,7 +119,7 @@ "refId": "A" } ], - "title": "Registerd Users", + "title": "Registered Users", "type": "stat" }, { diff --git a/helm/dendrite/templates/deployment.yaml b/helm/dendrite/templates/deployment.yaml index e3f84cdae..3952f4a7c 100644 --- a/helm/dendrite/templates/deployment.yaml +++ b/helm/dendrite/templates/deployment.yaml @@ -56,6 +56,9 @@ spec: args: - '--config' - '/etc/dendrite/dendrite.yaml' + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 10 }} + {{- end }} ports: - name: http containerPort: 8008 @@ -110,3 +113,19 @@ spec: httpGet: path: /_dendrite/monitor/up port: http + imagePullSecrets: + {{- with .Values.imagePullSecrets }} + {{ . | toYaml | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/helm/dendrite/templates/ingress.yaml b/helm/dendrite/templates/ingress.yaml index 4bcaee12d..eee762511 100644 --- a/helm/dendrite/templates/ingress.yaml +++ b/helm/dendrite/templates/ingress.yaml @@ -4,6 +4,7 @@ {{- $wellKnownServerHost := default $serverNameHost (regexFind "^(\\[.+\\])?[^:]*" .Values.dendrite_config.global.well_known_server_name) -}} {{- $wellKnownClientHost := default $serverNameHost (regexFind "//(\\[.+\\])?[^:/]*" .Values.dendrite_config.global.well_known_client_name | trimAll "/") -}} {{- $allHosts := list $serverNameHost $wellKnownServerHost $wellKnownClientHost | uniq -}} + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1 {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} @@ -56,7 +57,7 @@ spec: service: name: {{ $fullName }} port: - name: http + number: {{ $.Values.service.port }} {{- else }} serviceName: {{ $fullName }} servicePort: http @@ -72,7 +73,7 @@ spec: service: name: {{ $fullName }} port: - name: http + number: {{ $.Values.service.port }} {{- else }} serviceName: {{ $fullName }} servicePort: http @@ -88,7 +89,7 @@ spec: service: name: {{ $fullName }} port: - name: http + number: {{ $.Values.service.port }} {{- else }} serviceName: {{ $fullName }} servicePort: http @@ -105,7 +106,7 @@ spec: service: name: {{ $fullName }} port: - name: http + number: {{ $.Values.service.port }} {{- else }} serviceName: {{ $fullName }} servicePort: http diff --git a/helm/dendrite/templates/jobs.yaml b/helm/dendrite/templates/jobs.yaml index c10f358b0..7f96f2695 100644 --- a/helm/dendrite/templates/jobs.yaml +++ b/helm/dendrite/templates/jobs.yaml @@ -54,6 +54,10 @@ metadata: spec: template: spec: + imagePullSecrets: + {{- with .Values.imagePullSecrets }} + {{ . | toYaml | nindent 6 }} + {{- end }} restartPolicy: "Never" serviceAccount: {{ $name }} containers: @@ -94,6 +98,18 @@ spec: volumes: - name: signing-key emptyDir: {} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} parallelism: 1 completions: 1 backoffLimit: 1 diff --git a/helm/dendrite/templates/service.yaml b/helm/dendrite/templates/service.yaml index 3b571df1f..1b709c79c 100644 --- a/helm/dendrite/templates/service.yaml +++ b/helm/dendrite/templates/service.yaml @@ -14,4 +14,4 @@ spec: - name: http protocol: TCP port: {{ .Values.service.port }} - targetPort: 8008 \ No newline at end of file + targetPort: http \ No newline at end of file diff --git a/helm/dendrite/values.yaml b/helm/dendrite/values.yaml index afce1d930..02cd1aa13 100644 --- a/helm/dendrite/values.yaml +++ b/helm/dendrite/values.yaml @@ -6,6 +6,10 @@ image: # -- Overrides the image tag whose default is the chart appVersion. tag: "" +# -- Configure image pull secrets to use private container registry +# https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret +imagePullSecrets: [] +# - name: your-pull-secret-name # signing key to use signing_key: @@ -26,13 +30,13 @@ persistence: # -- The storage class to use for volume claims. # Used unless specified at the specific component. # Defaults to the cluster default storage class. - ## If defined, storageClassName: - ## If set to "-", storageClassName: "", which disables dynamic provisioning - ## If undefined (the default) or set to null, no storageClassName spec is - ## set, choosing the default provisioner. (gp2 on AWS, standard on - ## GKE, AWS & OpenStack) - ## - # storageClass: "" + # If defined, storageClassName: + # If set to "-", storageClassName: "", which disables dynamic provisioning + # If undefined (the default) or set to null, no storageClassName spec is + # set, choosing the default provisioner. (gp2 on AWS, standard on + # GKE, AWS & OpenStack) + # + storageClass: jetstream: # -- Use an existing volume claim for jetstream existingClaim: "" @@ -40,13 +44,12 @@ persistence: capacity: "1Gi" # -- The storage class to use for volume claims. # Defaults to persistence.storageClass - ## If defined, storageClassName: - ## If set to "-", storageClassName: "", which disables dynamic provisioning - ## If undefined (the default) or set to null, no storageClassName spec is - ## set, choosing the default provisioner. (gp2 on AWS, standard on - ## GKE, AWS & OpenStack) - ## - # storageClass: "" + # If defined, storageClassName: + # If set to "-", storageClassName: "", which disables dynamic provisioning + # If undefined (the default) or set to null, no storageClassName spec is + # set, choosing the default provisioner. (gp2 on AWS, standard on + # GKE, AWS & OpenStack) + storageClass: media: # -- Use an existing volume claim for media files existingClaim: "" @@ -54,13 +57,12 @@ persistence: capacity: "1Gi" # -- The storage class to use for volume claims. # Defaults to persistence.storageClass - ## If defined, storageClassName: - ## If set to "-", storageClassName: "", which disables dynamic provisioning - ## If undefined (the default) or set to null, no storageClassName spec is - ## set, choosing the default provisioner. (gp2 on AWS, standard on - ## GKE, AWS & OpenStack) - ## - # storageClass: "" + # If defined, storageClassName: + # If set to "-", storageClassName: "", which disables dynamic provisioning + # If undefined (the default) or set to null, no storageClassName spec is + # set, choosing the default provisioner. (gp2 on AWS, standard on + # GKE, AWS & OpenStack) + storageClass: search: # -- Use an existing volume claim for the fulltext search index existingClaim: "" @@ -68,13 +70,15 @@ persistence: capacity: "1Gi" # -- The storage class to use for volume claims. # Defaults to persistence.storageClass - ## If defined, storageClassName: - ## If set to "-", storageClassName: "", which disables dynamic provisioning - ## If undefined (the default) or set to null, no storageClassName spec is - ## set, choosing the default provisioner. (gp2 on AWS, standard on - ## GKE, AWS & OpenStack) - ## - # storageClass: "" + # If defined, storageClassName: + # If set to "-", storageClassName: "", which disables dynamic provisioning + # If undefined (the default) or set to null, no storageClassName spec is + # set, choosing the default provisioner. (gp2 on AWS, standard on + # GKE, AWS & OpenStack) + storageClass: + +# -- Add additional arguments to the dendrite command +extraArgs: [] # -- Add additional volumes to the Dendrite Pod extraVolumes: [] @@ -92,13 +96,22 @@ extraVolumeMounts: [] strategy: # -- Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) # If you are using ReadWriteOnce volumes, you should probably use Recreate - type: RollingUpdate + type: Recreate rollingUpdate: # -- Maximum number of pods that can be unavailable during the update process maxUnavailable: 25% # -- Maximum number of pods that can be scheduled above the desired number of pods maxSurge: 25% +# -- Node selector configuration +nodeSelector: {} + +# -- Tolerations configuration +tolerations: {} + +# -- Affinity configuration +affinity: {} + dendrite_config: version: 2 global: @@ -378,7 +391,7 @@ postgresql: enabled: false image: repository: bitnami/postgresql - tag: "15.1.0" + tag: "16.2.0" auth: username: dendrite password: changeme diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go index 1966e7546..034f19f1f 100644 --- a/internal/httputil/httpapi.go +++ b/internal/httputil/httpapi.go @@ -15,6 +15,7 @@ package httputil import ( + "encoding/json" "fmt" "io" "net/http" @@ -44,6 +45,7 @@ type BasicAuth struct { type AuthAPIOpts struct { GuestAccessAllowed bool + WithAuth bool } // AuthAPIOption is an option to MakeAuthAPI to add additional checks (e.g. guest access) to verify @@ -57,6 +59,13 @@ func WithAllowGuests() AuthAPIOption { } } +// WithAuth is an option to MakeHTTPAPI to add authentication. +func WithAuth() AuthAPIOption { + return func(opts *AuthAPIOpts) { + opts.WithAuth = true + } +} + // MakeAuthAPI turns a util.JSONRequestHandler function into an http.Handler which authenticates the request. func MakeAuthAPI( metricsName string, userAPI userapi.QueryAcccessTokenAPI, @@ -76,6 +85,8 @@ func MakeAuthAPI( // add the user to Sentry, if enabled hub := sentry.GetHubFromContext(req.Context()) if hub != nil { + // clone the hub, so we don't send garbage events with e.g. mismatching rooms/event_ids + hub = hub.Clone() hub.Scope().SetUser(sentry.User{ Username: device.UserID, }) @@ -195,13 +206,38 @@ func MakeExternalAPI(metricsName string, f func(*http.Request) util.JSONResponse return http.HandlerFunc(withSpan) } -// MakeHTMLAPI adds Span metrics to the HTML Handler function +// MakeHTTPAPI adds Span metrics to the HTML Handler function // This is used to serve HTML alongside JSON error messages -func MakeHTMLAPI(metricsName string, enableMetrics bool, f func(http.ResponseWriter, *http.Request)) http.Handler { +func MakeHTTPAPI(metricsName string, userAPI userapi.QueryAcccessTokenAPI, enableMetrics bool, f func(http.ResponseWriter, *http.Request), checks ...AuthAPIOption) http.Handler { withSpan := func(w http.ResponseWriter, req *http.Request) { + if req.Method == http.MethodOptions { + util.SetCORSHeaders(w) + w.WriteHeader(http.StatusOK) // Maybe http.StatusNoContent? + return + } + trace, ctx := internal.StartTask(req.Context(), metricsName) defer trace.EndTask() req = req.WithContext(ctx) + + // apply additional checks, if any + opts := AuthAPIOpts{} + for _, opt := range checks { + opt(&opts) + } + + if opts.WithAuth { + logger := util.GetLogger(req.Context()) + _, jsonErr := auth.VerifyUserFromRequest(req, userAPI) + if jsonErr != nil { + w.WriteHeader(jsonErr.Code) + if err := json.NewEncoder(w).Encode(jsonErr.JSON); err != nil { + logger.WithError(err).Error("failed to encode JSON response") + } + return + } + } + f(w, req) } diff --git a/internal/httputil/routing.go b/internal/httputil/routing.go index 2052c798f..f5f1c6528 100644 --- a/internal/httputil/routing.go +++ b/internal/httputil/routing.go @@ -66,15 +66,15 @@ func NewRouters() Routers { } var NotAllowedHandler = WrapHandlerInCORS(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusMethodNotAllowed) w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusMethodNotAllowed) unrecognizedErr, _ := json.Marshal(spec.Unrecognized("Unrecognized request")) // nolint:misspell _, _ = w.Write(unrecognizedErr) // nolint:misspell })) var NotFoundCORSHandler = WrapHandlerInCORS(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) unrecognizedErr, _ := json.Marshal(spec.Unrecognized("Unrecognized request")) // nolint:misspell _, _ = w.Write(unrecognizedErr) // nolint:misspell })) diff --git a/internal/httputil/routing_test.go b/internal/httputil/routing_test.go index 21e2bf48a..39ccd6213 100644 --- a/internal/httputil/routing_test.go +++ b/internal/httputil/routing_test.go @@ -17,7 +17,7 @@ func TestRoutersError(t *testing.T) { if rec.Code != http.StatusNotFound { t.Fatalf("unexpected status code: %d - %s", rec.Code, rec.Body.String()) } - if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + if ct := rec.Result().Header.Get("Content-Type"); ct != "application/json" { t.Fatalf("unexpected content-type: %s", ct) } @@ -32,7 +32,7 @@ func TestRoutersError(t *testing.T) { if rec.Code != http.StatusMethodNotAllowed { t.Fatalf("unexpected status code: %d - %s", rec.Code, rec.Body.String()) } - if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + if ct := rec.Result().Header.Get("Content-Type"); ct != "application/json" { t.Fatalf("unexpected content-type: %s", ct) } } diff --git a/internal/sqlutil/sqlutil_test.go b/internal/sqlutil/sqlutil_test.go index c40757893..93b84aa20 100644 --- a/internal/sqlutil/sqlutil_test.go +++ b/internal/sqlutil/sqlutil_test.go @@ -218,5 +218,5 @@ func assertNoError(t *testing.T, err error, msg string) { if err == nil { return } - t.Fatalf(msg) + t.Fatal(msg) } diff --git a/internal/sqlutil/writer_exclusive.go b/internal/sqlutil/writer_exclusive.go index c6a271c1c..69eb8609c 100644 --- a/internal/sqlutil/writer_exclusive.go +++ b/internal/sqlutil/writer_exclusive.go @@ -3,8 +3,7 @@ package sqlutil import ( "database/sql" "errors" - - "go.uber.org/atomic" + "sync/atomic" ) // ExclusiveWriter implements sqlutil.Writer. diff --git a/internal/transactionrequest_test.go b/internal/transactionrequest_test.go index ffc1cd89a..8dd100d11 100644 --- a/internal/transactionrequest_test.go +++ b/internal/transactionrequest_test.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" "strconv" + "sync/atomic" "testing" "time" @@ -26,7 +27,6 @@ import ( "github.com/matrix-org/gomatrixserverlib/spec" "github.com/nats-io/nats.go" "github.com/stretchr/testify/assert" - "go.uber.org/atomic" "gotest.tools/v3/poll" "github.com/matrix-org/dendrite/federationapi/producers" @@ -228,7 +228,7 @@ func TestProcessTransactionRequestEDUTyping(t *testing.T) { ctx := process.NewProcessContext() defer ctx.ShutdownDendrite() txn, js, cfg := createTransactionWithEDU(ctx, edus) - received := atomic.NewBool(false) + received := atomic.Bool{} onMessage := func(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called room := msg.Header.Get(jetstream.RoomID) @@ -294,7 +294,7 @@ func TestProcessTransactionRequestEDUToDevice(t *testing.T) { ctx := process.NewProcessContext() defer ctx.ShutdownDendrite() txn, js, cfg := createTransactionWithEDU(ctx, edus) - received := atomic.NewBool(false) + received := atomic.Bool{} onMessage := func(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called @@ -371,7 +371,7 @@ func TestProcessTransactionRequestEDUDeviceListUpdate(t *testing.T) { ctx := process.NewProcessContext() defer ctx.ShutdownDendrite() txn, js, cfg := createTransactionWithEDU(ctx, edus) - received := atomic.NewBool(false) + received := atomic.Bool{} onMessage := func(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called @@ -468,7 +468,7 @@ func TestProcessTransactionRequestEDUReceipt(t *testing.T) { ctx := process.NewProcessContext() defer ctx.ShutdownDendrite() txn, js, cfg := createTransactionWithEDU(ctx, edus) - received := atomic.NewBool(false) + received := atomic.Bool{} onMessage := func(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called @@ -512,7 +512,7 @@ func TestProcessTransactionRequestEDUSigningKeyUpdate(t *testing.T) { ctx := process.NewProcessContext() defer ctx.ShutdownDendrite() txn, js, cfg := createTransactionWithEDU(ctx, edus) - received := atomic.NewBool(false) + received := atomic.Bool{} onMessage := func(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called @@ -569,7 +569,7 @@ func TestProcessTransactionRequestEDUPresence(t *testing.T) { ctx := process.NewProcessContext() defer ctx.ShutdownDendrite() txn, js, cfg := createTransactionWithEDU(ctx, edus) - received := atomic.NewBool(false) + received := atomic.Bool{} onMessage := func(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called diff --git a/internal/validate.go b/internal/validate.go index 7f0d8b9e6..c831565f5 100644 --- a/internal/validate.go +++ b/internal/validate.go @@ -20,6 +20,9 @@ import ( "net/http" "regexp" + "github.com/matrix-org/dendrite/clientapi/userutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" ) @@ -35,9 +38,9 @@ var ( ErrPasswordTooLong = fmt.Errorf("password too long: max %d characters", maxPasswordLength) ErrPasswordWeak = fmt.Errorf("password too weak: min %d characters", minPasswordLength) ErrUsernameTooLong = fmt.Errorf("username exceeds the maximum length of %d characters", maxUsernameLength) - ErrUsernameInvalid = errors.New("username can only contain characters a-z, 0-9, or '_-./='") + ErrUsernameInvalid = errors.New("username can only contain characters a-z, 0-9, or '_+-./='") ErrUsernameUnderscore = errors.New("username cannot start with a '_'") - validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`) + validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-+=./]+$`) ) // ValidatePassword returns an error if the password is invalid @@ -100,10 +103,139 @@ func UsernameResponse(err error) *util.JSONResponse { // ValidateApplicationServiceUsername returns an error if the username is invalid for an application service func ValidateApplicationServiceUsername(localpart string, domain spec.ServerName) error { - if id := fmt.Sprintf("@%s:%s", localpart, domain); len(id) > maxUsernameLength { + userID := userutil.MakeUserID(localpart, domain) + return ValidateApplicationServiceUserID(userID) +} + +func ValidateApplicationServiceUserID(userID string) error { + if len(userID) > maxUsernameLength { return ErrUsernameTooLong - } else if !validUsernameRegex.MatchString(localpart) { + } + + localpart, _, err := gomatrixserverlib.SplitID('@', userID) + if err != nil || !validUsernameRegex.MatchString(localpart) { return ErrUsernameInvalid } + return nil } + +// userIDIsWithinApplicationServiceNamespace checks to see if a given userID +// falls within any of the namespaces of a given Application Service. If no +// Application Service is given, it will check to see if it matches any +// Application Service's namespace. +func userIDIsWithinApplicationServiceNamespace( + cfg *config.ClientAPI, + userID string, + appservice *config.ApplicationService, +) bool { + var localpart, domain, err = gomatrixserverlib.SplitID('@', userID) + if err != nil { + // Not a valid userID + return false + } + + if !cfg.Matrix.IsLocalServerName(domain) { + // This is a federated userID + return false + } + + if localpart == appservice.SenderLocalpart { + // This is the application service bot userID + return true + } + + // Loop through given application service's namespaces and see if any match + for _, namespace := range appservice.NamespaceMap["users"] { + // Application service namespaces are checked for validity in config + if namespace.RegexpObject.MatchString(userID) { + return true + } + } + + return false +} + +// usernameMatchesMultipleExclusiveNamespaces will check if a given username matches +// more than one exclusive namespace. More than one is not allowed +func userIDMatchesMultipleExclusiveNamespaces( + cfg *config.ClientAPI, + userID string, +) bool { + // Check namespaces and see if more than one match + matchCount := 0 + for _, appservice := range cfg.Derived.ApplicationServices { + if appservice.OwnsNamespaceCoveringUserId(userID) { + if matchCount++; matchCount > 1 { + return true + } + } + } + return false +} + +// ValidateApplicationServiceRequest checks if a provided application service +// token corresponds to one that is registered, and, if so, checks if the +// supplied userIDOrLocalpart is within that application service's namespace. +// +// As long as these two requirements are met, the matched application service +// ID will be returned. Otherwise, it will return a JSON response with the +// appropriate error message. +func ValidateApplicationServiceRequest( + cfg *config.ClientAPI, + userIDOrLocalpart string, + accessToken string, +) (string, *util.JSONResponse) { + localpart, domain, err := userutil.ParseUsernameParam(userIDOrLocalpart, cfg.Matrix) + if err != nil { + return "", &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: spec.InvalidUsername(err.Error()), + } + } + + userID := userutil.MakeUserID(localpart, domain) + + // Check if the token if the application service is valid with one we have + // registered in the config. + var matchedApplicationService *config.ApplicationService + for _, appservice := range cfg.Derived.ApplicationServices { + if appservice.ASToken == accessToken { + matchedApplicationService = &appservice + break + } + } + if matchedApplicationService == nil { + return "", &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: spec.UnknownToken("Supplied access_token does not match any known application service"), + } + } + + // Ensure the desired username is within at least one of the application service's namespaces. + if !userIDIsWithinApplicationServiceNamespace(cfg, userID, matchedApplicationService) { + // If we didn't find any matches, return M_EXCLUSIVE + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.ASExclusive(fmt.Sprintf( + "Supplied username %s did not match any namespaces for application service ID: %s", userIDOrLocalpart, matchedApplicationService.ID)), + } + } + + // Check this user does not fit multiple application service namespaces + if userIDMatchesMultipleExclusiveNamespaces(cfg, userID) { + return "", &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: spec.ASExclusive(fmt.Sprintf( + "Supplied username %s matches multiple exclusive application service namespaces. Only 1 match allowed", userIDOrLocalpart)), + } + } + + // Check username application service is trying to register is valid + if err := ValidateApplicationServiceUserID(userID); err != nil { + return "", UsernameResponse(err) + } + + // No errors, registration valid + return matchedApplicationService.ID, nil +} diff --git a/internal/validate_test.go b/internal/validate_test.go index e3a10178f..1019102df 100644 --- a/internal/validate_test.go +++ b/internal/validate_test.go @@ -3,9 +3,11 @@ package internal import ( "net/http" "reflect" + "regexp" "strings" "testing" + "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" ) @@ -38,7 +40,7 @@ func Test_validatePassword(t *testing.T) { t.Run(tt.name, func(t *testing.T) { gotErr := ValidatePassword(tt.password) if !reflect.DeepEqual(gotErr, tt.wantError) { - t.Errorf("validatePassword() = %v, wantJSON %v", gotErr, tt.wantError) + t.Errorf("validatePassword() = %v, wantError %v", gotErr, tt.wantError) } if got := PasswordResponse(gotErr); !reflect.DeepEqual(got, tt.wantJSON) { @@ -127,6 +129,11 @@ func Test_validateUsername(t *testing.T) { localpart: "i_am_allowed=1", domain: "localhost", }, + { + name: "special characters are allowed 3", + localpart: "+55555555555", + domain: "localhost", + }, { name: "not all special characters are allowed", localpart: "notallowed#", // contains # @@ -137,6 +144,16 @@ func Test_validateUsername(t *testing.T) { JSON: spec.InvalidUsername(ErrUsernameInvalid.Error()), }, }, + { + name: "not all special characters are allowed 2", + localpart: " 0 { - reader = io.NopCloser(io.LimitReader(*body, int64(maxFileSizeBytes))) + reader = io.NopCloser(io.LimitReader(reader, int64(maxFileSizeBytes))) } contentLength = 0 } @@ -724,6 +813,11 @@ func (r *downloadRequest) GetContentLengthAndReader(contentLengthHeader string, return contentLength, reader, nil } +// mediaMeta contains information about a multipart media response. +// TODO: extend once something is defined. +type mediaMeta struct{} + +// nolint: gocyclo func (r *downloadRequest) fetchRemoteFile( ctx context.Context, client *fclient.Client, @@ -732,19 +826,38 @@ func (r *downloadRequest) fetchRemoteFile( ) (types.Path, bool, error) { r.Logger.Debug("Fetching remote file") - // create request for remote file - resp, err := client.CreateMediaDownloadRequest(ctx, r.MediaMetadata.Origin, string(r.MediaMetadata.MediaID)) + // Attempt to download via authenticated media endpoint + isAuthed := true + resp, err := r.fedClient.DownloadMedia(ctx, r.origin, r.MediaMetadata.Origin, string(r.MediaMetadata.MediaID)) if err != nil || (resp != nil && resp.StatusCode != http.StatusOK) { - if resp != nil && resp.StatusCode == http.StatusNotFound { - return "", false, fmt.Errorf("File with media ID %q does not exist on %s", r.MediaMetadata.MediaID, r.MediaMetadata.Origin) + isAuthed = false + // try again on the unauthed endpoint + // create request for remote file + resp, err = client.CreateMediaDownloadRequest(ctx, r.MediaMetadata.Origin, string(r.MediaMetadata.MediaID)) + if err != nil || (resp != nil && resp.StatusCode != http.StatusOK) { + if resp != nil && resp.StatusCode == http.StatusNotFound { + return "", false, fmt.Errorf("File with media ID %q does not exist on %s", r.MediaMetadata.MediaID, r.MediaMetadata.Origin) + } + return "", false, fmt.Errorf("file with media ID %q could not be downloaded from %s: %w", r.MediaMetadata.MediaID, r.MediaMetadata.Origin, err) } - return "", false, fmt.Errorf("file with media ID %q could not be downloaded from %s", r.MediaMetadata.MediaID, r.MediaMetadata.Origin) } defer resp.Body.Close() // nolint: errcheck - // The reader returned here will be limited either by the Content-Length - // and/or the configured maximum media size. - contentLength, reader, parseErr := r.GetContentLengthAndReader(resp.Header.Get("Content-Length"), &resp.Body, maxFileSizeBytes) + // If this wasn't a multipart response, set the Content-Type now. Will be overwritten + // by the multipart Content-Type below. + r.MediaMetadata.ContentType = types.ContentType(resp.Header.Get("Content-Type")) + + var contentLength int64 + var reader io.Reader + var parseErr error + if isAuthed { + contentLength, reader, parseErr = parseMultipartResponse(r, resp, maxFileSizeBytes) + } else { + // The reader returned here will be limited either by the Content-Length + // and/or the configured maximum media size. + contentLength, reader, parseErr = r.GetContentLengthAndReader(resp.Header.Get("Content-Length"), resp.Body, maxFileSizeBytes) + } + if parseErr != nil { return "", false, parseErr } @@ -755,7 +868,6 @@ func (r *downloadRequest) fetchRemoteFile( } r.MediaMetadata.FileSizeBytes = types.FileSizeBytes(contentLength) - r.MediaMetadata.ContentType = types.ContentType(resp.Header.Get("Content-Type")) dispositionHeader := resp.Header.Get("Content-Disposition") if _, params, e := mime.ParseMediaType(dispositionHeader); e == nil { @@ -808,3 +920,56 @@ func (r *downloadRequest) fetchRemoteFile( return types.Path(finalPath), duplicate, nil } + +func parseMultipartResponse(r *downloadRequest, resp *http.Response, maxFileSizeBytes config.FileSizeBytes) (int64, io.Reader, error) { + _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if err != nil { + return 0, nil, err + } + if params["boundary"] == "" { + return 0, nil, fmt.Errorf("no boundary header found on media %s from %s", r.MediaMetadata.MediaID, r.MediaMetadata.Origin) + } + mr := multipart.NewReader(resp.Body, params["boundary"]) + + // Get the first, JSON, part + p, err := mr.NextPart() + if err != nil { + return 0, nil, err + } + defer p.Close() // nolint: errcheck + + if p.Header.Get("Content-Type") != "application/json" { + return 0, nil, fmt.Errorf("first part of the response must be application/json") + } + // Try to parse media meta information + meta := mediaMeta{} + if err = json.NewDecoder(p).Decode(&meta); err != nil { + return 0, nil, err + } + defer p.Close() // nolint: errcheck + + // Get the actual media content + p, err = mr.NextPart() + if err != nil { + return 0, nil, err + } + + redirect := p.Header.Get("Location") + if redirect != "" { + return 0, nil, fmt.Errorf("Location header is not yet supported") + } + + contentLength, reader, err := r.GetContentLengthAndReader(p.Header.Get("Content-Length"), p, maxFileSizeBytes) + // For multipart requests, we need to get the Content-Type of the second part, which is the actual media + r.MediaMetadata.ContentType = types.ContentType(p.Header.Get("Content-Type")) + return contentLength, reader, err +} + +// contentDispositionFor returns the Content-Disposition for a given +// content type. +func contentDispositionFor(contentType types.ContentType) string { + if _, ok := allowInlineTypes[contentType]; ok { + return "inline" + } + return "attachment" +} diff --git a/mediaapi/routing/download_test.go b/mediaapi/routing/download_test.go new file mode 100644 index 000000000..9654b7474 --- /dev/null +++ b/mediaapi/routing/download_test.go @@ -0,0 +1,43 @@ +package routing + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/matrix-org/dendrite/mediaapi/types" + "github.com/stretchr/testify/assert" +) + +func Test_dispositionFor(t *testing.T) { + assert.Equal(t, "attachment", contentDispositionFor(""), "empty content type") + assert.Equal(t, "attachment", contentDispositionFor("image/svg"), "image/svg") + assert.Equal(t, "inline", contentDispositionFor("image/jpeg"), "image/jpg") +} + +func Test_Multipart(t *testing.T) { + r := &downloadRequest{ + MediaMetadata: &types.MediaMetadata{}, + } + data := bytes.Buffer{} + responseBody := "This media is plain text. Maybe somebody used it as a paste bin." + data.WriteString(responseBody) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, err := multipartResponse(w, r, "text/plain", &data) + assert.NoError(t, err) + })) + defer srv.Close() + + resp, err := srv.Client().Get(srv.URL) + assert.NoError(t, err) + defer resp.Body.Close() + // contentLength is always 0, since there's no Content-Length header on the multipart part. + _, reader, err := parseMultipartResponse(r, resp, 1000) + assert.NoError(t, err) + gotResponse, err := io.ReadAll(reader) + assert.NoError(t, err) + assert.Equal(t, responseBody, string(gotResponse)) +} diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index e0af4a911..2867df605 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -20,11 +20,13 @@ import ( "strings" "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/matrix-org/util" @@ -45,15 +47,19 @@ type configResponse struct { // applied: // nolint: gocyclo func Setup( - publicAPIMux *mux.Router, + routers httputil.Routers, cfg *config.Dendrite, db storage.Database, userAPI userapi.MediaUserAPI, client *fclient.Client, + federationClient fclient.FederationClient, + keyRing gomatrixserverlib.JSONVerifier, ) { rateLimits := httputil.NewRateLimits(&cfg.ClientAPI.RateLimiting) - v3mux := publicAPIMux.PathPrefix("/{apiversion:(?:r0|v1|v3)}/").Subrouter() + v3mux := routers.Media.PathPrefix("/{apiversion:(?:r0|v1|v3)}/").Subrouter() + v1mux := routers.Client.PathPrefix("/v1/media/").Subrouter() + v1fedMux := routers.Federation.PathPrefix("/v1/media/").Subrouter() activeThumbnailGeneration := &types.ActiveThumbnailGeneration{ PathToResult: map[string]*types.ThumbnailGenerationResult{}, @@ -90,39 +96,110 @@ func Setup( MXCToResult: map[string]*types.RemoteRequestResult{}, } - downloadHandler := makeDownloadAPI("download", &cfg.MediaAPI, rateLimits, db, client, activeRemoteRequests, activeThumbnailGeneration) + downloadHandler := makeDownloadAPI("download_unauthed", &cfg.MediaAPI, rateLimits, db, client, federationClient, activeRemoteRequests, activeThumbnailGeneration, false) v3mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) v3mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) v3mux.Handle("/thumbnail/{serverName}/{mediaId}", - makeDownloadAPI("thumbnail", &cfg.MediaAPI, rateLimits, db, client, activeRemoteRequests, activeThumbnailGeneration), + makeDownloadAPI("thumbnail_unauthed", &cfg.MediaAPI, rateLimits, db, client, federationClient, activeRemoteRequests, activeThumbnailGeneration, false), ).Methods(http.MethodGet, http.MethodOptions) + + // v1 client endpoints requiring auth + downloadHandlerAuthed := httputil.MakeHTTPAPI("download", userAPI, cfg.Global.Metrics.Enabled, makeDownloadAPI("download_authed_client", &cfg.MediaAPI, rateLimits, db, client, federationClient, activeRemoteRequests, activeThumbnailGeneration, false), httputil.WithAuth()) + v1mux.Handle("/config", configHandler).Methods(http.MethodGet, http.MethodOptions) + v1mux.Handle("/download/{serverName}/{mediaId}", downloadHandlerAuthed).Methods(http.MethodGet, http.MethodOptions) + v1mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandlerAuthed).Methods(http.MethodGet, http.MethodOptions) + + v1mux.Handle("/thumbnail/{serverName}/{mediaId}", + httputil.MakeHTTPAPI("thumbnail", userAPI, cfg.Global.Metrics.Enabled, makeDownloadAPI("thumbnail_authed_client", &cfg.MediaAPI, rateLimits, db, client, federationClient, activeRemoteRequests, activeThumbnailGeneration, false), httputil.WithAuth()), + ).Methods(http.MethodGet, http.MethodOptions) + + // same, but for federation + v1fedMux.Handle("/download/{mediaId}", routing.MakeFedHTTPAPI(cfg.Global.ServerName, cfg.Global.IsLocalServerName, keyRing, + makeDownloadAPI("download_authed_federation", &cfg.MediaAPI, rateLimits, db, client, federationClient, activeRemoteRequests, activeThumbnailGeneration, true), + )).Methods(http.MethodGet, http.MethodOptions) + v1fedMux.Handle("/thumbnail/{mediaId}", routing.MakeFedHTTPAPI(cfg.Global.ServerName, cfg.Global.IsLocalServerName, keyRing, + makeDownloadAPI("thumbnail_authed_federation", &cfg.MediaAPI, rateLimits, db, client, federationClient, activeRemoteRequests, activeThumbnailGeneration, true), + )).Methods(http.MethodGet, http.MethodOptions) } +var thumbnailCounter = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "dendrite", + Subsystem: "mediaapi", + Name: "thumbnail", + Help: "Total number of media_api requests for thumbnails", + }, + []string{"code", "type"}, +) + +var thumbnailSize = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "dendrite", + Subsystem: "mediaapi", + Name: "thumbnail_size_bytes", + Help: "Total size of media_api requests for thumbnails", + Buckets: []float64{50, 100, 200, 500, 900, 1500, 3000, 6000}, + }, + []string{"code", "type"}, +) + +var downloadCounter = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "dendrite", + Subsystem: "mediaapi", + Name: "download", + Help: "Total size of media_api requests for full downloads", + }, + []string{"code", "type"}, +) + +var downloadSize = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "dendrite", + Subsystem: "mediaapi", + Name: "download_size_bytes", + Help: "Total size of media_api requests for full downloads", + Buckets: []float64{1500, 3000, 6000, 10_000, 50_000, 100_000}, + }, + []string{"code", "type"}, +) + func makeDownloadAPI( name string, cfg *config.MediaAPI, rateLimits *httputil.RateLimits, db storage.Database, client *fclient.Client, + fedClient fclient.FederationClient, activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, + forFederation bool, ) http.HandlerFunc { var counterVec *prometheus.CounterVec + var sizeVec *prometheus.HistogramVec + var requestType string if cfg.Matrix.Metrics.Enabled { - counterVec = promauto.NewCounterVec( - prometheus.CounterOpts{ - Name: name, - Help: "Total number of media_api requests for either thumbnails or full downloads", - }, - []string{"code"}, - ) + split := strings.Split(name, "_") + // The first part of the split is either "download" or "thumbnail" + name = split[0] + // The remainder of the split is something like "authed_download" or "unauthed_thumbnail", etc. + // This is used to curry the metrics with the given types. + requestType = strings.Join(split[1:], "_") + + counterVec = thumbnailCounter + sizeVec = thumbnailSize + if name != "thumbnail" { + counterVec = downloadCounter + sizeVec = downloadSize + } } httpHandler := func(w http.ResponseWriter, req *http.Request) { req = util.RequestWithLogging(req) // Set internal headers returned regardless of the outcome of the request util.SetCORSHeaders(w) + w.Header().Set("Cross-Origin-Resource-Policy", "cross-origin") // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors w.Header().Set("Content-Type", "application/json") @@ -163,16 +240,21 @@ func makeDownloadAPI( cfg, db, client, + fedClient, activeRemoteRequests, activeThumbnailGeneration, - name == "thumbnail", + strings.HasPrefix(name, "thumbnail"), vars["downloadName"], + forFederation, ) } var handlerFunc http.HandlerFunc if counterVec != nil { + counterVec = counterVec.MustCurryWith(prometheus.Labels{"type": requestType}) + sizeVec2 := sizeVec.MustCurryWith(prometheus.Labels{"type": requestType}) handlerFunc = promhttp.InstrumentHandlerCounter(counterVec, http.HandlerFunc(httpHandler)) + handlerFunc = promhttp.InstrumentHandlerResponseSize(sizeVec2, handlerFunc).ServeHTTP } else { handlerFunc = http.HandlerFunc(httpHandler) } diff --git a/mediaapi/storage/shared/mediaapi.go b/mediaapi/storage/shared/mediaapi.go index 867405fb3..bdd7f317b 100644 --- a/mediaapi/storage/shared/mediaapi.go +++ b/mediaapi/storage/shared/mediaapi.go @@ -17,6 +17,7 @@ package shared import ( "context" "database/sql" + "errors" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/storage/tables" @@ -33,7 +34,7 @@ type Database struct { // StoreMediaMetadata inserts the metadata about the uploaded media into the database. // Returns an error if the combination of MediaID and Origin are not unique in the table. -func (d Database) StoreMediaMetadata(ctx context.Context, mediaMetadata *types.MediaMetadata) error { +func (d *Database) StoreMediaMetadata(ctx context.Context, mediaMetadata *types.MediaMetadata) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { return d.MediaRepository.InsertMedia(ctx, txn, mediaMetadata) }) @@ -42,9 +43,9 @@ func (d Database) StoreMediaMetadata(ctx context.Context, mediaMetadata *types.M // GetMediaMetadata returns metadata about media stored on this server. // The media could have been uploaded to this server or fetched from another server and cached here. // Returns nil metadata if there is no metadata associated with this media. -func (d Database) GetMediaMetadata(ctx context.Context, mediaID types.MediaID, mediaOrigin spec.ServerName) (*types.MediaMetadata, error) { +func (d *Database) GetMediaMetadata(ctx context.Context, mediaID types.MediaID, mediaOrigin spec.ServerName) (*types.MediaMetadata, error) { mediaMetadata, err := d.MediaRepository.SelectMedia(ctx, nil, mediaID, mediaOrigin) - if err != nil && err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return nil, nil } return mediaMetadata, err @@ -53,9 +54,9 @@ func (d Database) GetMediaMetadata(ctx context.Context, mediaID types.MediaID, m // GetMediaMetadataByHash returns metadata about media stored on this server. // The media could have been uploaded to this server or fetched from another server and cached here. // Returns nil metadata if there is no metadata associated with this media. -func (d Database) GetMediaMetadataByHash(ctx context.Context, mediaHash types.Base64Hash, mediaOrigin spec.ServerName) (*types.MediaMetadata, error) { +func (d *Database) GetMediaMetadataByHash(ctx context.Context, mediaHash types.Base64Hash, mediaOrigin spec.ServerName) (*types.MediaMetadata, error) { mediaMetadata, err := d.MediaRepository.SelectMediaByHash(ctx, nil, mediaHash, mediaOrigin) - if err != nil && err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return nil, nil } return mediaMetadata, err @@ -63,7 +64,7 @@ func (d Database) GetMediaMetadataByHash(ctx context.Context, mediaHash types.Ba // StoreThumbnail inserts the metadata about the thumbnail into the database. // Returns an error if the combination of MediaID and Origin are not unique in the table. -func (d Database) StoreThumbnail(ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata) error { +func (d *Database) StoreThumbnail(ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { return d.Thumbnails.InsertThumbnail(ctx, txn, thumbnailMetadata) }) @@ -72,13 +73,10 @@ func (d Database) StoreThumbnail(ctx context.Context, thumbnailMetadata *types.T // GetThumbnail returns metadata about a specific thumbnail. // The media could have been uploaded to this server or fetched from another server and cached here. // Returns nil metadata if there is no metadata associated with this thumbnail. -func (d Database) GetThumbnail(ctx context.Context, mediaID types.MediaID, mediaOrigin spec.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error) { +func (d *Database) GetThumbnail(ctx context.Context, mediaID types.MediaID, mediaOrigin spec.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error) { metadata, err := d.Thumbnails.SelectThumbnail(ctx, nil, mediaID, mediaOrigin, width, height, resizeMethod) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, err + if errors.Is(err, sql.ErrNoRows) { + return nil, nil } return metadata, err } @@ -86,13 +84,10 @@ func (d Database) GetThumbnail(ctx context.Context, mediaID types.MediaID, media // GetThumbnails returns metadata about all thumbnails for a specific media stored on this server. // The media could have been uploaded to this server or fetched from another server and cached here. // Returns nil metadata if there are no thumbnails associated with this media. -func (d Database) GetThumbnails(ctx context.Context, mediaID types.MediaID, mediaOrigin spec.ServerName) ([]*types.ThumbnailMetadata, error) { +func (d *Database) GetThumbnails(ctx context.Context, mediaID types.MediaID, mediaOrigin spec.ServerName) ([]*types.ThumbnailMetadata, error) { metadatas, err := d.Thumbnails.SelectThumbnails(ctx, nil, mediaID, mediaOrigin) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, err + if errors.Is(err, sql.ErrNoRows) { + return nil, nil } return metadatas, err } diff --git a/relayapi/routing/routing.go b/relayapi/routing/routing.go index f11b0a7c5..92476d6c2 100644 --- a/relayapi/routing/routing.go +++ b/relayapi/routing/routing.go @@ -108,6 +108,8 @@ func MakeRelayAPI( // add the user to Sentry, if enabled hub := sentry.GetHubFromContext(req.Context()) if hub != nil { + // clone the hub, so we don't send garbage events with e.g. mismatching rooms/event_ids + hub = hub.Clone() hub.Scope().SetTag("origin", string(fedReq.Origin())) hub.Scope().SetTag("uri", fedReq.RequestURI()) } diff --git a/relayapi/storage/postgres/relay_queue_json_table.go b/relayapi/storage/postgres/relay_queue_json_table.go index 74410fc88..94ae41407 100644 --- a/relayapi/storage/postgres/relay_queue_json_table.go +++ b/relayapi/storage/postgres/relay_queue_json_table.go @@ -109,5 +109,5 @@ func (s *relayQueueJSONStatements) SelectQueueJSON( } blobs[nid] = blob } - return blobs, err + return blobs, rows.Err() } diff --git a/relayapi/storage/sqlite3/relay_queue_json_table.go b/relayapi/storage/sqlite3/relay_queue_json_table.go index 502da3b00..a1af82aa0 100644 --- a/relayapi/storage/sqlite3/relay_queue_json_table.go +++ b/relayapi/storage/sqlite3/relay_queue_json_table.go @@ -133,5 +133,5 @@ func (s *relayQueueJSONStatements) SelectQueueJSON( } blobs[nid] = blob } - return blobs, err + return blobs, rows.Err() } diff --git a/roomserver/acls/acls.go b/roomserver/acls/acls.go index 601ce9063..4950e6231 100644 --- a/roomserver/acls/acls.go +++ b/roomserver/acls/acls.go @@ -23,49 +23,57 @@ import ( "strings" "sync" - "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/sirupsen/logrus" ) +const MRoomServerACL = "m.room.server_acl" + type ServerACLDatabase interface { - // GetKnownRooms returns a list of all rooms we know about. - GetKnownRooms(ctx context.Context) ([]string, error) - // GetStateEvent returns the state event of a given type for a given room with a given state key - // If no event could be found, returns nil - // If there was an issue during the retrieval, returns an error - GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*types.HeaderedEvent, error) + // RoomsWithACLs returns all room IDs for rooms with ACLs + RoomsWithACLs(ctx context.Context) ([]string, error) + + // GetBulkStateContent returns all state events which match a given room ID and a given state key tuple. Both must be satisfied for a match. + // If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned. + GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error) } type ServerACLs struct { - acls map[string]*serverACL // room ID -> ACL - aclsMutex sync.RWMutex // protects the above + acls map[string]*serverACL // room ID -> ACL + aclsMutex sync.RWMutex // protects the above + aclRegexCache map[string]**regexp.Regexp // Cache from "serverName" -> pointer to a regex + aclRegexCacheMutex sync.RWMutex // protects the above } func NewServerACLs(db ServerACLDatabase) *ServerACLs { ctx := context.TODO() acls := &ServerACLs{ acls: make(map[string]*serverACL), + // Be generous when creating the cache, as in reality + // there are hundreds of servers in an ACL. + aclRegexCache: make(map[string]**regexp.Regexp, 100), } + // Look up all of the rooms that the current state server knows about. - rooms, err := db.GetKnownRooms(ctx) + rooms, err := db.RoomsWithACLs(ctx) if err != nil { logrus.WithError(err).Fatalf("Failed to get known rooms") } // For each room, let's see if we have a server ACL state event. If we // do then we'll process it into memory so that we have the regexes to // hand. - for _, room := range rooms { - state, err := db.GetStateEvent(ctx, room, "m.room.server_acl", "") - if err != nil { - logrus.WithError(err).Errorf("Failed to get server ACLs for room %q", room) - continue - } - if state != nil { - acls.OnServerACLUpdate(state.PDU) - } + + events, err := db.GetBulkStateContent(ctx, rooms, []gomatrixserverlib.StateKeyTuple{{EventType: MRoomServerACL, StateKey: ""}}, false) + if err != nil { + logrus.WithError(err).Errorf("Failed to get server ACLs for all rooms: %q", err) } + + for _, event := range events { + acls.OnServerACLUpdate(event) + } + return acls } @@ -77,8 +85,8 @@ type ServerACL struct { type serverACL struct { ServerACL - allowedRegexes []*regexp.Regexp - deniedRegexes []*regexp.Regexp + allowedRegexes []**regexp.Regexp + deniedRegexes []**regexp.Regexp } func compileACLRegex(orig string) (*regexp.Regexp, error) { @@ -88,9 +96,28 @@ func compileACLRegex(orig string) (*regexp.Regexp, error) { return regexp.Compile(escaped) } -func (s *ServerACLs) OnServerACLUpdate(state gomatrixserverlib.PDU) { +// cachedCompileACLRegex is a wrapper around compileACLRegex with added caching +func (s *ServerACLs) cachedCompileACLRegex(orig string) (**regexp.Regexp, error) { + s.aclRegexCacheMutex.RLock() + re, ok := s.aclRegexCache[orig] + if ok { + s.aclRegexCacheMutex.RUnlock() + return re, nil + } + s.aclRegexCacheMutex.RUnlock() + compiled, err := compileACLRegex(orig) + if err != nil { + return nil, err + } + s.aclRegexCacheMutex.Lock() + defer s.aclRegexCacheMutex.Unlock() + s.aclRegexCache[orig] = &compiled + return &compiled, nil +} + +func (s *ServerACLs) OnServerACLUpdate(strippedEvent tables.StrippedEvent) { acls := &serverACL{} - if err := json.Unmarshal(state.Content(), &acls.ServerACL); err != nil { + if err := json.Unmarshal([]byte(strippedEvent.ContentValue), &acls.ServerACL); err != nil { logrus.WithError(err).Errorf("Failed to unmarshal state content for server ACLs") return } @@ -99,14 +126,14 @@ func (s *ServerACLs) OnServerACLUpdate(state gomatrixserverlib.PDU) { // special characters and then replace * and ? with their regex counterparts. // https://matrix.org/docs/spec/client_server/r0.6.1#m-room-server-acl for _, orig := range acls.Allowed { - if expr, err := compileACLRegex(orig); err != nil { + if expr, err := s.cachedCompileACLRegex(orig); err != nil { logrus.WithError(err).Errorf("Failed to compile allowed regex") } else { acls.allowedRegexes = append(acls.allowedRegexes, expr) } } for _, orig := range acls.Denied { - if expr, err := compileACLRegex(orig); err != nil { + if expr, err := s.cachedCompileACLRegex(orig); err != nil { logrus.WithError(err).Errorf("Failed to compile denied regex") } else { acls.deniedRegexes = append(acls.deniedRegexes, expr) @@ -116,10 +143,15 @@ func (s *ServerACLs) OnServerACLUpdate(state gomatrixserverlib.PDU) { "allow_ip_literals": acls.AllowIPLiterals, "num_allowed": len(acls.allowedRegexes), "num_denied": len(acls.deniedRegexes), - }).Debugf("Updating server ACLs for %q", state.RoomID()) + }).Debugf("Updating server ACLs for %q", strippedEvent.RoomID) + + // Clear out Denied and Allowed, now that we have the compiled regexes. + // They are not needed anymore from this point on. + acls.Denied = nil + acls.Allowed = nil s.aclsMutex.Lock() defer s.aclsMutex.Unlock() - s.acls[state.RoomID().String()] = acls + s.acls[strippedEvent.RoomID] = acls } func (s *ServerACLs) IsServerBannedFromRoom(serverName spec.ServerName, roomID string) bool { @@ -149,14 +181,14 @@ func (s *ServerACLs) IsServerBannedFromRoom(serverName spec.ServerName, roomID s // Check if the hostname matches one of the denied regexes. If it does then // the server is banned from the room. for _, expr := range acls.deniedRegexes { - if expr.MatchString(string(serverName)) { + if (*expr).MatchString(string(serverName)) { return true } } // Check if the hostname matches one of the allowed regexes. If it does then // the server is NOT banned from the room. for _, expr := range acls.allowedRegexes { - if expr.MatchString(string(serverName)) { + if (*expr).MatchString(string(serverName)) { return false } } diff --git a/roomserver/acls/acls_test.go b/roomserver/acls/acls_test.go index 9fb6a5581..7fd20f114 100644 --- a/roomserver/acls/acls_test.go +++ b/roomserver/acls/acls_test.go @@ -15,19 +15,25 @@ package acls import ( + "context" "regexp" "testing" + + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/stretchr/testify/assert" ) func TestOpenACLsWithBlacklist(t *testing.T) { roomID := "!test:test.com" allowRegex, err := compileACLRegex("*") if err != nil { - t.Fatalf(err.Error()) + t.Fatal(err) } denyRegex, err := compileACLRegex("foo.com") if err != nil { - t.Fatalf(err.Error()) + t.Fatal(err) } acls := ServerACLs{ @@ -38,8 +44,8 @@ func TestOpenACLsWithBlacklist(t *testing.T) { ServerACL: ServerACL{ AllowIPLiterals: true, }, - allowedRegexes: []*regexp.Regexp{allowRegex}, - deniedRegexes: []*regexp.Regexp{denyRegex}, + allowedRegexes: []**regexp.Regexp{&allowRegex}, + deniedRegexes: []**regexp.Regexp{&denyRegex}, } if acls.IsServerBannedFromRoom("1.2.3.4", roomID) { @@ -66,7 +72,7 @@ func TestDefaultACLsWithWhitelist(t *testing.T) { roomID := "!test:test.com" allowRegex, err := compileACLRegex("foo.com") if err != nil { - t.Fatalf(err.Error()) + t.Fatal(err) } acls := ServerACLs{ @@ -77,8 +83,8 @@ func TestDefaultACLsWithWhitelist(t *testing.T) { ServerACL: ServerACL{ AllowIPLiterals: false, }, - allowedRegexes: []*regexp.Regexp{allowRegex}, - deniedRegexes: []*regexp.Regexp{}, + allowedRegexes: []**regexp.Regexp{&allowRegex}, + deniedRegexes: []**regexp.Regexp{}, } if !acls.IsServerBannedFromRoom("1.2.3.4", roomID) { @@ -103,3 +109,45 @@ func TestDefaultACLsWithWhitelist(t *testing.T) { t.Fatal("Expected qux.com:4567 to be allowed but wasn't") } } + +var ( + content1 = `{"allow":["*"],"allow_ip_literals":false,"deny":["hello.world", "*.hello.world"]}` +) + +type dummyACLDB struct{} + +func (d dummyACLDB) RoomsWithACLs(ctx context.Context) ([]string, error) { + return []string{"1", "2"}, nil +} + +func (d dummyACLDB) GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error) { + return []tables.StrippedEvent{ + { + RoomID: "1", + ContentValue: content1, + }, + { + RoomID: "2", + ContentValue: content1, + }, + }, nil +} + +func TestCachedRegex(t *testing.T) { + db := dummyACLDB{} + wantBannedServer := spec.ServerName("hello.world") + + acls := NewServerACLs(db) + + // Check that hello.world is banned in room 1 + banned := acls.IsServerBannedFromRoom(wantBannedServer, "1") + assert.True(t, banned) + + // Check that hello.world is banned in room 2 + banned = acls.IsServerBannedFromRoom(wantBannedServer, "2") + assert.True(t, banned) + + // Check that matrix.hello.world is banned in room 2 + banned = acls.IsServerBannedFromRoom("matrix."+wantBannedServer, "2") + assert.True(t, banned) +} diff --git a/roomserver/api/api.go b/roomserver/api/api.go index ef5bc3d17..b2b319244 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -86,6 +86,9 @@ type RoomserverInternalAPI interface { req *QueryAuthChainRequest, res *QueryAuthChainResponse, ) error + + // RoomsWithACLs returns all room IDs for rooms with ACLs + RoomsWithACLs(ctx context.Context) ([]string, error) } type UserRoomPrivateKeyCreator interface { @@ -138,7 +141,12 @@ type QueryRoomHierarchyAPI interface { // // If returned walker is nil, then there are no more rooms left to traverse. This method does not modify the provided walker, so it // can be cached. - QueryNextRoomHierarchyPage(ctx context.Context, walker RoomHierarchyWalker, limit int) ([]fclient.RoomHierarchyRoom, *RoomHierarchyWalker, error) + QueryNextRoomHierarchyPage(ctx context.Context, walker RoomHierarchyWalker, limit int) ( + hierarchyRooms []fclient.RoomHierarchyRoom, + inaccessibleRooms []string, + hierarchyWalker *RoomHierarchyWalker, + err error, + ) } type QueryMembershipAPI interface { @@ -220,6 +228,7 @@ type ClientRoomserverAPI interface { UserRoomPrivateKeyCreator QueryRoomHierarchyAPI DefaultRoomVersionAPI + QueryMembershipForUser(ctx context.Context, req *QueryMembershipForUserRequest, res *QueryMembershipForUserResponse) error QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error QueryRoomsForUser(ctx context.Context, userID spec.UserID, desiredMembership string) ([]spec.RoomID, error) @@ -261,6 +270,15 @@ type ClientRoomserverAPI interface { RemoveRoomAlias(ctx context.Context, senderID spec.SenderID, alias string) (aliasFound bool, aliasRemoved bool, err error) SigningIdentityFor(ctx context.Context, roomID spec.RoomID, senderID spec.UserID) (fclient.SigningIdentity, error) + + InsertReportedEvent( + ctx context.Context, + roomID, eventID, reportingUserID, reason string, + score int64, + ) (int64, error) + QueryAdminEventReports(ctx context.Context, from, limit uint64, backwards bool, userID, roomID string) ([]QueryAdminEventReportsResponse, int64, error) + QueryAdminEventReport(ctx context.Context, reportID uint64) (QueryAdminEventReportResponse, error) + PerformAdminDeleteEventReport(ctx context.Context, reportID uint64) error } type UserRoomserverAPI interface { diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index 2818efaa3..d6caec08c 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -8,7 +8,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" - "github.com/matrix-org/util" ) type PerformCreateRoomRequest struct { @@ -51,16 +50,14 @@ type PerformLeaveResponse struct { } type InviteInput struct { - RoomID spec.RoomID - Inviter spec.UserID - Invitee spec.UserID - DisplayName string - AvatarURL string - Reason string - IsDirect bool - KeyID gomatrixserverlib.KeyID - PrivateKey ed25519.PrivateKey - EventTime time.Time + RoomID spec.RoomID + Inviter spec.UserID + Invitee spec.UserID + Reason string + IsDirect bool + KeyID gomatrixserverlib.KeyID + PrivateKey ed25519.PrivateKey + EventTime time.Time } type PerformInviteRequest struct { @@ -91,14 +88,44 @@ type PerformBackfillRequest struct { VirtualHost spec.ServerName `json:"virtual_host"` } -// PrevEventIDs returns the prev_event IDs of all backwards extremities, de-duplicated in a lexicographically sorted order. +// limitPrevEventIDs is the maximum of eventIDs we +// return when calling PrevEventIDs. +const limitPrevEventIDs = 100 + +// PrevEventIDs returns the prev_event IDs of either 100 backwards extremities or +// len(r.BackwardsExtremities). Limited to 100, due to Synapse/Dendrite stopping after reaching +// this limit. (which sounds sane) func (r *PerformBackfillRequest) PrevEventIDs() []string { - var prevEventIDs []string - for _, pes := range r.BackwardsExtremities { - prevEventIDs = append(prevEventIDs, pes...) + var uniqueIDs map[string]struct{} + + // Create a unique eventID map of either 100 or len(r.BackwardsExtremities). + // 100 since Synapse/Dendrite stops after reaching 100 events. + if len(r.BackwardsExtremities) > limitPrevEventIDs { + uniqueIDs = make(map[string]struct{}, limitPrevEventIDs) + } else { + uniqueIDs = make(map[string]struct{}, len(r.BackwardsExtremities)) } - prevEventIDs = util.UniqueStrings(prevEventIDs) - return prevEventIDs + +outerLoop: + for _, pes := range r.BackwardsExtremities { + for _, evID := range pes { + uniqueIDs[evID] = struct{}{} + // We found enough unique eventIDs. + if len(uniqueIDs) >= limitPrevEventIDs { + break outerLoop + } + } + } + + // map -> []string + result := make([]string, len(uniqueIDs)) + i := 0 + for evID := range uniqueIDs { + result[i] = evID + i++ + } + + return result } // PerformBackfillResponse is a response to PerformBackfill. diff --git a/roomserver/api/perform_test.go b/roomserver/api/perform_test.go new file mode 100644 index 000000000..f26438d32 --- /dev/null +++ b/roomserver/api/perform_test.go @@ -0,0 +1,81 @@ +package api + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/stretchr/testify/assert" +) + +func BenchmarkPrevEventIDs(b *testing.B) { + for _, x := range []int64{1, 10, 100, 500, 1000, 2000} { + benchPrevEventIDs(b, int(x)) + } +} + +func benchPrevEventIDs(b *testing.B, count int) { + bwExtrems := generateBackwardsExtremities(b, count) + backfiller := PerformBackfillRequest{ + BackwardsExtremities: bwExtrems, + } + + b.Run(fmt.Sprintf("Original%d", count), func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + prevIDs := backfiller.PrevEventIDs() + _ = prevIDs + } + }) +} + +type testLike interface { + Helper() +} + +const randomIDCharsCount = 10 + +func generateBackwardsExtremities(t testLike, count int) map[string][]string { + t.Helper() + result := make(map[string][]string, count) + for i := 0; i < count; i++ { + eventID := randomEventId(int64(i)) + result[eventID] = []string{ + randomEventId(int64(i + 1)), + randomEventId(int64(i + 2)), + randomEventId(int64(i + 3)), + } + } + return result +} + +const alphanumerics = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +// randomEventId generates a pseudo-random string of length n. +func randomEventId(src int64) string { + randSrc := rand.NewSource(src) + b := make([]byte, randomIDCharsCount) + for i := range b { + b[i] = alphanumerics[randSrc.Int63()%int64(len(alphanumerics))] + } + return string(b) +} + +func TestPrevEventIDs(t *testing.T) { + // generate 10 backwards extremities + bwExtrems := generateBackwardsExtremities(t, 10) + backfiller := PerformBackfillRequest{ + BackwardsExtremities: bwExtrems, + } + + prevIDs := backfiller.PrevEventIDs() + // Given how "generateBackwardsExtremities" works, this + // generates 12 unique event IDs + assert.Equal(t, 12, len(prevIDs)) + + // generate 200 backwards extremities + backfiller.BackwardsExtremities = generateBackwardsExtremities(t, 200) + prevIDs = backfiller.PrevEventIDs() + // PrevEventIDs returns at max 100 event IDs + assert.Equal(t, 100, len(prevIDs)) +} diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 893d5dccf..c4c019f99 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -346,6 +346,28 @@ type QueryServerBannedFromRoomResponse struct { Banned bool `json:"banned"` } +type QueryAdminEventReportsResponse struct { + ID int64 `json:"id"` + Score int64 `json:"score"` + EventNID types.EventNID `json:"-"` // only used to query the state + RoomNID types.RoomNID `json:"-"` // only used to query the state + ReportingUserNID types.EventStateKeyNID `json:"-"` // only used in the DB + SenderNID types.EventStateKeyNID `json:"-"` // only used in the DB + RoomID string `json:"room_id"` + EventID string `json:"event_id"` + UserID string `json:"user_id"` // the user reporting the event + Reason string `json:"reason"` + Sender string `json:"sender"` // the user sending the reported event + CanonicalAlias string `json:"canonical_alias"` + RoomName string `json:"name"` + ReceivedTS spec.Timestamp `json:"received_ts"` +} + +type QueryAdminEventReportResponse struct { + QueryAdminEventReportsResponse + EventJSON json.RawMessage `json:"event_json"` +} + // MarshalJSON stringifies the room ID and StateKeyTuple keys so they can be sent over the wire in HTTP API mode. func (r *QueryBulkStateContentResponse) MarshalJSON() ([]byte, error) { se := make(map[string]string) diff --git a/roomserver/api/wrapper.go b/roomserver/api/wrapper.go index 0ad5d2013..4979d18c7 100644 --- a/roomserver/api/wrapper.go +++ b/roomserver/api/wrapper.go @@ -189,7 +189,7 @@ func PopulatePublicRooms(ctx context.Context, roomIDs []string, rsAPI QueryBulkS RoomID: roomID, } joinCount := 0 - var joinRule, guestAccess string + var guestAccess string for tuple, contentVal := range data { if tuple.EventType == spec.MRoomMember && contentVal == "join" { joinCount++ @@ -210,12 +210,12 @@ func PopulatePublicRooms(ctx context.Context, roomIDs []string, rsAPI QueryBulkS pub.WorldReadable = contentVal == "world_readable" // need both of these to determine whether guests can join case joinRuleTuple: - joinRule = contentVal + pub.JoinRule = contentVal case guestTuple: guestAccess = contentVal } } - if joinRule == spec.Public && guestAccess == "can_join" { + if pub.JoinRule == spec.Public && guestAccess == "can_join" { pub.GuestCanJoin = true } pub.JoinedMembersCount = joinCount diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index 1e08f6a3a..a71fd2d15 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -340,3 +340,11 @@ func (r *RoomserverInternalAPI) SigningIdentityFor(ctx context.Context, roomID s func (r *RoomserverInternalAPI) AssignRoomNID(ctx context.Context, roomID spec.RoomID, roomVersion gomatrixserverlib.RoomVersion) (roomNID types.RoomNID, err error) { return r.DB.AssignRoomNID(ctx, roomID, roomVersion) } + +func (r *RoomserverInternalAPI) InsertReportedEvent( + ctx context.Context, + roomID, eventID, reportingUserID, reason string, + score int64, +) (int64, error) { + return r.DB.InsertReportedEvent(ctx, roomID, eventID, reportingUserID, reason, score) +} diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go index 404751532..104ce94e5 100644 --- a/roomserver/internal/input/input.go +++ b/roomserver/internal/input/input.go @@ -108,20 +108,27 @@ type worker struct { r *Inputer roomID string subscription *nats.Subscription + sentryHub *sentry.Hub } func (r *Inputer) startWorkerForRoom(roomID string) { v, loaded := r.workers.LoadOrStore(roomID, &worker{ - r: r, - roomID: roomID, + r: r, + roomID: roomID, + sentryHub: sentry.CurrentHub().Clone(), }) w := v.(*worker) w.Lock() defer w.Unlock() if !loaded || w.subscription == nil { + streamName := r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent) consumer := r.Cfg.Matrix.JetStream.Prefixed("RoomInput" + jetstream.Tokenise(w.roomID)) subject := r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEventSubj(w.roomID)) + logger := logrus.WithFields(logrus.Fields{ + "stream_name": streamName, + "consumer": consumer, + }) // Create the consumer. We do this as a specific step rather than // letting PullSubscribe create it for us because we need the consumer // to outlive the subscription. If we do it this way, we can Bind in the @@ -135,21 +142,62 @@ func (r *Inputer) startWorkerForRoom(roomID string) { // before it. This is necessary because otherwise our consumer will never // acknowledge things we filtered out for other subjects and therefore they // will linger around forever. - if _, err := w.r.JetStream.AddConsumer( - r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent), - &nats.ConsumerConfig{ - Durable: consumer, - AckPolicy: nats.AckAllPolicy, - DeliverPolicy: nats.DeliverAllPolicy, - FilterSubject: subject, - AckWait: MaximumMissingProcessingTime + (time.Second * 10), - InactiveThreshold: inactiveThreshold, - }, - ); err != nil { - logrus.WithError(err).Errorf("Failed to create consumer for room %q", w.roomID) + + info, err := w.r.JetStream.ConsumerInfo(streamName, consumer) + if err != nil && !errors.Is(err, nats.ErrConsumerNotFound) { + // log and return, we will retry anyway + logger.WithError(err).Errorf("failed to get consumer info") return } + consumerConfig := &nats.ConsumerConfig{ + Durable: consumer, + AckPolicy: nats.AckExplicitPolicy, + DeliverPolicy: nats.DeliverAllPolicy, + FilterSubject: subject, + AckWait: MaximumMissingProcessingTime + (time.Second * 10), + InactiveThreshold: inactiveThreshold, + } + + // The consumer already exists, try to update if necessary. + if info != nil { + // Not using reflect.DeepEqual here, since consumerConfig does not explicitly set + // e.g. the consumerName, which is added by NATS later. So this would result + // in constantly updating/recreating the consumer. + switch { + case info.Config.AckWait.Nanoseconds() != consumerConfig.AckWait.Nanoseconds(): + // Initially we had a AckWait of 2m 10s, now we have 5m 10s, so we need to update + // existing consumers. + fallthrough + case info.Config.AckPolicy != consumerConfig.AckPolicy: + // We've changed the AckPolicy from AckAll to AckExplicit, this needs a + // recreation of the consumer. (Note: Only a few changes actually need a recreat) + logger.Warn("Consumer already exists, trying to update it.") + // Try updating the consumer first + if _, err = w.r.JetStream.UpdateConsumer(streamName, consumerConfig); err != nil { + // We failed to update the consumer, recreate it + logger.WithError(err).Warn("Unable to update consumer, recreating...") + if err = w.r.JetStream.DeleteConsumer(streamName, consumer); err != nil { + logger.WithError(err).Fatal("Unable to delete consumer") + return + } + // Set info to nil, so it can be recreated with the correct config. + info = nil + } + } + } + + if info == nil { + // Create the consumer with the correct config + if _, err = w.r.JetStream.AddConsumer( + r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent), + consumerConfig, + ); err != nil { + logger.WithError(err).Errorf("Failed to create consumer for room %q", w.roomID) + return + } + } + // Bind to our durable consumer. We want to receive all messages waiting // for this subject and we want to manually acknowledge them, so that we // can ensure they are only cleaned up when we are done processing them. @@ -162,7 +210,7 @@ func (r *Inputer) startWorkerForRoom(roomID string) { nats.InactiveThreshold(inactiveThreshold), ) if err != nil { - logrus.WithError(err).Errorf("Failed to subscribe to stream for room %q", w.roomID) + logger.WithError(err).Errorf("Failed to subscribe to stream for room %q", w.roomID) return } @@ -219,9 +267,9 @@ func (w *worker) _next() { // Look up what the next event is that's waiting to be processed. ctx, cancel := context.WithTimeout(w.r.ProcessContext.Context(), time.Minute) defer cancel() - if scope := sentry.CurrentHub().Scope(); scope != nil { + w.sentryHub.ConfigureScope(func(scope *sentry.Scope) { scope.SetTag("room_id", w.roomID) - } + }) msgs, err := w.subscription.Fetch(1, nats.Context(ctx)) switch err { case nil: @@ -263,21 +311,23 @@ func (w *worker) _next() { return } + // Since we either Ack() or Term() the message at this point, we can defer decrementing the room backpressure + defer roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Dec() + // Try to unmarshal the input room event. If the JSON unmarshalling // fails then we'll terminate the message — this notifies NATS that // we are done with the message and never want to see it again. msg := msgs[0] var inputRoomEvent api.InputRoomEvent if err = json.Unmarshal(msg.Data, &inputRoomEvent); err != nil { - _ = msg.Term() + // using AckWait here makes the call synchronous; 5 seconds is the default value used by NATS + _ = msg.Term(nats.AckWait(time.Second * 5)) return } - if scope := sentry.CurrentHub().Scope(); scope != nil { + w.sentryHub.ConfigureScope(func(scope *sentry.Scope) { scope.SetTag("event_id", inputRoomEvent.Event.EventID()) - } - roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Inc() - defer roomserverInputBackpressure.With(prometheus.Labels{"room_id": w.roomID}).Dec() + }) // Process the room event. If something goes wrong then we'll tell // NATS to terminate the message. We'll store the error result as @@ -299,7 +349,7 @@ func (w *worker) _next() { }).Warn("Roomserver rejected event") default: if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) { - sentry.CaptureException(err) + w.sentryHub.CaptureException(err) } logrus.WithError(err).WithFields(logrus.Fields{ "room_id": w.roomID, @@ -307,10 +357,15 @@ func (w *worker) _next() { "type": inputRoomEvent.Event.Type(), }).Warn("Roomserver failed to process event") } - _ = msg.Term() + // Even though we failed to process this message (e.g. due to Dendrite restarting and receiving a context canceled), + // the message may already have been queued for redelivery or will be, so this makes sure that we still reprocess the msg + // after restarting. We only Ack if the context was not yet canceled. + if w.r.ProcessContext.Context().Err() == nil { + _ = msg.AckSync() + } errString = err.Error() } else { - _ = msg.Ack() + _ = msg.AckSync() } // If it was a synchronous input request then the "sync" field @@ -381,6 +436,9 @@ func (r *Inputer) queueInputRoomEvents( }).Error("Roomserver failed to queue async event") return nil, fmt.Errorf("r.JetStream.PublishMsg: %w", err) } + + // Now that the event is queued, increment the room backpressure + roomserverInputBackpressure.With(prometheus.Labels{"room_id": roomID}).Inc() } return } diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 77b50d0e2..657ca8719 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -24,6 +24,7 @@ import ( "fmt" "time" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/tidwall/gjson" "github.com/matrix-org/gomatrixserverlib" @@ -33,6 +34,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/roomserver/acls" "github.com/matrix-org/dendrite/roomserver/internal/helpers" userAPI "github.com/matrix-org/dendrite/userapi/api" @@ -47,8 +49,10 @@ import ( "github.com/matrix-org/dendrite/roomserver/types" ) -// TODO: Does this value make sense? -const MaximumMissingProcessingTime = time.Minute * 2 +// MaximumMissingProcessingTime is the maximum time we allow "processRoomEvent" to fetch +// e.g. missing auth/prev events. This duration is used for AckWait, and if it is exceeded +// NATS queues the event for redelivery. +const MaximumMissingProcessingTime = time.Minute * 5 var processRoomEventDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -491,6 +495,33 @@ func (r *Inputer) processRoomEvent( } } + // If this is a membership event, it is possible we newly joined a federated room and eventually + // missed to update our m.room.server_acl - the following ensures we set the ACLs + // TODO: This probably performs badly in benchmarks + if event.Type() == spec.MRoomMember { + membership, _ := event.Membership() + if membership == spec.Join { + _, serverName, _ := gomatrixserverlib.SplitID('@', *event.StateKey()) + // only handle local membership events + if r.Cfg.Matrix.IsLocalServerName(serverName) { + var aclEvent *types.HeaderedEvent + aclEvent, err = r.DB.GetStateEvent(ctx, event.RoomID().String(), acls.MRoomServerACL, "") + if err != nil { + logrus.WithError(err).Error("failed to get server ACLs") + } + if aclEvent != nil { + strippedEvent := tables.StrippedEvent{ + RoomID: aclEvent.RoomID().String(), + EventType: aclEvent.Type(), + StateKey: *aclEvent.StateKey(), + ContentValue: string(aclEvent.Content()), + } + r.ACLs.OnServerACLUpdate(strippedEvent) + } + } + } + } + // Handle remote room upgrades, e.g. remove published room if event.Type() == "m.room.tombstone" && event.StateKeyEquals("") && !r.Cfg.Matrix.IsLocalServerName(senderDomain) { if err = r.handleRemoteRoomUpgrade(ctx, event); err != nil { diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index ec03d6f13..e9856cc5d 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -298,6 +298,7 @@ func (u *latestEventsUpdater) latestState() error { }).Warnf("State reset detected (removing %d events)", removed) sentry.WithScope(func(scope *sentry.Scope) { scope.SetLevel("warning") + scope.SetTag("room_id", u.event.RoomID().String()) scope.SetContext("State reset", map[string]interface{}{ "Event ID": u.event.EventID(), "Old state NID": fmt.Sprintf("%d", u.oldStateNID), diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go index ae203854b..1b8817234 100644 --- a/roomserver/internal/perform/perform_admin.go +++ b/roomserver/internal/perform/perform_admin.go @@ -354,3 +354,7 @@ func (r *Admin) PerformAdminDownloadState( return nil } + +func (r *Admin) PerformAdminDeleteEventReport(ctx context.Context, reportID uint64) error { + return r.DB.AdminDeleteEventReport(ctx, reportID) +} diff --git a/roomserver/internal/perform/perform_create_room.go b/roomserver/internal/perform/perform_create_room.go index eb8de7811..093082f90 100644 --- a/roomserver/internal/perform/perform_create_room.go +++ b/roomserver/internal/perform/perform_create_room.go @@ -503,16 +503,14 @@ func (c *Creator) PerformCreateRoom(ctx context.Context, userID spec.UserID, roo err = c.RSAPI.PerformInvite(ctx, &api.PerformInviteRequest{ InviteInput: api.InviteInput{ - RoomID: roomID, - Inviter: userID, - Invitee: *inviteeUserID, - DisplayName: createRequest.UserDisplayName, - AvatarURL: createRequest.UserAvatarURL, - Reason: "", - IsDirect: createRequest.IsDirect, - KeyID: createRequest.KeyID, - PrivateKey: createRequest.PrivateKey, - EventTime: createRequest.EventTime, + RoomID: roomID, + Inviter: userID, + Invitee: *inviteeUserID, + Reason: "", + IsDirect: createRequest.IsDirect, + KeyID: createRequest.KeyID, + PrivateKey: createRequest.PrivateKey, + EventTime: createRequest.EventTime, }, InviteRoomState: globalStrippedState, SendAsServer: string(userID.Domain()), diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go index 3abb69cb9..86563e8c3 100644 --- a/roomserver/internal/perform/perform_invite.go +++ b/roomserver/internal/perform/perform_invite.go @@ -144,11 +144,9 @@ func (r *Inviter) PerformInvite( } content := gomatrixserverlib.MemberContent{ - Membership: spec.Invite, - DisplayName: req.InviteInput.DisplayName, - AvatarURL: req.InviteInput.AvatarURL, - Reason: req.InviteInput.Reason, - IsDirect: req.InviteInput.IsDirect, + Membership: spec.Invite, + Reason: req.InviteInput.Reason, + IsDirect: req.InviteInput.IsDirect, } if err = proto.SetContent(content); err != nil { diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 74b010281..886d00492 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -1099,3 +1099,18 @@ func (r *Queryer) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, return nil, nil } + +// RoomsWithACLs returns all room IDs for rooms with ACLs +func (r *Queryer) RoomsWithACLs(ctx context.Context) ([]string, error) { + return r.DB.RoomsWithACLs(ctx) +} + +// QueryAdminEventReports returns event reports given a filter. +func (r *Queryer) QueryAdminEventReports(ctx context.Context, from uint64, limit uint64, backwards bool, userID, roomID string) ([]api.QueryAdminEventReportsResponse, int64, error) { + return r.DB.QueryAdminEventReports(ctx, from, limit, backwards, userID, roomID) +} + +// QueryAdminEventReport returns a single event report. +func (r *Queryer) QueryAdminEventReport(ctx context.Context, reportID uint64) (api.QueryAdminEventReportResponse, error) { + return r.DB.QueryAdminEventReport(ctx, reportID) +} diff --git a/roomserver/internal/query/query_room_hierarchy.go b/roomserver/internal/query/query_room_hierarchy.go index 76eba12be..3fc613192 100644 --- a/roomserver/internal/query/query_room_hierarchy.go +++ b/roomserver/internal/query/query_room_hierarchy.go @@ -17,6 +17,7 @@ package query import ( "context" "encoding/json" + "errors" "fmt" "sort" @@ -38,9 +39,14 @@ import ( // // If returned walker is nil, then there are no more rooms left to traverse. This method does not modify the provided walker, so it // can be cached. -func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker roomserver.RoomHierarchyWalker, limit int) ([]fclient.RoomHierarchyRoom, *roomserver.RoomHierarchyWalker, error) { - if authorised, _ := authorised(ctx, querier, walker.Caller, walker.RootRoomID, nil); !authorised { - return nil, nil, roomserver.ErrRoomUnknownOrNotAllowed{Err: fmt.Errorf("room is unknown/forbidden")} +func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker roomserver.RoomHierarchyWalker, limit int) ( + []fclient.RoomHierarchyRoom, + []string, + *roomserver.RoomHierarchyWalker, + error, +) { + if authorised, _, _ := authorised(ctx, querier, walker.Caller, walker.RootRoomID, nil); !authorised { + return nil, []string{walker.RootRoomID.String()}, nil, roomserver.ErrRoomUnknownOrNotAllowed{Err: fmt.Errorf("room is unknown/forbidden")} } discoveredRooms := []fclient.RoomHierarchyRoom{} @@ -49,6 +55,7 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r unvisited := make([]roomserver.RoomHierarchyWalkerQueuedRoom, len(walker.Unvisited)) copy(unvisited, walker.Unvisited) processed := walker.Processed.Copy() + inaccessible := []string{} // Depth first -> stack data structure for len(unvisited) > 0 { @@ -56,6 +63,12 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r break } + // If the context is canceled, we might still have discovered rooms + // return them to the client and let the client know there _may_ be more rooms. + if errors.Is(ctx.Err(), context.Canceled) { + break + } + // pop the stack queuedRoom := unvisited[len(unvisited)-1] unvisited = unvisited[:len(unvisited)-1] @@ -101,7 +114,7 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r // as these children may be rooms we do know about. roomType = spec.MSpace } - } else if authorised, isJoinedOrInvited := authorised(ctx, querier, walker.Caller, queuedRoom.RoomID, queuedRoom.ParentRoomID); authorised { + } else if authorised, isJoinedOrInvited, allowedRoomIDs := authorised(ctx, querier, walker.Caller, queuedRoom.RoomID, queuedRoom.ParentRoomID); authorised { // Get all `m.space.child` state events for this room events, err := childReferences(ctx, querier, walker.SuggestedOnly, queuedRoom.RoomID) if err != nil { @@ -112,15 +125,24 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r pubRoom := publicRoomsChunk(ctx, querier, queuedRoom.RoomID) + if pubRoom == nil { + util.GetLogger(ctx).WithField("room_id", queuedRoom.RoomID).Debug("unable to get publicRoomsChunk") + continue + } + discoveredRooms = append(discoveredRooms, fclient.RoomHierarchyRoom{ - PublicRoom: *pubRoom, - RoomType: roomType, - ChildrenState: events, + PublicRoom: *pubRoom, + RoomType: roomType, + ChildrenState: events, + AllowedRoomIDs: allowedRoomIDs, }) // don't walk children if the user is not joined/invited to the space if !isJoinedOrInvited { continue } + } else if !authorised { + inaccessible = append(inaccessible, queuedRoom.RoomID.String()) + continue } else { // room exists but user is not authorised continue @@ -137,6 +159,7 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r // We need to invert the order here because the child events are lo->hi on the timestamp, // so we need to ensure we pop in the same lo->hi order, which won't be the case if we // insert the highest timestamp last in a stack. + extendQueueLoop: for i := len(discoveredChildEvents) - 1; i >= 0; i-- { spaceContent := struct { Via []string `json:"via"` @@ -149,6 +172,12 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r if err != nil { util.GetLogger(ctx).WithError(err).WithField("invalid_room_id", ev.StateKey).WithField("parent_room_id", queuedRoom.RoomID).Warn("Invalid room ID in m.space.child state event") } else { + // Make sure not to queue inaccessible rooms + for _, inaccessibleRoomID := range inaccessible { + if inaccessibleRoomID == childRoomID.String() { + continue extendQueueLoop + } + } unvisited = append(unvisited, roomserver.RoomHierarchyWalkerQueuedRoom{ RoomID: *childRoomID, ParentRoomID: &queuedRoom.RoomID, @@ -161,7 +190,7 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r if len(unvisited) == 0 { // If no more rooms to walk, then don't return a walker for future pages - return discoveredRooms, nil, nil + return discoveredRooms, inaccessible, nil, nil } else { // If there are more rooms to walk, then return a new walker to resume walking from (for querying more pages) newWalker := roomserver.RoomHierarchyWalker{ @@ -173,22 +202,25 @@ func (querier *Queryer) QueryNextRoomHierarchyPage(ctx context.Context, walker r Processed: processed, } - return discoveredRooms, &newWalker, nil + return discoveredRooms, inaccessible, &newWalker, nil } } // authorised returns true iff the user is joined this room or the room is world_readable -func authorised(ctx context.Context, querier *Queryer, caller types.DeviceOrServerName, roomID spec.RoomID, parentRoomID *spec.RoomID) (authed, isJoinedOrInvited bool) { +func authorised(ctx context.Context, querier *Queryer, caller types.DeviceOrServerName, roomID spec.RoomID, parentRoomID *spec.RoomID) (authed, isJoinedOrInvited bool, resultAllowedRoomIDs []string) { if clientCaller := caller.Device(); clientCaller != nil { return authorisedUser(ctx, querier, clientCaller, roomID, parentRoomID) - } else { - return authorisedServer(ctx, querier, roomID, *caller.ServerName()), false } + if serverCaller := caller.ServerName(); serverCaller != nil { + authed, resultAllowedRoomIDs = authorisedServer(ctx, querier, roomID, *serverCaller) + return authed, false, resultAllowedRoomIDs + } + return false, false, resultAllowedRoomIDs } // authorisedServer returns true iff the server is joined this room or the room is world_readable, public, or knockable -func authorisedServer(ctx context.Context, querier *Queryer, roomID spec.RoomID, callerServerName spec.ServerName) bool { +func authorisedServer(ctx context.Context, querier *Queryer, roomID spec.RoomID, callerServerName spec.ServerName) (bool, []string) { // Check history visibility / join rules first hisVisTuple := gomatrixserverlib.StateKeyTuple{ EventType: spec.MRoomHistoryVisibility, @@ -207,13 +239,13 @@ func authorisedServer(ctx context.Context, querier *Queryer, roomID spec.RoomID, }, &queryRoomRes) if err != nil { util.GetLogger(ctx).WithError(err).Error("failed to QueryCurrentState") - return false + return false, []string{} } hisVisEv := queryRoomRes.StateEvents[hisVisTuple] if hisVisEv != nil { hisVis, _ := hisVisEv.HistoryVisibility() if hisVis == "world_readable" { - return true + return true, []string{} } } @@ -226,19 +258,23 @@ func authorisedServer(ctx context.Context, querier *Queryer, roomID spec.RoomID, rule, ruleErr := joinRuleEv.JoinRule() if ruleErr != nil { util.GetLogger(ctx).WithError(ruleErr).WithField("parent_room_id", roomID).Warn("failed to get join rule") - return false + return false, []string{} } if rule == spec.Public || rule == spec.Knock { - return true + return true, []string{} } - if rule == spec.Restricted { + if rule == spec.Restricted || rule == spec.KnockRestricted { allowJoinedToRoomIDs = append(allowJoinedToRoomIDs, restrictedJoinRuleAllowedRooms(ctx, joinRuleEv)...) } } // check if server is joined to any allowed room + resultAllowedRoomIDs := make([]string, 0, len(allowJoinedToRoomIDs)) + for _, allowedRoomID := range allowJoinedToRoomIDs { + resultAllowedRoomIDs = append(resultAllowedRoomIDs, allowedRoomID.String()) + } for _, allowedRoomID := range allowJoinedToRoomIDs { var queryRes fs.QueryJoinedHostServerNamesInRoomResponse err = querier.FSAPI.QueryJoinedHostServerNamesInRoom(ctx, &fs.QueryJoinedHostServerNamesInRoomRequest{ @@ -250,18 +286,18 @@ func authorisedServer(ctx context.Context, querier *Queryer, roomID spec.RoomID, } for _, srv := range queryRes.ServerNames { if srv == callerServerName { - return true + return true, resultAllowedRoomIDs[1:] } } } - return false + return false, resultAllowedRoomIDs[1:] } // authorisedUser returns true iff the user is invited/joined this room or the room is world_readable // or if the room has a public or knock join rule. // Failing that, if the room has a restricted join rule and belongs to the space parent listed, it will return true. -func authorisedUser(ctx context.Context, querier *Queryer, clientCaller *userapi.Device, roomID spec.RoomID, parentRoomID *spec.RoomID) (authed bool, isJoinedOrInvited bool) { +func authorisedUser(ctx context.Context, querier *Queryer, clientCaller *userapi.Device, roomID spec.RoomID, parentRoomID *spec.RoomID) (authed bool, isJoinedOrInvited bool, resultAllowedRoomIDs []string) { hisVisTuple := gomatrixserverlib.StateKeyTuple{ EventType: spec.MRoomHistoryVisibility, StateKey: "", @@ -283,20 +319,20 @@ func authorisedUser(ctx context.Context, querier *Queryer, clientCaller *userapi }, &queryRes) if err != nil { util.GetLogger(ctx).WithError(err).Error("failed to QueryCurrentState") - return false, false + return false, false, resultAllowedRoomIDs } memberEv := queryRes.StateEvents[roomMemberTuple] if memberEv != nil { membership, _ := memberEv.Membership() if membership == spec.Join || membership == spec.Invite { - return true, true + return true, true, resultAllowedRoomIDs } } hisVisEv := queryRes.StateEvents[hisVisTuple] if hisVisEv != nil { hisVis, _ := hisVisEv.HistoryVisibility() if hisVis == "world_readable" { - return true, false + return true, false, resultAllowedRoomIDs } } joinRuleEv := queryRes.StateEvents[joinRuleTuple] @@ -311,6 +347,7 @@ func authorisedUser(ctx context.Context, querier *Queryer, clientCaller *userapi allowedRoomIDs := restrictedJoinRuleAllowedRooms(ctx, joinRuleEv) // check parent is in the allowed set for _, a := range allowedRoomIDs { + resultAllowedRoomIDs = append(resultAllowedRoomIDs, a.String()) if *parentRoomID == a { allowed = true break @@ -333,13 +370,13 @@ func authorisedUser(ctx context.Context, querier *Queryer, clientCaller *userapi if memberEv != nil { membership, _ := memberEv.Membership() if membership == spec.Join { - return true, false + return true, false, resultAllowedRoomIDs } } } } } - return false, false + return false, false, resultAllowedRoomIDs } // helper function to fetch a state event diff --git a/roomserver/producers/roomevent.go b/roomserver/producers/roomevent.go index 165304d49..894e6d81b 100644 --- a/roomserver/producers/roomevent.go +++ b/roomserver/producers/roomevent.go @@ -17,6 +17,7 @@ package producers import ( "encoding/json" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/nats-io/nats.go" log "github.com/sirupsen/logrus" "github.com/tidwall/gjson" @@ -73,9 +74,15 @@ func (r *RoomEventProducer) ProduceRoomEvents(roomID string, updates []api.Outpu } } - if eventType == "m.room.server_acl" && update.NewRoomEvent.Event.StateKeyEquals("") { + if eventType == acls.MRoomServerACL && update.NewRoomEvent.Event.StateKeyEquals("") { ev := update.NewRoomEvent.Event.PDU - defer r.ACLs.OnServerACLUpdate(ev) + strippedEvent := tables.StrippedEvent{ + RoomID: ev.RoomID().String(), + EventType: ev.Type(), + StateKey: *ev.StateKey(), + ContentValue: string(ev.Content()), + } + defer r.ACLs.OnServerACLUpdate(strippedEvent) } } logger.Tracef("Producing to topic '%s'", r.Topic) diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index 90e67b699..85312efd9 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -7,14 +7,18 @@ import ( "testing" "time" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/internal/input" "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/nats-io/nats.go" "github.com/stretchr/testify/assert" "github.com/tidwall/gjson" + "github.com/matrix-org/dendrite/roomserver/acls" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/userapi" @@ -34,6 +38,10 @@ import ( "github.com/matrix-org/dendrite/test/testrig" ) +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + type FakeQuerier struct { api.QuerySenderIDAPI } @@ -58,7 +66,7 @@ func TestUsers(t *testing.T) { }) t.Run("kick users", func(t *testing.T) { - usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) rsAPI.SetUserAPI(usrAPI) testKickUsers(t, rsAPI, usrAPI) }) @@ -258,7 +266,7 @@ func TestPurgeRoom(t *testing.T) { fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(fsAPI, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, fsAPI.IsBlacklistedOrBackingOff) syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) // Create the room @@ -1050,7 +1058,7 @@ func TestUpgrade(t *testing.T) { rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) rsAPI.SetFederationAPI(nil, nil) - userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff) rsAPI.SetUserAPI(userAPI) for _, tc := range testCases { @@ -1185,3 +1193,129 @@ func TestStateReset(t *testing.T) { } }) } + +func TestNewServerACLs(t *testing.T) { + alice := test.NewUser(t) + roomWithACL := test.NewRoom(t, alice) + + roomWithACL.CreateAndInsert(t, alice, acls.MRoomServerACL, acls.ServerACL{ + Allowed: []string{"*"}, + Denied: []string{"localhost"}, + AllowIPLiterals: false, + }, test.WithStateKey("")) + + roomWithoutACL := test.NewRoom(t, alice) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType) + defer closeDB() + + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + natsInstance := &jetstream.NATSInstance{} + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + // start JetStream listeners + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + + // let the RS create the events + err := api.SendEvents(context.Background(), rsAPI, api.KindNew, roomWithACL.Events(), "test", "test", "test", nil, false) + assert.NoError(t, err) + err = api.SendEvents(context.Background(), rsAPI, api.KindNew, roomWithoutACL.Events(), "test", "test", "test", nil, false) + assert.NoError(t, err) + + db, err := storage.Open(processCtx.Context(), cm, &cfg.RoomServer.Database, caches) + assert.NoError(t, err) + // create new server ACLs and verify server is banned/not banned + serverACLs := acls.NewServerACLs(db) + banned := serverACLs.IsServerBannedFromRoom("localhost", roomWithACL.ID) + assert.Equal(t, true, banned) + banned = serverACLs.IsServerBannedFromRoom("localhost", roomWithoutACL.ID) + assert.Equal(t, false, banned) + }) +} + +// Validate that changing the AckPolicy/AckWait of room consumers +// results in their recreation +func TestRoomConsumerRecreation(t *testing.T) { + + alice := test.NewUser(t) + room := test.NewRoom(t, alice) + + // As this is DB unrelated, just use SQLite + cfg, processCtx, closeDB := testrig.CreateConfig(t, test.DBTypeSQLite) + defer closeDB() + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + natsInstance := &jetstream.NATSInstance{} + + // Prepare a stream and consumer using the old configuration + jsCtx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + + streamName := cfg.Global.JetStream.Prefixed(jetstream.InputRoomEvent) + consumer := cfg.Global.JetStream.Prefixed("RoomInput" + jetstream.Tokenise(room.ID)) + subject := cfg.Global.JetStream.Prefixed(jetstream.InputRoomEventSubj(room.ID)) + + consumerConfig := &nats.ConsumerConfig{ + Durable: consumer, + AckPolicy: nats.AckAllPolicy, + DeliverPolicy: nats.DeliverAllPolicy, + FilterSubject: subject, + AckWait: (time.Minute * 2) + (time.Second * 10), + InactiveThreshold: time.Hour * 24, + } + + // Create the consumer with the old config + _, err := jsCtx.AddConsumer(streamName, consumerConfig) + assert.NoError(t, err) + + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + // start JetStream listeners + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + + // let the RS create the events, this also recreates the Consumers + err = api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false) + assert.NoError(t, err) + + // Validate that AckPolicy and AckWait has changed + info, err := jsCtx.ConsumerInfo(streamName, consumer) + assert.NoError(t, err) + assert.Equal(t, nats.AckExplicitPolicy, info.Config.AckPolicy) + + wantAckWait := input.MaximumMissingProcessingTime + (time.Second * 10) + assert.Equal(t, wantAckWait, info.Config.AckWait) +} + +func TestRoomsWithACLs(t *testing.T) { + ctx := context.Background() + alice := test.NewUser(t) + noACLRoom := test.NewRoom(t, alice) + aclRoom := test.NewRoom(t, alice) + + aclRoom.CreateAndInsert(t, alice, "m.room.server_acl", map[string]any{ + "deny": []string{"evilhost.test"}, + "allow": []string{"*"}, + }, test.WithStateKey("")) + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType) + defer closeDB() + + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + natsInstance := &jetstream.NATSInstance{} + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + // start JetStream listeners + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + + for _, room := range []*test.Room{noACLRoom, aclRoom} { + // Create the rooms + err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false) + assert.NoError(t, err) + } + + // Validate that we only have one ACLd room. + roomsWithACLs, err := rsAPI.RoomsWithACLs(ctx) + assert.NoError(t, err) + assert.Equal(t, []string{aclRoom.ID}, roomsWithACLs) + }) +} diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 0638252b2..ab105e6f9 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -30,6 +30,7 @@ import ( type Database interface { UserRoomKeys + ReportedEvents // Do we support processing input events for more than one room at a time? SupportsConcurrentRoomInputs() bool AssignRoomNID(ctx context.Context, roomID spec.RoomID, roomVersion gomatrixserverlib.RoomVersion) (roomNID types.RoomNID, err error) @@ -170,8 +171,6 @@ type Database interface { GetServerInRoom(ctx context.Context, roomNID types.RoomNID, serverName spec.ServerName) (bool, error) // GetKnownUsers searches all users that userID knows about. GetKnownUsers(ctx context.Context, userID, searchString string, limit int) ([]string, error) - // GetKnownRooms returns a list of all rooms we know about. - GetKnownRooms(ctx context.Context) ([]string, error) // ForgetRoom sets a flag in the membership table, that the user wishes to forget a specific room ForgetRoom(ctx context.Context, userID, roomID string, forget bool) error @@ -193,6 +192,12 @@ type Database interface { MaybeRedactEvent( ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event gomatrixserverlib.PDU, plResolver state.PowerLevelResolver, querier api.QuerySenderIDAPI, ) (gomatrixserverlib.PDU, gomatrixserverlib.PDU, error) + + // RoomsWithACLs returns all room IDs for rooms with ACLs + RoomsWithACLs(ctx context.Context) ([]string, error) + QueryAdminEventReports(ctx context.Context, from uint64, limit uint64, backwards bool, userID string, roomID string) ([]api.QueryAdminEventReportsResponse, int64, error) + QueryAdminEventReport(ctx context.Context, reportID uint64) (api.QueryAdminEventReportResponse, error) + AdminDeleteEventReport(ctx context.Context, reportID uint64) error } type UserRoomKeys interface { @@ -256,3 +261,11 @@ type EventDatabase interface { ) (gomatrixserverlib.PDU, gomatrixserverlib.PDU, error) StoreEvent(ctx context.Context, event gomatrixserverlib.PDU, roomInfo *types.RoomInfo, eventTypeNID types.EventTypeNID, eventStateKeyNID types.EventStateKeyNID, authEventNIDs []types.EventNID, isRejected bool) (types.EventNID, types.StateAtEvent, error) } + +type ReportedEvents interface { + InsertReportedEvent( + ctx context.Context, + roomID, eventID, reportingUserID, reason string, + score int64, + ) (int64, error) +} diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index a00b4b1d7..180a03cd6 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -68,6 +68,10 @@ CREATE TABLE IF NOT EXISTS roomserver_events ( -- Create an index which helps in resolving membership events (event_type_nid = 5) - (used for history visibility) CREATE INDEX IF NOT EXISTS roomserver_events_memberships_idx ON roomserver_events (room_nid, event_state_key_nid) WHERE (event_type_nid = 5); + +-- The following indexes are used by bulkSelectStateEventByNIDSQL +CREATE INDEX IF NOT EXISTS roomserver_event_event_type_nid_idx ON roomserver_events (event_type_nid); +CREATE INDEX IF NOT EXISTS roomserver_event_state_key_nid_idx ON roomserver_events (event_state_key_nid); ` const insertEventSQL = "" + @@ -147,6 +151,8 @@ const selectRoomNIDsForEventNIDsSQL = "" + const selectEventRejectedSQL = "" + "SELECT is_rejected FROM roomserver_events WHERE room_nid = $1 AND event_id = $2" +const selectRoomsWithEventTypeNIDSQL = `SELECT DISTINCT room_nid FROM roomserver_events WHERE event_type_nid = $1` + type eventStatements struct { insertEventStmt *sql.Stmt selectEventStmt *sql.Stmt @@ -166,6 +172,7 @@ type eventStatements struct { selectMaxEventDepthStmt *sql.Stmt selectRoomNIDsForEventNIDsStmt *sql.Stmt selectEventRejectedStmt *sql.Stmt + selectRoomsWithEventTypeNIDStmt *sql.Stmt } func CreateEventsTable(db *sql.DB) error { @@ -206,6 +213,7 @@ func PrepareEventsTable(db *sql.DB) (tables.Events, error) { {&s.selectMaxEventDepthStmt, selectMaxEventDepthSQL}, {&s.selectRoomNIDsForEventNIDsStmt, selectRoomNIDsForEventNIDsSQL}, {&s.selectEventRejectedStmt, selectEventRejectedSQL}, + {&s.selectRoomsWithEventTypeNIDStmt, selectRoomsWithEventTypeNIDSQL}, }.Prepare(db) } @@ -249,6 +257,7 @@ func (s *eventStatements) BulkSelectSnapshotsFromEventIDs( if err != nil { return nil, err } + defer internal.CloseAndLogIfError(ctx, rows, "BulkSelectSnapshotsFromEventIDs: rows.close() failed") var eventID string var stateNID types.StateSnapshotNID @@ -563,7 +572,7 @@ func (s *eventStatements) SelectRoomNIDsForEventNIDs( } result[eventNID] = roomNID } - return result, nil + return result, rows.Err() } func eventNIDsAsArray(eventNIDs []types.EventNID) pq.Int64Array { @@ -581,3 +590,25 @@ func (s *eventStatements) SelectEventRejected( err = stmt.QueryRowContext(ctx, roomNID, eventID).Scan(&rejected) return } + +func (s *eventStatements) SelectRoomsWithEventTypeNID( + ctx context.Context, txn *sql.Tx, eventTypeNID types.EventTypeNID, +) ([]types.RoomNID, error) { + stmt := sqlutil.TxStmt(txn, s.selectRoomsWithEventTypeNIDStmt) + rows, err := stmt.QueryContext(ctx, eventTypeNID) + defer internal.CloseAndLogIfError(ctx, rows, "SelectRoomsWithEventTypeNID: rows.close() failed") + if err != nil { + return nil, err + } + + var roomNIDs []types.RoomNID + var roomNID types.RoomNID + for rows.Next() { + if err := rows.Scan(&roomNID); err != nil { + return nil, err + } + roomNIDs = append(roomNIDs, roomNID) + } + + return roomNIDs, rows.Err() +} diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index 835a43b2d..1a96e3527 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -363,7 +363,7 @@ func (s *membershipStatements) SelectRoomsWithMembership( } roomNIDs = append(roomNIDs, roomNID) } - return roomNIDs, nil + return roomNIDs, rows.Err() } func (s *membershipStatements) SelectJoinedUsersSetForRooms( diff --git a/roomserver/storage/postgres/purge_statements.go b/roomserver/storage/postgres/purge_statements.go index efba439bd..9ca8d0346 100644 --- a/roomserver/storage/postgres/purge_statements.go +++ b/roomserver/storage/postgres/purge_statements.go @@ -41,6 +41,11 @@ const purgePreviousEventsSQL = "" + " SELECT ARRAY_AGG(event_nid) FROM roomserver_events WHERE room_nid = $1" + ")" +// This removes the majority of prev events and is way faster than the above. +// The above query is still needed to delete the remaining prev events. +const purgePreviousEvents2SQL = "" + + "DELETE FROM roomserver_previous_events rpe WHERE EXISTS(SELECT event_id FROM roomserver_events re WHERE room_nid = $1 AND re.event_id = rpe.previous_event_id)" + const purgePublishedSQL = "" + "DELETE FROM roomserver_published WHERE room_id = $1" @@ -69,6 +74,7 @@ type purgeStatements struct { purgeInvitesStmt *sql.Stmt purgeMembershipsStmt *sql.Stmt purgePreviousEventsStmt *sql.Stmt + purgePreviousEvents2Stmt *sql.Stmt purgePublishedStmt *sql.Stmt purgeRedactionStmt *sql.Stmt purgeRoomAliasesStmt *sql.Stmt @@ -87,6 +93,7 @@ func PreparePurgeStatements(db *sql.DB) (*purgeStatements, error) { {&s.purgeMembershipsStmt, purgeMembershipsSQL}, {&s.purgePublishedStmt, purgePublishedSQL}, {&s.purgePreviousEventsStmt, purgePreviousEventsSQL}, + {&s.purgePreviousEvents2Stmt, purgePreviousEvents2SQL}, {&s.purgeRedactionStmt, purgeRedactionsSQL}, {&s.purgeRoomAliasesStmt, purgeRoomAliasesSQL}, {&s.purgeRoomStmt, purgeRoomSQL}, @@ -117,7 +124,8 @@ func (s *purgeStatements) PurgeRoom( s.purgeStateSnapshotEntriesStmt, s.purgeInvitesStmt, s.purgeMembershipsStmt, - s.purgePreviousEventsStmt, + s.purgePreviousEvents2Stmt, // Fast purge the majority of events + s.purgePreviousEventsStmt, // Slow purge the remaining events s.purgeEventJSONStmt, s.purgeRedactionStmt, s.purgeEventsStmt, diff --git a/roomserver/storage/postgres/reported_events_table.go b/roomserver/storage/postgres/reported_events_table.go new file mode 100644 index 000000000..c46f47b34 --- /dev/null +++ b/roomserver/storage/postgres/reported_events_table.go @@ -0,0 +1,221 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + "time" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib/spec" +) + +const reportedEventsScheme = ` +CREATE SEQUENCE IF NOT EXISTS roomserver_reported_events_id_seq; +CREATE TABLE IF NOT EXISTS roomserver_reported_events +( + id BIGINT PRIMARY KEY DEFAULT nextval('roomserver_reported_events_id_seq'), + room_nid BIGINT NOT NULL, + event_nid BIGINT NOT NULL, + reporting_user_nid BIGINT NOT NULL, -- the user reporting the event + event_sender_nid BIGINT NOT NULL, -- the user who sent the reported event + reason TEXT, + score INTEGER, + received_ts BIGINT NOT NULL +);` + +const insertReportedEventSQL = ` + INSERT INTO roomserver_reported_events (room_nid, event_nid, reporting_user_nid, event_sender_nid, reason, score, received_ts) + VALUES ($1, $2, $3, $4, $5, $6, $7) + RETURNING id +` + +const selectReportedEventsDescSQL = ` +WITH countReports AS ( + SELECT count(*) as report_count + FROM roomserver_reported_events + WHERE ($1::BIGINT IS NULL OR room_nid = $1::BIGINT) AND ($2::TEXT IS NULL OR reporting_user_nid = $2::BIGINT) +) +SELECT report_count, id, room_nid, event_nid, reporting_user_nid, event_sender_nid, reason, score, received_ts +FROM roomserver_reported_events, countReports +WHERE ($1::BIGINT IS NULL OR room_nid = $1::BIGINT) AND ($2::TEXT IS NULL OR reporting_user_nid = $2::BIGINT) +ORDER BY received_ts DESC +OFFSET $3 +LIMIT $4 +` + +const selectReportedEventsAscSQL = ` +WITH countReports AS ( + SELECT count(*) as report_count + FROM roomserver_reported_events + WHERE ($1::BIGINT IS NULL OR room_nid = $1::BIGINT) AND ($2::TEXT IS NULL OR reporting_user_nid = $2::BIGINT) +) +SELECT report_count, id, room_nid, event_nid, reporting_user_nid, event_sender_nid, reason, score, received_ts +FROM roomserver_reported_events, countReports +WHERE ($1::BIGINT IS NULL OR room_nid = $1::BIGINT) AND ($2::TEXT IS NULL OR reporting_user_nid = $2::BIGINT) +ORDER BY received_ts ASC +OFFSET $3 +LIMIT $4 +` + +const selectReportedEventSQL = ` +SELECT id, room_nid, event_nid, reporting_user_nid, event_sender_nid, reason, score, received_ts +FROM roomserver_reported_events +WHERE id = $1 +` + +const deleteReportedEventSQL = `DELETE FROM roomserver_reported_events WHERE id = $1` + +type reportedEventsStatements struct { + insertReportedEventsStmt *sql.Stmt + selectReportedEventsDescStmt *sql.Stmt + selectReportedEventsAscStmt *sql.Stmt + selectReportedEventStmt *sql.Stmt + deleteReportedEventStmt *sql.Stmt +} + +func CreateReportedEventsTable(db *sql.DB) error { + _, err := db.Exec(reportedEventsScheme) + return err +} + +func PrepareReportedEventsTable(db *sql.DB) (tables.ReportedEvents, error) { + s := &reportedEventsStatements{} + + return s, sqlutil.StatementList{ + {&s.insertReportedEventsStmt, insertReportedEventSQL}, + {&s.selectReportedEventsDescStmt, selectReportedEventsDescSQL}, + {&s.selectReportedEventsAscStmt, selectReportedEventsAscSQL}, + {&s.selectReportedEventStmt, selectReportedEventSQL}, + {&s.deleteReportedEventStmt, deleteReportedEventSQL}, + }.Prepare(db) +} + +func (r *reportedEventsStatements) InsertReportedEvent( + ctx context.Context, + txn *sql.Tx, + roomNID types.RoomNID, + eventNID types.EventNID, + reportingUserID types.EventStateKeyNID, + eventSenderID types.EventStateKeyNID, + reason string, + score int64, +) (int64, error) { + stmt := sqlutil.TxStmt(txn, r.insertReportedEventsStmt) + + var reportID int64 + err := stmt.QueryRowContext(ctx, + roomNID, + eventNID, + reportingUserID, + eventSenderID, + reason, + score, + spec.AsTimestamp(time.Now()), + ).Scan(&reportID) + return reportID, err +} + +func (r *reportedEventsStatements) SelectReportedEvents( + ctx context.Context, + txn *sql.Tx, + from, limit uint64, + backwards bool, + reportingUserID types.EventStateKeyNID, + roomNID types.RoomNID, +) ([]api.QueryAdminEventReportsResponse, int64, error) { + var stmt *sql.Stmt + if backwards { + stmt = sqlutil.TxStmt(txn, r.selectReportedEventsDescStmt) + } else { + stmt = sqlutil.TxStmt(txn, r.selectReportedEventsAscStmt) + } + + var qryRoomNID *types.RoomNID + if roomNID > 0 { + qryRoomNID = &roomNID + } + var qryReportingUser *types.EventStateKeyNID + if reportingUserID > 0 { + qryReportingUser = &reportingUserID + } + + rows, err := stmt.QueryContext(ctx, + qryRoomNID, + qryReportingUser, + from, + limit, + ) + if err != nil { + return nil, 0, err + } + defer internal.CloseAndLogIfError(ctx, rows, "SelectReportedEvents: failed to close rows") + + var result []api.QueryAdminEventReportsResponse + var row api.QueryAdminEventReportsResponse + var count int64 + for rows.Next() { + if err = rows.Scan( + &count, + &row.ID, + &row.RoomNID, + &row.EventNID, + &row.ReportingUserNID, + &row.SenderNID, + &row.Reason, + &row.Score, + &row.ReceivedTS, + ); err != nil { + return nil, 0, err + } + result = append(result, row) + } + + return result, count, rows.Err() +} + +func (r *reportedEventsStatements) SelectReportedEvent( + ctx context.Context, + txn *sql.Tx, + reportID uint64, +) (api.QueryAdminEventReportResponse, error) { + stmt := sqlutil.TxStmt(txn, r.selectReportedEventStmt) + + var row api.QueryAdminEventReportResponse + if err := stmt.QueryRowContext(ctx, reportID).Scan( + &row.ID, + &row.RoomNID, + &row.EventNID, + &row.ReportingUserNID, + &row.SenderNID, + &row.Reason, + &row.Score, + &row.ReceivedTS, + ); err != nil { + return api.QueryAdminEventReportResponse{}, err + } + return row, nil +} + +func (r *reportedEventsStatements) DeleteReportedEvent(ctx context.Context, txn *sql.Tx, reportID uint64) error { + stmt := sqlutil.TxStmt(txn, r.deleteReportedEventStmt) + _, err := stmt.ExecContext(ctx, reportID) + return err +} diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index c8346733d..4de6dee46 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -76,9 +76,6 @@ const selectRoomVersionsForRoomNIDsSQL = "" + const selectRoomInfoSQL = "" + "SELECT room_version, room_nid, state_snapshot_nid, latest_event_nids FROM roomserver_rooms WHERE room_id = $1" -const selectRoomIDsSQL = "" + - "SELECT room_id FROM roomserver_rooms WHERE array_length(latest_event_nids, 1) > 0" - const bulkSelectRoomIDsSQL = "" + "SELECT room_id FROM roomserver_rooms WHERE room_nid = ANY($1)" @@ -94,7 +91,6 @@ type roomStatements struct { updateLatestEventNIDsStmt *sql.Stmt selectRoomVersionsForRoomNIDsStmt *sql.Stmt selectRoomInfoStmt *sql.Stmt - selectRoomIDsStmt *sql.Stmt bulkSelectRoomIDsStmt *sql.Stmt bulkSelectRoomNIDsStmt *sql.Stmt } @@ -116,29 +112,11 @@ func PrepareRoomsTable(db *sql.DB) (tables.Rooms, error) { {&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL}, {&s.selectRoomVersionsForRoomNIDsStmt, selectRoomVersionsForRoomNIDsSQL}, {&s.selectRoomInfoStmt, selectRoomInfoSQL}, - {&s.selectRoomIDsStmt, selectRoomIDsSQL}, {&s.bulkSelectRoomIDsStmt, bulkSelectRoomIDsSQL}, {&s.bulkSelectRoomNIDsStmt, bulkSelectRoomNIDsSQL}, }.Prepare(db) } -func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error) { - stmt := sqlutil.TxStmt(txn, s.selectRoomIDsStmt) - rows, err := stmt.QueryContext(ctx) - if err != nil { - return nil, err - } - defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsStmt: rows.close() failed") - var roomIDs []string - var roomID string - for rows.Next() { - if err = rows.Scan(&roomID); err != nil { - return nil, err - } - roomIDs = append(roomIDs, roomID) - } - return roomIDs, nil -} func (s *roomStatements) InsertRoomNID( ctx context.Context, txn *sql.Tx, roomID string, roomVersion gomatrixserverlib.RoomVersion, @@ -255,7 +233,7 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs( } result[roomNID] = roomVersion } - return result, nil + return result, rows.Err() } func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error) { @@ -277,7 +255,7 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo } roomIDs = append(roomIDs, roomID) } - return roomIDs, nil + return roomIDs, rows.Err() } func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error) { @@ -299,7 +277,7 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro } roomNIDs = append(roomNIDs, roomNID) } - return roomNIDs, nil + return roomNIDs, rows.Err() } func roomNIDsAsArray(roomNIDs []types.RoomNID) pq.Int64Array { diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index c5c206cfb..1068230f7 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -134,6 +134,9 @@ func (d *Database) create(db *sql.DB) error { if err := CreateUserRoomKeysTable(db); err != nil { return err } + if err := CreateReportedEventsTable(db); err != nil { + return err + } return nil } @@ -199,6 +202,10 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room if err != nil { return err } + reportedEvents, err := PrepareReportedEventsTable(db) + if err != nil { + return err + } d.Database = shared.Database{ DB: db, @@ -212,6 +219,7 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room EventStateKeysTable: eventStateKeys, PrevEventsTable: prevEvents, RedactionsTable: redactions, + ReportedEventsTable: reportedEvents, }, Cache: cache, Writer: writer, diff --git a/roomserver/storage/postgres/user_room_keys_table.go b/roomserver/storage/postgres/user_room_keys_table.go index 217ee957f..f8befc46b 100644 --- a/roomserver/storage/postgres/user_room_keys_table.go +++ b/roomserver/storage/postgres/user_room_keys_table.go @@ -162,6 +162,10 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, if errors.Is(err, sql.ErrNoRows) { return nil, nil } + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "SelectAllPublicKeysForUser: failed to close rows") resultMap := make(map[types.RoomNID]ed25519.PublicKey) @@ -173,5 +177,5 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, } resultMap[roomNID] = pubkey } - return resultMap, err + return resultMap, rows.Err() } diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 3331c6029..7b04641bf 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -61,6 +61,7 @@ type EventDatabase struct { EventStateKeysTable tables.EventStateKeys PrevEventsTable tables.PreviousEvents RedactionsTable tables.Redactions + ReportedEventsTable tables.ReportedEvents } func (d *Database) SupportsConcurrentRoomInputs() bool { @@ -889,10 +890,10 @@ func (d *Database) assignRoomNID( } // Check if we already have a numeric ID in the database. roomNID, err := d.RoomsTable.SelectRoomNID(ctx, txn, roomID) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { // We don't have a numeric ID so insert one into the database. roomNID, err = d.RoomsTable.InsertRoomNID(ctx, txn, roomID, roomVersion) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { // We raced with another insert so run the select again. roomNID, err = d.RoomsTable.SelectRoomNID(ctx, txn, roomID) } @@ -914,10 +915,10 @@ func (d *Database) assignEventTypeNID( } // Check if we already have a numeric ID in the database. eventTypeNID, err := d.EventTypesTable.SelectEventTypeNID(ctx, txn, eventType) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { // We don't have a numeric ID so insert one into the database. eventTypeNID, err = d.EventTypesTable.InsertEventTypeNID(ctx, txn, eventType) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { // We raced with another insert so run the select again. eventTypeNID, err = d.EventTypesTable.SelectEventTypeNID(ctx, txn, eventType) } @@ -938,16 +939,19 @@ func (d *EventDatabase) assignStateKeyNID( } // Check if we already have a numeric ID in the database. eventStateKeyNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, txn, eventStateKey) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { // We don't have a numeric ID so insert one into the database. eventStateKeyNID, err = d.EventStateKeysTable.InsertEventStateKeyNID(ctx, txn, eventStateKey) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { // We raced with another insert so run the select again. eventStateKeyNID, err = d.EventStateKeysTable.SelectEventStateKeyNID(ctx, txn, eventStateKey) } } + if err != nil { + return 0, err + } d.Cache.StoreEventStateKey(eventStateKeyNID, eventStateKey) - return eventStateKeyNID, err + return eventStateKeyNID, nil } func extractRoomVersionFromCreateEvent(event gomatrixserverlib.PDU) ( @@ -1622,9 +1626,24 @@ func (d *Database) GetKnownUsers(ctx context.Context, userID, searchString strin return d.MembershipTable.SelectKnownUsers(ctx, nil, stateKeyNID, searchString, limit) } -// GetKnownRooms returns a list of all rooms we know about. -func (d *Database) GetKnownRooms(ctx context.Context) ([]string, error) { - return d.RoomsTable.SelectRoomIDsWithEvents(ctx, nil) +func (d *Database) RoomsWithACLs(ctx context.Context) ([]string, error) { + + eventTypeNID, err := d.GetOrCreateEventTypeNID(ctx, "m.room.server_acl") + if err != nil { + return nil, err + } + + roomNIDs, err := d.EventsTable.SelectRoomsWithEventTypeNID(ctx, nil, eventTypeNID) + if err != nil { + return nil, err + } + + roomIDs, err := d.RoomsTable.BulkSelectRoomIDs(ctx, nil, roomNIDs) + if err != nil { + return nil, err + } + + return roomIDs, nil } // ForgetRoom sets a users room to forgotten @@ -1864,6 +1883,252 @@ func (d *Database) SelectUserIDsForPublicKeys(ctx context.Context, publicKeys ma return result, err } +// InsertReportedEvent stores a reported event. +func (d *Database) InsertReportedEvent( + ctx context.Context, + roomID, eventID, reportingUserID, reason string, + score int64, +) (int64, error) { + roomInfo, err := d.roomInfo(ctx, nil, roomID) + if err != nil { + return 0, err + } + if roomInfo == nil { + return 0, fmt.Errorf("room does not exist") + } + + events, err := d.eventsFromIDs(ctx, nil, roomInfo, []string{eventID}, NoFilter) + if err != nil { + return 0, err + } + if len(events) == 0 { + return 0, fmt.Errorf("unable to find requested event") + } + + stateKeyNIDs, err := d.EventStateKeyNIDs(ctx, []string{reportingUserID, events[0].SenderID().ToUserID().String()}) + if err != nil { + return 0, fmt.Errorf("failed to query eventStateKeyNIDs: %w", err) + } + + // We expect exactly 2 stateKeyNIDs + if len(stateKeyNIDs) != 2 { + return 0, fmt.Errorf("expected 2 stateKeyNIDs, received %d", len(stateKeyNIDs)) + } + + var reportID int64 + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + reportID, err = d.ReportedEventsTable.InsertReportedEvent( + ctx, + txn, + roomInfo.RoomNID, + events[0].EventNID, + stateKeyNIDs[reportingUserID], + stateKeyNIDs[events[0].SenderID().ToUserID().String()], + reason, + score, + ) + if err != nil { + return err + } + return nil + }) + + return reportID, err +} + +// QueryAdminEventReports returns event reports given a filter. +func (d *Database) QueryAdminEventReports(ctx context.Context, from uint64, limit uint64, backwards bool, userID string, roomID string) ([]api.QueryAdminEventReportsResponse, int64, error) { + // Filter on roomID, if requested + var roomNID types.RoomNID + if roomID != "" { + roomInfo, err := d.RoomInfo(ctx, roomID) + if err != nil { + return nil, 0, err + } + roomNID = roomInfo.RoomNID + } + + // Same as above, but for userID + var userNID types.EventStateKeyNID + if userID != "" { + stateKeysMap, err := d.EventStateKeyNIDs(ctx, []string{userID}) + if err != nil { + return nil, 0, err + } + if len(stateKeysMap) != 1 { + return nil, 0, fmt.Errorf("failed to get eventStateKeyNID for %s", userID) + } + userNID = stateKeysMap[userID] + } + + // Query all reported events matching the filters + reports, count, err := d.ReportedEventsTable.SelectReportedEvents(ctx, nil, from, limit, backwards, userNID, roomNID) + if err != nil { + return nil, 0, fmt.Errorf("failed to SelectReportedEvents: %w", err) + } + + // TODO: The below code may be inefficient due to many DB round trips and needs to be revisited. + // For the time being, this is "good enough". + qryRoomNIDs := make([]types.RoomNID, 0, len(reports)) + qryEventNIDs := make([]types.EventNID, 0, len(reports)) + qryStateKeyNIDs := make([]types.EventStateKeyNID, 0, len(reports)) + for _, report := range reports { + qryRoomNIDs = append(qryRoomNIDs, report.RoomNID) + qryEventNIDs = append(qryEventNIDs, report.EventNID) + qryStateKeyNIDs = append(qryStateKeyNIDs, report.ReportingUserNID, report.SenderNID) + } + + // This also de-dupes the roomIDs, otherwise we would query the same + // roomIDs in GetBulkStateContent multiple times + roomIDs, err := d.RoomsTable.BulkSelectRoomIDs(ctx, nil, qryRoomNIDs) + if err != nil { + return nil, 0, err + } + + // TODO: replace this with something more efficient, as it loads the entire state snapshot. + stateContent, err := d.GetBulkStateContent(ctx, roomIDs, []gomatrixserverlib.StateKeyTuple{ + {EventType: spec.MRoomName, StateKey: ""}, + {EventType: spec.MRoomCanonicalAlias, StateKey: ""}, + }, false) + if err != nil { + return nil, 0, err + } + + eventIDMap, err := d.EventIDs(ctx, qryEventNIDs) + if err != nil { + logrus.WithError(err).Error("unable to map eventNIDs to eventIDs") + return nil, 0, err + } + if len(eventIDMap) != len(qryEventNIDs) { + return nil, 0, fmt.Errorf("expected %d eventIDs, got %d", len(qryEventNIDs), len(eventIDMap)) + } + + // Get a map from EventStateKeyNID to userID + userNIDMap, err := d.EventStateKeys(ctx, qryStateKeyNIDs) + if err != nil { + logrus.WithError(err).Error("unable to map userNIDs to userIDs") + return nil, 0, err + } + + // Create a cache from roomNID to roomID to avoid hitting the DB again + roomNIDIDCache := make(map[types.RoomNID]string, len(roomIDs)) + for i := 0; i < len(reports); i++ { + cachedRoomID := roomNIDIDCache[reports[i].RoomNID] + if cachedRoomID == "" { + // We need to query this again, as we otherwise don't have a way to match roomNID -> roomID + roomIDs, err = d.RoomsTable.BulkSelectRoomIDs(ctx, nil, []types.RoomNID{reports[i].RoomNID}) + if err != nil { + return nil, 0, err + } + if len(roomIDs) == 0 || len(roomIDs) > 1 { + logrus.Warnf("unable to map roomNID %d to a roomID, was this room deleted?", roomNID) + continue + } + roomNIDIDCache[reports[i].RoomNID] = roomIDs[0] + cachedRoomID = roomIDs[0] + } + + reports[i].EventID = eventIDMap[reports[i].EventNID] + reports[i].RoomID = cachedRoomID + roomName, canonicalAlias := findRoomNameAndCanonicalAlias(stateContent, cachedRoomID) + reports[i].RoomName = roomName + reports[i].CanonicalAlias = canonicalAlias + reports[i].Sender = userNIDMap[reports[i].SenderNID] + reports[i].UserID = userNIDMap[reports[i].ReportingUserNID] + } + + return reports, count, nil +} + +func (d *Database) QueryAdminEventReport(ctx context.Context, reportID uint64) (api.QueryAdminEventReportResponse, error) { + + report, err := d.ReportedEventsTable.SelectReportedEvent(ctx, nil, reportID) + if err != nil { + return api.QueryAdminEventReportResponse{}, err + } + + // Get a map from EventStateKeyNID to userID + userNIDMap, err := d.EventStateKeys(ctx, []types.EventStateKeyNID{report.ReportingUserNID, report.SenderNID}) + if err != nil { + logrus.WithError(err).Error("unable to map userNIDs to userIDs") + return report, err + } + + roomIDs, err := d.RoomsTable.BulkSelectRoomIDs(ctx, nil, []types.RoomNID{report.RoomNID}) + if err != nil { + return report, err + } + + if len(roomIDs) != 1 { + return report, fmt.Errorf("expected one roomID, got %d", len(roomIDs)) + } + + // TODO: replace this with something more efficient, as it loads the entire state snapshot. + stateContent, err := d.GetBulkStateContent(ctx, roomIDs, []gomatrixserverlib.StateKeyTuple{ + {EventType: spec.MRoomName, StateKey: ""}, + {EventType: spec.MRoomCanonicalAlias, StateKey: ""}, + }, false) + if err != nil { + return report, err + } + + eventIDMap, err := d.EventIDs(ctx, []types.EventNID{report.EventNID}) + if err != nil { + logrus.WithError(err).Error("unable to map eventNIDs to eventIDs") + return report, err + } + if len(eventIDMap) != 1 { + return report, fmt.Errorf("expected %d eventIDs, got %d", 1, len(eventIDMap)) + } + + eventJSONs, err := d.EventJSONTable.BulkSelectEventJSON(ctx, nil, []types.EventNID{report.EventNID}) + if err != nil { + return report, err + } + if len(eventJSONs) != 1 { + return report, fmt.Errorf("expected %d eventJSONs, got %d", 1, len(eventJSONs)) + } + + roomName, canonicalAlias := findRoomNameAndCanonicalAlias(stateContent, roomIDs[0]) + + report.Sender = userNIDMap[report.SenderNID] + report.UserID = userNIDMap[report.ReportingUserNID] + report.RoomID = roomIDs[0] + report.RoomName = roomName + report.CanonicalAlias = canonicalAlias + report.EventID = eventIDMap[report.EventNID] + report.EventJSON = eventJSONs[0].EventJSON + + return report, nil +} + +func (d *Database) AdminDeleteEventReport(ctx context.Context, reportID uint64) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + return d.ReportedEventsTable.DeleteReportedEvent(ctx, txn, reportID) + }) +} + +// findRoomNameAndCanonicalAlias loops over events to find the corresponding room name and canonicalAlias +// for a given roomID. +func findRoomNameAndCanonicalAlias(events []tables.StrippedEvent, roomID string) (name, canonicalAlias string) { + for _, ev := range events { + if ev.RoomID != roomID { + continue + } + if ev.EventType == spec.MRoomName { + name = ev.ContentValue + } + if ev.EventType == spec.MRoomCanonicalAlias { + canonicalAlias = ev.ContentValue + } + // We found both wanted values, break the loop + if name != "" && canonicalAlias != "" { + break + } + } + return name, canonicalAlias +} + // FIXME TODO: Remove all this - horrible dupe with roomserver/state. Can't use the original impl because of circular loops // it should live in this package! diff --git a/roomserver/storage/sqlite3/event_json_table.go b/roomserver/storage/sqlite3/event_json_table.go index dc26885bb..325951c7f 100644 --- a/roomserver/storage/sqlite3/event_json_table.go +++ b/roomserver/storage/sqlite3/event_json_table.go @@ -109,5 +109,5 @@ func (s *eventJSONStatements) BulkSelectEventJSON( } result.EventNID = types.EventNID(eventNID) } - return results[:i], nil + return results[:i], rows.Err() } diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go index 347524a81..a052d69ed 100644 --- a/roomserver/storage/sqlite3/event_state_keys_table.go +++ b/roomserver/storage/sqlite3/event_state_keys_table.go @@ -136,7 +136,7 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKeyNID( } result[stateKey] = types.EventStateKeyNID(stateKeyNID) } - return result, nil + return result, rows.Err() } func (s *eventStateKeyStatements) BulkSelectEventStateKey( @@ -167,5 +167,5 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKey( } result[types.EventStateKeyNID(stateKeyNID)] = stateKey } - return result, nil + return result, rows.Err() } diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go index 0581ec194..c030fffea 100644 --- a/roomserver/storage/sqlite3/event_types_table.go +++ b/roomserver/storage/sqlite3/event_types_table.go @@ -147,5 +147,5 @@ func (s *eventTypeStatements) BulkSelectEventTypeNID( } result[eventType] = types.EventTypeNID(eventTypeNID) } - return result, nil + return result, rows.Err() } diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index c49c6dc38..26401e45d 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -44,6 +44,14 @@ const eventsSchema = ` auth_event_nids TEXT NOT NULL DEFAULT '[]', is_rejected BOOLEAN NOT NULL DEFAULT FALSE ); + +-- Create an index which helps in resolving membership events (event_type_nid = 5) - (used for history visibility) +CREATE INDEX IF NOT EXISTS roomserver_events_memberships_idx ON roomserver_events (room_nid, event_state_key_nid) WHERE (event_type_nid = 5); + +-- The following indexes are used by bulkSelectStateEventByNIDSQL +CREATE INDEX IF NOT EXISTS roomserver_event_event_type_nid_idx ON roomserver_events (event_type_nid); +CREATE INDEX IF NOT EXISTS roomserver_event_state_key_nid_idx ON roomserver_events (event_state_key_nid); + ` const insertEventSQL = ` @@ -120,6 +128,8 @@ const selectRoomNIDsForEventNIDsSQL = "" + const selectEventRejectedSQL = "" + "SELECT is_rejected FROM roomserver_events WHERE room_nid = $1 AND event_id = $2" +const selectRoomsWithEventTypeNIDSQL = `SELECT DISTINCT room_nid FROM roomserver_events WHERE event_type_nid = $1` + type eventStatements struct { db *sql.DB insertEventStmt *sql.Stmt @@ -135,6 +145,7 @@ type eventStatements struct { bulkSelectStateAtEventAndReferenceStmt *sql.Stmt bulkSelectEventIDStmt *sql.Stmt selectEventRejectedStmt *sql.Stmt + selectRoomsWithEventTypeNIDStmt *sql.Stmt //bulkSelectEventNIDStmt *sql.Stmt //bulkSelectUnsentEventNIDStmt *sql.Stmt //selectRoomNIDsForEventNIDsStmt *sql.Stmt @@ -192,6 +203,7 @@ func PrepareEventsTable(db *sql.DB) (tables.Events, error) { //{&s.bulkSelectUnsentEventNIDStmt, bulkSelectUnsentEventNIDSQL}, //{&s.selectRoomNIDForEventNIDStmt, selectRoomNIDForEventNIDSQL}, {&s.selectEventRejectedStmt, selectEventRejectedSQL}, + {&s.selectRoomsWithEventTypeNIDStmt, selectRoomsWithEventTypeNIDSQL}, }.Prepare(db) } @@ -310,6 +322,9 @@ func (s *eventStatements) BulkSelectStateEventByID( } results = append(results, result) } + if err = rows.Err(); err != nil { + return nil, err + } if !excludeRejected && i != len(eventIDs) { // If there are fewer rows returned than IDs then we were asked to lookup event IDs we don't have. // We don't know which ones were missing because we don't return the string IDs in the query. @@ -377,7 +392,7 @@ func (s *eventStatements) BulkSelectStateEventByNID( return nil, err } } - return results[:i], err + return results[:i], rows.Err() } // bulkSelectStateAtEventByID lookups the state at a list of events by event ID. @@ -425,6 +440,9 @@ func (s *eventStatements) BulkSelectStateAtEventByID( ) } } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventIDs) { return nil, types.MissingEventError( fmt.Sprintf("storage: event IDs missing from the database (%d != %d)", i, len(eventIDs)), @@ -507,6 +525,9 @@ func (s *eventStatements) BulkSelectStateAtEventAndReference( result.BeforeStateSnapshotNID = types.StateSnapshotNID(stateSnapshotNID) result.EventID = eventID } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventNIDs) { return nil, fmt.Errorf("storage: event NIDs missing from the database (%d != %d)", i, len(eventNIDs)) } @@ -544,6 +565,9 @@ func (s *eventStatements) BulkSelectEventID(ctx context.Context, txn *sql.Tx, ev } results[types.EventNID(eventNID)] = eventID } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventNIDs) { return nil, fmt.Errorf("storage: event NIDs missing from the database (%d != %d)", i, len(eventNIDs)) } @@ -602,7 +626,7 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, e RoomNID: types.RoomNID(roomNID), } } - return results, nil + return results, rows.Err() } func (s *eventStatements) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) { @@ -652,7 +676,7 @@ func (s *eventStatements) SelectRoomNIDsForEventNIDs( } result[eventNID] = roomNID } - return result, nil + return result, rows.Err() } func eventNIDsAsArray(eventNIDs []types.EventNID) string { @@ -670,3 +694,25 @@ func (s *eventStatements) SelectEventRejected( err = stmt.QueryRowContext(ctx, roomNID, eventID).Scan(&rejected) return } + +func (s *eventStatements) SelectRoomsWithEventTypeNID( + ctx context.Context, txn *sql.Tx, eventTypeNID types.EventTypeNID, +) ([]types.RoomNID, error) { + stmt := sqlutil.TxStmt(txn, s.selectRoomsWithEventTypeNIDStmt) + rows, err := stmt.QueryContext(ctx, eventTypeNID) + defer internal.CloseAndLogIfError(ctx, rows, "SelectRoomsWithEventTypeNID: rows.close() failed") + if err != nil { + return nil, err + } + + var roomNIDs []types.RoomNID + var roomNID types.RoomNID + for rows.Next() { + if err := rows.Scan(&roomNID); err != nil { + return nil, err + } + roomNIDs = append(roomNIDs, roomNID) + } + + return roomNIDs, rows.Err() +} diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index ca6e7c511..b678d8add 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -126,6 +126,9 @@ func (s *inviteStatements) UpdateInviteRetired( } eventIDs = append(eventIDs, inviteEventID) } + if err = rows.Err(); err != nil { + return + } // now retire the invites stmt = sqlutil.TxStmt(txn, s.updateInviteRetiredStmt) _, err = stmt.ExecContext(ctx, roomNID, targetUserNID) @@ -157,5 +160,5 @@ func (s *inviteStatements) SelectInviteActiveForUserInRoom( result = append(result, types.EventStateKeyNID(senderUserNID)) eventIDs = append(eventIDs, eventID) } - return result, eventIDs, eventJSON, nil + return result, eventIDs, eventJSON, rows.Err() } diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index 977788d50..1012c074a 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -250,6 +250,7 @@ func (s *membershipStatements) SelectMembershipsFromRoom( } eventNIDs = append(eventNIDs, eNID) } + err = rows.Err() return } @@ -277,6 +278,7 @@ func (s *membershipStatements) SelectMembershipsFromRoomAndMembership( } eventNIDs = append(eventNIDs, eNID) } + err = rows.Err() return } @@ -313,7 +315,7 @@ func (s *membershipStatements) SelectRoomsWithMembership( } roomNIDs = append(roomNIDs, roomNID) } - return roomNIDs, nil + return roomNIDs, rows.Err() } func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, localOnly bool) (map[types.EventStateKeyNID]int, error) { diff --git a/roomserver/storage/sqlite3/purge_statements.go b/roomserver/storage/sqlite3/purge_statements.go index c7b4d27a5..cb21515b8 100644 --- a/roomserver/storage/sqlite3/purge_statements.go +++ b/roomserver/storage/sqlite3/purge_statements.go @@ -41,6 +41,11 @@ const purgePreviousEventsSQL = "" + " SELECT event_nid FROM roomserver_events WHERE room_nid = $1" + ")" +// This removes the majority of prev events and is way faster than the above. +// The above query is still needed to delete the remaining prev events. +const purgePreviousEvents2SQL = "" + + "DELETE FROM roomserver_previous_events AS rpe WHERE EXISTS(SELECT event_id FROM roomserver_events AS re WHERE room_nid = $1 AND re.event_id = rpe.previous_event_id)" + const purgePublishedSQL = "" + "DELETE FROM roomserver_published WHERE room_id = $1" @@ -64,6 +69,7 @@ type purgeStatements struct { purgeInvitesStmt *sql.Stmt purgeMembershipsStmt *sql.Stmt purgePreviousEventsStmt *sql.Stmt + purgePreviousEvents2Stmt *sql.Stmt purgePublishedStmt *sql.Stmt purgeRedactionStmt *sql.Stmt purgeRoomAliasesStmt *sql.Stmt @@ -81,6 +87,7 @@ func PreparePurgeStatements(db *sql.DB, stateSnapshot *stateSnapshotStatements) {&s.purgeMembershipsStmt, purgeMembershipsSQL}, {&s.purgePublishedStmt, purgePublishedSQL}, {&s.purgePreviousEventsStmt, purgePreviousEventsSQL}, + {&s.purgePreviousEvents2Stmt, purgePreviousEvents2SQL}, {&s.purgeRedactionStmt, purgeRedactionsSQL}, {&s.purgeRoomAliasesStmt, purgeRoomAliasesSQL}, {&s.purgeRoomStmt, purgeRoomSQL}, @@ -114,7 +121,8 @@ func (s *purgeStatements) PurgeRoom( s.purgeStateSnapshotEntriesStmt, s.purgeInvitesStmt, s.purgeMembershipsStmt, - s.purgePreviousEventsStmt, + s.purgePreviousEvents2Stmt, // Fast purge the majority of events + s.purgePreviousEventsStmt, // Slow purge the remaining events s.purgeEventJSONStmt, s.purgeRedactionStmt, s.purgeEventsStmt, diff --git a/roomserver/storage/sqlite3/reported_events_table.go b/roomserver/storage/sqlite3/reported_events_table.go new file mode 100644 index 000000000..b72cb0685 --- /dev/null +++ b/roomserver/storage/sqlite3/reported_events_table.go @@ -0,0 +1,221 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "time" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib/spec" +) + +const reportedEventsScheme = ` +CREATE TABLE IF NOT EXISTS roomserver_reported_events +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + room_nid INTEGER NOT NULL, + event_nid INTEGER NOT NULL, + reporting_user_nid INTEGER NOT NULL, -- the user reporting the event + event_sender_nid INTEGER NOT NULL, -- the user who sent the reported event + reason TEXT, + score INTEGER, + received_ts INTEGER NOT NULL +);` + +const insertReportedEventSQL = ` + INSERT INTO roomserver_reported_events (room_nid, event_nid, reporting_user_nid, event_sender_nid, reason, score, received_ts) + VALUES ($1, $2, $3, $4, $5, $6, $7) + RETURNING id +` + +const selectReportedEventsDescSQL = ` +WITH countReports AS ( + SELECT count(*) as report_count + FROM roomserver_reported_events + WHERE ($1 IS NULL OR room_nid = $1) AND ($2 IS NULL OR reporting_user_nid = $2) +) +SELECT report_count, id, room_nid, event_nid, reporting_user_nid, event_sender_nid, reason, score, received_ts +FROM roomserver_reported_events, countReports +WHERE ($1 IS NULL OR room_nid = $1) AND ($2 IS NULL OR reporting_user_nid = $2) +ORDER BY received_ts DESC +LIMIT $3 +OFFSET $4 +` + +const selectReportedEventsAscSQL = ` +WITH countReports AS ( + SELECT count(*) as report_count + FROM roomserver_reported_events + WHERE ($1 IS NULL OR room_nid = $1) AND ($2 IS NULL OR reporting_user_nid = $2) +) +SELECT report_count, id, room_nid, event_nid, reporting_user_nid, event_sender_nid, reason, score, received_ts +FROM roomserver_reported_events, countReports +WHERE ($1 IS NULL OR room_nid = $1) AND ($2 IS NULL OR reporting_user_nid = $2) +ORDER BY received_ts ASC +LIMIT $3 +OFFSET $4 +` + +const selectReportedEventSQL = ` +SELECT id, room_nid, event_nid, reporting_user_nid, event_sender_nid, reason, score, received_ts +FROM roomserver_reported_events +WHERE id = $1 +` + +const deleteReportedEventSQL = `DELETE FROM roomserver_reported_events WHERE id = $1` + +type reportedEventsStatements struct { + insertReportedEventsStmt *sql.Stmt + selectReportedEventsDescStmt *sql.Stmt + selectReportedEventsAscStmt *sql.Stmt + selectReportedEventStmt *sql.Stmt + deleteReportedEventStmt *sql.Stmt +} + +func CreateReportedEventsTable(db *sql.DB) error { + _, err := db.Exec(reportedEventsScheme) + return err +} + +func PrepareReportedEventsTable(db *sql.DB) (tables.ReportedEvents, error) { + s := &reportedEventsStatements{} + + return s, sqlutil.StatementList{ + {&s.insertReportedEventsStmt, insertReportedEventSQL}, + {&s.selectReportedEventsDescStmt, selectReportedEventsDescSQL}, + {&s.selectReportedEventsAscStmt, selectReportedEventsAscSQL}, + {&s.selectReportedEventStmt, selectReportedEventSQL}, + {&s.deleteReportedEventStmt, deleteReportedEventSQL}, + }.Prepare(db) +} + +func (r *reportedEventsStatements) InsertReportedEvent( + ctx context.Context, + txn *sql.Tx, + roomNID types.RoomNID, + eventNID types.EventNID, + reportingUserID types.EventStateKeyNID, + eventSenderID types.EventStateKeyNID, + reason string, + score int64, +) (int64, error) { + stmt := sqlutil.TxStmt(txn, r.insertReportedEventsStmt) + + var reportID int64 + err := stmt.QueryRowContext(ctx, + roomNID, + eventNID, + reportingUserID, + eventSenderID, + reason, + score, + spec.AsTimestamp(time.Now()), + ).Scan(&reportID) + return reportID, err +} + +func (r *reportedEventsStatements) SelectReportedEvents( + ctx context.Context, + txn *sql.Tx, + from, limit uint64, + backwards bool, + reportingUserID types.EventStateKeyNID, + roomNID types.RoomNID, +) ([]api.QueryAdminEventReportsResponse, int64, error) { + + var stmt *sql.Stmt + if backwards { + stmt = sqlutil.TxStmt(txn, r.selectReportedEventsDescStmt) + } else { + stmt = sqlutil.TxStmt(txn, r.selectReportedEventsAscStmt) + } + + var qryRoomNID *types.RoomNID + if roomNID > 0 { + qryRoomNID = &roomNID + } + var qryReportingUser *types.EventStateKeyNID + if reportingUserID > 0 { + qryReportingUser = &reportingUserID + } + + rows, err := stmt.QueryContext(ctx, + qryRoomNID, + qryReportingUser, + limit, + from, + ) + if err != nil { + return nil, 0, err + } + defer internal.CloseAndLogIfError(ctx, rows, "SelectReportedEvents: failed to close rows") + + var result []api.QueryAdminEventReportsResponse + var row api.QueryAdminEventReportsResponse + var count int64 + for rows.Next() { + if err = rows.Scan( + &count, + &row.ID, + &row.RoomNID, + &row.EventNID, + &row.ReportingUserNID, + &row.SenderNID, + &row.Reason, + &row.Score, + &row.ReceivedTS, + ); err != nil { + return nil, 0, err + } + result = append(result, row) + } + + return result, count, rows.Err() +} + +func (r *reportedEventsStatements) SelectReportedEvent( + ctx context.Context, + txn *sql.Tx, + reportID uint64, +) (api.QueryAdminEventReportResponse, error) { + stmt := sqlutil.TxStmt(txn, r.selectReportedEventStmt) + + var row api.QueryAdminEventReportResponse + if err := stmt.QueryRowContext(ctx, reportID).Scan( + &row.ID, + &row.RoomNID, + &row.EventNID, + &row.ReportingUserNID, + &row.SenderNID, + &row.Reason, + &row.Score, + &row.ReceivedTS, + ); err != nil { + return api.QueryAdminEventReportResponse{}, err + } + return row, nil +} + +func (r *reportedEventsStatements) DeleteReportedEvent(ctx context.Context, txn *sql.Tx, reportID uint64) error { + stmt := sqlutil.TxStmt(txn, r.deleteReportedEventStmt) + _, err := stmt.ExecContext(ctx, reportID) + return err +} diff --git a/roomserver/storage/sqlite3/room_aliases_table.go b/roomserver/storage/sqlite3/room_aliases_table.go index 3bdbbaa35..815b42a27 100644 --- a/roomserver/storage/sqlite3/room_aliases_table.go +++ b/roomserver/storage/sqlite3/room_aliases_table.go @@ -121,7 +121,7 @@ func (s *roomAliasesStatements) SelectAliasesFromRoomID( aliases = append(aliases, alias) } - + err = rows.Err() return } diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index 7556b3461..5034b2425 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -65,9 +65,6 @@ const selectRoomVersionsForRoomNIDsSQL = "" + const selectRoomInfoSQL = "" + "SELECT room_version, room_nid, state_snapshot_nid, latest_event_nids FROM roomserver_rooms WHERE room_id = $1" -const selectRoomIDsSQL = "" + - "SELECT room_id FROM roomserver_rooms WHERE latest_event_nids != '[]'" - const bulkSelectRoomIDsSQL = "" + "SELECT room_id FROM roomserver_rooms WHERE room_nid IN ($1)" @@ -87,7 +84,6 @@ type roomStatements struct { updateLatestEventNIDsStmt *sql.Stmt //selectRoomVersionForRoomNIDStmt *sql.Stmt selectRoomInfoStmt *sql.Stmt - selectRoomIDsStmt *sql.Stmt } func CreateRoomsTable(db *sql.DB) error { @@ -108,29 +104,10 @@ func PrepareRoomsTable(db *sql.DB) (tables.Rooms, error) { {&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL}, //{&s.selectRoomVersionForRoomNIDsStmt, selectRoomVersionForRoomNIDsSQL}, {&s.selectRoomInfoStmt, selectRoomInfoSQL}, - {&s.selectRoomIDsStmt, selectRoomIDsSQL}, {&s.selectRoomNIDForUpdateStmt, selectRoomNIDForUpdateSQL}, }.Prepare(db) } -func (s *roomStatements) SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error) { - stmt := sqlutil.TxStmt(txn, s.selectRoomIDsStmt) - rows, err := stmt.QueryContext(ctx) - if err != nil { - return nil, err - } - defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsStmt: rows.close() failed") - var roomIDs []string - var roomID string - for rows.Next() { - if err = rows.Scan(&roomID); err != nil { - return nil, err - } - roomIDs = append(roomIDs, roomID) - } - return roomIDs, nil -} - func (s *roomStatements) SelectRoomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) { var info types.RoomInfo var latestNIDsJSON string @@ -265,7 +242,7 @@ func (s *roomStatements) SelectRoomVersionsForRoomNIDs( } result[roomNID] = roomVersion } - return result, nil + return result, rows.Err() } func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error) { @@ -293,7 +270,7 @@ func (s *roomStatements) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roo } roomIDs = append(roomIDs, roomID) } - return roomIDs, nil + return roomIDs, rows.Err() } func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error) { @@ -321,5 +298,5 @@ func (s *roomStatements) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, ro } roomNIDs = append(roomNIDs, roomNID) } - return roomNIDs, nil + return roomNIDs, rows.Err() } diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index 2edff0ba8..dcac0b07c 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -133,13 +133,16 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs( var stateBlockNIDsJSON string for ; rows.Next(); i++ { result := &results[i] - if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil { + if err = rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil { return nil, err } - if err := json.Unmarshal([]byte(stateBlockNIDsJSON), &result.StateBlockNIDs); err != nil { + if err = json.Unmarshal([]byte(stateBlockNIDsJSON), &result.StateBlockNIDs); err != nil { return nil, err } } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(stateNIDs) { return nil, types.MissingStateError(fmt.Sprintf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs))) } diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 98d88f923..191c07223 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -141,7 +141,9 @@ func (d *Database) create(db *sql.DB) error { if err := CreateUserRoomKeysTable(db); err != nil { return err } - + if err := CreateReportedEventsTable(db); err != nil { + return err + } return nil } @@ -206,6 +208,10 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room if err != nil { return err } + reportedEvents, err := PrepareReportedEventsTable(db) + if err != nil { + return err + } d.Database = shared.Database{ DB: db, @@ -219,6 +225,7 @@ func (d *Database) prepare(db *sql.DB, writer sqlutil.Writer, cache caching.Room EventJSONTable: eventJSON, PrevEventsTable: prevEvents, RedactionsTable: redactions, + ReportedEventsTable: reportedEvents, }, Cache: cache, Writer: writer, diff --git a/roomserver/storage/sqlite3/user_room_keys_table.go b/roomserver/storage/sqlite3/user_room_keys_table.go index 434bad295..ef3b8fe20 100644 --- a/roomserver/storage/sqlite3/user_room_keys_table.go +++ b/roomserver/storage/sqlite3/user_room_keys_table.go @@ -177,6 +177,10 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, if errors.Is(err, sql.ErrNoRows) { return nil, nil } + if err != nil { + return nil, err + } + defer internal.CloseAndLogIfError(ctx, rows, "SelectAllPublicKeysForUser: failed to close rows") resultMap := make(map[types.RoomNID]ed25519.PublicKey) @@ -188,5 +192,5 @@ func (s *userRoomKeysStatements) SelectAllPublicKeysForUser(ctx context.Context, } resultMap[roomNID] = pubkey } - return resultMap, err + return resultMap, rows.Err() } diff --git a/roomserver/storage/tables/events_table_test.go b/roomserver/storage/tables/events_table_test.go index 5ed805648..52aeacc2f 100644 --- a/roomserver/storage/tables/events_table_test.go +++ b/roomserver/storage/tables/events_table_test.go @@ -2,6 +2,7 @@ package tables_test import ( "context" + "fmt" "testing" "github.com/matrix-org/dendrite/internal/sqlutil" @@ -147,3 +148,38 @@ func Test_EventsTable(t *testing.T) { assert.Equal(t, int64(len(room.Events())+1), maxDepth) }) } + +func TestRoomsWithACL(t *testing.T) { + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + eventStateKeys, closeEventStateKeys := mustCreateEventTypesTable(t, dbType) + defer closeEventStateKeys() + + eventsTable, closeEventsTable := mustCreateEventsTable(t, dbType) + defer closeEventsTable() + + ctx := context.Background() + + // insert the m.room.server_acl event type + eventTypeNID, err := eventStateKeys.InsertEventTypeNID(ctx, nil, "m.room.server_acl") + assert.Nil(t, err) + + // Create ACL'd rooms + var wantRoomNIDs []types.RoomNID + for i := 0; i < 10; i++ { + _, _, err = eventsTable.InsertEvent(ctx, nil, types.RoomNID(i), eventTypeNID, types.EmptyStateKeyNID, fmt.Sprintf("$1337+%d", i), nil, 0, false) + assert.Nil(t, err) + wantRoomNIDs = append(wantRoomNIDs, types.RoomNID(i)) + } + + // Create non-ACL'd rooms (eventTypeNID+1) + for i := 10; i < 20; i++ { + _, _, err = eventsTable.InsertEvent(ctx, nil, types.RoomNID(i), eventTypeNID+1, types.EmptyStateKeyNID, fmt.Sprintf("$1337+%d", i), nil, 0, false) + assert.Nil(t, err) + } + + gotRoomNIDs, err := eventsTable.SelectRoomsWithEventTypeNID(ctx, nil, eventTypeNID) + assert.Nil(t, err) + assert.Equal(t, wantRoomNIDs, gotRoomNIDs) + }) +} diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index 0ae064e6b..02f6992c4 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -6,6 +6,7 @@ import ( "database/sql" "errors" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/tidwall/gjson" @@ -69,6 +70,8 @@ type Events interface { SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) SelectRoomNIDsForEventNIDs(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (roomNIDs map[types.EventNID]types.RoomNID, err error) SelectEventRejected(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, eventID string) (rejected bool, err error) + + SelectRoomsWithEventTypeNID(ctx context.Context, txn *sql.Tx, eventTypeNID types.EventTypeNID) ([]types.RoomNID, error) } type Rooms interface { @@ -80,7 +83,6 @@ type Rooms interface { UpdateLatestEventNIDs(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, eventNIDs []types.EventNID, lastEventSentNID types.EventNID, stateSnapshotNID types.StateSnapshotNID) error SelectRoomVersionsForRoomNIDs(ctx context.Context, txn *sql.Tx, roomNID []types.RoomNID) (map[types.RoomNID]gomatrixserverlib.RoomVersion, error) SelectRoomInfo(ctx context.Context, txn *sql.Tx, roomID string) (*types.RoomInfo, error) - SelectRoomIDsWithEvents(ctx context.Context, txn *sql.Tx) ([]string, error) BulkSelectRoomIDs(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID) ([]string, error) BulkSelectRoomNIDs(ctx context.Context, txn *sql.Tx, roomIDs []string) ([]types.RoomNID, error) } @@ -126,6 +128,33 @@ type Invites interface { SelectInviteActiveForUserInRoom(ctx context.Context, txn *sql.Tx, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID) ([]types.EventStateKeyNID, []string, []byte, error) } +type ReportedEvents interface { + InsertReportedEvent( + ctx context.Context, + txn *sql.Tx, + roomNID types.RoomNID, + eventNID types.EventNID, + reportingUserID types.EventStateKeyNID, + eventSenderID types.EventStateKeyNID, + reason string, + score int64, + ) (int64, error) + SelectReportedEvents( + ctx context.Context, + txn *sql.Tx, + from, limit uint64, + backwards bool, + reportingUserID types.EventStateKeyNID, + roomNID types.RoomNID, + ) ([]api.QueryAdminEventReportsResponse, int64, error) + SelectReportedEvent( + ctx context.Context, + txn *sql.Tx, + reportID uint64, + ) (api.QueryAdminEventReportResponse, error) + DeleteReportedEvent(ctx context.Context, txn *sql.Tx, reportID uint64) error +} + type MembershipState int64 const ( @@ -235,6 +264,10 @@ func ExtractContentValue(ev *types.HeaderedEvent) string { key = "topic" case "m.room.guest_access": key = "guest_access" + case "m.room.server_acl": + // We need the entire content and not only one key, so we can use it + // on startup to generate the ACLs. This is merely a workaround. + return string(content) } result := gjson.GetBytes(content, key) if !result.Exists() { diff --git a/roomserver/storage/tables/interface_test.go b/roomserver/storage/tables/interface_test.go new file mode 100644 index 000000000..8727e2436 --- /dev/null +++ b/roomserver/storage/tables/interface_test.go @@ -0,0 +1,76 @@ +package tables + +import ( + "testing" + + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/gomatrixserverlib/spec" + "github.com/stretchr/testify/assert" +) + +func TestExtractContentValue(t *testing.T) { + alice := test.NewUser(t) + room := test.NewRoom(t, alice) + + tests := []struct { + name string + event *types.HeaderedEvent + want string + }{ + { + name: "returns creator ID for create events", + event: room.Events()[0], + want: alice.ID, + }, + { + name: "returns the alias for canonical alias events", + event: room.CreateEvent(t, alice, spec.MRoomCanonicalAlias, map[string]string{"alias": "#test:test"}), + want: "#test:test", + }, + { + name: "returns the history_visibility for history visibility events", + event: room.CreateEvent(t, alice, spec.MRoomHistoryVisibility, map[string]string{"history_visibility": "shared"}), + want: "shared", + }, + { + name: "returns the join rules for join_rules events", + event: room.CreateEvent(t, alice, spec.MRoomJoinRules, map[string]string{"join_rule": "public"}), + want: "public", + }, + { + name: "returns the membership for room_member events", + event: room.CreateEvent(t, alice, spec.MRoomMember, map[string]string{"membership": "join"}, test.WithStateKey(alice.ID)), + want: "join", + }, + { + name: "returns the room name for room_name events", + event: room.CreateEvent(t, alice, spec.MRoomName, map[string]string{"name": "testing"}, test.WithStateKey(alice.ID)), + want: "testing", + }, + { + name: "returns the room avatar for avatar events", + event: room.CreateEvent(t, alice, spec.MRoomAvatar, map[string]string{"url": "mxc://testing"}, test.WithStateKey(alice.ID)), + want: "mxc://testing", + }, + { + name: "returns the room topic for topic events", + event: room.CreateEvent(t, alice, spec.MRoomTopic, map[string]string{"topic": "testing"}, test.WithStateKey(alice.ID)), + want: "testing", + }, + { + name: "returns guest_access for guest access events", + event: room.CreateEvent(t, alice, "m.room.guest_access", map[string]string{"guest_access": "forbidden"}, test.WithStateKey(alice.ID)), + want: "forbidden", + }, + { + name: "returns empty string if key can't be found or unknown event", + event: room.CreateEvent(t, alice, "idontexist", nil), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, ExtractContentValue(tt.event), "ExtractContentValue(%v)", tt.event) + }) + } +} diff --git a/roomserver/storage/tables/rooms_table_test.go b/roomserver/storage/tables/rooms_table_test.go index eddd012c8..e97e3e339 100644 --- a/roomserver/storage/tables/rooms_table_test.go +++ b/roomserver/storage/tables/rooms_table_test.go @@ -74,11 +74,6 @@ func TestRoomsTable(t *testing.T) { assert.NoError(t, err) assert.Nil(t, roomInfo) - // There are no rooms with latestEventNIDs yet - roomIDs, err := tab.SelectRoomIDsWithEvents(ctx, nil) - assert.NoError(t, err) - assert.Equal(t, 0, len(roomIDs)) - roomVersions, err := tab.SelectRoomVersionsForRoomNIDs(ctx, nil, []types.RoomNID{wantRoomNID, 1337}) assert.NoError(t, err) assert.Equal(t, roomVersions[wantRoomNID], room.Version) @@ -86,7 +81,7 @@ func TestRoomsTable(t *testing.T) { _, ok := roomVersions[1337] assert.False(t, ok) - roomIDs, err = tab.BulkSelectRoomIDs(ctx, nil, []types.RoomNID{wantRoomNID, 1337}) + roomIDs, err := tab.BulkSelectRoomIDs(ctx, nil, []types.RoomNID{wantRoomNID, 1337}) assert.NoError(t, err) assert.Equal(t, []string{room.ID}, roomIDs) diff --git a/setup/base/base.go b/setup/base/base.go index ea342054c..26615fc08 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -28,13 +28,13 @@ import ( _ "net/http/pprof" "os" "os/signal" + "sync/atomic" "syscall" "time" sentryhttp "github.com/getsentry/sentry-go/http" "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/prometheus/client_golang/prometheus/promhttp" - "go.uber.org/atomic" "github.com/gorilla/mux" "github.com/kardianos/minwinsvc" @@ -50,6 +50,10 @@ import ( //go:embed static/*.gotmpl var staticContent embed.FS +//go:embed static/client/login +var loginFallback embed.FS +var StaticContent = staticContent + const HTTPServerTimeout = time.Minute * 5 // CreateClient creates a new client (normally used for media fetch requests). @@ -158,6 +162,14 @@ func SetupAndServeHTTP( _, _ = w.Write(landingPage.Bytes()) }) + // We only need the files beneath the static/client/login folder. + sub, err := fs.Sub(loginFallback, "static/client/login") + if err != nil { + logrus.Panicf("unable to read embedded files, this should never happen: %s", err) + } + // Serve a static page for login fallback + routers.Static.PathPrefix("/client/login/").Handler(http.StripPrefix("/_matrix/static/client/login/", http.FileServer(http.FS(sub)))) + var clientHandler http.Handler clientHandler = routers.Client if cfg.Global.Sentry.Enabled { @@ -224,7 +236,6 @@ func SetupAndServeHTTP( logrus.WithError(err).Fatal("failed to serve unix socket") } } - } else { if err := externalServ.ListenAndServe(); err != nil { if err != http.ErrServerClosed { diff --git a/setup/base/sanity_other.go b/setup/base/sanity_other.go index d35c2e872..38e2b941f 100644 --- a/setup/base/sanity_other.go +++ b/setup/base/sanity_other.go @@ -1,5 +1,5 @@ -//go:build !linux && !darwin && !netbsd && !freebsd && !openbsd && !solaris && !dragonfly && !aix -// +build !linux,!darwin,!netbsd,!freebsd,!openbsd,!solaris,!dragonfly,!aix +//go:build !unix +// +build !unix package base diff --git a/setup/base/sanity_unix.go b/setup/base/sanity_unix.go index 0403df1a8..90e38a6db 100644 --- a/setup/base/sanity_unix.go +++ b/setup/base/sanity_unix.go @@ -1,5 +1,5 @@ -//go:build linux || darwin || netbsd || freebsd || openbsd || solaris || dragonfly || aix -// +build linux darwin netbsd freebsd openbsd solaris dragonfly aix +//go:build unix +// +build unix package base diff --git a/setup/base/static/client/login/index.html b/setup/base/static/client/login/index.html new file mode 100644 index 000000000..7d3b109a1 --- /dev/null +++ b/setup/base/static/client/login/index.html @@ -0,0 +1,47 @@ + + + + + Login + + + + + + + +
+

+ + + +
+ +
+ + + + + + +
+ + diff --git a/setup/base/static/client/login/js/jquery-3.4.1.min.js b/setup/base/static/client/login/js/jquery-3.4.1.min.js new file mode 100644 index 000000000..a1c07fd80 --- /dev/null +++ b/setup/base/static/client/login/js/jquery-3.4.1.min.js @@ -0,0 +1,2 @@ +/*! jQuery v3.4.1 | (c) JS Foundation and other contributors | jquery.org/license */ +!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],E=C.document,r=Object.getPrototypeOf,s=t.slice,g=t.concat,u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType},x=function(e){return null!=e&&e===e.window},c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.4.1",k=function(e,t){return new k.fn.init(e,t)},p=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;function d(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp($),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+$),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\([\\da-f]{1,6}"+M+"?|("+M+")|.)","ig"),ne=function(e,t,n){var r="0x"+t-65536;return r!=r||n?t:r<0?String.fromCharCode(r+65536):String.fromCharCode(r>>10|55296,1023&r|56320)},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(m.childNodes),m.childNodes),t[m.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&((e?e.ownerDocument||e:m)!==C&&T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!A[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&U.test(t)){(s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=k),o=(l=h(t)).length;while(o--)l[o]="#"+s+" "+xe(l[o]);c=l.join(","),f=ee.test(t)&&ye(e.parentNode)||e}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){A(t,!0)}finally{s===k&&e.removeAttribute("id")}}}return g(t.replace(B,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[k]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e.namespaceURI,n=(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:m;return r!==C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),m!==C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=k,!C.getElementsByName||!C.getElementsByName(k).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){a.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+k+"-]").length||v.push("~="),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+k+"+*").length||v.push(".#.+[+~]")}),ce(function(e){e.innerHTML="";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",$)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)===(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e===C||e.ownerDocument===m&&y(m,e)?-1:t===C||t.ownerDocument===m&&y(m,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e===C?-1:t===C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]===m?-1:s[r]===m?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if((e.ownerDocument||e)!==C&&T(e),d.matchesSelector&&E&&!A[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){A(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=p[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&p(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,n,r){return m(n)?k.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?k.grep(e,function(e){return e===n!==r}):"string"!=typeof n?k.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(k.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||q,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:L.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof k?t[0]:t,k.merge(this,k.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),D.test(r[1])&&k.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(k):k.makeArray(e,this)}).prototype=k.fn,q=k(E);var H=/^(?:parents|prev(?:Until|All))/,O={children:!0,contents:!0,next:!0,prev:!0};function P(e,t){while((e=e[t])&&1!==e.nodeType);return e}k.fn.extend({has:function(e){var t=k(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i,ge={option:[1,""],thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?k.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;nx",y.noCloneChecked=!!me.cloneNode(!0).lastChild.defaultValue;var Te=/^key/,Ce=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,Ee=/^([^.]*)(?:\.(.+)|)/;function ke(){return!0}function Se(){return!1}function Ne(e,t){return e===function(){try{return E.activeElement}catch(e){}}()==("focus"===t)}function Ae(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"!=typeof n&&(r=r||n,n=void 0),t)Ae(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=Se;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return k().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=k.guid++)),e.each(function(){k.event.add(this,t,i,r,n)})}function De(e,i,o){o?(Q.set(e,i,!1),k.event.add(e,i,{namespace:!1,handler:function(e){var t,n,r=Q.get(this,i);if(1&e.isTrigger&&this[i]){if(r.length)(k.event.special[i]||{}).delegateType&&e.stopPropagation();else if(r=s.call(arguments),Q.set(this,i,r),t=o(this,i),this[i](),r!==(n=Q.get(this,i))||t?Q.set(this,i,!1):n={},r!==n)return e.stopImmediatePropagation(),e.preventDefault(),n.value}else r.length&&(Q.set(this,i,{value:k.event.trigger(k.extend(r[0],k.Event.prototype),r.slice(1),this)}),e.stopImmediatePropagation())}})):void 0===Q.get(e,i)&&k.event.add(e,i,ke)}k.event={global:{},add:function(t,e,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Q.get(t);if(v){n.handler&&(n=(o=n).handler,i=o.selector),i&&k.find.matchesSelector(ie,i),n.guid||(n.guid=k.guid++),(u=v.events)||(u=v.events={}),(a=v.handle)||(a=v.handle=function(e){return"undefined"!=typeof k&&k.event.triggered!==e.type?k.event.dispatch.apply(t,arguments):void 0}),l=(e=(e||"").match(R)||[""]).length;while(l--)d=g=(s=Ee.exec(e[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=k.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=k.event.special[d]||{},c=k.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&k.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(t,r,h,a)||t.addEventListener&&t.addEventListener(d,a)),f.add&&(f.add.call(t,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),k.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Q.hasData(e)&&Q.get(e);if(v&&(u=v.events)){l=(t=(t||"").match(R)||[""]).length;while(l--)if(d=g=(s=Ee.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d){f=k.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,v.handle)||k.removeEvent(e,d,v.handle),delete u[d])}else for(d in u)k.event.remove(e,d+t[l],n,r,!0);k.isEmptyObject(u)&&Q.remove(e,"handle events")}},dispatch:function(e){var t,n,r,i,o,a,s=k.event.fix(e),u=new Array(arguments.length),l=(Q.get(this,"events")||{})[s.type]||[],c=k.event.special[s.type]||{};for(u[0]=s,t=1;t\x20\t\r\n\f]*)[^>]*)\/>/gi,qe=/\s*$/g;function Oe(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&k(e).children("tbody")[0]||e}function Pe(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function Re(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Me(e,t){var n,r,i,o,a,s,u,l;if(1===t.nodeType){if(Q.hasData(e)&&(o=Q.access(e),a=Q.set(t,o),l=o.events))for(i in delete a.handle,a.events={},l)for(n=0,r=l[i].length;n")},clone:function(e,t,n){var r,i,o,a,s,u,l,c=e.cloneNode(!0),f=oe(e);if(!(y.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||k.isXMLDoc(e)))for(a=ve(c),r=0,i=(o=ve(e)).length;r").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var Vt,Gt=[],Yt=/(=)\?(?=&|$)|\?\?/;k.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Gt.pop()||k.expando+"_"+kt++;return this[e]=!0,e}}),k.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Yt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Yt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Yt,"$1"+r):!1!==e.jsonp&&(e.url+=(St.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||k.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?k(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Gt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((Vt=E.implementation.createHTMLDocument("").body).innerHTML="
",2===Vt.childNodes.length),k.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=D.exec(e))?[t.createElement(i[1])]:(i=we([e],t,o),o&&o.length&&k(o).remove(),k.merge([],i.childNodes)));var r,i,o},k.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1").append(k.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},k.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){k.fn[t]=function(e){return this.on(t,e)}}),k.expr.pseudos.animated=function(t){return k.grep(k.timers,function(e){return t===e.elem}).length},k.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=k.css(e,"position"),c=k(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=k.css(e,"top"),u=k.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,k.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},k.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){k.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===k.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===k.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=k(e).offset()).top+=k.css(e,"borderTopWidth",!0),i.left+=k.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-k.css(r,"marginTop",!0),left:t.left-i.left-k.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===k.css(e,"position"))e=e.offsetParent;return e||ie})}}),k.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;k.fn[t]=function(e){return _(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),k.each(["top","left"],function(e,n){k.cssHooks[n]=ze(y.pixelPosition,function(e,t){if(t)return t=_e(e,n),$e.test(t)?k(e).position()[n]+"px":t})}),k.each({Height:"height",Width:"width"},function(a,s){k.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){k.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return _(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?k.css(e,t,i):k.style(e,t,n,i)},s,n?e:void 0,n)}})}),k.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){k.fn[n]=function(e,t){return 0 * { + display: block; + margin-left: auto; + margin-right: auto; + text-align: center; +} + +/* + * A wrapper around each login flow. + */ +.login_flow { + width: 300px; + text-align: left; + padding: 10px; + margin-bottom: 40px; + + border-radius: 10px; + box-shadow: 0px 0px 20px 0px rgba(0,0,0,0.15); + + background-color: #f8f8f8; + border: 1px #ccc solid; +} + +/* + * Used to show error content. + */ +#feedback { + /* Red text. */ + color: #ff0000; + /* A little space to not overlap the box-shadow. */ + margin-bottom: 20px; +} diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go index ef10649d2..a95cec046 100644 --- a/setup/config/config_appservice.go +++ b/setup/config/config_appservice.go @@ -40,6 +40,9 @@ type AppServiceAPI struct { // on appservice endpoints. This is not recommended in production! DisableTLSValidation bool `yaml:"disable_tls_validation"` + LegacyAuth bool `yaml:"legacy_auth"` + LegacyPaths bool `yaml:"legacy_paths"` + ConfigFiles []string `yaml:"config_files"` } diff --git a/setup/config/config_federationapi.go b/setup/config/config_federationapi.go index a72eee369..073c46e03 100644 --- a/setup/config/config_federationapi.go +++ b/setup/config/config_federationapi.go @@ -18,6 +18,13 @@ type FederationAPI struct { // The default value is 16 if not specified, which is circa 18 hours. FederationMaxRetries uint32 `yaml:"send_max_retries"` + // P2P Feature: Whether relaying to specific nodes should be enabled. + // Defaults to false. + // Note: Enabling relays introduces a huge startup delay, if you are not using + // relays and have many servers to re-hydrate on start. Only enable this + // if you are using relays! + EnableRelays bool `yaml:"enable_relays"` + // P2P Feature: How many consecutive failures that we should tolerate when // sending federation requests to a specific server until we should assume they // are offline. If we assume they are offline then we will attempt to send diff --git a/setup/config/config_global.go b/setup/config/config_global.go index 5b4ccf400..3234dadb4 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -114,6 +114,11 @@ func (c *Global) Verify(configErrs *ConfigErrors) { checkNotEmpty(configErrs, "global.server_name", string(c.ServerName)) checkNotEmpty(configErrs, "global.private_key", string(c.PrivateKeyPath)) + // Check that client well-known has a proper format + if c.WellKnownClientName != "" && !strings.HasPrefix(c.WellKnownClientName, "http://") && !strings.HasPrefix(c.WellKnownClientName, "https://") { + configErrs.Add("The configuration for well_known_client_name does not have a proper format, consider adding http:// or https://. Some clients may fail to connect.") + } + for _, v := range c.VirtualHosts { v.Verify(configErrs) } diff --git a/setup/config/config_jetstream.go b/setup/config/config_jetstream.go index b8abed25c..a048e4d09 100644 --- a/setup/config/config_jetstream.go +++ b/setup/config/config_jetstream.go @@ -21,6 +21,9 @@ type JetStream struct { NoLog bool `yaml:"-"` // Disables TLS validation. This should NOT be used in production DisableTLSValidation bool `yaml:"disable_tls_validation"` + // A credentials file to be used for authentication, example: + // https://docs.nats.io/using-nats/developer/connecting/creds + Credentials Path `yaml:"credentials_path"` } func (c *JetStream) Prefixed(name string) string { @@ -38,6 +41,7 @@ func (c *JetStream) Defaults(opts DefaultOpts) { c.StoragePath = Path("./") c.NoLog = true c.DisableTLSValidation = true + c.Credentials = Path("") } } diff --git a/setup/config/config_test.go b/setup/config/config_test.go index 8a65c990f..eeefb425f 100644 --- a/setup/config/config_test.go +++ b/setup/config/config_test.go @@ -54,7 +54,7 @@ global: key_id: ed25519:auto key_validity_period: 168h0m0s well_known_server_name: "localhost:443" - well_known_client_name: "localhost:443" + well_known_client_name: "https://localhost" trusted_third_party_id_servers: - matrix.org - vector.im diff --git a/setup/config/config_userapi.go b/setup/config/config_userapi.go index e64a3910c..559de72ac 100644 --- a/setup/config/config_userapi.go +++ b/setup/config/config_userapi.go @@ -21,6 +21,10 @@ type UserAPI struct { // Users who register on this homeserver will automatically // be joined to the rooms listed under this option. AutoJoinRooms []string `yaml:"auto_join_rooms"` + + // The number of workers to start for the DeviceListUpdater. Defaults to 8. + // This only needs updating if the "InputDeviceListUpdate" stream keeps growing indefinitely. + WorkerCount int `yaml:"worker_count"` } const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes @@ -28,6 +32,7 @@ const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes func (c *UserAPI) Defaults(opts DefaultOpts) { c.BCryptCost = bcrypt.DefaultCost c.OpenIDTokenLifetimeMS = DefaultOpenIDTokenLifetimeMS + c.WorkerCount = 8 if opts.Generate { if !opts.SingleDatabase { c.AccountDatabase.ConnectionString = "file:userapi_accounts.db" diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go index 8820e86b2..09048cc94 100644 --- a/setup/jetstream/nats.go +++ b/setup/jetstream/nats.go @@ -38,7 +38,12 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS defer natsLock.Unlock() // check if we need an in-process NATS Server if len(cfg.Addresses) != 0 { - return setupNATS(process, cfg, nil) + // reuse existing connections + if s.nc != nil { + return s.js, s.nc + } + s.js, s.nc = setupNATS(process, cfg, nil) + return s.js, s.nc } if s.Server == nil { var err error @@ -51,6 +56,7 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS MaxPayload: 16 * 1024 * 1024, NoSigs: true, NoLog: cfg.NoLog, + SyncAlways: true, } s.Server, err = natsserver.NewServer(opts) if err != nil { @@ -97,6 +103,9 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc InsecureSkipVerify: true, })) } + if string(cfg.Credentials) != "" { + opts = append(opts, natsclient.UserCredentials(string(cfg.Credentials))) + } nc, err = natsclient.Connect(strings.Join(cfg.Addresses, ","), opts...) if err != nil { logrus.WithError(err).Panic("Unable to connect to NATS") diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go index 741407926..1dc9f4cec 100644 --- a/setup/jetstream/streams.go +++ b/setup/jetstream/streams.go @@ -20,6 +20,7 @@ var ( InputDeviceListUpdate = "InputDeviceListUpdate" InputSigningKeyUpdate = "InputSigningKeyUpdate" OutputRoomEvent = "OutputRoomEvent" + OutputAppserviceEvent = "OutputAppserviceEvent" OutputSendToDeviceEvent = "OutputSendToDeviceEvent" OutputKeyChangeEvent = "OutputKeyChangeEvent" OutputTypingEvent = "OutputTypingEvent" @@ -65,6 +66,11 @@ var streams = []*nats.StreamConfig{ Retention: nats.InterestPolicy, Storage: nats.FileStorage, }, + { + Name: OutputAppserviceEvent, + Retention: nats.InterestPolicy, + Storage: nats.FileStorage, + }, { Name: OutputSendToDeviceEvent, Retention: nats.InterestPolicy, diff --git a/setup/monolith.go b/setup/monolith.go index 4856d6e83..72750354b 100644 --- a/setup/monolith.go +++ b/setup/monolith.go @@ -78,7 +78,7 @@ func (m *Monolith) AddAllPublicRoutes( federationapi.AddPublicRoutes( processCtx, routers, cfg, natsInstance, m.UserAPI, m.FedClient, m.KeyRing, m.RoomserverAPI, m.FederationAPI, enableMetrics, ) - mediaapi.AddPublicRoutes(routers.Media, cm, cfg, m.UserAPI, m.Client) + mediaapi.AddPublicRoutes(routers, cm, cfg, m.UserAPI, m.Client, m.FedClient, m.KeyRing) syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, natsInstance, m.UserAPI, m.RoomserverAPI, caches, enableMetrics) if m.RelayAPI != nil { diff --git a/setup/mscs/msc2836/storage.go b/setup/mscs/msc2836/storage.go index ade2a1616..696d0b0da 100644 --- a/setup/mscs/msc2836/storage.go +++ b/setup/mscs/msc2836/storage.go @@ -301,7 +301,7 @@ func (p *DB) ChildrenForParent(ctx context.Context, eventID, relType string, rec } children = append(children, evInfo) } - return children, nil + return children, rows.Err() } func (p *DB) ParentForChild(ctx context.Context, eventID, relType string) (*eventInfo, error) { diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 666f900d7..abf888829 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -31,6 +31,7 @@ import ( "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/syncapi/notifier" + "github.com/matrix-org/dendrite/syncapi/producers" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/streams" "github.com/matrix-org/dendrite/syncapi/synctypes" @@ -55,6 +56,7 @@ type OutputRoomEventConsumer struct { inviteStream streams.StreamProvider notifier *notifier.Notifier fts fulltext.Indexer + asProducer *producers.AppserviceEventProducer } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -68,6 +70,7 @@ func NewOutputRoomEventConsumer( inviteStream streams.StreamProvider, rsAPI api.SyncRoomserverAPI, fts *fulltext.Search, + asProducer *producers.AppserviceEventProducer, ) *OutputRoomEventConsumer { return &OutputRoomEventConsumer{ ctx: process.Context(), @@ -81,6 +84,7 @@ func NewOutputRoomEventConsumer( inviteStream: inviteStream, rsAPI: rsAPI, fts: fts, + asProducer: asProducer, } } @@ -119,6 +123,11 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms } } err = s.onNewRoomEvent(s.ctx, *output.NewRoomEvent) + if err == nil && s.asProducer != nil { + if err = s.asProducer.ProduceRoomEvents(msg); err != nil { + log.WithError(err).Warn("failed to produce OutputAppserviceEvent") + } + } case api.OutputTypeOldRoomEvent: err = s.onOldRoomEvent(s.ctx, *output.OldRoomEvent) case api.OutputTypeNewInviteEvent: @@ -592,9 +601,11 @@ func (s *OutputRoomEventConsumer) writeFTS(ev *rstypes.HeaderedEvent, pduPositio } e.SetContentType(ev.Type()) + var relatesTo gjson.Result switch ev.Type() { case "m.room.message": e.Content = gjson.GetBytes(ev.Content(), "body").String() + relatesTo = gjson.GetBytes(ev.Content(), "m\\.relates_to") case spec.MRoomName: e.Content = gjson.GetBytes(ev.Content(), "name").String() case spec.MRoomTopic: @@ -613,6 +624,22 @@ func (s *OutputRoomEventConsumer) writeFTS(ev *rstypes.HeaderedEvent, pduPositio if err := s.fts.Index(e); err != nil { return err } + // If the event is an edited message we remove the original event from the index + // to avoid duplicates in the search results. + if relatesTo.Exists() { + relatedData := relatesTo.Map() + if _, ok := relatedData["rel_type"]; ok && relatedData["rel_type"].Str == "m.replace" { + // We remove the original event from the index + if srcEventID, ok := relatedData["event_id"]; ok { + if err := s.fts.Delete(srcEventID.Str); err != nil { + log.WithFields(log.Fields{ + "event_id": ev.EventID(), + "src_id": srcEventID.Str, + }).WithError(err).Error("Failed to delete edited message from the fulltext index") + } + } + } + } } return nil } diff --git a/syncapi/producers/appservices.go b/syncapi/producers/appservices.go new file mode 100644 index 000000000..bb0dbb9cf --- /dev/null +++ b/syncapi/producers/appservices.go @@ -0,0 +1,33 @@ +// Copyright 2023 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package producers + +import ( + "github.com/nats-io/nats.go" +) + +// AppserviceEventProducer produces events for the appservice API to consume +type AppserviceEventProducer struct { + Topic string + JetStream nats.JetStreamContext +} + +func (a *AppserviceEventProducer) ProduceRoomEvents( + msg *nats.Msg, +) error { + msg.Subject = a.Topic + _, err := a.JetStream.PublishMsg(msg) + return err +} diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go index b0c91c40b..b136c69a0 100644 --- a/syncapi/routing/context.go +++ b/syncapi/routing/context.go @@ -110,6 +110,7 @@ func Context( } stateFilter := synctypes.StateFilter{ + Limit: filter.Limit, NotSenders: filter.NotSenders, NotTypes: filter.NotTypes, Senders: filter.Senders, @@ -157,6 +158,11 @@ func Context( } } + // Limit is split up for before/after events + if filter.Limit > 1 { + filter.Limit = filter.Limit / 2 + } + eventsBefore, err := snapshot.SelectContextBeforeEvent(ctx, id, roomID, filter) if err != nil && err != sql.ErrNoRows { logrus.WithError(err).Error("unable to fetch before events") diff --git a/syncapi/routing/getevent.go b/syncapi/routing/getevent.go index c089539f0..d0227f4ea 100644 --- a/syncapi/routing/getevent.go +++ b/syncapi/routing/getevent.go @@ -44,7 +44,7 @@ func GetEvent( rsAPI api.SyncRoomserverAPI, ) util.JSONResponse { ctx := req.Context() - db, err := syncDB.NewDatabaseTransaction(ctx) + db, err := syncDB.NewDatabaseSnapshot(ctx) logger := util.GetLogger(ctx).WithFields(logrus.Fields{ "event_id": eventID, "room_id": rawRoomID, @@ -56,6 +56,7 @@ func GetEvent( JSON: spec.InternalServerError{}, } } + defer db.Rollback() // nolint: errcheck roomID, err := spec.NewRoomID(rawRoomID) if err != nil { diff --git a/syncapi/routing/memberships.go b/syncapi/routing/memberships.go index e849adf6d..9cc937d88 100644 --- a/syncapi/routing/memberships.go +++ b/syncapi/routing/memberships.go @@ -15,7 +15,6 @@ package routing import ( - "encoding/json" "math" "net/http" @@ -33,31 +32,13 @@ type getMembershipResponse struct { Chunk []synctypes.ClientEvent `json:"chunk"` } -// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members -type getJoinedMembersResponse struct { - Joined map[string]joinedMember `json:"joined"` -} - -type joinedMember struct { - DisplayName string `json:"display_name"` - AvatarURL string `json:"avatar_url"` -} - -// The database stores 'displayname' without an underscore. -// Deserialize into this and then change to the actual API response -type databaseJoinedMember struct { - DisplayName string `json:"displayname"` - AvatarURL string `json:"avatar_url"` -} - // GetMemberships implements // // GET /rooms/{roomId}/members -// GET /rooms/{roomId}/joined_members func GetMemberships( req *http.Request, device *userapi.Device, roomID string, syncDB storage.Database, rsAPI api.SyncRoomserverAPI, - joinedOnly bool, membership, notMembership *string, at string, + membership, notMembership *string, at string, ) util.JSONResponse { userID, err := spec.NewUserID(device.UserID, true) if err != nil { @@ -87,13 +68,6 @@ func GetMemberships( } } - if joinedOnly && !queryRes.IsInRoom { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."), - } - } - db, err := syncDB.NewDatabaseSnapshot(req.Context()) if err != nil { return util.JSONResponse{ @@ -139,40 +113,6 @@ func GetMemberships( result := qryRes.Events - if joinedOnly { - var res getJoinedMembersResponse - res.Joined = make(map[string]joinedMember) - for _, ev := range result { - var content databaseJoinedMember - if err := json.Unmarshal(ev.Content(), &content); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - - userID, err := rsAPI.QueryUserIDForSender(req.Context(), ev.RoomID(), ev.SenderID()) - if err != nil || userID == nil { - util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryUserIDForSender failed") - return util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: spec.InternalServerError{}, - } - } - if err != nil { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: spec.Forbidden("You don't have permission to kick this user, unknown senderID"), - } - } - res.Joined[userID.String()] = joinedMember(content) - } - return util.JSONResponse{ - Code: http.StatusOK, - JSON: res, - } - } return util.JSONResponse{ Code: http.StatusOK, JSON: getMembershipResponse{synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(result), synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) { diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 3333cb54d..7ea01c7dc 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -135,13 +135,6 @@ func OnIncomingMessagesRequest( var fromStream *types.StreamingToken fromQuery := req.URL.Query().Get("from") toQuery := req.URL.Query().Get("to") - emptyFromSupplied := fromQuery == "" - if emptyFromSupplied { - // NOTSPEC: We will pretend they used the latest sync token if no ?from= was provided. - // We do this to allow clients to get messages without having to call `/sync` e.g Cerulean - currPos := srp.Notifier.CurrentPosition() - fromQuery = currPos.String() - } // Direction to return events from. dir := req.URL.Query().Get("dir") @@ -155,6 +148,23 @@ func OnIncomingMessagesRequest( // to have one of the two accepted values (so dir == "f" <=> !backwardOrdering). backwardOrdering := (dir == "b") + emptyFromSupplied := fromQuery == "" + if emptyFromSupplied { + // If "from" isn't provided, it defaults to either the earliest stream + // position (if we're going forward) or to the latest one (if we're + // going backward). + + var from types.TopologyToken + if backwardOrdering { + from = types.TopologyToken{Depth: math.MaxInt64, PDUPosition: math.MaxInt64} + } else { + // go 1 earlier than the first event so we correctly fetch the earliest event + // this is because Database.GetEventsInTopologicalRange is exclusive of the lower-bound. + from = types.TopologyToken{} + } + fromQuery = from.String() + } + from, err := types.NewTopologyTokenFromString(fromQuery) if err != nil { var streamToken types.StreamingToken diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index a837e1696..78188d1b6 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -197,19 +197,7 @@ func Setup( } at := req.URL.Query().Get("at") - return GetMemberships(req, device, vars["roomID"], syncDB, rsAPI, false, membership, notMembership, at) + return GetMemberships(req, device, vars["roomID"], syncDB, rsAPI, membership, notMembership, at) }, httputil.WithAllowGuests()), ).Methods(http.MethodGet, http.MethodOptions) - - v3mux.Handle("/rooms/{roomID}/joined_members", - httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - at := req.URL.Query().Get("at") - membership := spec.Join - return GetMemberships(req, device, vars["roomID"], syncDB, rsAPI, true, &membership, nil, at) - }), - ).Methods(http.MethodGet, http.MethodOptions) } diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index b0148bef5..ec0b27adc 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -392,7 +392,7 @@ func currentRoomStateRowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, er }) } - return events, nil + return events, rows.Err() } func rowsToEvents(rows *sql.Rows) ([]*rstypes.HeaderedEvent, error) { diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go index 4fe4260da..e5208b891 100644 --- a/syncapi/storage/postgres/memberships_table.go +++ b/syncapi/storage/postgres/memberships_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "fmt" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -160,6 +161,7 @@ func (s *membershipsStatements) SelectMemberships( if err != nil { return } + defer internal.CloseAndLogIfError(ctx, rows, "SelectMemberships: failed to close rows") var ( eventID string ) diff --git a/syncapi/storage/postgres/peeks_table.go b/syncapi/storage/postgres/peeks_table.go index 64183073d..1120dce0b 100644 --- a/syncapi/storage/postgres/peeks_table.go +++ b/syncapi/storage/postgres/peeks_table.go @@ -164,7 +164,7 @@ func (s *peekStatements) SelectPeekingDevices( devices = append(devices, types.PeekingDevice{UserID: userID, DeviceID: deviceID}) result[roomID] = devices } - return result, nil + return result, rows.Err() } func (s *peekStatements) SelectMaxPeekID( diff --git a/syncapi/storage/postgres/presence_table.go b/syncapi/storage/postgres/presence_table.go index f37b5331e..53acecce5 100644 --- a/syncapi/storage/postgres/presence_table.go +++ b/syncapi/storage/postgres/presence_table.go @@ -144,7 +144,7 @@ func (p *presenceStatements) GetPresenceForUsers( presence.ClientFields.Presence = presence.Presence.String() result = append(result, presence) } - return result, err + return result, rows.Err() } func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 78b2e397c..f430fcc05 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -177,7 +177,7 @@ func (s *currentRoomStateStatements) SelectJoinedUsers( users = append(users, userID) result[roomID] = users } - return result, nil + return result, rows.Err() } // SelectJoinedUsersInRoom returns a map of room ID to a list of joined user IDs for a given room. @@ -236,7 +236,7 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( } result = append(result, roomID) } - return result, nil + return result, rows.Err() } // SelectRoomIDsWithAnyMembership returns a map of all memberships for the given user. @@ -419,7 +419,7 @@ func currentRoomStateRowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, er }) } - return events, nil + return events, rows.Err() } func rowsToEvents(rows *sql.Rows) ([]*rstypes.HeaderedEvent, error) { diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index ebb469d24..e50b5cbf8 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -176,7 +176,7 @@ func (s *inviteEventsStatements) SelectInviteEventsInRange( if lastPos == 0 { lastPos = r.To } - return result, retired, lastPos, nil + return result, retired, lastPos, rows.Err() } func (s *inviteEventsStatements) SelectMaxInviteID( diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go index a1b16306c..9e50422e5 100644 --- a/syncapi/storage/sqlite3/memberships_table.go +++ b/syncapi/storage/sqlite3/memberships_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "fmt" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -163,6 +164,7 @@ func (s *membershipsStatements) SelectMemberships( if err != nil { return } + defer internal.CloseAndLogIfError(ctx, rows, "SelectMemberships: failed to close rows") var eventID string for rows.Next() { if err = rows.Scan(&eventID); err != nil { diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 93caee806..c7b11d3ef 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -274,7 +274,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( } } - return stateNeeded, eventIDToEvent, nil + return stateNeeded, eventIDToEvent, rows.Err() } // MaxID returns the ID of the last inserted event in this table. 'txn' is optional. If it is not supplied, @@ -520,7 +520,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { ExcludeFromSync: excludeFromSync, }) } - return result, nil + return result, rows.Err() } func (s *outputRoomEventsStatements) SelectContextEvent( ctx context.Context, txn *sql.Tx, roomID, eventID string, diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index 36967d1e7..c00fb7a79 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" rstypes "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/syncapi/storage/tables" @@ -137,6 +138,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( } else if err != nil { return } + defer internal.CloseAndLogIfError(ctx, rows, "SelectEventIDsInRange: failed to close rows") // Return the IDs. var eventID string @@ -155,7 +157,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( start = tokens[0] end = tokens[len(tokens)-1] } - + err = rows.Err() return } diff --git a/syncapi/storage/sqlite3/peeks_table.go b/syncapi/storage/sqlite3/peeks_table.go index 5d5200abc..d8998e2b8 100644 --- a/syncapi/storage/sqlite3/peeks_table.go +++ b/syncapi/storage/sqlite3/peeks_table.go @@ -184,7 +184,7 @@ func (s *peekStatements) SelectPeekingDevices( devices = append(devices, types.PeekingDevice{UserID: userID, DeviceID: deviceID}) result[roomID] = devices } - return result, nil + return result, rows.Err() } func (s *peekStatements) SelectMaxPeekID( diff --git a/syncapi/storage/sqlite3/presence_table.go b/syncapi/storage/sqlite3/presence_table.go index 573fbad6c..40b57e75d 100644 --- a/syncapi/storage/sqlite3/presence_table.go +++ b/syncapi/storage/sqlite3/presence_table.go @@ -169,7 +169,7 @@ func (p *presenceStatements) GetPresenceForUsers( presence.ClientFields.Presence = presence.Presence.String() result = append(result, presence) } - return result, err + return result, rows.Err() } func (p *presenceStatements) GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 3abb0b3c6..790f5bd1b 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -203,6 +203,12 @@ func (p *PDUStreamProvider) IncrementalSync( req.Log.WithError(err).Error("unable to update event filter with ignored users") } + dbEvents, err := p.getRecentEvents(ctx, stateDeltas, r, eventFilter, snapshot) + if err != nil { + req.Log.WithError(err).Error("unable to get recent events") + return r.From + } + newPos = from for _, delta := range stateDeltas { newRange := r @@ -218,7 +224,7 @@ func (p *PDUStreamProvider) IncrementalSync( } } var pos types.StreamPosition - if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req); err != nil { + if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req, dbEvents); err != nil { req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed") if err == context.DeadlineExceeded || err == context.Canceled || err == sql.ErrTxDone { return newPos @@ -240,6 +246,66 @@ func (p *PDUStreamProvider) IncrementalSync( return newPos } +func (p *PDUStreamProvider) getRecentEvents(ctx context.Context, stateDeltas []types.StateDelta, r types.Range, eventFilter synctypes.RoomEventFilter, snapshot storage.DatabaseTransaction) (map[string]types.RecentEvents, error) { + var roomIDs []string + var newlyJoinedRoomIDs []string + for _, delta := range stateDeltas { + if delta.NewlyJoined { + newlyJoinedRoomIDs = append(newlyJoinedRoomIDs, delta.RoomID) + } else { + roomIDs = append(roomIDs, delta.RoomID) + } + } + dbEvents := make(map[string]types.RecentEvents) + if len(roomIDs) > 0 { + events, err := snapshot.RecentEvents( + ctx, roomIDs, r, + &eventFilter, true, true, + ) + if err != nil { + if err != sql.ErrNoRows { + return nil, err + } + } + for k, v := range events { + dbEvents[k] = v + } + } + if len(newlyJoinedRoomIDs) > 0 { + // For rooms that were joined in this sync, try to fetch + // as much timeline events as allowed by the filter. + + filter := eventFilter + // If we're going backwards, grep at least X events, this is mostly to satisfy Sytest + if eventFilter.Limit < recentEventBackwardsLimit { + filter.Limit = recentEventBackwardsLimit // TODO: Figure out a better way + diff := r.From - r.To + if diff > 0 && diff < recentEventBackwardsLimit { + filter.Limit = int(diff) + } + } + + events, err := snapshot.RecentEvents( + ctx, newlyJoinedRoomIDs, types.Range{ + From: r.To, + To: 0, + Backwards: true, + }, + &filter, true, true, + ) + if err != nil { + if err != sql.ErrNoRows { + return nil, err + } + } + for k, v := range events { + dbEvents[k] = v + } + } + + return dbEvents, nil +} + // Limit the recent events to X when going backwards const recentEventBackwardsLimit = 100 @@ -253,29 +319,9 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( eventFilter *synctypes.RoomEventFilter, stateFilter *synctypes.StateFilter, req *types.SyncRequest, + dbEvents map[string]types.RecentEvents, ) (types.StreamPosition, error) { var err error - originalLimit := eventFilter.Limit - // If we're going backwards, grep at least X events, this is mostly to satisfy Sytest - if r.Backwards && originalLimit < recentEventBackwardsLimit { - eventFilter.Limit = recentEventBackwardsLimit // TODO: Figure out a better way - diff := r.From - r.To - if diff > 0 && diff < recentEventBackwardsLimit { - eventFilter.Limit = int(diff) - } - } - - dbEvents, err := snapshot.RecentEvents( - ctx, []string{delta.RoomID}, r, - eventFilter, true, true, - ) - if err != nil { - if err == sql.ErrNoRows { - return r.To, nil - } - return r.From, fmt.Errorf("p.DB.RecentEvents: %w", err) - } - recentStreamEvents := dbEvents[delta.RoomID].Events limited := dbEvents[delta.RoomID].Limited @@ -337,9 +383,9 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( logrus.WithError(err).Error("unable to apply history visibility filter") } - if r.Backwards && len(events) > originalLimit { + if r.Backwards && len(events) > eventFilter.Limit { // We're going backwards and the events are ordered chronologically, so take the last `limit` events - events = events[len(events)-originalLimit:] + events = events[len(events)-eventFilter.Limit:] limited = true } diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 5a92c70e1..494be05f7 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -120,11 +120,34 @@ func (rp *RequestPool) cleanPresence(db storage.Presence, cleanupTime time.Durat } } +// set a unix timestamp of when it last saw the types +// this way it can filter based on time +type PresenceMap struct { + mu sync.Mutex + seen map[string]map[types.Presence]time.Time +} + +var lastPresence PresenceMap + +// how long before the online status expires +// should be long enough that any client will have another sync before expiring +const presenceTimeout = time.Second * 10 + // updatePresence sends presence updates to the SyncAPI and FederationAPI func (rp *RequestPool) updatePresence(db storage.Presence, presence string, userID string) { + // allow checking back on presence to set offline if needed + rp.updatePresenceInternal(db, presence, userID, true) +} + +func (rp *RequestPool) updatePresenceInternal(db storage.Presence, presence string, userID string, checkAgain bool) { if !rp.cfg.Matrix.Presence.EnableOutbound { return } + + // lock the map to this thread + lastPresence.mu.Lock() + defer lastPresence.mu.Unlock() + if presence == "" { presence = types.PresenceOnline.String() } @@ -140,6 +163,41 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user LastActiveTS: spec.AsTimestamp(time.Now()), } + // make sure that the map is defined correctly as needed + if lastPresence.seen == nil { + lastPresence.seen = make(map[string]map[types.Presence]time.Time) + } + if lastPresence.seen[userID] == nil { + lastPresence.seen[userID] = make(map[types.Presence]time.Time) + } + + now := time.Now() + // update time for each presence + lastPresence.seen[userID][presenceID] = now + + // Default to unknown presence + presenceToSet := types.PresenceUnknown + switch { + case now.Sub(lastPresence.seen[userID][types.PresenceOnline]) < presenceTimeout: + // online will always get priority + presenceToSet = types.PresenceOnline + case now.Sub(lastPresence.seen[userID][types.PresenceUnavailable]) < presenceTimeout: + // idle gets secondary priority because your presence shouldnt be idle if you are on a different device + // kinda copying discord presence + presenceToSet = types.PresenceUnavailable + case now.Sub(lastPresence.seen[userID][types.PresenceOffline]) < presenceTimeout: + // only set offline status if there is no known online devices + // clients may set offline to attempt to not alter the online status of the user + presenceToSet = types.PresenceOffline + + if checkAgain { + // after a timeout, check presence again to make sure it gets set as offline sooner or later + time.AfterFunc(presenceTimeout, func() { + rp.updatePresenceInternal(db, types.PresenceOffline.String(), userID, false) + }) + } + } + // ensure we also send the current status_msg to federated servers and not nil dbPresence, err := db.GetPresences(context.Background(), []string{userID}) if err != nil && err != sql.ErrNoRows { @@ -148,7 +206,7 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user if len(dbPresence) > 0 && dbPresence[0] != nil { newPresence.ClientFields = dbPresence[0].ClientFields } - newPresence.ClientFields.Presence = presenceID.String() + newPresence.ClientFields.Presence = presenceToSet.String() defer rp.presence.Store(userID, newPresence) // avoid spamming presence updates when syncing @@ -160,7 +218,7 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user } } - if err := rp.producer.SendPresence(userID, presenceID, newPresence.ClientFields.StatusMsg); err != nil { + if err := rp.producer.SendPresence(userID, presenceToSet, newPresence.ClientFields.StatusMsg); err != nil { logrus.WithError(err).Error("Unable to publish presence message from sync") return } @@ -168,9 +226,10 @@ func (rp *RequestPool) updatePresence(db storage.Presence, presence string, user // now synchronously update our view of the world. It's critical we do this before calculating // the /sync response else we may not return presence: online immediately. rp.consumer.EmitPresence( - context.Background(), userID, presenceID, newPresence.ClientFields.StatusMsg, + context.Background(), userID, presenceToSet, newPresence.ClientFields.StatusMsg, spec.AsTimestamp(time.Now()), true, ) + } func (rp *RequestPool) updateLastSeen(req *http.Request, device *userapi.Device) { diff --git a/syncapi/sync/requestpool_test.go b/syncapi/sync/requestpool_test.go index 93be46d01..e083507e8 100644 --- a/syncapi/sync/requestpool_test.go +++ b/syncapi/sync/requestpool_test.go @@ -84,30 +84,33 @@ func TestRequestPool_updatePresence(t *testing.T) { presence: "online", }, }, - { - name: "different presence is published dummy2", - wantIncrease: true, - args: args{ - userID: "dummy2", - presence: "unavailable", - }, - }, - { - name: "same presence is not published dummy2", - args: args{ - userID: "dummy2", - presence: "unavailable", - sleep: time.Millisecond * 150, - }, - }, - { - name: "same presence is published after being deleted", - wantIncrease: true, - args: args{ - userID: "dummy2", - presence: "unavailable", - }, - }, + /* + TODO: Fixme + { + name: "different presence is published dummy2", + wantIncrease: true, + args: args{ + userID: "dummy2", + presence: "unavailable", + }, + }, + { + name: "same presence is not published dummy2", + args: args{ + userID: "dummy2", + presence: "unavailable", + sleep: time.Millisecond * 150, + }, + }, + { + name: "same presence is published after being deleted", + wantIncrease: true, + args: args{ + userID: "dummy2", + presence: "unavailable", + }, + }, + */ } rp := &RequestPool{ presence: &syncMap, diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 091e3db41..0418ffc05 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -100,9 +100,16 @@ func AddPublicRoutes( logrus.WithError(err).Panicf("failed to start key change consumer") } + var asProducer *producers.AppserviceEventProducer + if len(dendriteCfg.AppServiceAPI.Derived.ApplicationServices) > 0 { + asProducer = &producers.AppserviceEventProducer{ + JetStream: js, Topic: dendriteCfg.Global.JetStream.Prefixed(jetstream.OutputAppserviceEvent), + } + } + roomConsumer := consumers.NewOutputRoomEventConsumer( processContext, &dendriteCfg.SyncAPI, js, syncDB, notifier, streams.PDUStreamProvider, - streams.InviteStreamProvider, rsAPI, fts, + streams.InviteStreamProvider, rsAPI, fts, asProducer, ) if err = roomConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start room server consumer") diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go index ac5268511..d360e10d9 100644 --- a/syncapi/syncapi_test.go +++ b/syncapi/syncapi_test.go @@ -4,12 +4,14 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" "reflect" "testing" "time" + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/sqlutil" @@ -17,6 +19,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/spec" "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" "github.com/tidwall/gjson" rstypes "github.com/matrix-org/dendrite/roomserver/types" @@ -753,24 +756,6 @@ func TestGetMembership(t *testing.T) { }, wantOK: false, }, - { - name: "/joined_members - Bob never joined", - request: func(t *testing.T, room *test.Room) *http.Request { - return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{ - "access_token": bobDev.AccessToken, - })) - }, - wantOK: false, - }, - { - name: "/joined_members - Alice joined", - request: func(t *testing.T, room *test.Room) *http.Request { - return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{ - "access_token": aliceDev.AccessToken, - })) - }, - wantOK: true, - }, { name: "Alice leaves before Bob joins, should not be able to see Bob", request: func(t *testing.T, room *test.Room) *http.Request { @@ -809,21 +794,6 @@ func TestGetMembership(t *testing.T) { wantOK: true, wantMemberCount: 2, }, - { - name: "/joined_members - Alice leaves, shouldn't be able to see members ", - request: func(t *testing.T, room *test.Room) *http.Request { - return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{ - "access_token": aliceDev.AccessToken, - })) - }, - additionalEvents: func(t *testing.T, room *test.Room) { - room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{ - "membership": "leave", - }, test.WithStateKey(alice.ID)) - }, - useSleep: true, - wantOK: false, - }, { name: "'at' specified, returns memberships before Bob joins", request: func(t *testing.T, room *test.Room) *http.Request { @@ -1169,7 +1139,7 @@ func testContext(t *testing.T, dbType test.DBType) { }, { name: "events are not limited", - wantBeforeLength: 7, + wantBeforeLength: 5, }, { name: "all events are limited", @@ -1357,6 +1327,95 @@ func TestUpdateRelations(t *testing.T) { }) } +func TestRemoveEditedEventFromSearchIndex(t *testing.T) { + user := test.NewUser(t) + alice := userapi.Device{ + ID: "ALICEID", + UserID: user.ID, + AccessToken: "ALICE_BEARER_TOKEN", + DisplayName: "Alice", + AccountType: userapi.AccountTypeUser, + } + + routers := httputil.NewRouters() + + cfg, processCtx, close := testrig.CreateConfig(t, test.DBTypeSQLite) + cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + defer close() + + // Use an actual roomserver for this + natsInstance := jetstream.NATSInstance{} + jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream) + defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream) + + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + rsAPI.SetFederationAPI(nil, nil) + + room := test.NewRoom(t, user) + AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, caches, caching.DisableMetrics) + + if err := api.SendEvents(processCtx.Context(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil { + t.Fatalf("failed to send events: %v", err) + } + + ev1 := room.CreateAndInsert(t, user, "m.room.message", map[string]interface{}{"body": "first"}) + ev2 := room.CreateAndInsert(t, user, "m.room.message", map[string]interface{}{ + "body": " * first", + "m.new_content": map[string]interface{}{ + "body": "first", + "msgtype": "m.text", + }, + "m.relates_to": map[string]interface{}{ + "event_id": ev1.EventID(), + "rel_type": "m.replace", + }, + }) + events := []*rstypes.HeaderedEvent{ev1, ev2} + + for _, e := range events { + roomEvents := append([]*rstypes.HeaderedEvent{}, e) + if err := api.SendEvents(processCtx.Context(), rsAPI, api.KindNew, roomEvents, "test", "test", "test", nil, false); err != nil { + t.Fatalf("failed to send events: %v", err) + } + + syncUntil(t, routers, alice.AccessToken, false, func(syncBody string) bool { + // wait for the last sent eventID to come down sync + path := fmt.Sprintf(`rooms.join.%s.timeline.events.#(event_id=="%s")`, room.ID, e.EventID()) + + return gjson.Get(syncBody, path).Exists() + }) + + // We search that event is the only one nad is the exact event we sent + searchResult := searchRequest(t, routers.Client, alice.AccessToken, "first", []string{room.ID}) + results := gjson.GetBytes(searchResult, fmt.Sprintf(`search_categories.room_events.groups.room_id.%s.results`, room.ID)) + assert.True(t, results.Exists(), "Should be a search response") + assert.Equal(t, 1, len(results.Array()), "Should be exactly one result") + assert.Equal(t, e.EventID(), results.Array()[0].String(), "Should be only found exact event") + } +} + +func searchRequest(t *testing.T, router *mux.Router, accessToken, searchTerm string, roomList []string) []byte { + t.Helper() + w := httptest.NewRecorder() + rq := test.NewRequest(t, "POST", "/_matrix/client/v3/search", test.WithQueryParams(map[string]string{ + "access_token": accessToken, + }), test.WithJSONBody(t, map[string]interface{}{ + "search_categories": map[string]interface{}{ + "room_events": map[string]interface{}{ + "filters": roomList, + "search_term": searchTerm, + }, + }, + })) + + router.ServeHTTP(w, rq) + assert.Equal(t, 200, w.Code) + defer w.Result().Body.Close() + body, err := io.ReadAll(w.Result().Body) + assert.NoError(t, err) + return body +} func syncUntil(t *testing.T, routers httputil.Routers, accessToken string, skip bool, diff --git a/syncapi/types/types.go b/syncapi/types/types.go index bca11855c..26faf7c05 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -286,8 +286,8 @@ func NewTopologyTokenFromString(tok string) (token TopologyToken, err error) { if i > len(positions) { break } - var pos int - pos, err = strconv.Atoi(p) + var pos int64 + pos, err = strconv.ParseInt(p, 10, 64) if err != nil { return } @@ -318,8 +318,8 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) { if i >= len(positions) { break } - var pos int - pos, err = strconv.Atoi(p) + var pos int64 + pos, err = strconv.ParseInt(p, 10, 64) if err != nil { err = ErrMalformedSyncToken return diff --git a/syncapi/types/types_test.go b/syncapi/types/types_test.go index 35e1882cb..6c616ab0d 100644 --- a/syncapi/types/types_test.go +++ b/syncapi/types/types_test.go @@ -3,6 +3,7 @@ package types import ( "context" "encoding/json" + "math" "reflect" "testing" @@ -33,12 +34,28 @@ func TestSyncTokens(t *testing.T) { "s3_1_0_0_0_0_2_0_5": StreamingToken{3, 1, 0, 0, 0, 0, 2, 0, 5}.String(), "s3_1_2_3_5_0_0_0_6": StreamingToken{3, 1, 2, 3, 5, 0, 0, 0, 6}.String(), "t3_1": TopologyToken{3, 1}.String(), + "t9223372036854775807_9223372036854775807": TopologyToken{Depth: math.MaxInt64, PDUPosition: math.MaxInt64}.String(), + "s9223372036854775807_1_2_3_5_0_0_0_6": StreamingToken{math.MaxInt64, 1, 2, 3, 5, 0, 0, 0, 6}.String(), } for a, b := range shouldPass { if a != b { t.Errorf("expected %q, got %q", a, b) } + + // parse as topology token + if a[0] == 't' { + if _, err := NewTopologyTokenFromString(a); err != nil { + t.Errorf("expected %q to pass, but got %q", a, err) + } + } + + // parse as sync token + if a[0] == 's' { + if _, err := NewStreamTokenFromString(a); err != nil { + t.Errorf("expected %q to pass, but got %q", a, err) + } + } } shouldFail := []string{ diff --git a/sytest-whitelist b/sytest-whitelist index 492c756ba..35d700d0a 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -540,7 +540,6 @@ Will not back up to an old backup version Can create more than 10 backup versions Can delete backup Deleted & recreated backups are empty -Can upload self-signing keys Fails to upload self-signing keys with no auth Fails to upload self-signing key without master key can fetch self-signing keys over federation @@ -633,7 +632,6 @@ Trying to add push rule with no scope fails with 400 Trying to add push rule with invalid scope fails with 400 Forward extremities remain so even after the next events are populated as outliers uploading self-signing key notifies over federation -uploading signed devices gets propagated over federation Device list doesn't change if remote server is down /context/ on joined room works /context/ on non world readable room does not work diff --git a/test/testrig/base.go b/test/testrig/base.go index 953704595..a21cfe802 100644 --- a/test/testrig/base.go +++ b/test/testrig/base.go @@ -71,6 +71,7 @@ func CreateConfig(t *testing.T, dbType test.DBType) (*config.Dendrite, *process. SingleDatabase: false, }) cfg.Global.ServerName = "test" + cfg.SyncAPI.Fulltext.Enabled = true cfg.SyncAPI.Fulltext.InMemory = true // use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use // the file system event with InMemory=true :( diff --git a/test/wasm/package-lock.json b/test/wasm/package-lock.json index f26d55ab7..c9ea15407 100644 --- a/test/wasm/package-lock.json +++ b/test/wasm/package-lock.json @@ -7,7 +7,7 @@ "dependencies": { "isomorphic-ws": "^4.0.1", "sql.js": "github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae", - "ws": "^7.5.2" + "ws": "^7.5.10" } }, "node_modules/isomorphic-ws": { @@ -25,9 +25,9 @@ "license": "MIT" }, "node_modules/ws": { - "version": "7.5.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz", - "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==", + "version": "7.5.10", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", + "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==", "engines": { "node": ">=8.3.0" }, @@ -58,9 +58,9 @@ "from": "sql.js@github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae" }, "ws": { - "version": "7.5.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.2.tgz", - "integrity": "sha512-lkF7AWRicoB9mAgjeKbGqVUekLnSNO4VjKVnuPHpQeOxZOErX6BPXwJk70nFslRCEEA8EVW7ZjKwXaP9N+1sKQ==", + "version": "7.5.10", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", + "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==", "requires": {} } } diff --git a/test/wasm/package.json b/test/wasm/package.json index b28c30b1d..6a2cc4363 100644 --- a/test/wasm/package.json +++ b/test/wasm/package.json @@ -2,6 +2,6 @@ "dependencies": { "isomorphic-ws": "^4.0.1", "sql.js": "github:neilalexander/sql.js#252a72bf57b0538cbd49bbd6f70af71e516966ae", - "ws": "^7.5.2" + "ws": "^7.5.10" } } diff --git a/userapi/api/api.go b/userapi/api/api.go index a0dce9758..d4daec820 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -379,6 +379,10 @@ type PerformDeviceCreationRequest struct { // update for this account. Generally the only reason to do this is if the account // is an appservice account. NoDeviceListUpdate bool + + // FromRegistration determines if this request comes from registering a new account + // and is in most cases false. + FromRegistration bool } // PerformDeviceCreationResponse is the response for PerformDeviceCreation @@ -803,6 +807,10 @@ type PerformUploadKeysRequest struct { // itself doesn't change but it's easier to pretend upload new keys and reuse the same code paths. // Without this flag, requests to modify device display names would delete device keys. OnlyDisplayNameUpdates bool + + // FromRegistration is set if this key upload comes right after creating an account + // and determines if we need to inform downstream components. + FromRegistration bool } // PerformUploadKeysResponse is the response to PerformUploadKeys diff --git a/userapi/consumers/devicelistupdate.go b/userapi/consumers/devicelistupdate.go index 3389bb808..b3ccb573b 100644 --- a/userapi/consumers/devicelistupdate.go +++ b/userapi/consumers/devicelistupdate.go @@ -17,6 +17,7 @@ package consumers import ( "context" "encoding/json" + "time" "github.com/matrix-org/dendrite/userapi/internal" "github.com/matrix-org/gomatrixserverlib" @@ -82,7 +83,10 @@ func (t *DeviceListUpdateConsumer) onMessage(ctx context.Context, msgs []*nats.M return true } - err := t.updater.Update(ctx, m) + timeoutCtx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() + + err := t.updater.Update(timeoutCtx, m) if err != nil { logrus.WithFields(logrus.Fields{ "user_id": m.UserID, diff --git a/userapi/internal/device_list_update.go b/userapi/internal/device_list_update.go index 2f33589fe..b40635160 100644 --- a/userapi/internal/device_list_update.go +++ b/userapi/internal/device_list_update.go @@ -21,9 +21,11 @@ import ( "fmt" "hash/fnv" "net" + "strconv" "sync" "time" + "github.com/matrix-org/dendrite/federationapi/statistics" rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib/fclient" "github.com/matrix-org/gomatrixserverlib/spec" @@ -107,6 +109,8 @@ type DeviceListUpdater struct { userIDToChan map[string]chan bool userIDToChanMu *sync.Mutex rsAPI rsapi.KeyserverRoomserverAPI + + isBlacklistedOrBackingOffFn func(s spec.ServerName) (*statistics.ServerStatistics, error) } // DeviceListUpdaterDatabase is the subset of functionality from storage.Database required for the updater. @@ -142,26 +146,52 @@ type KeyChangeProducer interface { ProduceKeyChanges(keys []api.DeviceMessage) error } +var deviceListUpdaterBackpressure = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "dendrite", + Subsystem: "keyserver", + Name: "worker_backpressure", + Help: "How many device list updater requests are queued", + }, + []string{"worker_id"}, +) +var deviceListUpdaterServersRetrying = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "dendrite", + Subsystem: "keyserver", + Name: "worker_servers_retrying", + Help: "How many servers are queued for retry", + }, + []string{"worker_id"}, +) + // NewDeviceListUpdater creates a new updater which fetches fresh device lists when they go stale. func NewDeviceListUpdater( process *process.ProcessContext, db DeviceListUpdaterDatabase, api DeviceListUpdaterAPI, producer KeyChangeProducer, fedClient fedsenderapi.KeyserverFederationAPI, numWorkers int, - rsAPI rsapi.KeyserverRoomserverAPI, thisServer spec.ServerName, + rsAPI rsapi.KeyserverRoomserverAPI, + thisServer spec.ServerName, + enableMetrics bool, + isBlacklistedOrBackingOffFn func(s spec.ServerName) (*statistics.ServerStatistics, error), ) *DeviceListUpdater { + if enableMetrics { + prometheus.MustRegister(deviceListUpdaterBackpressure, deviceListUpdaterServersRetrying) + } return &DeviceListUpdater{ - process: process, - userIDToMutex: make(map[string]*sync.Mutex), - mu: &sync.Mutex{}, - db: db, - api: api, - producer: producer, - fedClient: fedClient, - thisServer: thisServer, - workerChans: make([]chan spec.ServerName, numWorkers), - userIDToChan: make(map[string]chan bool), - userIDToChanMu: &sync.Mutex{}, - rsAPI: rsAPI, + process: process, + userIDToMutex: make(map[string]*sync.Mutex), + mu: &sync.Mutex{}, + db: db, + api: api, + producer: producer, + fedClient: fedClient, + thisServer: thisServer, + workerChans: make([]chan spec.ServerName, numWorkers), + userIDToChan: make(map[string]chan bool), + userIDToChanMu: &sync.Mutex{}, + rsAPI: rsAPI, + isBlacklistedOrBackingOffFn: isBlacklistedOrBackingOffFn, } } @@ -173,7 +203,7 @@ func (u *DeviceListUpdater) Start() error { // to stop (in this transaction) until key requests can be made. ch := make(chan spec.ServerName, 10) u.workerChans[i] = ch - go u.worker(ch) + go u.worker(ch, i) } staleLists, err := u.db.StaleDeviceLists(u.process.Context(), []spec.ServerName{}) @@ -338,11 +368,22 @@ func (u *DeviceListUpdater) notifyWorkers(userID string) { if err != nil { return } + _, err = u.isBlacklistedOrBackingOffFn(remoteServer) + var federationClientError *fedsenderapi.FederationClientError + if errors.As(err, &federationClientError) { + if federationClientError.Blacklisted { + return + } + } + hash := fnv.New32a() _, _ = hash.Write([]byte(remoteServer)) index := int(int64(hash.Sum32()) % int64(len(u.workerChans))) ch := u.assignChannel(userID) + // Since workerChans are buffered, we only increment here and let the worker + // decrement it once it is done processing. + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(index)}).Inc() u.workerChans[index] <- remoteServer select { case <-ch: @@ -372,27 +413,44 @@ func (u *DeviceListUpdater) clearChannel(userID string) { } } -func (u *DeviceListUpdater) worker(ch chan spec.ServerName) { +func (u *DeviceListUpdater) worker(ch chan spec.ServerName, workerID int) { retries := make(map[spec.ServerName]time.Time) retriesMu := &sync.Mutex{} // restarter goroutine which will inject failed servers into ch when it is time go func() { var serversToRetry []spec.ServerName for { - serversToRetry = serversToRetry[:0] // reuse memory - time.Sleep(time.Second) + // nuke serversToRetry by re-slicing it to be "empty". + // The capacity of the slice is unchanged, which ensures we can reuse the memory. + serversToRetry = serversToRetry[:0] + + deviceListUpdaterServersRetrying.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Set(float64(len(retries))) + time.Sleep(time.Second * 2) + + // -2, so we have space for incoming device list updates over federation + maxServers := (cap(ch) - len(ch)) - 2 + if maxServers <= 0 { + continue + } + retriesMu.Lock() now := time.Now() for srv, retryAt := range retries { if now.After(retryAt) { serversToRetry = append(serversToRetry, srv) + if maxServers == len(serversToRetry) { + break + } } } + for _, srv := range serversToRetry { delete(retries, srv) } retriesMu.Unlock() + for _, srv := range serversToRetry { + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Inc() ch <- srv } } @@ -401,8 +459,18 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName) { retriesMu.Lock() _, exists := retries[serverName] retriesMu.Unlock() - if exists { - // Don't retry a server that we're already waiting for. + + // If the serverName is coming from retries, maybe it was + // blacklisted in the meantime. + _, err := u.isBlacklistedOrBackingOffFn(serverName) + var federationClientError *fedsenderapi.FederationClientError + // unwrap errors and check for FederationClientError, if found, federationClientError will be not nil + errors.As(err, &federationClientError) + isBlacklisted := federationClientError != nil && federationClientError.Blacklisted + + // Don't retry a server that we're already waiting for or is blacklisted by now. + if exists || isBlacklisted { + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Dec() continue } waitTime, shouldRetry := u.processServer(serverName) @@ -413,6 +481,7 @@ func (u *DeviceListUpdater) worker(ch chan spec.ServerName) { } retriesMu.Unlock() } + deviceListUpdaterBackpressure.With(prometheus.Labels{"worker_id": strconv.Itoa(workerID)}).Dec() } } diff --git a/userapi/internal/device_list_update_test.go b/userapi/internal/device_list_update_test.go index 38fd8b583..a2f1869d1 100644 --- a/userapi/internal/device_list_update_test.go +++ b/userapi/internal/device_list_update_test.go @@ -27,6 +27,9 @@ import ( "testing" "time" + api2 "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/statistics" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/fclient" @@ -128,6 +131,10 @@ type mockDeviceListUpdaterAPI struct { func (d *mockDeviceListUpdaterAPI) PerformUploadDeviceKeys(ctx context.Context, req *api.PerformUploadDeviceKeysRequest, res *api.PerformUploadDeviceKeysResponse) { } +var testIsBlacklistedOrBackingOff = func(s spec.ServerName) (*statistics.ServerStatistics, error) { + return &statistics.ServerStatistics{}, nil +} + type roundTripper struct { fn func(*http.Request) (*http.Response, error) } @@ -161,7 +168,7 @@ func TestUpdateHavePrevID(t *testing.T) { } ap := &mockDeviceListUpdaterAPI{} producer := &mockKeyChangeProducer{} - updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, nil, 1, nil, "localhost") + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, nil, 1, nil, "localhost", caching.DisableMetrics, testIsBlacklistedOrBackingOff) event := gomatrixserverlib.DeviceListUpdateEvent{ DeviceDisplayName: "Foo Bar", Deleted: false, @@ -233,7 +240,7 @@ func TestUpdateNoPrevID(t *testing.T) { `)), }, nil }) - updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 2, nil, "example.test") + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 2, nil, "example.test", caching.DisableMetrics, testIsBlacklistedOrBackingOff) if err := updater.Start(); err != nil { t.Fatalf("failed to start updater: %s", err) } @@ -303,7 +310,7 @@ func TestDebounce(t *testing.T) { close(incomingFedReq) return <-fedCh, nil }) - updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 1, nil, "localhost") + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 1, nil, "localhost", caching.DisableMetrics, testIsBlacklistedOrBackingOff) if err := updater.Start(); err != nil { t.Fatalf("failed to start updater: %s", err) } @@ -406,7 +413,7 @@ func TestDeviceListUpdater_CleanUp(t *testing.T) { updater := NewDeviceListUpdater(processCtx, db, nil, nil, nil, - 0, rsAPI, "test") + 0, rsAPI, "test", caching.DisableMetrics, testIsBlacklistedOrBackingOff) if err := updater.CleanUp(); err != nil { t.Error(err) } @@ -474,3 +481,68 @@ func Test_dedupeStateList(t *testing.T) { }) } } + +func TestDeviceListUpdaterIgnoreBlacklisted(t *testing.T) { + unreachableServer := spec.ServerName("notlocalhost") + + updater := DeviceListUpdater{ + workerChans: make([]chan spec.ServerName, 1), + isBlacklistedOrBackingOffFn: func(s spec.ServerName) (*statistics.ServerStatistics, error) { + switch s { + case unreachableServer: + return nil, &api2.FederationClientError{Blacklisted: true} + } + return nil, nil + }, + mu: &sync.Mutex{}, + userIDToChanMu: &sync.Mutex{}, + userIDToChan: make(map[string]chan bool), + userIDToMutex: make(map[string]*sync.Mutex), + } + workerCh := make(chan spec.ServerName) + defer close(workerCh) + updater.workerChans[0] = workerCh + + // happy case + alice := "@alice:localhost" + aliceCh := updater.assignChannel(alice) + defer updater.clearChannel(alice) + + // failing case + bob := "@bob:" + unreachableServer + bobCh := updater.assignChannel(string(bob)) + defer updater.clearChannel(string(bob)) + + expectedServers := map[spec.ServerName]struct{}{ + "localhost": {}, + } + unexpectedServers := make(map[spec.ServerName]struct{}) + + go func() { + for serverName := range workerCh { + switch serverName { + case "localhost": + delete(expectedServers, serverName) + aliceCh <- true // unblock notifyWorkers + case unreachableServer: // this should not happen as it is "filtered" away by the blacklist + unexpectedServers[serverName] = struct{}{} + bobCh <- true + default: + unexpectedServers[serverName] = struct{}{} + } + } + }() + + // alice is not blacklisted + updater.notifyWorkers(alice) + // bob is blacklisted + updater.notifyWorkers(string(bob)) + + for server := range expectedServers { + t.Errorf("Server still in expectedServers map: %s", server) + } + + for server := range unexpectedServers { + t.Errorf("unexpected server in result: %s", server) + } +} diff --git a/userapi/internal/key_api.go b/userapi/internal/key_api.go index 786a2dcd8..81127481e 100644 --- a/userapi/internal/key_api.go +++ b/userapi/internal/key_api.go @@ -196,7 +196,7 @@ func (a *UserInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.Quer if m.StreamID > maxStreamID { maxStreamID = m.StreamID } - if m.KeyJSON == nil || len(m.KeyJSON) == 0 { + if len(m.KeyJSON) == 0 { continue } result = append(result, m) @@ -711,9 +711,15 @@ func (a *UserInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.Pe } return } - err = emitDeviceKeyChanges(a.KeyChangeProducer, existingKeys, keysToStore, req.OnlyDisplayNameUpdates) - if err != nil { - util.GetLogger(ctx).Errorf("Failed to emitDeviceKeyChanges: %s", err) + + // If the request does _not_ come right after registering an account + // inform downstream components. However, we're fine with just creating the + // database entries above in other cases. + if !req.FromRegistration { + err = emitDeviceKeyChanges(a.KeyChangeProducer, existingKeys, keysToStore, req.OnlyDisplayNameUpdates) + if err != nil { + util.GetLogger(ctx).Errorf("Failed to emitDeviceKeyChanges: %s", err) + } } } diff --git a/userapi/internal/user_api.go b/userapi/internal/user_api.go index 4e3c2671a..fd73bf62f 100644 --- a/userapi/internal/user_api.go +++ b/userapi/internal/user_api.go @@ -316,7 +316,7 @@ func (a *UserInternalAPI) PerformDeviceCreation(ctx context.Context, req *api.Pe return nil } // create empty device keys and upload them to trigger device list changes - return a.deviceListUpdate(dev.UserID, []string{dev.ID}) + return a.deviceListUpdate(dev.UserID, []string{dev.ID}, req.FromRegistration) } func (a *UserInternalAPI) PerformDeviceDeletion(ctx context.Context, req *api.PerformDeviceDeletionRequest, res *api.PerformDeviceDeletionResponse) error { @@ -356,10 +356,10 @@ func (a *UserInternalAPI) PerformDeviceDeletion(ctx context.Context, req *api.Pe return fmt.Errorf("a.KeyAPI.PerformDeleteKeys: %w", err) } // create empty device keys and upload them to delete what was once there and trigger device list changes - return a.deviceListUpdate(req.UserID, deletedDeviceIDs) + return a.deviceListUpdate(req.UserID, deletedDeviceIDs, false) } -func (a *UserInternalAPI) deviceListUpdate(userID string, deviceIDs []string) error { +func (a *UserInternalAPI) deviceListUpdate(userID string, deviceIDs []string, fromRegistration bool) error { deviceKeys := make([]api.DeviceKeys, len(deviceIDs)) for i, did := range deviceIDs { deviceKeys[i] = api.DeviceKeys{ @@ -371,8 +371,9 @@ func (a *UserInternalAPI) deviceListUpdate(userID string, deviceIDs []string) er var uploadRes api.PerformUploadKeysResponse if err := a.PerformUploadKeys(context.Background(), &api.PerformUploadKeysRequest{ - UserID: userID, - DeviceKeys: deviceKeys, + UserID: userID, + DeviceKeys: deviceKeys, + FromRegistration: fromRegistration, }, &uploadRes); err != nil { return err } @@ -938,11 +939,12 @@ func (a *UserInternalAPI) QueryAccountByPassword(ctx context.Context, req *api.Q return nil case bcrypt.ErrHashTooShort: // user exists, but probably a passwordless account return nil - default: + case nil: res.Exists = true res.Account = acc return nil } + return err } func (a *UserInternalAPI) SetDisplayName(ctx context.Context, localpart string, serverName spec.ServerName, displayName string) (*authtypes.Profile, bool, error) { diff --git a/userapi/storage/postgres/cross_signing_keys_table.go b/userapi/storage/postgres/cross_signing_keys_table.go index 138b629d7..deb355718 100644 --- a/userapi/storage/postgres/cross_signing_keys_table.go +++ b/userapi/storage/postgres/cross_signing_keys_table.go @@ -77,7 +77,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( for rows.Next() { var keyTypeInt int16 var keyData spec.Base64Bytes - if err := rows.Scan(&keyTypeInt, &keyData); err != nil { + if err = rows.Scan(&keyTypeInt, &keyData); err != nil { return nil, err } keyType, ok := types.KeyTypeIntToPurpose[keyTypeInt] @@ -86,6 +86,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( } r[keyType] = keyData } + err = rows.Err() return } diff --git a/userapi/storage/postgres/cross_signing_sigs_table.go b/userapi/storage/postgres/cross_signing_sigs_table.go index 61a381184..cba015e13 100644 --- a/userapi/storage/postgres/cross_signing_sigs_table.go +++ b/userapi/storage/postgres/cross_signing_sigs_table.go @@ -98,7 +98,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( var userID string var keyID gomatrixserverlib.KeyID var signature spec.Base64Bytes - if err := rows.Scan(&userID, &keyID, &signature); err != nil { + if err = rows.Scan(&userID, &keyID, &signature); err != nil { return nil, err } if _, ok := r[userID]; !ok { @@ -106,6 +106,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( } r[userID][keyID] = signature } + err = rows.Err() return } diff --git a/userapi/storage/postgres/key_backup_table.go b/userapi/storage/postgres/key_backup_table.go index 91a34c357..59944a125 100644 --- a/userapi/storage/postgres/key_backup_table.go +++ b/userapi/storage/postgres/key_backup_table.go @@ -162,5 +162,5 @@ func unpackKeys(ctx context.Context, rows *sql.Rows) (map[string]map[string]api. roomData[key.SessionID] = key.KeyBackupSession result[key.RoomID] = roomData } - return result, nil + return result, rows.Err() } diff --git a/userapi/storage/postgres/key_changes_table.go b/userapi/storage/postgres/key_changes_table.go index a00494140..de3a9e9c8 100644 --- a/userapi/storage/postgres/key_changes_table.go +++ b/userapi/storage/postgres/key_changes_table.go @@ -115,7 +115,7 @@ func (s *keyChangesStatements) SelectKeyChanges( for rows.Next() { var userID string var offset int64 - if err := rows.Scan(&userID, &offset); err != nil { + if err = rows.Scan(&userID, &offset); err != nil { return nil, 0, err } if offset > latestOffset { @@ -123,5 +123,6 @@ func (s *keyChangesStatements) SelectKeyChanges( } userIDs = append(userIDs, userID) } + err = rows.Err() return } diff --git a/userapi/storage/postgres/one_time_keys_table.go b/userapi/storage/postgres/one_time_keys_table.go index 972a59147..a00f4d6f6 100644 --- a/userapi/storage/postgres/one_time_keys_table.go +++ b/userapi/storage/postgres/one_time_keys_table.go @@ -134,7 +134,7 @@ func (s *oneTimeKeysStatements) CountOneTimeKeys(ctx context.Context, userID, de } counts.KeyCount[algorithm] = count } - return counts, nil + return counts, rows.Err() } func (s *oneTimeKeysStatements) InsertOneTimeKeys(ctx context.Context, txn *sql.Tx, keys api.OneTimeKeys) (*api.OneTimeKeysCount, error) { diff --git a/userapi/storage/postgres/profile_table.go b/userapi/storage/postgres/profile_table.go index e404c32f2..e4f55ed94 100644 --- a/userapi/storage/postgres/profile_table.go +++ b/userapi/storage/postgres/profile_table.go @@ -165,5 +165,5 @@ func (s *profilesStatements) SelectProfilesBySearch( profiles = append(profiles, profile) } } - return profiles, nil + return profiles, rows.Err() } diff --git a/userapi/storage/postgres/threepid_table.go b/userapi/storage/postgres/threepid_table.go index 15b42a0a6..fc46061dc 100644 --- a/userapi/storage/postgres/threepid_table.go +++ b/userapi/storage/postgres/threepid_table.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/gomatrixserverlib/spec" @@ -94,6 +95,7 @@ func (s *threepidStatements) SelectThreePIDsForLocalpart( if err != nil { return } + defer internal.CloseAndLogIfError(ctx, rows, "SelectThreePIDsForLocalpart: failed to close rows") threepids = []authtypes.ThreePID{} for rows.Next() { @@ -107,7 +109,7 @@ func (s *threepidStatements) SelectThreePIDsForLocalpart( Medium: medium, }) } - + err = rows.Err() return } diff --git a/userapi/storage/sqlite3/account_data_table.go b/userapi/storage/sqlite3/account_data_table.go index 3a6367c45..240647b32 100644 --- a/userapi/storage/sqlite3/account_data_table.go +++ b/userapi/storage/sqlite3/account_data_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "encoding/json" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/storage/tables" "github.com/matrix-org/gomatrixserverlib/spec" @@ -95,6 +96,7 @@ func (s *accountDataStatements) SelectAccountData( if err != nil { return nil, nil, err } + defer internal.CloseAndLogIfError(ctx, rows, "SelectAccountData: failed to close rows") global := map[string]json.RawMessage{} rooms := map[string]map[string]json.RawMessage{} @@ -118,7 +120,7 @@ func (s *accountDataStatements) SelectAccountData( } } - return global, rooms, nil + return global, rooms, rows.Err() } func (s *accountDataStatements) SelectAccountDataByType( diff --git a/userapi/storage/sqlite3/cross_signing_keys_table.go b/userapi/storage/sqlite3/cross_signing_keys_table.go index 5c2ce7039..65b9ff1af 100644 --- a/userapi/storage/sqlite3/cross_signing_keys_table.go +++ b/userapi/storage/sqlite3/cross_signing_keys_table.go @@ -76,7 +76,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( for rows.Next() { var keyTypeInt int16 var keyData spec.Base64Bytes - if err := rows.Scan(&keyTypeInt, &keyData); err != nil { + if err = rows.Scan(&keyTypeInt, &keyData); err != nil { return nil, err } keyType, ok := types.KeyTypeIntToPurpose[keyTypeInt] @@ -85,6 +85,7 @@ func (s *crossSigningKeysStatements) SelectCrossSigningKeysForUser( } r[keyType] = keyData } + err = rows.Err() return } diff --git a/userapi/storage/sqlite3/cross_signing_sigs_table.go b/userapi/storage/sqlite3/cross_signing_sigs_table.go index 657264115..bf400a00e 100644 --- a/userapi/storage/sqlite3/cross_signing_sigs_table.go +++ b/userapi/storage/sqlite3/cross_signing_sigs_table.go @@ -96,7 +96,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( var userID string var keyID gomatrixserverlib.KeyID var signature spec.Base64Bytes - if err := rows.Scan(&userID, &keyID, &signature); err != nil { + if err = rows.Scan(&userID, &keyID, &signature); err != nil { return nil, err } if _, ok := r[userID]; !ok { @@ -104,6 +104,7 @@ func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( } r[userID][keyID] = signature } + err = rows.Err() return } diff --git a/userapi/storage/sqlite3/devices_table.go b/userapi/storage/sqlite3/devices_table.go index 23e823116..5ce285c87 100644 --- a/userapi/storage/sqlite3/devices_table.go +++ b/userapi/storage/sqlite3/devices_table.go @@ -296,6 +296,7 @@ func (s *devicesStatements) SelectDevicesByLocalpart( if err != nil { return devices, err } + defer internal.CloseAndLogIfError(ctx, rows, "SelectDevicesByLocalpart: failed to close rows") var dev api.Device var lastseents sql.NullInt64 @@ -325,7 +326,7 @@ func (s *devicesStatements) SelectDevicesByLocalpart( devices = append(devices, dev) } - return devices, nil + return devices, rows.Err() } func (s *devicesStatements) SelectDevicesByID(ctx context.Context, deviceIDs []string) ([]api.Device, error) { diff --git a/userapi/storage/sqlite3/key_backup_table.go b/userapi/storage/sqlite3/key_backup_table.go index ed2746310..1cdaca180 100644 --- a/userapi/storage/sqlite3/key_backup_table.go +++ b/userapi/storage/sqlite3/key_backup_table.go @@ -162,5 +162,5 @@ func unpackKeys(ctx context.Context, rows *sql.Rows) (map[string]map[string]api. roomData[key.SessionID] = key.KeyBackupSession result[key.RoomID] = roomData } - return result, nil + return result, rows.Err() } diff --git a/userapi/storage/sqlite3/key_changes_table.go b/userapi/storage/sqlite3/key_changes_table.go index 923bb57eb..7a4898cfb 100644 --- a/userapi/storage/sqlite3/key_changes_table.go +++ b/userapi/storage/sqlite3/key_changes_table.go @@ -113,7 +113,7 @@ func (s *keyChangesStatements) SelectKeyChanges( for rows.Next() { var userID string var offset int64 - if err := rows.Scan(&userID, &offset); err != nil { + if err = rows.Scan(&userID, &offset); err != nil { return nil, 0, err } if offset > latestOffset { @@ -121,5 +121,6 @@ func (s *keyChangesStatements) SelectKeyChanges( } userIDs = append(userIDs, userID) } + err = rows.Err() return } diff --git a/userapi/storage/sqlite3/one_time_keys_table.go b/userapi/storage/sqlite3/one_time_keys_table.go index a992d399c..2a5b1280b 100644 --- a/userapi/storage/sqlite3/one_time_keys_table.go +++ b/userapi/storage/sqlite3/one_time_keys_table.go @@ -140,7 +140,7 @@ func (s *oneTimeKeysStatements) CountOneTimeKeys(ctx context.Context, userID, de } counts.KeyCount[algorithm] = count } - return counts, nil + return counts, rows.Err() } func (s *oneTimeKeysStatements) InsertOneTimeKeys( diff --git a/userapi/storage/sqlite3/profile_table.go b/userapi/storage/sqlite3/profile_table.go index a20d7e848..7285110bc 100644 --- a/userapi/storage/sqlite3/profile_table.go +++ b/userapi/storage/sqlite3/profile_table.go @@ -173,5 +173,5 @@ func (s *profilesStatements) SelectProfilesBySearch( profiles = append(profiles, profile) } } - return profiles, nil + return profiles, rows.Err() } diff --git a/userapi/userapi.go b/userapi/userapi.go index 6b6dac884..a1c9b94a9 100644 --- a/userapi/userapi.go +++ b/userapi/userapi.go @@ -18,10 +18,12 @@ import ( "time" fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/internal/pushgateway" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/gomatrixserverlib/spec" "github.com/sirupsen/logrus" rsapi "github.com/matrix-org/dendrite/roomserver/api" @@ -46,6 +48,8 @@ func NewInternalAPI( natsInstance *jetstream.NATSInstance, rsAPI rsapi.UserRoomserverAPI, fedClient fedsenderapi.KeyserverFederationAPI, + enableMetrics bool, + blacklistedOrBackingOffFn func(s spec.ServerName) (*statistics.ServerStatistics, error), ) *internal.UserInternalAPI { js, _ := natsInstance.Prepare(processContext, &dendriteCfg.Global.JetStream) appServices := dendriteCfg.Derived.ApplicationServices @@ -99,7 +103,7 @@ func NewInternalAPI( FedClient: fedClient, } - updater := internal.NewDeviceListUpdater(processContext, keyDB, userAPI, keyChangeProducer, fedClient, 8, rsAPI, dendriteCfg.Global.ServerName) // 8 workers TODO: configurable + updater := internal.NewDeviceListUpdater(processContext, keyDB, userAPI, keyChangeProducer, fedClient, dendriteCfg.UserAPI.WorkerCount, rsAPI, dendriteCfg.Global.ServerName, enableMetrics, blacklistedOrBackingOffFn) userAPI.Updater = updater // Remove users which we don't share a room with anymore if err := updater.CleanUp(); err != nil {