sigb.us/dendrite
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'main' into helm-chart

Browse source
This commit is contained in:
Till 2022-12-05 07:27:48 +01:00 committed by GitHub
parent 6bf7146728 b65f89e61e
commit c355fe49fb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
357 changed files with 13788 additions and 5272 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
.github/ISSUE_TEMPLATE/BUG_REPORT.md vendored
View file

@ -7,24 +7,28 @@ about: Create a report to help us improve
<!--
All bug reports must provide the following background information
Text between <!-- and --​> marks will be invisible in the report.
IF YOUR ISSUE IS CONSIDERED A SECURITY VULNERABILITY THEN PLEASE STOP
AND DO NOT POST IT AS A GITHUB ISSUE! Please report the issue responsibly by
disclosing in private by email to security@matrix.org instead. For more details, please
see: https://www.matrix.org/security-disclosure-policy/
-->
### Background information
<!-- Please include versions of all software when known e.g database versions, docker versions, client versions -->
- **Dendrite version or git SHA**:
- **Monolith or Polylith?**:
- **SQLite3 or Postgres?**:
- **Running in Docker?**:
- **Dendrite version or git SHA**:
- **Monolith or Polylith?**:
- **SQLite3 or Postgres?**:
- **Running in Docker?**:
- **`go version`**:
- **Client used (if applicable)**:
### Description
- **What** is the problem:
- **Who** is affected:
- **How** is this bug manifesting:
- **When** did this first appear:
- **What** is the problem:
- **Who** is affected:
- **How** is this bug manifesting:
- **When** did this first appear:
<!--
Examples of good descriptions:
@ -38,7 +42,6 @@ Examples of good descriptions:
- How: "Lots of logs about device change updates"
- When: "After my server joined Matrix HQ"
Examples of bad descriptions:
- What: "Can't send messages" - This is bad because it isn't specfic enough. Which endpoint isn't working and what is the response code? Does the message send but encryption fail?
- Who: "Me" - Who are you? Running the server or a user on a Dendrite server?

6
.github/PULL_REQUEST_TEMPLATE.md vendored
View file

@ -1,8 +1,8 @@
### Pull Request Checklist
<!-- Please read docs/CONTRIBUTING.md before submitting your pull request -->
<!-- Please read https://matrix-org.github.io/dendrite/development/contributing before submitting your pull request -->
* [ ] I have added tests for PR _or_ I have justified why this PR doesn't need tests.
* [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off)
* [ ] I have added Go unit tests or [Complement integration tests](https://github.com/matrix-org/complement) for this PR _or_ I have justified why this PR doesn't need tests
* [ ] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately
Signed-off-by: `Your Name <your@email.example.org>`

187
.github/workflows/dendrite.yml vendored
View file

@ -26,22 +26,14 @@ jobs:
uses: actions/setup-go@v3
with:
go-version: 1.18
- uses: actions/cache@v2
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-wasm-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-wasm
cache: true
- name: Install Node
uses: actions/setup-node@v2
with:
node-version: 14
- uses: actions/cache@v2
- uses: actions/cache@v3
with:
path: ~/.npm
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@ -76,7 +68,7 @@ jobs:
# run go test with different go versions
test:
timeout-minutes: 5
timeout-minutes: 10
name: Unit tests (Go ${{ matrix.go }})
runs-on: ubuntu-latest
# Service containers to run with `container-job`
@ -102,7 +94,7 @@ jobs:
strategy:
fail-fast: false
matrix:
go: ["1.18", "1.19"]
go: ["1.19"]
steps:
- uses: actions/checkout@v3
- name: Setup go
@ -110,14 +102,20 @@ jobs:
with:
go-version: ${{ matrix.go }}
- uses: actions/cache@v3
# manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}-test-${{ hashFiles('**/go.sum') }}
key: ${{ runner.os }}-go${{ matrix.go }}-unit-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go${{ matrix.go }}-test-
- run: go test ./...
${{ runner.os }}-go${{ matrix.go }}-unit-
- name: Set up gotestfmt
uses: gotesttools/gotestfmt-action@v2
with:
# Optional: pass GITHUB_TOKEN to avoid rate limiting.
token: ${{ secrets.GITHUB_TOKEN }}
- run: go test -json -v ./... 2>&1 | gotestfmt
env:
POSTGRES_HOST: localhost
POSTGRES_USER: postgres
@ -141,17 +139,17 @@ jobs:
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- name: Install dependencies x86
if: ${{ matrix.goarch == '386' }}
run: sudo apt update && sudo apt-get install -y gcc-multilib
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goarch }}-
key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-
- name: Install dependencies x86
if: ${{ matrix.goarch == '386' }}
run: sudo apt update && sudo apt-get install -y gcc-multilib
- env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
@ -175,16 +173,16 @@ jobs:
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- name: Install dependencies
run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }}-${{ hashFiles('**/go.sum') }}
key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }}
key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-
- name: Install dependencies
run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc
- env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
@ -204,6 +202,66 @@ jobs:
with:
jobs: ${{ toJSON(needs) }}
# run go test with different go versions
integration:
timeout-minutes: 20
needs: initial-tests-done
name: Integration tests (Go ${{ matrix.go }})
runs-on: ubuntu-latest
# Service containers to run with `container-job`
services:
# Label used to access the service container
postgres:
# Docker Hub image
image: postgres:13-alpine
# Provide the password for postgres
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: dendrite
ports:
# Maps tcp port 5432 on service container to the host
- 5432:5432
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
strategy:
fail-fast: false
matrix:
go: ["1.19"]
steps:
- uses: actions/checkout@v3
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- name: Set up gotestfmt
uses: gotesttools/gotestfmt-action@v2
with:
# Optional: pass GITHUB_TOKEN to avoid rate limiting.
token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go${{ matrix.go }}-test-race-
- run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt
env:
POSTGRES_HOST: localhost
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: dendrite
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
flags: unittests
# run database upgrade tests
upgrade_test:
name: Upgrade tests
@ -216,18 +274,13 @@ jobs:
uses: actions/setup-go@v3
with:
go-version: "1.18"
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-upgrade
cache: true
- name: Build upgrade-tests
run: go build ./cmd/dendrite-upgrade-tests
- name: Test upgrade
- name: Test upgrade (PostgreSQL)
run: ./dendrite-upgrade-tests --head .
- name: Test upgrade (SQLite)
run: ./dendrite-upgrade-tests --sqlite --head .
# run database upgrade tests, skipping over one version
upgrade_test_direct:
@ -241,17 +294,12 @@ jobs:
uses: actions/setup-go@v3
with:
go-version: "1.18"
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-upgrade
cache: true
- name: Build upgrade-tests
run: go build ./cmd/dendrite-upgrade-tests
- name: Test upgrade
- name: Test upgrade (PostgreSQL)
run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
- name: Test upgrade (SQLite)
run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
# run Sytest in different variations
@ -264,11 +312,18 @@ jobs:
fail-fast: false
matrix:
include:
- label: SQLite
- label: SQLite native
- label: SQLite, full HTTP APIs
- label: SQLite Cgo
cgo: 1
- label: SQLite native, full HTTP APIs
api: full-http
- label: SQLite Cgo, full HTTP APIs
api: full-http
cgo: 1
- label: PostgreSQL
postgres: postgres
@ -279,12 +334,23 @@ jobs:
image: matrixdotorg/sytest-dendrite:latest
volumes:
- ${{ github.workspace }}:/src
- /root/.cache/go-build:/github/home/.cache/go-build
- /root/.cache/go-mod:/gopath/pkg/mod
env:
POSTGRES: ${{ matrix.postgres && 1}}
API: ${{ matrix.api && 1 }}
SYTEST_BRANCH: ${{ github.head_ref }}
CGO_ENABLED: ${{ matrix.cgo && 1 }}
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
/gopath/pkg/mod
key: ${{ runner.os }}-go-sytest-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-sytest-
- name: Run Sytest
run: /bootstrap.sh dendrite
working-directory: /src
@ -318,17 +384,28 @@ jobs:
fail-fast: false
matrix:
include:
- label: SQLite
- label: SQLite native
cgo: 0
- label: SQLite, full HTTP APIs
- label: SQLite Cgo
cgo: 1
- label: SQLite native, full HTTP APIs
api: full-http
cgo: 0
- label: SQLite Cgo, full HTTP APIs
api: full-http
cgo: 1
- label: PostgreSQL
postgres: Postgres
cgo: 0
- label: PostgreSQL, full HTTP APIs
postgres: Postgres
api: full-http
cgo: 0
steps:
# Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
# See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
@ -336,16 +413,14 @@ jobs:
run: |
echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
echo "~/go/bin" >> $GITHUB_PATH
- name: "Install Complement Dependencies"
# We don't need to install Go because it is included on the Ubuntu 20.04 image:
# See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
run: |
sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
- name: Run actions/checkout@v2 for dendrite
uses: actions/checkout@v2
go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
- name: Run actions/checkout@v3 for dendrite
uses: actions/checkout@v3
with:
path: dendrite
@ -369,12 +444,10 @@ jobs:
if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
continue
fi
(wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
done
# Build initial Dendrite image
- run: docker build -t complement-dendrite -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
- run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
working-directory: dendrite
env:
DOCKER_BUILDKIT: 1
@ -386,7 +459,7 @@ jobs:
shell: bash
name: Run Complement Tests
env:
COMPLEMENT_BASE_IMAGE: complement-dendrite:latest
COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }}
API: ${{ matrix.api && 1 }}
working-directory: complement
@ -399,6 +472,7 @@ jobs:
upgrade_test_direct,
sytest,
complement,
integration
]
runs-on: ubuntu-latest
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
@ -413,6 +487,7 @@ jobs:
permissions:
packages: write
contents: read
security-events: write # To upload Trivy sarif files
if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main'
needs: [integration-tests-done]
uses: matrix-org/dendrite/.github/workflows/docker.yml@main

182
.github/workflows/docker.yml vendored
View file

@ -24,23 +24,29 @@ jobs:
permissions:
contents: read
packages: write
security-events: write # To upload Trivy sarif files
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Get release tag
uses: actions/checkout@v3
- name: Get release tag & build flags
if: github.event_name == 'release' # Only for GitHub releases
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub
uses: docker/login-action@v1
uses: docker/login-action@v2
with:
username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers
uses: docker/login-action@v1
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
@ -49,12 +55,13 @@ jobs:
- name: Build main monolith image
if: github.ref_name == 'main'
id: docker_build_monolith
uses: docker/build-push-action@v2
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
file: ./build/docker/Dockerfile.monolith
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: monolith
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
@ -64,12 +71,13 @@ jobs:
- name: Build release monolith image
if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_monolith_release
uses: docker/build-push-action@v2
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
file: ./build/docker/Dockerfile.monolith
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: monolith
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
@ -78,29 +86,47 @@ jobs:
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
format: "sarif"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
polylith:
name: Polylith image
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
security-events: write # To upload Trivy sarif files
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Get release tag
uses: actions/checkout@v3
- name: Get release tag & build flags
if: github.event_name == 'release' # Only for GitHub releases
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub
uses: docker/login-action@v1
uses: docker/login-action@v2
with:
username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers
uses: docker/login-action@v1
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
@ -109,12 +135,13 @@ jobs:
- name: Build main polylith image
if: github.ref_name == 'main'
id: docker_build_polylith
uses: docker/build-push-action@v2
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
file: ./build/docker/Dockerfile.polylith
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: polylith
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
@ -124,12 +151,13 @@ jobs:
- name: Build release polylith image
if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_polylith_release
uses: docker/build-push-action@v2
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
file: ./build/docker/Dockerfile.polylith
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
target: polylith
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
@ -138,6 +166,18 @@ jobs:
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
format: "sarif"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
demo-pinecone:
name: Pinecone demo image
runs-on: ubuntu-latest
@ -146,34 +186,40 @@ jobs:
packages: write
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Get release tag
uses: actions/checkout@v3
- name: Get release tag & build flags
if: github.event_name == 'release' # Only for GitHub releases
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub
uses: docker/login-action@v1
uses: docker/login-action@v2
with:
username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers
uses: docker/login-action@v1
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build main pinecone demo image
- name: Build main Pinecone demo image
if: github.ref_name == 'main'
id: docker_build_demo_pinecone
uses: docker/build-push-action@v2
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
file: ./build/docker/Dockerfile.demo-pinecone
platforms: ${{ env.PLATFORMS }}
push: true
@ -181,19 +227,87 @@ jobs:
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }}
- name: Build release pinecone demo image
- name: Build release Pinecone demo image
if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_demo_pinecone_release
uses: docker/build-push-action@v2
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
file: ./build/docker/Dockerfile.demo-pinecone
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:latest
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ env.RELEASE_VERSION }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ env.RELEASE_VERSION }}
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}
demo-yggdrasil:
name: Yggdrasil demo image
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Get release tag & build flags
if: github.event_name == 'release' # Only for GitHub releases
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build main Yggdrasil demo image
if: github.ref_name == 'main'
id: docker_build_demo_yggdrasil
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
file: ./build/docker/Dockerfile.demo-yggdrasil
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ github.ref_name }}
- name: Build release Yggdrasil demo image
if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_demo_yggdrasil_release
uses: docker/build-push-action@v3
with:
cache-from: type=gha
cache-to: type=gha,mode=max
context: .
build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
file: ./build/docker/Dockerfile.demo-yggdrasil
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:latest
${{ env.DOCKER_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-yggdrasil:${{ env.RELEASE_VERSION }}

75
.github/workflows/schedules.yaml vendored Normal file
View file

@ -0,0 +1,75 @@
name: Scheduled
on:
schedule:
- cron: '0 0 * * *' # every day at midnight
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
# run Sytest in different variations
sytest:
timeout-minutes: 60
name: "Sytest (${{ matrix.label }})"
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- label: SQLite
- label: SQLite, full HTTP APIs
api: full-http
- label: PostgreSQL
postgres: postgres
- label: PostgreSQL, full HTTP APIs
postgres: postgres
api: full-http
container:
image: matrixdotorg/sytest-dendrite:latest
volumes:
- ${{ github.workspace }}:/src
- /root/.cache/go-build:/github/home/.cache/go-build
- /root/.cache/go-mod:/gopath/pkg/mod
env:
POSTGRES: ${{ matrix.postgres && 1}}
API: ${{ matrix.api && 1 }}
SYTEST_BRANCH: ${{ github.head_ref }}
RACE_DETECTION: 1
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
/gopath/pkg/mod
key: ${{ runner.os }}-go-sytest-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-sytest-
- name: Run Sytest
run: /bootstrap.sh dendrite
working-directory: /src
- name: Summarise results.tap
if: ${{ always() }}
run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
- name: Sytest List Maintenance
if: ${{ always() }}
run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist
continue-on-error: true # not fatal
- name: Are We Synapse Yet?
if: ${{ always() }}
run: /src/are-we-synapse-yet.py /logs/results.tap -v
continue-on-error: true # not fatal
- name: Upload Sytest logs
uses: actions/upload-artifact@v2
if: ${{ always() }}
with:
name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
path: |
/logs/results.tap
/logs/**/*.log*

164
CHANGES.md
View file

@ -1,5 +1,169 @@
# Changelog
## Dendrite 0.10.8 (2022-11-29)
### Features
* The built-in NATS Server has been updated to version 2.9.8
* A number of under-the-hood changes have been merged for future virtual hosting support in Dendrite (running multiple domain names on the same Dendrite deployment)
### Fixes
* Event auth handling of invites has been refactored, which should fix some edge cases being handled incorrectly
* Fix a bug when returning an empty protocol list, which could cause Element to display "The homeserver may be too old to support third party networks" when opening the public room directory
* The sync API will no longer filter out the user's own membership when using lazy-loading
* Dendrite will now correctly detect JetStream consumers being deleted, stopping the consumer goroutine as needed
* A panic in the federation API where the server list could go out of bounds has been fixed
* Blacklisted servers will now be excluded when querying joined servers, which improves CPU usage and performs less unnecessary outbound requests
* A database writer will now be used to assign state key NIDs when requesting NIDs that may not exist yet
* Dendrite will now correctly move local aliases for an upgraded room when the room is upgraded remotely
* Dendrite will now correctly move account data for an upgraded room when the room is upgraded remotely
* Missing state key NIDs will now be allocated on request rather than returning an error
* Guest access is now correctly denied on a number of endpoints
* Presence information will now be correctly sent for new private chats
* A number of unspecced fields have been removed from outbound `/send` transactions
## Dendrite 0.10.7 (2022-11-04)
### Features
* Dendrite will now use a native SQLite port when building with `CGO_ENABLED=0`
* A number of `thirdparty` endpoints have been added, improving support for appservices
### Fixes
* The `"state"` section of the `/sync` response is no longer limited, so state events should not be dropped unexpectedly
* The deduplication of the `"timeline"` and `"state"` sections in `/sync` is now performed after applying history visibility, so state events should not be dropped unexpectedly
* The `prev_batch` token returned by `/sync` is now calculated after applying history visibility, so that the pagination boundaries are correct
* The room summary membership counts in `/sync` should now be calculated properly in more cases
* A false membership leave event should no longer be sent down `/sync` as a result of retiring an accepted invite (contributed by [tak-hntlabs](https://github.com/tak-hntlabs))
* Presence updates are now only sent to other servers for which the user shares rooms
* A bug which could cause a panic when converting events into the `ClientEvent` format has been fixed
## Dendrite 0.10.6 (2022-11-01)
### Features
* History visibility checks have been optimised, which should speed up response times on a variety of endpoints (including `/sync`, `/messages`, `/context` and others) and reduce database load
* The built-in NATS Server has been updated to version 2.9.4
* Some other minor dependencies have been updated
### Fixes
* A panic has been fixed in the sync API PDU stream which could cause requests to fail
* The `/members` response now contains the `room_id` field, which may fix some E2EE problems with clients using the JS SDK (contributed by [ashkitten](https://github.com/ashkitten))
* The auth difference calculation in state resolution v2 has been tweaked for clarity (and moved into gomatrixserverlib with the rest of the state resolution code)
## Dendrite 0.10.5 (2022-10-31)
### Features
* It is now possible to use hCaptcha instead of reCAPTCHA for protecting registration
* A new `auto_join_rooms` configuration option has been added for automatically joining new users to a set of rooms
* A new `/_dendrite/admin/downloadState/{serverName}/{roomID}` endpoint has been added, which allows a server administrator to attempt to repair a room with broken room state by downloading a state snapshot from another federated server in the room
### Fixes
* Querying cross-signing keys for users should now be considerably faster
* A bug in state resolution where some events were not correctly selected for third-party invites has been fixed
* A bug in state resolution which could result in `not in room` event rejections has been fixed
* When accepting a DM invite, it should now be possible to see messages that were sent before the invite was accepted
* Claiming remote E2EE one-time keys has been refactored and should be more reliable now
* Various fixes have been made to the `/members` endpoint, which may help with E2EE reliability and clients rendering memberships
* A race condition in the federation API destination queues has been fixed when associating queued events with remote server destinations
* A bug in the sync API where too many events were selected resulting in high CPU usage has been fixed
* Configuring the avatar URL for the Server Notices user should work correctly now
## Dendrite 0.10.4 (2022-10-21)
### Features
* Various tables belonging to the user API will be renamed so that they are namespaced with the `userapi_` prefix
* Note that, after upgrading to this version, you should not revert to an older version of Dendrite as the database changes **will not** be reverted automatically
* The backoff and retry behaviour in the federation API has been refactored and improved
### Fixes
* Private read receipt support is now advertised in the client `/versions` endpoint
* Private read receipts will now clear notification counts properly
* A bug where a false `leave` membership transition was inserted into the timeline after accepting an invite has been fixed
* Some panics caused by concurrent map writes in the key server have been fixed
* The sync API now calculates membership transitions from state deltas more accurately
* Transaction IDs are now scoped to endpoints, which should fix some bugs where transaction ID reuse could cause nonsensical cached responses from some endpoints
* The length of the `type`, `sender`, `state_key` and `room_id` fields in events are now verified by number of bytes rather than codepoints after a spec clarification, reverting a change made in Dendrite 0.9.6
## Dendrite 0.10.3 (2022-10-14)
### Features
* Event relations are now tracked and support for the `/room/{roomID}/relations/...` client API endpoints have been added
* Support has been added for private read receipts
* The built-in NATS Server has been updated to version 2.9.3
### Fixes
* The `unread_notifications` are now always populated in joined room responses
* The `/get_missing_events` federation API endpoint should now work correctly for rooms with `joined` and `invited` visibility settings, returning redacted events for events that other servers are not allowed to see
* The `/event` client API endpoint now applies history visibility correctly
* Read markers should now be updated much more reliably
* A rare bug in the sync API which could cause some `join` memberships to be incorrectly overwritten by other memberships when working out which rooms to populate has been fixed
* The federation API now correctly updates the joined hosts table during a state rewrite
## Dendrite 0.10.2 (2022-10-07)
### Features
* Dendrite will now fail to start if there is an obvious problem with the configured `max_open_conns` when using PostgreSQL database backends, since this can lead to instability and performance issues
* More information on this is available [in the documentation](https://matrix-org.github.io/dendrite/installation/start/optimisation#postgresql-connection-limit)
* Unnecessary/empty fields will no longer be sent in `/sync` responses
* It is now possible to configure `old_private_keys` from previous Matrix installations on the same domain if only public key is known, to make it easier to expire old keys correctly
* You can configure either just the `private_key` path, or you can supply both the `public_key` and `key_id`
### Fixes
* The sync transaction behaviour has been modified further so that errors in one stream should not propagate to other streams unnecessarily
* Rooms should now be classified as DM rooms correctly by passing through `is_direct` and unsigned hints
* A bug which caused marking device lists as stale to consume lots of CPU has been fixed
* Users accepting invites should no longer cause unnecessary federated joins if there are already other local users in the room
* The sync API state range queries have been optimised by adding missing indexes
* It should now be possible to configure non-English languages for full-text search in `search.language`
* The roomserver will no longer attempt to perform federated requests to the local server when trying to fetch missing events
* The `/keys/upload` endpoint will now always return the `one_time_keys_counts`, which may help with E2EE reliability
* The sync API will now retrieve the latest stream position before processing each stream rather than at the beginning of the request, to hopefully reduce the number of round-trips to `/sync`
## Dendrite 0.10.1 (2022-09-30)
### Features
* The built-in NATS Server has been updated to version 2.9.2
### Fixes
* A regression introduced in 0.10.0 in `/sync` as a result of transaction errors has been fixed
* Account data updates will no longer send duplicate output events
## Dendrite 0.10.0 (2022-09-30)
### Features
* High performance full-text searching has been added to Dendrite
* Search must be enabled in the [`search` section of the `sync_api` config](https://github.com/matrix-org/dendrite/blob/6348486a1365c7469a498101f5035a9b6bd16d22/dendrite-sample.monolith.yaml#L279-L290) before it can be used
* The search index is stored on the filesystem rather than the sync API database, so a path to a suitable storage location on disk must be configured
* Sync requests should now complete faster and use considerably less database connections as a result of better transactional isolation
* The notifications code has been refactored to hopefully make notifications more reliable
* A new `/_dendrite/admin/refreshDevices/{userID}` admin endpoint has been added for forcing a refresh of a remote user's device lists without having to modify the database by hand
* A new `/_dendrite/admin/fulltext/reindex` admin endpoint has been added for rebuilding the search index (although this may take some time)
### Fixes
* A number of bugs in the device list updater have been fixed, which should help considerably with federated device list synchronisation and E2EE reliability
* A state resolution bug has been fixed which should help to prevent unexpected state resets
* The deprecated `"origin"` field in events will now be correctly ignored in all cases
* Room versions 8 and 9 will now correctly evaluate `"knock"` join rules and membership states
* A database index has been added to speed up finding room memberships in the sync API (contributed by [PiotrKozimor](https://github.com/PiotrKozimor))
* The client API will now return an `M_UNRECOGNIZED` error for unknown endpoints/methods, which should help with client error handling
* A bug has been fixed when updating push rules which could result in `database is locked` on SQLite
## Dendrite 0.9.9 (2022-09-22)
### Features

92
Dockerfile Normal file
View file

@ -0,0 +1,92 @@
#syntax=docker/dockerfile:1.2
#
# base installs required dependencies and runs go mod download to cache dependencies
#
FROM --platform=${BUILDPLATFORM} docker.io/golang:1.19-alpine AS base
RUN apk --update --no-cache add bash build-base curl
#
# build creates all needed binaries
#
FROM --platform=${BUILDPLATFORM} base AS build
WORKDIR /src
ARG TARGETOS
ARG TARGETARCH
ARG FLAGS
RUN --mount=target=. \
--mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/go/pkg/mod \
USERARCH=`go env GOARCH` \
GOARCH="$TARGETARCH" \
GOOS="linux" \
CGO_ENABLED=$([ "$TARGETARCH" = "$USERARCH" ] && echo "1" || echo "0") \
go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/...
#
# The dendrite base image
#
FROM alpine:latest AS dendrite-base
LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
LABEL org.opencontainers.image.licenses="Apache-2.0"
LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/"
LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C."
#
# Builds the polylith image and only contains the polylith binary
#
FROM dendrite-base AS polylith
LABEL org.opencontainers.image.title="Dendrite (Polylith)"
COPY --from=build /out/dendrite-polylith-multi /usr/bin/
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]
#
# Builds the monolith image and contains all required binaries
#
FROM dendrite-base AS monolith
LABEL org.opencontainers.image.title="Dendrite (Monolith)"
COPY --from=build /out/create-account /usr/bin/create-account
COPY --from=build /out/generate-config /usr/bin/generate-config
COPY --from=build /out/generate-keys /usr/bin/generate-keys
COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
EXPOSE 8008 8448
#
# Builds the Complement image, used for integration tests
#
FROM base AS complement
LABEL org.opencontainers.image.title="Dendrite (Complement)"
RUN apk add --no-cache sqlite openssl ca-certificates
COPY --from=build /out/generate-config /usr/bin/generate-config
COPY --from=build /out/generate-keys /usr/bin/generate-keys
COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server
WORKDIR /dendrite
RUN /usr/bin/generate-keys --private-key matrix_key.pem && \
mkdir /ca && \
openssl genrsa -out /ca/ca.key 2048 && \
openssl req -new -x509 -key /ca/ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out /ca/ca.crt
ENV SERVER_NAME=localhost
ENV API=0
EXPOSE 8008 8448
# At runtime, generate TLS cert based on the CA now mounted at /ca
# At runtime, replace the SERVER_NAME with what we are told
CMD /usr/bin/generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /ca/ca.crt --tls-authority-key /ca/ca.key && \
/usr/bin/generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
cp /ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
/usr/bin/dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}

5
README.md
View file

@ -79,7 +79,7 @@ $ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --co
# Create an user account (add -admin for an admin user).
# Specify the localpart only, e.g. 'alice' for '@alice:domain.com'
$ ./bin/create-account --config dendrite.yaml --url http://localhost:8008 --username alice
$ ./bin/create-account --config dendrite.yaml --username alice
```
Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`.
@ -90,7 +90,7 @@ We use a script called Are We Synapse Yet which checks Sytest compliance rates.
test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it
updates with CI. As of August 2022 we're at around 90% CS API coverage and 95% Federation coverage, though check
CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse
servers such as matrix.org reasonably well, although there are still some missing features (like Search).
servers such as matrix.org reasonably well, although there are still some missing features (like SSO and Third-party ID APIs).
We are prioritising features that will benefit single-user homeservers first (e.g Receipts, E2E) rather
than features that massive deployments may be interested in (OpenID, Guests, Admin APIs, AS API).
@ -112,6 +112,7 @@ This means Dendrite supports amongst others:
- Guests
- User Directory
- Presence
- Fulltext search
## Contributing

75
appservice/api/query.go
View file

@ -19,11 +19,13 @@ package api
import (
"context"
"encoding/json"
"errors"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
)
// AppServiceInternalAPI is used to query user and room alias data from application
@ -41,6 +43,10 @@ type AppServiceInternalAPI interface {
req *UserIDExistsRequest,
resp *UserIDExistsResponse,
) error
Locations(ctx context.Context, req *LocationRequest, resp *LocationResponse) error
User(ctx context.Context, request *UserRequest, response *UserResponse) error
Protocols(ctx context.Context, req *ProtocolRequest, resp *ProtocolResponse) error
}
// RoomAliasExistsRequest is a request to an application service
@ -77,6 +83,73 @@ type UserIDExistsResponse struct {
UserIDExists bool `json:"exists"`
}
const (
ASProtocolPath = "/_matrix/app/unstable/thirdparty/protocol/"
ASUserPath = "/_matrix/app/unstable/thirdparty/user"
ASLocationPath = "/_matrix/app/unstable/thirdparty/location"
)
type ProtocolRequest struct {
Protocol string `json:"protocol,omitempty"`
}
type ProtocolResponse struct {
Protocols map[string]ASProtocolResponse `json:"protocols"`
Exists bool `json:"exists"`
}
type ASProtocolResponse struct {
FieldTypes map[string]FieldType `json:"field_types,omitempty"` // NOTSPEC: field_types is required by the spec
Icon string `json:"icon"`
Instances []ProtocolInstance `json:"instances"`
LocationFields []string `json:"location_fields"`
UserFields []string `json:"user_fields"`
}
type FieldType struct {
Placeholder string `json:"placeholder"`
Regexp string `json:"regexp"`
}
type ProtocolInstance struct {
Description string `json:"desc"`
Icon string `json:"icon,omitempty"`
NetworkID string `json:"network_id,omitempty"` // NOTSPEC: network_id is required by the spec
Fields json.RawMessage `json:"fields,omitempty"` // NOTSPEC: fields is required by the spec
}
type UserRequest struct {
Protocol string `json:"protocol"`
Params string `json:"params"`
}
type UserResponse struct {
Users []ASUserResponse `json:"users,omitempty"`
Exists bool `json:"exists,omitempty"`
}
type ASUserResponse struct {
Protocol string `json:"protocol"`
UserID string `json:"userid"`
Fields json.RawMessage `json:"fields"`
}
type LocationRequest struct {
Protocol string `json:"protocol"`
Params string `json:"params"`
}
type LocationResponse struct {
Locations []ASLocationResponse `json:"locations,omitempty"`
Exists bool `json:"exists,omitempty"`
}
type ASLocationResponse struct {
Alias string `json:"alias"`
Protocol string `json:"protocol"`
Fields json.RawMessage `json:"fields"`
}
// RetrieveUserProfile is a wrapper that queries both the local database and
// application services for a given user's profile
// TODO: Remove this, it's called from federationapi and clientapi but is a pure function

13
appservice/appservice.go
View file

@ -18,6 +18,7 @@ import (
"context"
"crypto/tls"
"net/http"
"sync"
"time"
"github.com/gorilla/mux"
@ -31,6 +32,7 @@ import (
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
)
// AddInternalRoutes registers HTTP handlers for internal API calls
@ -58,8 +60,10 @@ func NewInternalAPI(
// Create appserivce query API with an HTTP client that will be used for all
// outbound and inbound requests (inbound only for the internal API)
appserviceQueryAPI := &query.AppServiceQueryAPI{
HTTPClient: client,
Cfg: &base.Cfg.AppServiceAPI,
HTTPClient: client,
Cfg: &base.Cfg.AppServiceAPI,
ProtocolCache: map[string]appserviceAPI.ASProtocolResponse{},
CacheMu: sync.Mutex{},
}
if len(base.Cfg.Derived.ApplicationServices) == 0 {
@ -71,7 +75,7 @@ func NewInternalAPI(
// events to be sent out.
for _, appservice := range base.Cfg.Derived.ApplicationServices {
// Create bot account for this AS if it doesn't already exist
if err := generateAppServiceAccount(userAPI, appservice); err != nil {
if err := generateAppServiceAccount(userAPI, appservice, base.Cfg.Global.ServerName); err != nil {
logrus.WithFields(logrus.Fields{
"appservice": appservice.ID,
}).WithError(err).Panicf("failed to generate bot account for appservice")
@ -98,11 +102,13 @@ func NewInternalAPI(
func generateAppServiceAccount(
userAPI userapi.AppserviceUserAPI,
as config.ApplicationService,
serverName gomatrixserverlib.ServerName,
) error {
var accRes userapi.PerformAccountCreationResponse
err := userAPI.PerformAccountCreation(context.Background(), &userapi.PerformAccountCreationRequest{
AccountType: userapi.AccountTypeAppService,
Localpart: as.SenderLocalpart,
ServerName: serverName,
AppServiceID: as.ID,
OnConflict: userapi.ConflictUpdate,
}, &accRes)
@ -112,6 +118,7 @@ func generateAppServiceAccount(
var devRes userapi.PerformDeviceCreationResponse
err = userAPI.PerformDeviceCreation(context.Background(), &userapi.PerformDeviceCreationRequest{
Localpart: as.SenderLocalpart,
ServerName: serverName,
AccessToken: as.ASToken,
DeviceID: &as.SenderLocalpart,
DeviceDisplayName: &as.SenderLocalpart,

224
appservice/appservice_test.go Normal file
View file

@ -0,0 +1,224 @@
package appservice_test
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"reflect"
"regexp"
"strings"
"testing"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/appservice/inthttp"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/userapi"
"github.com/matrix-org/dendrite/test/testrig"
)
func TestAppserviceInternalAPI(t *testing.T) {
// Set expected results
existingProtocol := "irc"
wantLocationResponse := []api.ASLocationResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
wantUserResponse := []api.ASUserResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
wantProtocolResponse := api.ASProtocolResponse{Instances: []api.ProtocolInstance{{Fields: []byte("{}")}}}
wantProtocolResult := map[string]api.ASProtocolResponse{
existingProtocol: wantProtocolResponse,
}
// create a dummy AS url, handling some cases
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.Contains(r.URL.Path, "location"):
// Check if we've got an existing protocol, if so, return a proper response.
if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
if err := json.NewEncoder(w).Encode(wantLocationResponse); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
}
if err := json.NewEncoder(w).Encode([]api.ASLocationResponse{}); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
case strings.Contains(r.URL.Path, "user"):
if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
if err := json.NewEncoder(w).Encode(wantUserResponse); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
}
if err := json.NewEncoder(w).Encode([]api.UserResponse{}); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
case strings.Contains(r.URL.Path, "protocol"):
if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
if err := json.NewEncoder(w).Encode(wantProtocolResponse); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
}
if err := json.NewEncoder(w).Encode(nil); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
default:
t.Logf("hit location: %s", r.URL.Path)
}
}))
// TODO: use test.WithAllDatabases
// only one DBType, since appservice.AddInternalRoutes complains about multiple prometheus counters added
base, closeBase := testrig.CreateBaseDendrite(t, test.DBTypeSQLite)
defer closeBase()
// Create a dummy application service
base.Cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{
{
ID: "someID",
URL: srv.URL,
ASToken: "",
HSToken: "",
SenderLocalpart: "senderLocalPart",
NamespaceMap: map[string][]config.ApplicationServiceNamespace{
"users": {{RegexpObject: regexp.MustCompile("as-.*")}},
"aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}},
},
Protocols: []string{existingProtocol},
},
}
// Create required internal APIs
rsAPI := roomserver.NewInternalAPI(base)
usrAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, nil, rsAPI, nil)
asAPI := appservice.NewInternalAPI(base, usrAPI, rsAPI)
// The test cases to run
runCases := func(t *testing.T, testAPI api.AppServiceInternalAPI) {
t.Run("UserIDExists", func(t *testing.T) {
testUserIDExists(t, testAPI, "@as-testing:test", true)
testUserIDExists(t, testAPI, "@as1-testing:test", false)
})
t.Run("AliasExists", func(t *testing.T) {
testAliasExists(t, testAPI, "@asroom-testing:test", true)
testAliasExists(t, testAPI, "@asroom1-testing:test", false)
})
t.Run("Locations", func(t *testing.T) {
testLocations(t, testAPI, existingProtocol, wantLocationResponse)
testLocations(t, testAPI, "abc", nil)
})
t.Run("User", func(t *testing.T) {
testUser(t, testAPI, existingProtocol, wantUserResponse)
testUser(t, testAPI, "abc", nil)
})
t.Run("Protocols", func(t *testing.T) {
testProtocol(t, testAPI, existingProtocol, wantProtocolResult)
testProtocol(t, testAPI, existingProtocol, wantProtocolResult) // tests the cache
testProtocol(t, testAPI, "", wantProtocolResult) // tests getting all protocols
testProtocol(t, testAPI, "abc", nil)
})
}
// Finally execute the tests
t.Run("HTTP API", func(t *testing.T) {
router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter()
appservice.AddInternalRoutes(router, asAPI)
apiURL, cancel := test.ListenAndServe(t, router, false)
defer cancel()
asHTTPApi, err := inthttp.NewAppserviceClient(apiURL, &http.Client{})
if err != nil {
t.Fatalf("failed to create HTTP client: %s", err)
}
runCases(t, asHTTPApi)
})
t.Run("Monolith", func(t *testing.T) {
runCases(t, asAPI)
})
}
func testUserIDExists(t *testing.T, asAPI api.AppServiceInternalAPI, userID string, wantExists bool) {
ctx := context.Background()
userResp := &api.UserIDExistsResponse{}
if err := asAPI.UserIDExists(ctx, &api.UserIDExistsRequest{
UserID: userID,
}, userResp); err != nil {
t.Errorf("failed to get userID: %s", err)
}
if userResp.UserIDExists != wantExists {
t.Errorf("unexpected result for UserIDExists(%s): %v, expected %v", userID, userResp.UserIDExists, wantExists)
}
}
func testAliasExists(t *testing.T, asAPI api.AppServiceInternalAPI, alias string, wantExists bool) {
ctx := context.Background()
aliasResp := &api.RoomAliasExistsResponse{}
if err := asAPI.RoomAliasExists(ctx, &api.RoomAliasExistsRequest{
Alias: alias,
}, aliasResp); err != nil {
t.Errorf("failed to get alias: %s", err)
}
if aliasResp.AliasExists != wantExists {
t.Errorf("unexpected result for RoomAliasExists(%s): %v, expected %v", alias, aliasResp.AliasExists, wantExists)
}
}
func testLocations(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult []api.ASLocationResponse) {
ctx := context.Background()
locationResp := &api.LocationResponse{}
if err := asAPI.Locations(ctx, &api.LocationRequest{
Protocol: proto,
}, locationResp); err != nil {
t.Errorf("failed to get locations: %s", err)
}
if !reflect.DeepEqual(locationResp.Locations, wantResult) {
t.Errorf("unexpected result for Locations(%s): %+v, expected %+v", proto, locationResp.Locations, wantResult)
}
}
func testUser(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult []api.ASUserResponse) {
ctx := context.Background()
userResp := &api.UserResponse{}
if err := asAPI.User(ctx, &api.UserRequest{
Protocol: proto,
}, userResp); err != nil {
t.Errorf("failed to get user: %s", err)
}
if !reflect.DeepEqual(userResp.Users, wantResult) {
t.Errorf("unexpected result for User(%s): %+v, expected %+v", proto, userResp.Users, wantResult)
}
}
func testProtocol(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult map[string]api.ASProtocolResponse) {
ctx := context.Background()
protoResp := &api.ProtocolResponse{}
if err := asAPI.Protocols(ctx, &api.ProtocolRequest{
Protocol: proto,
}, protoResp); err != nil {
t.Errorf("failed to get Protocols: %s", err)
}
if !reflect.DeepEqual(protoResp.Protocols, wantResult) {
t.Errorf("unexpected result for Protocols(%s): %+v, expected %+v", proto, protoResp.Protocols[proto], wantResult)
}
}

5
appservice/consumers/roomserver.go
View file

@ -101,6 +101,11 @@ func (s *OutputRoomEventConsumer) onMessage(
log.WithField("appservice", state.ID).Tracef("Appservice worker received %d message(s) from roomserver", len(msgs))
events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(msgs))
for _, msg := range msgs {
// Only handle events we care about
receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType))
if receivedType != api.OutputTypeNewRoomEvent && receivedType != api.OutputTypeNewInviteEvent {
continue
}
// Parse out the event JSON
var output api.OutputEvent
if err := json.Unmarshal(msg.Data, &output); err != nil {

24
appservice/inthttp/client.go
View file

@ -13,6 +13,9 @@ import (
const (
AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists"
AppServiceUserIDExistsPath = "/appservice/UserIDExists"
AppServiceLocationsPath = "/appservice/locations"
AppServiceUserPath = "/appservice/users"
AppServiceProtocolsPath = "/appservice/protocols"
)
// httpAppServiceQueryAPI contains the URL to an appservice query API and a
@ -58,3 +61,24 @@ func (h *httpAppServiceQueryAPI) UserIDExists(
h.httpClient, ctx, request, response,
)
}
func (h *httpAppServiceQueryAPI) Locations(ctx context.Context, request *api.LocationRequest, response *api.LocationResponse) error {
return httputil.CallInternalRPCAPI(
"ASLocation", h.appserviceURL+AppServiceLocationsPath,
h.httpClient, ctx, request, response,
)
}
func (h *httpAppServiceQueryAPI) User(ctx context.Context, request *api.UserRequest, response *api.UserResponse) error {
return httputil.CallInternalRPCAPI(
"ASUser", h.appserviceURL+AppServiceUserPath,
h.httpClient, ctx, request, response,
)
}
func (h *httpAppServiceQueryAPI) Protocols(ctx context.Context, request *api.ProtocolRequest, response *api.ProtocolResponse) error {
return httputil.CallInternalRPCAPI(
"ASProtocols", h.appserviceURL+AppServiceProtocolsPath,
h.httpClient, ctx, request, response,
)
}

16
appservice/inthttp/server.go
View file

@ -2,6 +2,7 @@ package inthttp
import (
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/internal/httputil"
)
@ -17,4 +18,19 @@ func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) {
AppServiceUserIDExistsPath,
httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists),
)
internalAPIMux.Handle(
AppServiceProtocolsPath,
httputil.MakeInternalRPCAPI("AppserviceProtocols", a.Protocols),
)
internalAPIMux.Handle(
AppServiceLocationsPath,
httputil.MakeInternalRPCAPI("AppserviceLocations", a.Locations),
)
internalAPIMux.Handle(
AppServiceUserPath,
httputil.MakeInternalRPCAPI("AppserviceUser", a.User),
)
}

190
appservice/query/query.go
View file

@ -18,13 +18,18 @@ package query
import (
"context"
"encoding/json"
"io"
"net/http"
"net/url"
"strings"
"sync"
"github.com/opentracing/opentracing-go"
log "github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/setup/config"
opentracing "github.com/opentracing/opentracing-go"
log "github.com/sirupsen/logrus"
)
const roomAliasExistsPath = "/rooms/"
@ -32,8 +37,10 @@ const userIDExistsPath = "/users/"
// AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI
type AppServiceQueryAPI struct {
HTTPClient *http.Client
Cfg *config.AppServiceAPI
HTTPClient *http.Client
Cfg *config.AppServiceAPI
ProtocolCache map[string]api.ASProtocolResponse
CacheMu sync.Mutex
}
// RoomAliasExists performs a request to '/room/{roomAlias}' on all known
@ -165,3 +172,178 @@ func (a *AppServiceQueryAPI) UserIDExists(
response.UserIDExists = false
return nil
}
type thirdpartyResponses interface {
api.ASProtocolResponse | []api.ASUserResponse | []api.ASLocationResponse
}
func requestDo[T thirdpartyResponses](client *http.Client, url string, response *T) (err error) {
origURL := url
// try v1 and unstable appservice endpoints
for _, version := range []string{"v1", "unstable"} {
var resp *http.Response
var body []byte
asURL := strings.Replace(origURL, "unstable", version, 1)
resp, err = client.Get(asURL)
if err != nil {
continue
}
defer resp.Body.Close() // nolint: errcheck
body, err = io.ReadAll(resp.Body)
if err != nil {
continue
}
return json.Unmarshal(body, &response)
}
return err
}
func (a *AppServiceQueryAPI) Locations(
ctx context.Context,
req *api.LocationRequest,
resp *api.LocationResponse,
) error {
params, err := url.ParseQuery(req.Params)
if err != nil {
return err
}
for _, as := range a.Cfg.Derived.ApplicationServices {
var asLocations []api.ASLocationResponse
params.Set("access_token", as.HSToken)
url := as.URL + api.ASLocationPath
if req.Protocol != "" {
url += "/" + req.Protocol
}
if err := requestDo[[]api.ASLocationResponse](a.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil {
log.WithError(err).Error("unable to get 'locations' from application service")
continue
}
resp.Locations = append(resp.Locations, asLocations...)
}
if len(resp.Locations) == 0 {
resp.Exists = false
return nil
}
resp.Exists = true
return nil
}
func (a *AppServiceQueryAPI) User(
ctx context.Context,
req *api.UserRequest,
resp *api.UserResponse,
) error {
params, err := url.ParseQuery(req.Params)
if err != nil {
return err
}
for _, as := range a.Cfg.Derived.ApplicationServices {
var asUsers []api.ASUserResponse
params.Set("access_token", as.HSToken)
url := as.URL + api.ASUserPath
if req.Protocol != "" {
url += "/" + req.Protocol
}
if err := requestDo[[]api.ASUserResponse](a.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil {
log.WithError(err).Error("unable to get 'user' from application service")
continue
}
resp.Users = append(resp.Users, asUsers...)
}
if len(resp.Users) == 0 {
resp.Exists = false
return nil
}
resp.Exists = true
return nil
}
func (a *AppServiceQueryAPI) Protocols(
ctx context.Context,
req *api.ProtocolRequest,
resp *api.ProtocolResponse,
) error {
// get a single protocol response
if req.Protocol != "" {
a.CacheMu.Lock()
defer a.CacheMu.Unlock()
if proto, ok := a.ProtocolCache[req.Protocol]; ok {
resp.Exists = true
resp.Protocols = map[string]api.ASProtocolResponse{
req.Protocol: proto,
}
return nil
}
response := api.ASProtocolResponse{}
for _, as := range a.Cfg.Derived.ApplicationServices {
var proto api.ASProtocolResponse
if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+req.Protocol, &proto); err != nil {
log.WithError(err).Error("unable to get 'protocol' from application service")
continue
}
if len(response.Instances) != 0 {
response.Instances = append(response.Instances, proto.Instances...)
} else {
response = proto
}
}
if len(response.Instances) == 0 {
resp.Exists = false
return nil
}
resp.Exists = true
resp.Protocols = map[string]api.ASProtocolResponse{
req.Protocol: response,
}
a.ProtocolCache[req.Protocol] = response
return nil
}
response := make(map[string]api.ASProtocolResponse, len(a.Cfg.Derived.ApplicationServices))
for _, as := range a.Cfg.Derived.ApplicationServices {
for _, p := range as.Protocols {
var proto api.ASProtocolResponse
if err := requestDo[api.ASProtocolResponse](a.HTTPClient, as.URL+api.ASProtocolPath+p, &proto); err != nil {
log.WithError(err).Error("unable to get 'protocol' from application service")
continue
}
existing, ok := response[p]
if !ok {
response[p] = proto
continue
}
existing.Instances = append(existing.Instances, proto.Instances...)
response[p] = existing
}
}
if len(response) == 0 {
resp.Exists = false
return nil
}
a.CacheMu.Lock()
defer a.CacheMu.Unlock()
a.ProtocolCache = response
resp.Exists = true
resp.Protocols = response
return nil
}

2
are-we-synapse-yet.list
View file

@ -643,7 +643,7 @@ fed Inbound federation redacts events from erased users
fme Outbound federation can request missing events
fme Inbound federation can return missing events for world_readable visibility
fme Inbound federation can return missing events for shared visibility
fme Inbound federation can return missing events for invite visibility
fme Inbound federation can return missing events for invited visibility
fme Inbound federation can return missing events for joined visibility
fme outliers whose auth_events are in a different room are correctly rejected
fbk Outbound federation can backfill events

0
cmd/dendritejs-pinecone/jsServer.go → build/dendritejs-pinecone/jsServer.go
View file

0
cmd/dendritejs-pinecone/main.go → build/dendritejs-pinecone/main.go
View file

0
cmd/dendritejs-pinecone/main_noop.go → build/dendritejs-pinecone/main_noop.go
View file

0
cmd/dendritejs-pinecone/main_test.go → build/dendritejs-pinecone/main_test.go
View file

5
build/docker/Dockerfile.demo-pinecone
View file

@ -1,5 +1,10 @@
FROM docker.io/golang:1.19-alpine AS base
#
# Needs to be separate from the main Dockerfile for OpenShift,
# as --target is not supported there.
#
RUN apk --update --no-cache add bash build-base
WORKDIR /build

11
build/docker/Dockerfile.monolith → build/docker/Dockerfile.demo-yggdrasil
View file

@ -1,5 +1,10 @@
FROM docker.io/golang:1.19-alpine AS base
#
# Needs to be separate from the main Dockerfile for OpenShift,
# as --target is not supported there.
#
RUN apk --update --no-cache add bash build-base
WORKDIR /build
@ -7,12 +12,12 @@ WORKDIR /build
COPY . /build
RUN mkdir -p bin
RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server
RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-yggdrasil
RUN go build -trimpath -o bin/ ./cmd/create-account
RUN go build -trimpath -o bin/ ./cmd/generate-keys
FROM alpine:latest
LABEL org.opencontainers.image.title="Dendrite (Monolith)"
LABEL org.opencontainers.image.title="Dendrite (Yggdrasil demo)"
LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
LABEL org.opencontainers.image.licenses="Apache-2.0"
@ -22,4 +27,4 @@ COPY --from=base /build/bin/* /usr/bin/
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
ENTRYPOINT ["/usr/bin/dendrite-demo-yggdrasil"]

25
build/docker/Dockerfile.polylith
View file

@ -1,25 +0,0 @@
FROM docker.io/golang:1.19-alpine AS base
RUN apk --update --no-cache add bash build-base
WORKDIR /build
COPY . /build
RUN mkdir -p bin
RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi
RUN go build -trimpath -o bin/ ./cmd/create-account
RUN go build -trimpath -o bin/ ./cmd/generate-keys
FROM alpine:latest
LABEL org.opencontainers.image.title="Dendrite (Polylith)"
LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
LABEL org.opencontainers.image.licenses="Apache-2.0"
COPY --from=base /build/bin/* /usr/bin/
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]

17
build/docker/README.md
View file

@ -9,15 +9,20 @@ They can be found on Docker Hub:
## Dockerfiles
The `Dockerfile` builds the base image which contains all of the Dendrite
components. The `Dockerfile.component` file takes the given component, as
specified with `--buildarg component=` from the base image and produce
smaller component-specific images, which are substantially smaller and do
not contain the Go toolchain etc.
The `Dockerfile` is a multistage file which can build all four Dendrite
images depending on the supplied `--target`. From the root of the Dendrite
repository, run:
```
docker build . --target monolith -t matrixdotorg/dendrite-monolith
docker build . --target polylith -t matrixdotorg/dendrite-monolith
docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone
docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil
```
## Compose files
There are three sample `docker-compose` files:
There are two sample `docker-compose` files:
- `docker-compose.monolith.yml` which runs a monolith Dendrite deployment
- `docker-compose.polylith.yml` which runs a polylith Dendrite deployment

6
build/docker/images-build.sh
View file

@ -6,5 +6,7 @@ TAG=${1:-latest}
echo "Building tag '${TAG}'"
docker build -t matrixdotorg/dendrite-monolith:${TAG} -f build/docker/Dockerfile.monolith .
docker build -t matrixdotorg/dendrite-polylith:${TAG} -f build/docker/Dockerfile.polylith .
docker build . --target monolith -t matrixdotorg/dendrite-monolith:${TAG}
docker build . --target polylith -t matrixdotorg/dendrite-monolith:${TAG}
docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone:${TAG}
docker build . --target demo-yggdrasil -t matrixdotorg/dendrite-demo-yggdrasil:${TAG}

166
build/gobind-pinecone/monolith.go
View file

@ -30,6 +30,8 @@ import (
"sync"
"time"
"go.uber.org/atomic"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/clientapi/userutil"
@ -38,6 +40,7 @@ import (
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/roomserver"
@ -56,6 +59,7 @@ import (
pineconeConnections "github.com/matrix-org/pinecone/connections"
pineconeMulticast "github.com/matrix-org/pinecone/multicast"
pineconeRouter "github.com/matrix-org/pinecone/router"
pineconeEvents "github.com/matrix-org/pinecone/router/events"
pineconeSessions "github.com/matrix-org/pinecone/sessions"
"github.com/matrix-org/pinecone/types"
@ -66,6 +70,7 @@ const (
PeerTypeRemote = pineconeRouter.PeerTypeRemote
PeerTypeMulticast = pineconeRouter.PeerTypeMulticast
PeerTypeBluetooth = pineconeRouter.PeerTypeBluetooth
PeerTypeBonjour = pineconeRouter.PeerTypeBonjour
)
type DendriteMonolith struct {
@ -82,6 +87,10 @@ type DendriteMonolith struct {
userAPI userapiAPI.UserInternalAPI
}
func (m *DendriteMonolith) PublicKey() string {
return m.PineconeRouter.PublicKey().String()
}
func (m *DendriteMonolith) BaseURL() string {
return fmt.Sprintf("http://%s", m.listener.Addr().String())
}
@ -94,6 +103,48 @@ func (m *DendriteMonolith) SessionCount() int {
return len(m.PineconeQUIC.Protocol("matrix").Sessions())
}
type InterfaceInfo struct {
Name string
Index int
Mtu int
Up bool
Broadcast bool
Loopback bool
PointToPoint bool
Multicast bool
Addrs string
}
type InterfaceRetriever interface {
CacheCurrentInterfaces() int
GetCachedInterface(index int) *InterfaceInfo
}
func (m *DendriteMonolith) RegisterNetworkCallback(intfCallback InterfaceRetriever) {
callback := func() []pineconeMulticast.InterfaceInfo {
count := intfCallback.CacheCurrentInterfaces()
intfs := []pineconeMulticast.InterfaceInfo{}
for i := 0; i < count; i++ {
iface := intfCallback.GetCachedInterface(i)
if iface != nil {
intfs = append(intfs, pineconeMulticast.InterfaceInfo{
Name: iface.Name,
Index: iface.Index,
Mtu: iface.Mtu,
Up: iface.Up,
Broadcast: iface.Broadcast,
Loopback: iface.Loopback,
PointToPoint: iface.PointToPoint,
Multicast: iface.Multicast,
Addrs: iface.Addrs,
})
}
}
return intfs
}
m.PineconeMulticast.RegisterNetworkCallback(callback)
}
func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) {
if enabled {
m.PineconeMulticast.Start()
@ -105,7 +156,9 @@ func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) {
func (m *DendriteMonolith) SetStaticPeer(uri string) {
m.PineconeManager.RemovePeers()
m.PineconeManager.AddPeer(strings.TrimSpace(uri))
for _, uri := range strings.Split(uri, ",") {
m.PineconeManager.AddPeer(strings.TrimSpace(uri))
}
}
func (m *DendriteMonolith) DisconnectType(peertype int) {
@ -134,32 +187,21 @@ func (m *DendriteMonolith) Conduit(zone string, peertype int) (*Conduit, error)
go func() {
conduit.portMutex.Lock()
defer conduit.portMutex.Unlock()
loop:
for i := 1; i <= 10; i++ {
logrus.Errorf("Attempting authenticated connect (attempt %d)", i)
var err error
conduit.port, err = m.PineconeRouter.Connect(
l,
pineconeRouter.ConnectionZone(zone),
pineconeRouter.ConnectionPeerType(peertype),
)
switch err {
case io.ErrClosedPipe:
logrus.Errorf("Authenticated connect failed due to closed pipe (attempt %d)", i)
return
case io.EOF:
logrus.Errorf("Authenticated connect failed due to EOF (attempt %d)", i)
break loop
case nil:
logrus.Errorf("Authenticated connect succeeded, connected to port %d (attempt %d)", conduit.port, i)
return
default:
logrus.WithError(err).Errorf("Authenticated connect failed (attempt %d)", i)
time.Sleep(time.Second)
}
logrus.Errorf("Attempting authenticated connect")
var err error
if conduit.port, err = m.PineconeRouter.Connect(
l,
pineconeRouter.ConnectionZone(zone),
pineconeRouter.ConnectionPeerType(peertype),
); err != nil {
logrus.Errorf("Authenticated connect failed: %s", err)
_ = l.Close()
_ = r.Close()
_ = conduit.Close()
return
}
_ = l.Close()
_ = r.Close()
logrus.Infof("Authenticated connect succeeded (port %d)", conduit.port)
}()
return conduit, nil
}
@ -255,7 +297,12 @@ func (m *DendriteMonolith) Start() {
m.logger.SetOutput(BindLogger{})
logrus.SetOutput(BindLogger{})
pineconeEventChannel := make(chan pineconeEvents.Event)
m.PineconeRouter = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk)
m.PineconeRouter.EnableHopLimiting()
m.PineconeRouter.EnableWakeupBroadcasts()
m.PineconeRouter.Subscribe(pineconeEventChannel)
m.PineconeQUIC = pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), m.PineconeRouter, []string{"matrix"})
m.PineconeMulticast = pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), m.PineconeRouter)
m.PineconeManager = pineconeConnections.NewConnectionManager(m.PineconeRouter, nil)
@ -269,24 +316,27 @@ func (m *DendriteMonolith) Start() {
cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
cfg.Global.PrivateKey = sk
cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
cfg.Global.JetStream.InMemory = true
cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/%s", m.StorageDirectory, prefix))
cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-account.db", m.StorageDirectory, prefix))
cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-mediaapi.db", m.StorageDirectory))
cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-syncapi.db", m.StorageDirectory, prefix))
cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-roomserver.db", m.StorageDirectory, prefix))
cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-keyserver.db", m.StorageDirectory, prefix))
cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-federationsender.db", m.StorageDirectory, prefix))
cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory))
cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory))
cfg.Global.JetStream.InMemory = false
cfg.Global.JetStream.StoragePath = config.Path(filepath.Join(m.CacheDirectory, prefix))
cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(m.StorageDirectory, prefix)))
cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(m.StorageDirectory, prefix)))
cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(m.StorageDirectory, prefix)))
cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(m.StorageDirectory, prefix)))
cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(m.StorageDirectory, prefix)))
cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", filepath.Join(m.StorageDirectory, prefix)))
cfg.MediaAPI.BasePath = config.Path(filepath.Join(m.CacheDirectory, "media"))
cfg.MediaAPI.AbsBasePath = config.Path(filepath.Join(m.CacheDirectory, "media"))
cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
cfg.ClientAPI.RegistrationDisabled = false
cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
cfg.SyncAPI.Fulltext.Enabled = true
cfg.SyncAPI.Fulltext.IndexPath = config.Path(filepath.Join(m.CacheDirectory, "search"))
if err = cfg.Derive(); err != nil {
panic(err)
}
base := base.NewBaseDendrite(cfg, "Monolith")
base.ConfigureAdminEndpoints()
defer base.Close() // nolint: errcheck
federation := conn.CreateFederationClient(base, m.PineconeQUIC)
@ -333,6 +383,8 @@ func (m *DendriteMonolith) Start() {
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
httpRouter.HandleFunc("/pinecone", m.PineconeRouter.ManholeHandler)
pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()
@ -381,6 +433,34 @@ func (m *DendriteMonolith) Start() {
m.logger.Fatal(err)
}
}()
go func(ch <-chan pineconeEvents.Event) {
eLog := logrus.WithField("pinecone", "events")
for event := range ch {
switch e := event.(type) {
case pineconeEvents.PeerAdded:
case pineconeEvents.PeerRemoved:
case pineconeEvents.TreeParentUpdate:
case pineconeEvents.SnakeDescUpdate:
case pineconeEvents.TreeRootAnnUpdate:
case pineconeEvents.SnakeEntryAdded:
case pineconeEvents.SnakeEntryRemoved:
case pineconeEvents.BroadcastReceived:
eLog.Info("Broadcast received from: ", e.PeerID)
req := &api.PerformWakeupServersRequest{
ServerNames: []gomatrixserverlib.ServerName{gomatrixserverlib.ServerName(e.PeerID)},
}
res := &api.PerformWakeupServersResponse{}
if err := fsAPI.PerformWakeupServers(base.Context(), req, res); err != nil {
logrus.WithError(err).Error("Failed to wakeup destination", e.PeerID)
}
case pineconeEvents.BandwidthReport:
default:
}
}
}(pineconeEventChannel)
}
func (m *DendriteMonolith) Stop() {
@ -395,6 +475,7 @@ func (m *DendriteMonolith) Stop() {
const MaxFrameSize = types.MaxFrameSize
type Conduit struct {
closed atomic.Bool
conn net.Conn
port types.SwitchPortID
portMutex sync.Mutex
@ -407,10 +488,16 @@ func (c *Conduit) Port() int {
}
func (c *Conduit) Read(b []byte) (int, error) {
if c.closed.Load() {
return 0, io.EOF
}
return c.conn.Read(b)
}
func (c *Conduit) ReadCopy() ([]byte, error) {
if c.closed.Load() {
return nil, io.EOF
}
var buf [65535 * 2]byte
n, err := c.conn.Read(buf[:])
if err != nil {
@ -420,9 +507,16 @@ func (c *Conduit) ReadCopy() ([]byte, error) {
}
func (c *Conduit) Write(b []byte) (int, error) {
if c.closed.Load() {
return 0, io.EOF
}
return c.conn.Write(b)
}
func (c *Conduit) Close() error {
if c.closed.Load() {
return io.ErrClosedPipe
}
c.closed.Store(true)
return c.conn.Close()
}

3
build/gobind-yggdrasil/monolith.go
View file

@ -150,6 +150,7 @@ func (m *DendriteMonolith) Start() {
}
base := base.NewBaseDendrite(cfg, "Monolith")
base.ConfigureAdminEndpoints()
m.processContext = base.ProcessContext
defer base.Close() // nolint: errcheck
@ -196,6 +197,8 @@ func (m *DendriteMonolith) Start() {
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
yggRouter := mux.NewRouter()
yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)

7
build/scripts/Complement.Dockerfile
View file

@ -10,12 +10,13 @@ RUN mkdir /dendrite
# Utilise Docker caching when downloading dependencies, this stops us needlessly
# downloading dependencies every time.
ARG CGO
RUN --mount=target=. \
--mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
go build -o /dendrite ./cmd/generate-config && \
go build -o /dendrite ./cmd/generate-keys && \
go build -o /dendrite ./cmd/dendrite-monolith-server
CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/dendrite-monolith-server
WORKDIR /dendrite
RUN ./generate-keys --private-key matrix_key.pem

7
build/scripts/ComplementPostgres.Dockerfile
View file

@ -28,12 +28,13 @@ RUN mkdir /dendrite
# Utilise Docker caching when downloading dependencies, this stops us needlessly
# downloading dependencies every time.
ARG CGO
RUN --mount=target=. \
--mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
go build -o /dendrite ./cmd/generate-config && \
go build -o /dendrite ./cmd/generate-keys && \
go build -o /dendrite ./cmd/dendrite-monolith-server
CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-config && \
CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/generate-keys && \
CGO_ENABLED=${CGO} go build -o /dendrite ./cmd/dendrite-monolith-server
WORKDIR /dendrite
RUN ./generate-keys --private-key matrix_key.pem

9
clientapi/auth/login_test.go
View file

@ -24,6 +24,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/setup/config"
uapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
@ -66,7 +67,9 @@ func TestLoginFromJSONReader(t *testing.T) {
var userAPI fakeUserInternalAPI
cfg := &config.ClientAPI{
Matrix: &config.Global{
ServerName: serverName,
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: serverName,
},
},
}
login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)
@ -144,7 +147,9 @@ func TestBadLoginFromJSONReader(t *testing.T) {
var userAPI fakeUserInternalAPI
cfg := &config.ClientAPI{
Matrix: &config.Global{
ServerName: serverName,
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: serverName,
},
},
}
_, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, cfg)

27
clientapi/auth/password.go
View file

@ -61,39 +61,56 @@ func (t *LoginTypePassword) LoginFromJSON(ctx context.Context, reqBytes []byte)
func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) {
r := req.(*PasswordRequest)
username := strings.ToLower(r.Username())
username := r.Username()
if username == "" {
return nil, &util.JSONResponse{
Code: http.StatusUnauthorized,
JSON: jsonerror.BadJSON("A username must be supplied."),
}
}
localpart, err := userutil.ParseUsernameParam(username, &t.Config.Matrix.ServerName)
if len(r.Password) == 0 {
return nil, &util.JSONResponse{
Code: http.StatusUnauthorized,
JSON: jsonerror.BadJSON("A password must be supplied."),
}
}
localpart, domain, err := userutil.ParseUsernameParam(username, t.Config.Matrix)
if err != nil {
return nil, &util.JSONResponse{
Code: http.StatusUnauthorized,
JSON: jsonerror.InvalidUsername(err.Error()),
}
}
if !t.Config.Matrix.IsLocalServerName(domain) {
return nil, &util.JSONResponse{
Code: http.StatusUnauthorized,
JSON: jsonerror.InvalidUsername("The server name is not known."),
}
}
// Squash username to all lowercase letters
res := &api.QueryAccountByPasswordResponse{}
err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{Localpart: strings.ToLower(localpart), PlaintextPassword: r.Password}, res)
err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
Localpart: strings.ToLower(localpart),
ServerName: domain,
PlaintextPassword: r.Password,
}, res)
if err != nil {
return nil, &util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: jsonerror.Unknown("unable to fetch account by password"),
JSON: jsonerror.Unknown("Unable to fetch account by password."),
}
}
if !res.Exists {
err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
Localpart: localpart,
ServerName: domain,
PlaintextPassword: r.Password,
}, res)
if err != nil {
return nil, &util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: jsonerror.Unknown("unable to fetch account by password"),
JSON: jsonerror.Unknown("Unable to fetch account by password."),
}
}
// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows

4
clientapi/auth/user_interactive_test.go
View file

@ -47,7 +47,9 @@ func (d *fakeAccountDatabase) QueryAccountByPassword(ctx context.Context, req *a
func setup() *UserInteractive {
cfg := &config.ClientAPI{
Matrix: &config.Global{
ServerName: serverName,
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: serverName,
},
},
}
return NewUserInteractive(&fakeAccountDatabase{}, cfg)

44
clientapi/routing/account_data.go
View file

@ -154,33 +154,31 @@ func SaveReadMarker(
return *resErr
}
if r.FullyRead == "" {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON("Missing m.fully_read mandatory field"),
if r.FullyRead != "" {
data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead})
if err != nil {
return jsonerror.InternalServerError()
}
dataReq := api.InputAccountDataRequest{
UserID: device.UserID,
DataType: "m.fully_read",
RoomID: roomID,
AccountData: data,
}
dataRes := api.InputAccountDataResponse{}
if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed")
return util.ErrorResponse(err)
}
}
data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead})
if err != nil {
return jsonerror.InternalServerError()
}
dataReq := api.InputAccountDataRequest{
UserID: device.UserID,
DataType: "m.fully_read",
RoomID: roomID,
AccountData: data,
}
dataRes := api.InputAccountDataResponse{}
if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed")
return util.ErrorResponse(err)
}
// Handle the read receipt that may be included in the read marker
// Handle the read receipts that may be included in the read marker.
if r.Read != "" {
return SetReceipt(req, syncProducer, device, roomID, "m.read", r.Read)
return SetReceipt(req, userAPI, syncProducer, device, roomID, "m.read", r.Read)
}
if r.ReadPrivate != "" {
return SetReceipt(req, userAPI, syncProducer, device, roomID, "m.read.private", r.ReadPrivate)
}
return util.JSONResponse{

102
clientapi/routing/admin.go
View file

@ -2,20 +2,23 @@ package routing
import (
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/httputil"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/jetstream"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/nats-io/nats.go"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/keyserver/api"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/jetstream"
userapi "github.com/matrix-org/dendrite/userapi/api"
)
func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
@ -67,7 +70,7 @@ func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi
if err != nil {
return util.MessageResponse(http.StatusBadRequest, err.Error())
}
if domain != cfg.Matrix.ServerName {
if !cfg.Matrix.IsLocalServerName(domain) {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.MissingArgument("User ID must belong to this server."),
@ -99,6 +102,7 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap
if err != nil {
return util.ErrorResponse(err)
}
serverName := cfg.Matrix.ServerName
localpart, ok := vars["localpart"]
if !ok {
return util.JSONResponse{
@ -106,6 +110,9 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap
JSON: jsonerror.MissingArgument("Expecting user localpart."),
}
}
if l, s, err := cfg.Matrix.SplitLocalID('@', localpart); err == nil {
localpart, serverName = l, s
}
request := struct {
Password string `json:"password"`
}{}
@ -123,6 +130,7 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap
}
updateReq := &userapi.PerformPasswordUpdateRequest{
Localpart: localpart,
ServerName: serverName,
Password: request.Password,
LogoutDevices: true,
}
@ -144,12 +152,6 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap
}
func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, natsClient *nats.Conn) util.JSONResponse {
if device.AccountType != userapi.AccountTypeAdmin {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("This API can only be used by admin users."),
}
}
_, err := natsClient.RequestMsg(nats.NewMsg(cfg.Matrix.JetStream.Prefixed(jetstream.InputFulltextReindex)), time.Second*10)
if err != nil {
logrus.WithError(err).Error("failed to publish nats message")
@ -160,3 +162,77 @@ func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *userapi.Devi
JSON: struct{}{},
}
}
func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.ClientKeyAPI) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
userID := vars["userID"]
_, domain, err := gomatrixserverlib.SplitID('@', userID)
if err != nil {
return util.MessageResponse(http.StatusBadRequest, err.Error())
}
if cfg.Matrix.IsLocalServerName(domain) {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.InvalidParam("Can not mark local device list as stale"),
}
}
err = keyAPI.PerformMarkAsStaleIfNeeded(req.Context(), &api.PerformMarkAsStaleRequest{
UserID: userID,
Domain: domain,
}, &struct{}{})
if err != nil {
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: jsonerror.Unknown(fmt.Sprintf("Failed to mark device list as stale: %s", err)),
}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
}
}
func AdminDownloadState(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
roomID, ok := vars["roomID"]
if !ok {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.MissingArgument("Expecting room ID."),
}
}
serverName, ok := vars["serverName"]
if !ok {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.MissingArgument("Expecting remote server name."),
}
}
res := &roomserverAPI.PerformAdminDownloadStateResponse{}
if err := rsAPI.PerformAdminDownloadState(
req.Context(),
&roomserverAPI.PerformAdminDownloadStateRequest{
UserID: device.UserID,
RoomID: roomID,
ServerName: gomatrixserverlib.ServerName(serverName),
},
res,
); err != nil {
return jsonerror.InternalAPIError(req.Context(), err)
}
if err := res.Error; err != nil {
return err.JSONResponse()
}
return util.JSONResponse{
Code: 200,
JSON: map[string]interface{}{},
}
}

18
clientapi/routing/auth_fallback.go
View file

@ -31,8 +31,7 @@ const recaptchaTemplate = `
<title>Authentication</title>
<meta name='viewport' content='width=device-width, initial-scale=1,
user-scalable=no, minimum-scale=1.0, maximum-scale=1.0'>
<script src="https://www.google.com/recaptcha/api.js"
async defer></script>
<script src="{{.apiJsUrl}}" async defer></script>
<script src="//code.jquery.com/jquery-1.11.2.min.js"></script>
<script>
function captchaDone() {
@ -51,8 +50,8 @@ function captchaDone() {
Please verify that you're not a robot.
</p>
<input type="hidden" name="session" value="{{.session}}" />
<div class="g-recaptcha"
data-sitekey="{{.siteKey}}"
<div class="{{.sitekeyClass}}"
data-sitekey="{{.sitekey}}"
data-callback="captchaDone">
</div>
<noscript>
@ -114,9 +113,12 @@ func AuthFallback(
serveRecaptcha := func() {
data := map[string]string{
"myUrl": req.URL.String(),
"session": sessionID,
"siteKey": cfg.RecaptchaPublicKey,
"myUrl": req.URL.String(),
"session": sessionID,
"apiJsUrl": cfg.RecaptchaApiJsUrl,
"sitekey": cfg.RecaptchaPublicKey,
"sitekeyClass": cfg.RecaptchaSitekeyClass,
"formField": cfg.RecaptchaFormField,
}
serveTemplate(w, recaptchaTemplate, data)
}
@ -155,7 +157,7 @@ func AuthFallback(
return &res
}
response := req.Form.Get("g-recaptcha-response")
response := req.Form.Get(cfg.RecaptchaFormField)
if err := validateRecaptcha(cfg, response, clientIP); err != nil {
util.GetLogger(req.Context()).Error(err)
return err

27
clientapi/routing/createroom.go
View file

@ -169,9 +169,21 @@ func createRoom(
asAPI appserviceAPI.AppServiceInternalAPI,
evTime time.Time,
) util.JSONResponse {
_, userDomain, err := gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.SplitID failed")
return jsonerror.InternalServerError()
}
if !cfg.Matrix.IsLocalServerName(userDomain) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden(fmt.Sprintf("User domain %q not configured locally", userDomain)),
}
}
// TODO (#267): Check room ID doesn't clash with an existing one, and we
// probably shouldn't be using pseudo-random strings, maybe GUIDs?
roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.Matrix.ServerName)
roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), userDomain)
logger := util.GetLogger(ctx)
userID := device.UserID
@ -314,7 +326,7 @@ func createRoom(
var roomAlias string
if r.RoomAliasName != "" {
roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, cfg.Matrix.ServerName)
roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, userDomain)
// check it's free TODO: This races but is better than nothing
hasAliasReq := roomserverAPI.GetRoomIDForAliasRequest{
Alias: roomAlias,
@ -436,7 +448,7 @@ func createRoom(
builder.PrevEvents = []gomatrixserverlib.EventReference{builtEvents[i-1].EventReference()}
}
var ev *gomatrixserverlib.Event
ev, err = buildEvent(&builder, &authEvents, cfg, evTime, roomVersion)
ev, err = buildEvent(&builder, userDomain, &authEvents, cfg, evTime, roomVersion)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("buildEvent failed")
return jsonerror.InternalServerError()
@ -461,11 +473,11 @@ func createRoom(
inputs = append(inputs, roomserverAPI.InputRoomEvent{
Kind: roomserverAPI.KindNew,
Event: event,
Origin: cfg.Matrix.ServerName,
Origin: userDomain,
SendAsServer: roomserverAPI.DoNotSendToOtherServers,
})
}
if err = roomserverAPI.SendInputRoomEvents(ctx, rsAPI, inputs, false); err != nil {
if err = roomserverAPI.SendInputRoomEvents(ctx, rsAPI, device.UserDomain(), inputs, false); err != nil {
util.GetLogger(ctx).WithError(err).Error("roomserverAPI.SendInputRoomEvents failed")
return jsonerror.InternalServerError()
}
@ -548,7 +560,7 @@ func createRoom(
Event: event,
InviteRoomState: inviteStrippedState,
RoomVersion: event.RoomVersion,
SendAsServer: string(cfg.Matrix.ServerName),
SendAsServer: string(userDomain),
}, &inviteRes); err != nil {
util.GetLogger(ctx).WithError(err).Error("PerformInvite failed")
return util.JSONResponse{
@ -591,6 +603,7 @@ func createRoom(
// buildEvent fills out auth_events for the builder then builds the event
func buildEvent(
builder *gomatrixserverlib.EventBuilder,
serverName gomatrixserverlib.ServerName,
provider gomatrixserverlib.AuthEventProvider,
cfg *config.ClientAPI,
evTime time.Time,
@ -606,7 +619,7 @@ func buildEvent(
}
builder.AuthEvents = refs
event, err := builder.Build(
evTime, cfg.Matrix.ServerName, cfg.Matrix.KeyID,
evTime, serverName, cfg.Matrix.KeyID,
cfg.Matrix.PrivateKey, roomVersion,
)
if err != nil {

51
clientapi/routing/directory.go
View file

@ -18,14 +18,15 @@ import (
"fmt"
"net/http"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
federationAPI "github.com/matrix-org/dendrite/federationapi/api"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
type roomDirectoryResponse struct {
@ -75,8 +76,8 @@ func DirectoryRoom(
if res.RoomID == "" {
// If we don't know it locally, do a federation query.
// But don't send the query to ourselves.
if domain != cfg.Matrix.ServerName {
fedRes, fedErr := federation.LookupRoomAlias(req.Context(), domain, roomAlias)
if !cfg.Matrix.IsLocalServerName(domain) {
fedRes, fedErr := federation.LookupRoomAlias(req.Context(), cfg.Matrix.ServerName, domain, roomAlias)
if fedErr != nil {
// TODO: Return 502 if the remote server errored.
// TODO: Return 504 if the remote server timed out.
@ -127,7 +128,7 @@ func SetLocalAlias(
}
}
if domain != cfg.Matrix.ServerName {
if !cfg.Matrix.IsLocalServerName(domain) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Alias must be on local homeserver"),
@ -318,3 +319,43 @@ func SetVisibility(
JSON: struct{}{},
}
}
func SetVisibilityAS(
req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, dev *userapi.Device,
networkID, roomID string,
) util.JSONResponse {
if dev.AccountType != userapi.AccountTypeAppService {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Only appservice may use this endpoint"),
}
}
var v roomVisibility
// If the method is delete, we simply mark the visibility as private
if req.Method == http.MethodDelete {
v.Visibility = "private"
} else {
if reqErr := httputil.UnmarshalJSONRequest(req, &v); reqErr != nil {
return *reqErr
}
}
var publishRes roomserverAPI.PerformPublishResponse
if err := rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{
RoomID: roomID,
Visibility: v.Visibility,
NetworkID: networkID,
AppserviceID: dev.AppserviceID,
}, &publishRes); err != nil {
return jsonerror.InternalAPIError(req.Context(), err)
}
if publishRes.Error != nil {
util.GetLogger(req.Context()).WithError(publishRes.Error).Error("PerformPublish failed")
return publishRes.Error.JSONResponse()
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
}
}

45
clientapi/routing/directory_public.go
View file

@ -39,14 +39,17 @@ var (
)
type PublicRoomReq struct {
Since string `json:"since,omitempty"`
Limit int16 `json:"limit,omitempty"`
Filter filter `json:"filter,omitempty"`
Server string `json:"server,omitempty"`
Since string `json:"since,omitempty"`
Limit int64 `json:"limit,omitempty"`
Filter filter `json:"filter,omitempty"`
Server string `json:"server,omitempty"`
IncludeAllNetworks bool `json:"include_all_networks,omitempty"`
NetworkID string `json:"third_party_instance_id,omitempty"`
}
type filter struct {
SearchTerms string `json:"generic_search_term,omitempty"`
SearchTerms string `json:"generic_search_term,omitempty"`
RoomTypes []string `json:"room_types,omitempty"` // TODO: Implement filter on this
}
// GetPostPublicRooms implements GET and POST /publicRooms
@ -61,11 +64,17 @@ func GetPostPublicRooms(
return *fillErr
}
serverName := gomatrixserverlib.ServerName(request.Server)
if request.IncludeAllNetworks && request.NetworkID != "" {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.InvalidParam("include_all_networks and third_party_instance_id can not be used together"),
}
}
if serverName != "" && serverName != cfg.Matrix.ServerName {
serverName := gomatrixserverlib.ServerName(request.Server)
if serverName != "" && !cfg.Matrix.IsLocalServerName(serverName) {
res, err := federation.GetPublicRoomsFiltered(
req.Context(), serverName,
req.Context(), cfg.Matrix.ServerName, serverName,
int(request.Limit), request.Since,
request.Filter.SearchTerms, false,
"",
@ -98,7 +107,7 @@ func publicRooms(
response := gomatrixserverlib.RespPublicRooms{
Chunk: []gomatrixserverlib.PublicRoom{},
}
var limit int16
var limit int64
var offset int64
limit = request.Limit
if limit == 0 {
@ -115,7 +124,7 @@ func publicRooms(
var rooms []gomatrixserverlib.PublicRoom
if request.Since == "" {
rooms = refreshPublicRoomCache(ctx, rsAPI, extRoomsProvider)
rooms = refreshPublicRoomCache(ctx, rsAPI, extRoomsProvider, request)
} else {
rooms = getPublicRoomsFromCache()
}
@ -177,7 +186,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO
JSON: jsonerror.BadJSON("limit param is not a number"),
}
}
request.Limit = int16(limit)
request.Limit = int64(limit)
request.Since = httpReq.FormValue("since")
request.Server = httpReq.FormValue("server")
} else {
@ -205,7 +214,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO
// limit=3&since=6 => G (prev='3', next='')
//
// A value of '-1' for prev/next indicates no position.
func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int16) (subset []gomatrixserverlib.PublicRoom, prev, next int) {
func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int64) (subset []gomatrixserverlib.PublicRoom, prev, next int) {
prev = -1
next = -1
@ -231,6 +240,7 @@ func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int16) (
func refreshPublicRoomCache(
ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, extRoomsProvider api.ExtraPublicRoomsProvider,
request PublicRoomReq,
) []gomatrixserverlib.PublicRoom {
cacheMu.Lock()
defer cacheMu.Unlock()
@ -239,8 +249,17 @@ func refreshPublicRoomCache(
extraRooms = extRoomsProvider.Rooms()
}
// TODO: this is only here to make Sytest happy, for now.
ns := strings.Split(request.NetworkID, "|")
if len(ns) == 2 {
request.NetworkID = ns[1]
}
var queryRes roomserverAPI.QueryPublishedRoomsResponse
err := rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes)
err := rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{
NetworkID: request.NetworkID,
IncludeAllNetworks: request.IncludeAllNetworks,
}, &queryRes)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed")
return publicRoomsCache

2
clientapi/routing/directory_public_test.go
View file

@ -17,7 +17,7 @@ func TestSliceInto(t *testing.T) {
slice := []gomatrixserverlib.PublicRoom{
pubRoom("a"), pubRoom("b"), pubRoom("c"), pubRoom("d"), pubRoom("e"), pubRoom("f"), pubRoom("g"),
}
limit := int16(3)
limit := int64(3)
testCases := []struct {
since int64
wantPrev int

138
clientapi/routing/getevent.go
View file

@ -1,138 +0,0 @@
// Copyright 2019 Alex Chen
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package routing
import (
"net/http"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
type getEventRequest struct {
req *http.Request
device *userapi.Device
roomID string
eventID string
cfg *config.ClientAPI
requestedEvent *gomatrixserverlib.Event
}
// GetEvent implements GET /_matrix/client/r0/rooms/{roomId}/event/{eventId}
// https://matrix.org/docs/spec/client_server/r0.4.0.html#get-matrix-client-r0-rooms-roomid-event-eventid
func GetEvent(
req *http.Request,
device *userapi.Device,
roomID string,
eventID string,
cfg *config.ClientAPI,
rsAPI api.ClientRoomserverAPI,
) util.JSONResponse {
eventsReq := api.QueryEventsByIDRequest{
EventIDs: []string{eventID},
}
var eventsResp api.QueryEventsByIDResponse
err := rsAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryEventsByID failed")
return jsonerror.InternalServerError()
}
if len(eventsResp.Events) == 0 {
// Event not found locally
return util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"),
}
}
requestedEvent := eventsResp.Events[0].Event
r := getEventRequest{
req: req,
device: device,
roomID: roomID,
eventID: eventID,
cfg: cfg,
requestedEvent: requestedEvent,
}
stateReq := api.QueryStateAfterEventsRequest{
RoomID: r.requestedEvent.RoomID(),
PrevEventIDs: r.requestedEvent.PrevEventIDs(),
StateToFetch: []gomatrixserverlib.StateKeyTuple{{
EventType: gomatrixserverlib.MRoomMember,
StateKey: device.UserID,
}},
}
var stateResp api.QueryStateAfterEventsResponse
if err := rsAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryStateAfterEvents failed")
return jsonerror.InternalServerError()
}
if !stateResp.RoomExists {
util.GetLogger(req.Context()).Errorf("Expected to find room for event %s but failed", r.requestedEvent.EventID())
return jsonerror.InternalServerError()
}
if !stateResp.PrevEventsExist {
// Missing some events locally; stateResp.StateEvents unavailable.
return util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"),
}
}
var appService *config.ApplicationService
if device.AppserviceID != "" {
for _, as := range cfg.Derived.ApplicationServices {
if as.ID == device.AppserviceID {
appService = &as
break
}
}
}
for _, stateEvent := range stateResp.StateEvents {
if appService != nil {
if !appService.IsInterestedInUserID(*stateEvent.StateKey()) {
continue
}
} else if !stateEvent.StateKeyEquals(device.UserID) {
continue
}
membership, err := stateEvent.Membership()
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("stateEvent.Membership failed")
return jsonerror.InternalServerError()
}
if membership == gomatrixserverlib.Join {
return util.JSONResponse{
Code: http.StatusOK,
JSON: gomatrixserverlib.ToClientEvent(r.requestedEvent, gomatrixserverlib.FormatAll),
}
}
}
return util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"),
}
}

52
clientapi/routing/joined_rooms.go Normal file
View file

@ -0,0 +1,52 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package routing
import (
"net/http"
"github.com/matrix-org/util"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/roomserver/api"
userapi "github.com/matrix-org/dendrite/userapi/api"
)
type getJoinedRoomsResponse struct {
JoinedRooms []string `json:"joined_rooms"`
}
func GetJoinedRooms(
req *http.Request,
device *userapi.Device,
rsAPI api.ClientRoomserverAPI,
) util.JSONResponse {
var res api.QueryRoomsForUserResponse
err := rsAPI.QueryRoomsForUser(req.Context(), &api.QueryRoomsForUserRequest{
UserID: device.UserID,
WantMembership: "join",
}, &res)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed")
return jsonerror.InternalServerError()
}
if res.RoomIDs == nil {
res.RoomIDs = []string{}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: getJoinedRoomsResponse{res.RoomIDs},
}
}

10
clientapi/routing/keys.go
View file

@ -19,11 +19,12 @@ import (
"net/http"
"time"
"github.com/matrix-org/util"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/keyserver/api"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/util"
)
type uploadKeysRequest struct {
@ -77,7 +78,6 @@ func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Devi
}
}
keyCount := make(map[string]int)
// we only return key counts when the client uploads OTKs
if len(uploadRes.OneTimeKeyCounts) > 0 {
keyCount = uploadRes.OneTimeKeyCounts[0].KeyCount
}
@ -99,7 +99,11 @@ func (r *queryKeysRequest) GetTimeout() time.Duration {
if r.Timeout == 0 {
return 10 * time.Second
}
return time.Duration(r.Timeout) * time.Millisecond
timeout := time.Duration(r.Timeout) * time.Millisecond
if timeout > time.Second*20 {
timeout = time.Second * 20
}
return timeout
}
func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse {

7
clientapi/routing/login.go
View file

@ -68,7 +68,7 @@ func Login(
return *authErr
}
// make a device/access token
authErr2 := completeAuth(req.Context(), cfg.Matrix.ServerName, userAPI, login, req.RemoteAddr, req.UserAgent())
authErr2 := completeAuth(req.Context(), cfg.Matrix, userAPI, login, req.RemoteAddr, req.UserAgent())
cleanup(req.Context(), &authErr2)
return authErr2
}
@ -79,7 +79,7 @@ func Login(
}
func completeAuth(
ctx context.Context, serverName gomatrixserverlib.ServerName, userAPI userapi.ClientUserAPI, login *auth.Login,
ctx context.Context, cfg *config.Global, userAPI userapi.ClientUserAPI, login *auth.Login,
ipAddr, userAgent string,
) util.JSONResponse {
token, err := auth.GenerateAccessToken()
@ -88,7 +88,7 @@ func completeAuth(
return jsonerror.InternalServerError()
}
localpart, err := userutil.ParseUsernameParam(login.Username(), &serverName)
localpart, serverName, err := userutil.ParseUsernameParam(login.Username(), cfg)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("auth.ParseUsernameParam failed")
return jsonerror.InternalServerError()
@ -100,6 +100,7 @@ func completeAuth(
DeviceID: login.DeviceID,
AccessToken: token,
Localpart: localpart,
ServerName: serverName,
IPAddr: ipAddr,
UserAgent: userAgent,
}, &performRes)

17
clientapi/routing/membership.go
View file

@ -105,12 +105,14 @@ func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, devic
return jsonerror.InternalServerError()
}
serverName := device.UserDomain()
if err = roomserverAPI.SendEvents(
ctx, rsAPI,
roomserverAPI.KindNew,
[]*gomatrixserverlib.HeaderedEvent{event.Event.Headered(roomVer)},
cfg.Matrix.ServerName,
cfg.Matrix.ServerName,
device.UserDomain(),
serverName,
serverName,
nil,
false,
); err != nil {
@ -271,7 +273,7 @@ func sendInvite(
Event: event,
InviteRoomState: nil, // ask the roomserver to draw up invite room state for us
RoomVersion: event.RoomVersion,
SendAsServer: string(cfg.Matrix.ServerName),
SendAsServer: string(device.UserDomain()),
}, &inviteRes); err != nil {
util.GetLogger(ctx).WithError(err).Error("PerformInvite failed")
return util.JSONResponse{
@ -321,7 +323,12 @@ func buildMembershipEvent(
return nil, err
}
return eventutil.QueryAndBuildEvent(ctx, &builder, cfg.Matrix, evTime, rsAPI, nil)
identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
if err != nil {
return nil, err
}
return eventutil.QueryAndBuildEvent(ctx, &builder, cfg.Matrix, identity, evTime, rsAPI, nil)
}
// loadProfile lookups the profile of a given user from the database and returns
@ -341,7 +348,7 @@ func loadProfile(
}
var profile *authtypes.Profile
if serverName == cfg.Matrix.ServerName {
if cfg.Matrix.IsLocalServerName(serverName) {
profile, err = appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, profileAPI)
} else {
profile = &authtypes.Profile{}

11
clientapi/routing/notification.go
View file

@ -40,16 +40,17 @@ func GetNotifications(
}
var queryRes userapi.QueryNotificationsResponse
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("SplitID failed")
return jsonerror.InternalServerError()
}
err = userAPI.QueryNotifications(req.Context(), &userapi.QueryNotificationsRequest{
Localpart: localpart,
From: req.URL.Query().Get("from"),
Limit: int(limit),
Only: req.URL.Query().Get("only"),
Localpart: localpart,
ServerName: domain,
From: req.URL.Query().Get("from"),
Limit: int(limit),
Only: req.URL.Query().Get("only"),
}, &queryRes)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("QueryNotifications failed")

2
clientapi/routing/openid.go
View file

@ -63,7 +63,7 @@ func CreateOpenIDToken(
JSON: openIDTokenResponse{
AccessToken: response.Token.Token,
TokenType: "Bearer",
MatrixServerName: string(cfg.Matrix.ServerName),
MatrixServerName: string(device.UserDomain()),
ExpiresIn: response.Token.ExpiresAtMS / 1000, // convert ms to s
},
}

12
clientapi/routing/password.go
View file

@ -86,7 +86,7 @@ func Password(
}
// Get the local part.
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
return jsonerror.InternalServerError()
@ -94,8 +94,9 @@ func Password(
// Ask the user API to perform the password change.
passwordReq := &api.PerformPasswordUpdateRequest{
Localpart: localpart,
Password: r.NewPassword,
Localpart: localpart,
ServerName: domain,
Password: r.NewPassword,
}
passwordRes := &api.PerformPasswordUpdateResponse{}
if err := userAPI.PerformPasswordUpdate(req.Context(), passwordReq, passwordRes); err != nil {
@ -122,8 +123,9 @@ func Password(
}
pushersReq := &api.PerformPusherDeletionRequest{
Localpart: localpart,
SessionID: device.SessionID,
Localpart: localpart,
ServerName: domain,
SessionID: device.SessionID,
}
if err := userAPI.PerformPusherDeletion(req.Context(), pushersReq, &struct{}{}); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("PerformPusherDeletion failed")

165
clientapi/routing/profile.go
View file

@ -19,6 +19,8 @@ import (
"net/http"
"time"
"github.com/matrix-org/gomatrixserverlib"
appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/httputil"
@ -27,7 +29,6 @@ import (
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/util"
@ -112,12 +113,19 @@ func SetAvatarURL(
}
}
localpart, _, err := gomatrixserverlib.SplitID('@', userID)
localpart, domain, err := gomatrixserverlib.SplitID('@', userID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
return jsonerror.InternalServerError()
}
if !cfg.Matrix.IsLocalServerName(domain) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("userID does not belong to a locally configured domain"),
}
}
evTime, err := httputil.ParseTSParam(req)
if err != nil {
return util.JSONResponse{
@ -126,63 +134,26 @@ func SetAvatarURL(
}
}
res := &userapi.QueryProfileResponse{}
err = profileAPI.QueryProfile(req.Context(), &userapi.QueryProfileRequest{
UserID: userID,
}, res)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("profileAPI.QueryProfile failed")
return jsonerror.InternalServerError()
}
oldProfile := &authtypes.Profile{
Localpart: localpart,
DisplayName: res.DisplayName,
AvatarURL: res.AvatarURL,
}
setRes := &userapi.PerformSetAvatarURLResponse{}
if err = profileAPI.SetAvatarURL(req.Context(), &userapi.PerformSetAvatarURLRequest{
Localpart: localpart,
AvatarURL: r.AvatarURL,
Localpart: localpart,
ServerName: domain,
AvatarURL: r.AvatarURL,
}, setRes); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetAvatarURL failed")
return jsonerror.InternalServerError()
}
var roomsRes api.QueryRoomsForUserResponse
err = rsAPI.QueryRoomsForUser(req.Context(), &api.QueryRoomsForUserRequest{
UserID: device.UserID,
WantMembership: "join",
}, &roomsRes)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed")
return jsonerror.InternalServerError()
}
newProfile := authtypes.Profile{
Localpart: localpart,
DisplayName: oldProfile.DisplayName,
AvatarURL: r.AvatarURL,
}
events, err := buildMembershipEvents(
req.Context(), roomsRes.RoomIDs, newProfile, userID, cfg, evTime, rsAPI,
)
switch e := err.(type) {
case nil:
case gomatrixserverlib.BadJSONError:
// No need to build new membership events, since nothing changed
if !setRes.Changed {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON(e.Error()),
Code: http.StatusOK,
JSON: struct{}{},
}
default:
util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed")
return jsonerror.InternalServerError()
}
if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, events, cfg.Matrix.ServerName, cfg.Matrix.ServerName, nil, true); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
return jsonerror.InternalServerError()
response, err := updateProfile(req.Context(), rsAPI, device, setRes.Profile, userID, cfg, evTime)
if err != nil {
return response
}
return util.JSONResponse{
@ -241,12 +212,19 @@ func SetDisplayName(
}
}
localpart, _, err := gomatrixserverlib.SplitID('@', userID)
localpart, domain, err := gomatrixserverlib.SplitID('@', userID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
return jsonerror.InternalServerError()
}
if !cfg.Matrix.IsLocalServerName(domain) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("userID does not belong to a locally configured domain"),
}
}
evTime, err := httputil.ParseTSParam(req)
if err != nil {
return util.JSONResponse{
@ -255,47 +233,58 @@ func SetDisplayName(
}
}
pRes := &userapi.QueryProfileResponse{}
err = profileAPI.QueryProfile(req.Context(), &userapi.QueryProfileRequest{
UserID: userID,
}, pRes)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("profileAPI.QueryProfile failed")
return jsonerror.InternalServerError()
}
oldProfile := &authtypes.Profile{
Localpart: localpart,
DisplayName: pRes.DisplayName,
AvatarURL: pRes.AvatarURL,
}
profileRes := &userapi.PerformUpdateDisplayNameResponse{}
err = profileAPI.SetDisplayName(req.Context(), &userapi.PerformUpdateDisplayNameRequest{
Localpart: localpart,
ServerName: domain,
DisplayName: r.DisplayName,
}, &struct{}{})
}, profileRes)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetDisplayName failed")
return jsonerror.InternalServerError()
}
// No need to build new membership events, since nothing changed
if !profileRes.Changed {
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
}
}
response, err := updateProfile(req.Context(), rsAPI, device, profileRes.Profile, userID, cfg, evTime)
if err != nil {
return response
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
}
}
func updateProfile(
ctx context.Context, rsAPI api.ClientRoomserverAPI, device *userapi.Device,
profile *authtypes.Profile,
userID string, cfg *config.ClientAPI, evTime time.Time,
) (util.JSONResponse, error) {
var res api.QueryRoomsForUserResponse
err = rsAPI.QueryRoomsForUser(req.Context(), &api.QueryRoomsForUserRequest{
err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
UserID: device.UserID,
WantMembership: "join",
}, &res)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed")
return jsonerror.InternalServerError()
util.GetLogger(ctx).WithError(err).Error("QueryRoomsForUser failed")
return jsonerror.InternalServerError(), err
}
newProfile := authtypes.Profile{
Localpart: localpart,
DisplayName: r.DisplayName,
AvatarURL: oldProfile.AvatarURL,
_, domain, err := gomatrixserverlib.SplitID('@', userID)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.SplitID failed")
return jsonerror.InternalServerError(), err
}
events, err := buildMembershipEvents(
req.Context(), res.RoomIDs, newProfile, userID, cfg, evTime, rsAPI,
ctx, device, res.RoomIDs, *profile, userID, cfg, evTime, rsAPI,
)
switch e := err.(type) {
case nil:
@ -303,21 +292,17 @@ func SetDisplayName(
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON(e.Error()),
}
}, e
default:
util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed")
return jsonerror.InternalServerError()
util.GetLogger(ctx).WithError(err).Error("buildMembershipEvents failed")
return jsonerror.InternalServerError(), e
}
if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, events, cfg.Matrix.ServerName, cfg.Matrix.ServerName, nil, true); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
return jsonerror.InternalServerError()
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, true); err != nil {
util.GetLogger(ctx).WithError(err).Error("SendEvents failed")
return jsonerror.InternalServerError(), err
}
return util.JSONResponse{}, nil
}
// getProfile gets the full profile of a user by querying the database or a
@ -335,8 +320,8 @@ func getProfile(
return nil, err
}
if domain != cfg.Matrix.ServerName {
profile, fedErr := federation.LookupProfile(ctx, domain, userID, "")
if !cfg.Matrix.IsLocalServerName(domain) {
profile, fedErr := federation.LookupProfile(ctx, cfg.Matrix.ServerName, domain, userID, "")
if fedErr != nil {
if x, ok := fedErr.(gomatrix.HTTPError); ok {
if x.Code == http.StatusNotFound {
@ -364,6 +349,7 @@ func getProfile(
func buildMembershipEvents(
ctx context.Context,
device *userapi.Device,
roomIDs []string,
newProfile authtypes.Profile, userID string, cfg *config.ClientAPI,
evTime time.Time, rsAPI api.ClientRoomserverAPI,
@ -395,7 +381,12 @@ func buildMembershipEvents(
return nil, err
}
event, err := eventutil.QueryAndBuildEvent(ctx, &builder, cfg.Matrix, evTime, rsAPI, nil)
identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
if err != nil {
return nil, err
}
event, err := eventutil.QueryAndBuildEvent(ctx, &builder, cfg.Matrix, identity, evTime, rsAPI, nil)
if err != nil {
return nil, err
}

8
clientapi/routing/pusher.go
View file

@ -31,13 +31,14 @@ func GetPushers(
userAPI userapi.ClientUserAPI,
) util.JSONResponse {
var queryRes userapi.QueryPushersResponse
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("SplitID failed")
return jsonerror.InternalServerError()
}
err = userAPI.QueryPushers(req.Context(), &userapi.QueryPushersRequest{
Localpart: localpart,
Localpart: localpart,
ServerName: domain,
}, &queryRes)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("QueryPushers failed")
@ -59,7 +60,7 @@ func SetPusher(
req *http.Request, device *userapi.Device,
userAPI userapi.ClientUserAPI,
) util.JSONResponse {
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("SplitID failed")
return jsonerror.InternalServerError()
@ -93,6 +94,7 @@ func SetPusher(
}
body.Localpart = localpart
body.ServerName = domain
body.SessionID = device.SessionID
err = userAPI.PerformPusherSet(req.Context(), &body, &struct{}{})
if err != nil {

36
clientapi/routing/receipt.go
View file

@ -15,19 +15,22 @@
package routing
import (
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/userapi/api"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/util"
"github.com/sirupsen/logrus"
)
func SetReceipt(req *http.Request, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse {
func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse {
timestamp := gomatrixserverlib.AsTimestamp(time.Now())
logrus.WithFields(logrus.Fields{
"roomID": roomID,
@ -37,13 +40,32 @@ func SetReceipt(req *http.Request, syncProducer *producers.SyncAPIProducer, devi
"timestamp": timestamp,
}).Debug("Setting receipt")
// currently only m.read is accepted
if receiptType != "m.read" {
return util.MessageResponse(400, fmt.Sprintf("receipt type must be m.read not '%s'", receiptType))
}
switch receiptType {
case "m.read", "m.read.private":
if err := syncProducer.SendReceipt(req.Context(), device.UserID, roomID, eventID, receiptType, timestamp); err != nil {
return util.ErrorResponse(err)
}
if err := syncProducer.SendReceipt(req.Context(), device.UserID, roomID, eventID, receiptType, timestamp); err != nil {
return util.ErrorResponse(err)
case "m.fully_read":
data, err := json.Marshal(fullyReadEvent{EventID: eventID})
if err != nil {
return jsonerror.InternalServerError()
}
dataReq := api.InputAccountDataRequest{
UserID: device.UserID,
DataType: "m.fully_read",
RoomID: roomID,
AccountData: data,
}
dataRes := api.InputAccountDataResponse{}
if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed")
return util.ErrorResponse(err)
}
default:
return util.MessageResponse(400, fmt.Sprintf("Receipt type '%s' not known", receiptType))
}
return util.JSONResponse{

19
clientapi/routing/redaction.go
View file

@ -19,6 +19,9 @@ import (
"net/http"
"time"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/eventutil"
@ -26,8 +29,6 @@ import (
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
type redactionContent struct {
@ -51,7 +52,7 @@ func SendRedaction(
if txnID != nil {
// Try to fetch response from transactionsCache
if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok {
if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID, req.URL); ok {
return *res
}
}
@ -122,15 +123,21 @@ func SendRedaction(
return jsonerror.InternalServerError()
}
identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
if err != nil {
return jsonerror.InternalServerError()
}
var queryRes roomserverAPI.QueryLatestEventsAndStateResponse
e, err := eventutil.QueryAndBuildEvent(req.Context(), &builder, cfg.Matrix, time.Now(), rsAPI, &queryRes)
e, err := eventutil.QueryAndBuildEvent(req.Context(), &builder, cfg.Matrix, identity, time.Now(), rsAPI, &queryRes)
if err == eventutil.ErrRoomNoExists {
return util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound("Room does not exist"),
}
}
if err = roomserverAPI.SendEvents(context.Background(), rsAPI, roomserverAPI.KindNew, []*gomatrixserverlib.HeaderedEvent{e}, cfg.Matrix.ServerName, cfg.Matrix.ServerName, nil, false); err != nil {
domain := device.UserDomain()
if err = roomserverAPI.SendEvents(context.Background(), rsAPI, roomserverAPI.KindNew, []*gomatrixserverlib.HeaderedEvent{e}, device.UserDomain(), domain, domain, nil, false); err != nil {
util.GetLogger(req.Context()).WithError(err).Errorf("failed to SendEvents")
return jsonerror.InternalServerError()
}
@ -144,7 +151,7 @@ func SendRedaction(
// Add response to transactionsCache
if txnID != nil {
txnCache.AddTransaction(device.AccessToken, *txnID, &res)
txnCache.AddTransaction(device.AccessToken, *txnID, req.URL, &res)
}
return res

103
clientapi/routing/register.go
View file

@ -20,6 +20,7 @@ import (
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/url"
"regexp"
@ -210,9 +211,10 @@ var (
// previous parameters with the ones supplied. This mean you cannot "build up" request params.
type registerRequest struct {
// registration parameters
Password string `json:"password"`
Username string `json:"username"`
Admin bool `json:"admin"`
Password string `json:"password"`
Username string `json:"username"`
ServerName gomatrixserverlib.ServerName `json:"-"`
Admin bool `json:"admin"`
// user-interactive auth params
Auth authDict `json:"auth"`
@ -336,6 +338,7 @@ func validateRecaptcha(
response string,
clientip string,
) *util.JSONResponse {
ip, _, _ := net.SplitHostPort(clientip)
if !cfg.RecaptchaEnabled {
return &util.JSONResponse{
Code: http.StatusConflict,
@ -355,7 +358,7 @@ func validateRecaptcha(
url.Values{
"secret": {cfg.RecaptchaPrivateKey},
"response": {response},
"remoteip": {clientip},
"remoteip": {ip},
},
)
@ -412,7 +415,7 @@ func UserIDIsWithinApplicationServiceNamespace(
return false
}
if domain != cfg.Matrix.ServerName {
if !cfg.Matrix.IsLocalServerName(domain) {
return false
}
@ -548,6 +551,12 @@ func Register(
}
var r registerRequest
host := gomatrixserverlib.ServerName(req.Host)
if v := cfg.Matrix.VirtualHostForHTTPHost(host); v != nil {
r.ServerName = v.ServerName
} else {
r.ServerName = cfg.Matrix.ServerName
}
sessionID := gjson.GetBytes(reqBody, "auth.session").String()
if sessionID == "" {
// Generate a new, random session ID
@ -557,6 +566,7 @@ func Register(
// Some of these might end up being overwritten if the
// values are specified again in the request body.
r.Username = data.Username
r.ServerName = data.ServerName
r.Password = data.Password
r.DeviceID = data.DeviceID
r.InitialDisplayName = data.InitialDisplayName
@ -568,11 +578,13 @@ func Register(
JSON: response,
}
}
}
if resErr := httputil.UnmarshalJSON(reqBody, &r); resErr != nil {
return *resErr
}
if l, d, err := cfg.Matrix.SplitLocalID('@', r.Username); err == nil {
r.Username, r.ServerName = l, d
}
if req.URL.Query().Get("kind") == "guest" {
return handleGuestRegistration(req, r, cfg, userAPI)
}
@ -586,12 +598,15 @@ func Register(
}
// Auto generate a numeric username if r.Username is empty
if r.Username == "" {
res := &userapi.QueryNumericLocalpartResponse{}
if err := userAPI.QueryNumericLocalpart(req.Context(), res); err != nil {
nreq := &userapi.QueryNumericLocalpartRequest{
ServerName: r.ServerName,
}
nres := &userapi.QueryNumericLocalpartResponse{}
if err := userAPI.QueryNumericLocalpart(req.Context(), nreq, nres); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryNumericLocalpart failed")
return jsonerror.InternalServerError()
}
r.Username = strconv.FormatInt(res.ID, 10)
r.Username = strconv.FormatInt(nres.ID, 10)
}
// Is this an appservice registration? It will be if the access
@ -604,7 +619,7 @@ func Register(
case r.Type == authtypes.LoginTypeApplicationService && accessTokenErr == nil:
// Spec-compliant case (the access_token is specified and the login type
// is correctly set, so it's an appservice registration)
if resErr := validateApplicationServiceUsername(r.Username, cfg.Matrix.ServerName); resErr != nil {
if resErr := validateApplicationServiceUsername(r.Username, r.ServerName); resErr != nil {
return *resErr
}
case accessTokenErr == nil:
@ -617,7 +632,7 @@ func Register(
default:
// Spec-compliant case (neither the access_token nor the login type are
// specified, so it's a normal user registration)
if resErr := validateUsername(r.Username, cfg.Matrix.ServerName); resErr != nil {
if resErr := validateUsername(r.Username, r.ServerName); resErr != nil {
return *resErr
}
}
@ -641,16 +656,25 @@ func handleGuestRegistration(
cfg *config.ClientAPI,
userAPI userapi.ClientUserAPI,
) util.JSONResponse {
if cfg.RegistrationDisabled || cfg.GuestsDisabled {
registrationEnabled := !cfg.RegistrationDisabled
guestsEnabled := !cfg.GuestsDisabled
if v := cfg.Matrix.VirtualHost(r.ServerName); v != nil {
registrationEnabled, guestsEnabled = v.RegistrationAllowed()
}
if !registrationEnabled || !guestsEnabled {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Guest registration is disabled"),
JSON: jsonerror.Forbidden(
fmt.Sprintf("Guest registration is disabled on %q", r.ServerName),
),
}
}
var res userapi.PerformAccountCreationResponse
err := userAPI.PerformAccountCreation(req.Context(), &userapi.PerformAccountCreationRequest{
AccountType: userapi.AccountTypeGuest,
ServerName: r.ServerName,
}, &res)
if err != nil {
return util.JSONResponse{
@ -674,6 +698,7 @@ func handleGuestRegistration(
var devRes userapi.PerformDeviceCreationResponse
err = userAPI.PerformDeviceCreation(req.Context(), &userapi.PerformDeviceCreationRequest{
Localpart: res.Account.Localpart,
ServerName: res.Account.ServerName,
DeviceDisplayName: r.InitialDisplayName,
AccessToken: token,
IPAddr: req.RemoteAddr,
@ -726,10 +751,16 @@ func handleRegistrationFlow(
)
}
if cfg.RegistrationDisabled && r.Auth.Type != authtypes.LoginTypeSharedSecret {
registrationEnabled := !cfg.RegistrationDisabled
if v := cfg.Matrix.VirtualHost(r.ServerName); v != nil {
registrationEnabled, _ = v.RegistrationAllowed()
}
if !registrationEnabled && r.Auth.Type != authtypes.LoginTypeSharedSecret {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Registration is disabled"),
JSON: jsonerror.Forbidden(
fmt.Sprintf("Registration is disabled on %q", r.ServerName),
),
}
}
@ -817,8 +848,9 @@ func handleApplicationServiceRegistration(
// Don't need to worry about appending to registration stages as
// application service registration is entirely separate.
return completeRegistration(
req.Context(), userAPI, r.Username, "", appserviceID, req.RemoteAddr, req.UserAgent(), r.Auth.Session,
r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeAppService,
req.Context(), userAPI, r.Username, r.ServerName, "", appserviceID, req.RemoteAddr,
req.UserAgent(), r.Auth.Session, r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
userapi.AccountTypeAppService,
)
}
@ -836,8 +868,9 @@ func checkAndCompleteFlow(
if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
// This flow was completed, registration can continue
return completeRegistration(
req.Context(), userAPI, r.Username, r.Password, "", req.RemoteAddr, req.UserAgent(), sessionID,
r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeUser,
req.Context(), userAPI, r.Username, r.ServerName, r.Password, "", req.RemoteAddr,
req.UserAgent(), sessionID, r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
userapi.AccountTypeUser,
)
}
sessions.addParams(sessionID, r)
@ -859,7 +892,8 @@ func checkAndCompleteFlow(
func completeRegistration(
ctx context.Context,
userAPI userapi.ClientUserAPI,
username, password, appserviceID, ipAddr, userAgent, sessionID string,
username string, serverName gomatrixserverlib.ServerName,
password, appserviceID, ipAddr, userAgent, sessionID string,
inhibitLogin eventutil.WeakBoolean,
displayName, deviceID *string,
accType userapi.AccountType,
@ -881,6 +915,7 @@ func completeRegistration(
err := userAPI.PerformAccountCreation(ctx, &userapi.PerformAccountCreationRequest{
AppServiceID: appserviceID,
Localpart: username,
ServerName: serverName,
Password: password,
AccountType: accType,
OnConflict: userapi.ConflictAbort,
@ -924,6 +959,7 @@ func completeRegistration(
var devRes userapi.PerformDeviceCreationResponse
err = userAPI.PerformDeviceCreation(ctx, &userapi.PerformDeviceCreationRequest{
Localpart: username,
ServerName: serverName,
AccessToken: token,
DeviceDisplayName: displayName,
DeviceID: deviceID,
@ -1017,13 +1053,31 @@ func RegisterAvailable(
// Squash username to all lowercase letters
username = strings.ToLower(username)
domain := cfg.Matrix.ServerName
host := gomatrixserverlib.ServerName(req.Host)
if v := cfg.Matrix.VirtualHostForHTTPHost(host); v != nil {
domain = v.ServerName
}
if u, l, err := cfg.Matrix.SplitLocalID('@', username); err == nil {
username, domain = u, l
}
for _, v := range cfg.Matrix.VirtualHosts {
if v.ServerName == domain && !v.AllowRegistration {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden(
fmt.Sprintf("Registration is not allowed on %q", string(v.ServerName)),
),
}
}
}
if err := validateUsername(username, cfg.Matrix.ServerName); err != nil {
if err := validateUsername(username, domain); err != nil {
return *err
}
// Check if this username is reserved by an application service
userID := userutil.MakeUserID(username, cfg.Matrix.ServerName)
userID := userutil.MakeUserID(username, domain)
for _, appservice := range cfg.Derived.ApplicationServices {
if appservice.OwnsNamespaceCoveringUserId(userID) {
return util.JSONResponse{
@ -1035,7 +1089,8 @@ func RegisterAvailable(
res := &userapi.QueryAccountAvailabilityResponse{}
err := registerAPI.QueryAccountAvailability(req.Context(), &userapi.QueryAccountAvailabilityRequest{
Localpart: username,
Localpart: username,
ServerName: domain,
}, res)
if err != nil {
return util.JSONResponse{
@ -1092,5 +1147,5 @@ func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.Clien
if ssrr.Admin {
accType = userapi.AccountTypeAdmin
}
return completeRegistration(req.Context(), userAPI, ssrr.User, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType)
return completeRegistration(req.Context(), userAPI, ssrr.User, cfg.Matrix.ServerName, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType)
}

179
clientapi/routing/routing.go
View file

@ -70,6 +70,7 @@ func Setup(
unstableFeatures := map[string]bool{
"org.matrix.e2e_cross_signing": true,
"org.matrix.msc2285.stable": true,
}
for _, msc := range cfg.MSCs.MSCs {
unstableFeatures["org.matrix."+msc] = true
@ -156,22 +157,34 @@ func Setup(
}),
).Methods(http.MethodGet, http.MethodOptions)
dendriteAdminRouter.Handle("/admin/resetPassword/{localpart}",
dendriteAdminRouter.Handle("/admin/resetPassword/{userID}",
httputil.MakeAdminAPI("admin_reset_password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return AdminResetPassword(req, cfg, device, userAPI)
}),
).Methods(http.MethodPost, http.MethodOptions)
dendriteAdminRouter.Handle("/admin/downloadState/{serverName}/{roomID}",
httputil.MakeAdminAPI("admin_download_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return AdminDownloadState(req, cfg, device, rsAPI)
}),
).Methods(http.MethodGet, http.MethodOptions)
dendriteAdminRouter.Handle("/admin/fulltext/reindex",
httputil.MakeAuthAPI("admin_fultext_reindex", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
httputil.MakeAdminAPI("admin_fultext_reindex", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return AdminReindex(req, cfg, device, natsClient)
}),
).Methods(http.MethodGet, http.MethodOptions)
dendriteAdminRouter.Handle("/admin/refreshDevices/{userID}",
httputil.MakeAdminAPI("admin_refresh_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return AdminMarkAsStale(req, cfg, keyAPI)
}),
).Methods(http.MethodPost, http.MethodOptions)
// server notifications
if cfg.Matrix.ServerNotices.Enabled {
logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
serverNotificationSender, err := getSenderDevice(context.Background(), userAPI, cfg)
serverNotificationSender, err := getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
if err != nil {
logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
}
@ -239,7 +252,7 @@ func Setup(
return JoinRoomByIDOrAlias(
req, device, rsAPI, userAPI, vars["roomIDOrAlias"],
)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPost, http.MethodOptions)
if mscCfg.Enabled("msc2753") {
@ -261,7 +274,7 @@ func Setup(
v3mux.Handle("/joined_rooms",
httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return GetJoinedRooms(req, device, rsAPI)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/join",
httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@ -275,7 +288,7 @@ func Setup(
return JoinRoomByIDOrAlias(
req, device, rsAPI, userAPI, vars["roomID"],
)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/leave",
httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@ -289,7 +302,7 @@ func Setup(
return LeaveRoomByID(
req, device, rsAPI, vars["roomID"],
)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/unpeek",
httputil.MakeAuthAPI("unpeek", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@ -348,7 +361,7 @@ func Setup(
return util.ErrorResponse(err)
}
return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, rsAPI, nil)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}",
httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@ -359,17 +372,8 @@ func Setup(
txnID := vars["txnID"]
return SendEvent(req, device, vars["roomID"], vars["eventType"], &txnID,
nil, cfg, rsAPI, transactionsCache)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPut, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/event/{eventID}",
httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI)
}),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -377,7 +381,7 @@ func Setup(
return util.ErrorResponse(err)
}
return OnIncomingStateRequest(req.Context(), device, rsAPI, vars["roomID"])
})).Methods(http.MethodGet, http.MethodOptions)
}, httputil.WithAllowGuests())).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/aliases", httputil.MakeAuthAPI("aliases", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -396,7 +400,7 @@ func Setup(
eventType := strings.TrimSuffix(vars["type"], "/")
eventFormat := req.URL.Query().Get("format") == "event"
return OnIncomingStateTypeRequest(req.Context(), device, rsAPI, vars["roomID"], eventType, "", eventFormat)
})).Methods(http.MethodGet, http.MethodOptions)
}, httputil.WithAllowGuests())).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -405,7 +409,7 @@ func Setup(
}
eventFormat := req.URL.Query().Get("format") == "event"
return OnIncomingStateTypeRequest(req.Context(), device, rsAPI, vars["roomID"], vars["type"], vars["stateKey"], eventFormat)
})).Methods(http.MethodGet, http.MethodOptions)
}, httputil.WithAllowGuests())).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}",
httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@ -416,7 +420,7 @@ func Setup(
emptyString := ""
eventType := strings.TrimSuffix(vars["eventType"], "/")
return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, rsAPI, nil)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPut, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}",
@ -427,7 +431,7 @@ func Setup(
}
stateKey := vars["stateKey"]
return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, rsAPI, nil)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPut, http.MethodOptions)
v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
@ -482,7 +486,7 @@ func Setup(
return GetVisibility(req, rsAPI, vars["roomID"])
}),
).Methods(http.MethodGet, http.MethodOptions)
// TODO: Add AS support
v3mux.Handle("/directory/list/room/{roomID}",
httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -492,6 +496,27 @@ func Setup(
return SetVisibility(req, rsAPI, device, vars["roomID"])
}),
).Methods(http.MethodPut, http.MethodOptions)
v3mux.Handle("/directory/list/appservice/{networkID}/{roomID}",
httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return SetVisibilityAS(req, rsAPI, device, vars["networkID"], vars["roomID"])
}),
).Methods(http.MethodPut, http.MethodOptions)
// Undocumented endpoint
v3mux.Handle("/directory/list/appservice/{networkID}/{roomID}",
httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return SetVisibilityAS(req, rsAPI, device, vars["networkID"], vars["roomID"])
}),
).Methods(http.MethodDelete, http.MethodOptions)
v3mux.Handle("/publicRooms",
httputil.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse {
return GetPostPublicRooms(req, rsAPI, extRoomsProvider, federation, cfg)
@ -550,7 +575,7 @@ func Setup(
}
txnID := vars["txnID"]
return SendToDevice(req, device, syncProducer, transactionsCache, vars["eventType"], &txnID)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPut, http.MethodOptions)
// This is only here because sytest refers to /unstable for this endpoint
@ -564,7 +589,7 @@ func Setup(
}
txnID := vars["txnID"]
return SendToDevice(req, device, syncProducer, transactionsCache, vars["eventType"], &txnID)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPut, http.MethodOptions)
v3mux.Handle("/account/whoami",
@ -573,7 +598,7 @@ func Setup(
return *r
}
return Whoami(req, device)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/account/password",
@ -805,7 +830,7 @@ func Setup(
return util.ErrorResponse(err)
}
return SetDisplayName(req, userAPI, device, vars["userID"], cfg, rsAPI)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPut, http.MethodOptions)
// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
// PUT requests, so we need to allow this method
@ -844,13 +869,51 @@ func Setup(
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/thirdparty/protocols",
httputil.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse {
// TODO: Return the third party protcols
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
httputil.MakeAuthAPI("thirdparty_protocols", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return Protocols(req, asAPI, device, "")
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/thirdparty/protocol/{protocolID}",
httputil.MakeAuthAPI("thirdparty_protocols", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
}),
return Protocols(req, asAPI, device, vars["protocolID"])
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/thirdparty/user/{protocolID}",
httputil.MakeAuthAPI("thirdparty_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return User(req, asAPI, device, vars["protocolID"], req.URL.Query())
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/thirdparty/user",
httputil.MakeAuthAPI("thirdparty_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return User(req, asAPI, device, "", req.URL.Query())
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/thirdparty/location/{protocolID}",
httputil.MakeAuthAPI("thirdparty_location", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return Location(req, asAPI, device, vars["protocolID"], req.URL.Query())
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/thirdparty/location",
httputil.MakeAuthAPI("thirdparty_location", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return Location(req, asAPI, device, "", req.URL.Query())
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/initialSync",
@ -952,26 +1015,6 @@ func Setup(
}),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/members",
httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return GetMemberships(req, device, vars["roomID"], false, cfg, rsAPI)
}),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/joined_members",
httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return GetMemberships(req, device, vars["roomID"], true, cfg, rsAPI)
}),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/read_markers",
httputil.MakeAuthAPI("rooms_read_markers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.Limit(req, device); r != nil {
@ -1011,7 +1054,7 @@ func Setup(
v3mux.Handle("/devices",
httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return GetDevicesByLocalpart(req, userAPI, device)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/devices/{deviceID}",
@ -1021,7 +1064,7 @@ func Setup(
return util.ErrorResponse(err)
}
return GetDeviceByID(req, userAPI, device, vars["deviceID"])
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/devices/{deviceID}",
@ -1031,7 +1074,7 @@ func Setup(
return util.ErrorResponse(err)
}
return UpdateDeviceByID(req, userAPI, device, vars["deviceID"])
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPut, http.MethodOptions)
v3mux.Handle("/devices/{deviceID}",
@ -1073,21 +1116,21 @@ func Setup(
// Stub implementations for sytest
v3mux.Handle("/events",
httputil.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse {
httputil.MakeAuthAPI("events", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{
"chunk": []interface{}{},
"start": "",
"end": "",
}}
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/initialSync",
httputil.MakeExternalAPI("initial_sync", func(req *http.Request) util.JSONResponse {
httputil.MakeAuthAPI("initial_sync", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{
"end": "",
}}
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/user/{userId}/rooms/{roomId}/tags",
@ -1126,7 +1169,7 @@ func Setup(
return *r
}
return GetCapabilities(req, rsAPI)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
// Key Backup Versions (Metadata)
@ -1307,7 +1350,7 @@ func Setup(
postDeviceSigningSignatures := httputil.MakeAuthAPI("post_device_signing_signatures", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return UploadCrossSigningDeviceSignatures(req, keyAPI, device)
})
}, httputil.WithAllowGuests())
v3mux.Handle("/keys/device_signing/upload", postDeviceSigningKeys).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/keys/signatures/upload", postDeviceSigningSignatures).Methods(http.MethodPost, http.MethodOptions)
@ -1319,22 +1362,22 @@ func Setup(
v3mux.Handle("/keys/upload/{deviceID}",
httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return UploadKeys(req, keyAPI, device)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/keys/upload",
httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return UploadKeys(req, keyAPI, device)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/keys/query",
httputil.MakeAuthAPI("keys_query", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return QueryKeys(req, keyAPI, device)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/keys/claim",
httputil.MakeAuthAPI("keys_claim", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return ClaimKeys(req, keyAPI)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}",
httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@ -1346,7 +1389,7 @@ func Setup(
return util.ErrorResponse(err)
}
return SetReceipt(req, syncProducer, device, vars["roomId"], vars["receiptType"], vars["eventId"])
return SetReceipt(req, userAPI, syncProducer, device, vars["roomId"], vars["receiptType"], vars["eventId"])
}),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/presence/{userId}/status",

18
clientapi/routing/sendevent.go
View file

@ -86,7 +86,7 @@ func SendEvent(
if txnID != nil {
// Try to fetch response from transactionsCache
if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok {
if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID, req.URL); ok {
return *res
}
}
@ -94,6 +94,7 @@ func SendEvent(
// create a mutex for the specific user in the specific room
// this avoids a situation where events that are received in quick succession are sent to the roomserver in a jumbled order
userID := device.UserID
domain := device.UserDomain()
mutex, _ := userRoomSendMutexes.LoadOrStore(roomID+userID, &sync.Mutex{})
mutex.(*sync.Mutex).Lock()
defer mutex.(*sync.Mutex).Unlock()
@ -185,8 +186,9 @@ func SendEvent(
[]*gomatrixserverlib.HeaderedEvent{
e.Headered(verRes.RoomVersion),
},
cfg.Matrix.ServerName,
cfg.Matrix.ServerName,
device.UserDomain(),
domain,
domain,
txnAndSessionID,
false,
); err != nil {
@ -206,7 +208,7 @@ func SendEvent(
}
// Add response to transactionsCache
if txnID != nil {
txnCache.AddTransaction(device.AccessToken, *txnID, &res)
txnCache.AddTransaction(device.AccessToken, *txnID, req.URL, &res)
}
// Take a note of how long it took to generate the event vs submit
@ -274,8 +276,14 @@ func generateSendEvent(
return nil, &resErr
}
identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
if err != nil {
resErr := jsonerror.InternalServerError()
return nil, &resErr
}
var queryRes api.QueryLatestEventsAndStateResponse
e, err := eventutil.QueryAndBuildEvent(ctx, &builder, cfg.Matrix, evTime, rsAPI, &queryRes)
e, err := eventutil.QueryAndBuildEvent(ctx, &builder, cfg.Matrix, identity, evTime, rsAPI, &queryRes)
if err == eventutil.ErrRoomNoExists {
return nil, &util.JSONResponse{
Code: http.StatusNotFound,

7
clientapi/routing/sendtodevice.go
View file

@ -16,12 +16,13 @@ import (
"encoding/json"
"net/http"
"github.com/matrix-org/util"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/dendrite/internal/transactions"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/util"
)
// SendToDevice handles PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId}
@ -33,7 +34,7 @@ func SendToDevice(
eventType string, txnID *string,
) util.JSONResponse {
if txnID != nil {
if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok {
if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID, req.URL); ok {
return *res
}
}
@ -63,7 +64,7 @@ func SendToDevice(
}
if txnID != nil {
txnCache.AddTransaction(device.AccessToken, *txnID, &res)
txnCache.AddTransaction(device.AccessToken, *txnID, req.URL, &res)
}
return res

47
clientapi/routing/server_notices.go
View file

@ -21,7 +21,6 @@ import (
"net/http"
"time"
"github.com/matrix-org/dendrite/roomserver/version"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/gomatrixserverlib/tokens"
@ -29,6 +28,8 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/roomserver/version"
appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
@ -73,7 +74,7 @@ func SendServerNotice(
if txnID != nil {
// Try to fetch response from transactionsCache
if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok {
if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID, req.URL); ok {
return *res
}
}
@ -230,6 +231,7 @@ func SendServerNotice(
[]*gomatrixserverlib.HeaderedEvent{
e.Headered(roomVersion),
},
device.UserDomain(),
cfgClient.Matrix.ServerName,
cfgClient.Matrix.ServerName,
txnAndSessionID,
@ -251,7 +253,7 @@ func SendServerNotice(
}
// Add response to transactionsCache
if txnID != nil {
txnCache.AddTransaction(device.AccessToken, *txnID, &res)
txnCache.AddTransaction(device.AccessToken, *txnID, req.URL, &res)
}
// Take a note of how long it took to generate the event vs submit
@ -276,6 +278,7 @@ func (r sendServerNoticeRequest) valid() (ok bool) {
// It returns an userapi.Device, which is used for building the event
func getSenderDevice(
ctx context.Context,
rsAPI api.ClientRoomserverAPI,
userAPI userapi.ClientUserAPI,
cfg *config.ClientAPI,
) (*userapi.Device, error) {
@ -284,22 +287,41 @@ func getSenderDevice(
err := userAPI.PerformAccountCreation(ctx, &userapi.PerformAccountCreationRequest{
AccountType: userapi.AccountTypeUser,
Localpart: cfg.Matrix.ServerNotices.LocalPart,
ServerName: cfg.Matrix.ServerName,
OnConflict: userapi.ConflictUpdate,
}, &accRes)
if err != nil {
return nil, err
}
// set the avatarurl for the user
res := &userapi.PerformSetAvatarURLResponse{}
// Set the avatarurl for the user
avatarRes := &userapi.PerformSetAvatarURLResponse{}
if err = userAPI.SetAvatarURL(ctx, &userapi.PerformSetAvatarURLRequest{
Localpart: cfg.Matrix.ServerNotices.LocalPart,
AvatarURL: cfg.Matrix.ServerNotices.AvatarURL,
}, res); err != nil {
Localpart: cfg.Matrix.ServerNotices.LocalPart,
ServerName: cfg.Matrix.ServerName,
AvatarURL: cfg.Matrix.ServerNotices.AvatarURL,
}, avatarRes); err != nil {
util.GetLogger(ctx).WithError(err).Error("userAPI.SetAvatarURL failed")
return nil, err
}
profile := avatarRes.Profile
// Set the displayname for the user
displayNameRes := &userapi.PerformUpdateDisplayNameResponse{}
if err = userAPI.SetDisplayName(ctx, &userapi.PerformUpdateDisplayNameRequest{
Localpart: cfg.Matrix.ServerNotices.LocalPart,
ServerName: cfg.Matrix.ServerName,
DisplayName: cfg.Matrix.ServerNotices.DisplayName,
}, displayNameRes); err != nil {
util.GetLogger(ctx).WithError(err).Error("userAPI.SetDisplayName failed")
return nil, err
}
if displayNameRes.Changed {
profile.DisplayName = cfg.Matrix.ServerNotices.DisplayName
}
// Check if we got existing devices
deviceRes := &userapi.QueryDevicesResponse{}
err = userAPI.QueryDevices(ctx, &userapi.QueryDevicesRequest{
@ -309,7 +331,15 @@ func getSenderDevice(
return nil, err
}
// We've got an existing account, return the first device of it
if len(deviceRes.Devices) > 0 {
// If there were changes to the profile, create a new membership event
if displayNameRes.Changed || avatarRes.Changed {
_, err = updateProfile(ctx, rsAPI, &deviceRes.Devices[0], profile, accRes.Account.UserID, cfg, time.Now())
if err != nil {
return nil, err
}
}
return &deviceRes.Devices[0], nil
}
@ -327,6 +357,7 @@ func getSenderDevice(
var devRes userapi.PerformDeviceCreationResponse
err = userAPI.PerformDeviceCreation(ctx, &userapi.PerformDeviceCreationRequest{
Localpart: cfg.Matrix.ServerNotices.LocalPart,
ServerName: cfg.Matrix.ServerName,
DeviceDisplayName: &cfg.Matrix.ServerNotices.LocalPart,
AccessToken: token,
NoDeviceListUpdate: true,

112
clientapi/routing/thirdparty.go Normal file
View file

@ -0,0 +1,112 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package routing
import (
"net/http"
"net/url"
"github.com/matrix-org/util"
appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/userapi/api"
)
// Protocols implements
//
// GET /_matrix/client/v3/thirdparty/protocols/{protocol}
// GET /_matrix/client/v3/thirdparty/protocols
func Protocols(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, device *api.Device, protocol string) util.JSONResponse {
resp := &appserviceAPI.ProtocolResponse{}
if err := asAPI.Protocols(req.Context(), &appserviceAPI.ProtocolRequest{Protocol: protocol}, resp); err != nil {
return jsonerror.InternalServerError()
}
if !resp.Exists {
if protocol != "" {
return util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound("The protocol is unknown."),
}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
}
}
if protocol != "" {
return util.JSONResponse{
Code: http.StatusOK,
JSON: resp.Protocols[protocol],
}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: resp.Protocols,
}
}
// User implements
//
// GET /_matrix/client/v3/thirdparty/user
// GET /_matrix/client/v3/thirdparty/user/{protocol}
func User(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, device *api.Device, protocol string, params url.Values) util.JSONResponse {
resp := &appserviceAPI.UserResponse{}
params.Del("access_token")
if err := asAPI.User(req.Context(), &appserviceAPI.UserRequest{
Protocol: protocol,
Params: params.Encode(),
}, resp); err != nil {
return jsonerror.InternalServerError()
}
if !resp.Exists {
return util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound("The Matrix User ID was not found"),
}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: resp.Users,
}
}
// Location implements
//
// GET /_matrix/client/v3/thirdparty/location
// GET /_matrix/client/v3/thirdparty/location/{protocol}
func Location(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, device *api.Device, protocol string, params url.Values) util.JSONResponse {
resp := &appserviceAPI.LocationResponse{}
params.Del("access_token")
if err := asAPI.Locations(req.Context(), &appserviceAPI.LocationRequest{
Protocol: protocol,
Params: params.Encode(),
}, resp); err != nil {
return jsonerror.InternalServerError()
}
if !resp.Exists {
return util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound("No portal rooms were found."),
}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: resp.Locations,
}
}

14
clientapi/routing/threepid.go
View file

@ -136,16 +136,17 @@ func CheckAndSave3PIDAssociation(
}
// Save the association in the database
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
return jsonerror.InternalServerError()
}
if err = threePIDAPI.PerformSaveThreePIDAssociation(req.Context(), &api.PerformSaveThreePIDAssociationRequest{
ThreePID: address,
Localpart: localpart,
Medium: medium,
ThreePID: address,
Localpart: localpart,
ServerName: domain,
Medium: medium,
}, &struct{}{}); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("threePIDAPI.PerformSaveThreePIDAssociation failed")
return jsonerror.InternalServerError()
@ -161,7 +162,7 @@ func CheckAndSave3PIDAssociation(
func GetAssociated3PIDs(
req *http.Request, threepidAPI api.ClientUserAPI, device *api.Device,
) util.JSONResponse {
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
return jsonerror.InternalServerError()
@ -169,7 +170,8 @@ func GetAssociated3PIDs(
res := &api.QueryThreePIDsForLocalpartResponse{}
err = threepidAPI.QueryThreePIDsForLocalpart(req.Context(), &api.QueryThreePIDsForLocalpartRequest{
Localpart: localpart,
Localpart: localpart,
ServerName: domain,
}, res)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("threepidAPI.QueryThreePIDsForLocalpart failed")

2
clientapi/routing/userdirectory.go
View file

@ -106,7 +106,7 @@ knownUsersLoop:
continue
}
// TODO: We should probably cache/store this
fedProfile, fedErr := federation.LookupProfile(ctx, serverName, userID, "")
fedProfile, fedErr := federation.LookupProfile(ctx, localServerName, serverName, userID, "")
if fedErr != nil {
if x, ok := fedErr.(gomatrix.HTTPError); ok {
if x.Code == http.StatusNotFound {

10
clientapi/threepid/invites.go
View file

@ -215,7 +215,7 @@ func queryIDServerStoreInvite(
}
var profile *authtypes.Profile
if serverName == cfg.Matrix.ServerName {
if cfg.Matrix.IsLocalServerName(serverName) {
res := &userapi.QueryProfileResponse{}
err = userAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: device.UserID}, res)
if err != nil {
@ -359,8 +359,13 @@ func emit3PIDInviteEvent(
return err
}
identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
if err != nil {
return err
}
queryRes := api.QueryLatestEventsAndStateResponse{}
event, err := eventutil.QueryAndBuildEvent(ctx, builder, cfg.Matrix, evTime, rsAPI, &queryRes)
event, err := eventutil.QueryAndBuildEvent(ctx, builder, cfg.Matrix, identity, evTime, rsAPI, &queryRes)
if err != nil {
return err
}
@ -371,6 +376,7 @@ func emit3PIDInviteEvent(
[]*gomatrixserverlib.HeaderedEvent{
event.Headered(queryRes.RoomVersion),
},
device.UserDomain(),
cfg.Matrix.ServerName,
cfg.Matrix.ServerName,
nil,

13
clientapi/userutil/userutil.go
View file

@ -17,6 +17,7 @@ import (
"fmt"
"strings"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
)
@ -24,23 +25,23 @@ import (
// usernameParam can either be a user ID or just the localpart/username.
// If serverName is passed, it is verified against the domain obtained from usernameParam (if present)
// Returns error in case of invalid usernameParam.
func ParseUsernameParam(usernameParam string, expectedServerName *gomatrixserverlib.ServerName) (string, error) {
func ParseUsernameParam(usernameParam string, cfg *config.Global) (string, gomatrixserverlib.ServerName, error) {
localpart := usernameParam
if strings.HasPrefix(usernameParam, "@") {
lp, domain, err := gomatrixserverlib.SplitID('@', usernameParam)
if err != nil {
return "", errors.New("invalid username")
return "", "", errors.New("invalid username")
}
if expectedServerName != nil && domain != *expectedServerName {
return "", errors.New("user ID does not belong to this server")
if !cfg.IsLocalServerName(domain) {
return "", "", errors.New("user ID does not belong to this server")
}
localpart = lp
return lp, domain, nil
}
return localpart, nil
return localpart, cfg.ServerName, nil
}
// MakeUserID generates user ID from localpart & server name

33
clientapi/userutil/userutil_test.go
View file

@ -15,6 +15,7 @@ package userutil
import (
"testing"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
)
@ -28,7 +29,13 @@ var (
// TestGoodUserID checks that correct localpart is returned for a valid user ID.
func TestGoodUserID(t *testing.T) {
lp, err := ParseUsernameParam(goodUserID, &serverName)
cfg := &config.Global{
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: serverName,
},
}
lp, _, err := ParseUsernameParam(goodUserID, cfg)
if err != nil {
t.Error("User ID Parsing failed for ", goodUserID, " with error: ", err.Error())
@ -41,7 +48,13 @@ func TestGoodUserID(t *testing.T) {
// TestWithLocalpartOnly checks that localpart is returned when usernameParam contains only localpart.
func TestWithLocalpartOnly(t *testing.T) {
lp, err := ParseUsernameParam(localpart, &serverName)
cfg := &config.Global{
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: serverName,
},
}
lp, _, err := ParseUsernameParam(localpart, cfg)
if err != nil {
t.Error("User ID Parsing failed for ", localpart, " with error: ", err.Error())
@ -54,7 +67,13 @@ func TestWithLocalpartOnly(t *testing.T) {
// TestIncorrectDomain checks for error when there's server name mismatch.
func TestIncorrectDomain(t *testing.T) {
_, err := ParseUsernameParam(goodUserID, &invalidServerName)
cfg := &config.Global{
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: invalidServerName,
},
}
_, _, err := ParseUsernameParam(goodUserID, cfg)
if err == nil {
t.Error("Invalid Domain should return an error")
@ -63,7 +82,13 @@ func TestIncorrectDomain(t *testing.T) {
// TestBadUserID checks that ParseUsernameParam fails for invalid user ID
func TestBadUserID(t *testing.T) {
_, err := ParseUsernameParam(badUserID, &serverName)
cfg := &config.Global{
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: serverName,
},
}
_, _, err := ParseUsernameParam(badUserID, cfg)
if err == nil {
t.Error("Illegal User ID should return an error")

9
cmd/create-account/main.go
View file

@ -64,7 +64,7 @@ var (
pwdStdin = flag.Bool("passwordstdin", false, "Reads the password from stdin")
isAdmin = flag.Bool("admin", false, "Create an admin account")
resetPassword = flag.Bool("reset-password", false, "Deprecated")
serverURL = flag.String("url", "https://localhost:8448", "The URL to connect to.")
serverURL = flag.String("url", "http://localhost:8008", "The URL to connect to.")
validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
timeout = flag.Duration("timeout", time.Second*30, "Timeout for the http client when connecting to the server")
)
@ -177,9 +177,12 @@ func sharedSecretRegister(sharedSecret, serverURL, localpart, password string, a
defer regResp.Body.Close() // nolint: errcheck
if regResp.StatusCode < 200 || regResp.StatusCode >= 300 {
body, _ = io.ReadAll(regResp.Body)
return "", fmt.Errorf(gjson.GetBytes(body, "error").Str)
return "", fmt.Errorf("got HTTP %d error from server: %s", regResp.StatusCode, string(body))
}
r, err := io.ReadAll(regResp.Body)
if err != nil {
return "", fmt.Errorf("failed to read response body (HTTP %d): %w", regResp.StatusCode, err)
}
r, _ := io.ReadAll(regResp.Body)
return gjson.GetBytes(r, "access_token").Str, nil
}

26
cmd/dendrite-demo-pinecone/README.md Normal file
View file

@ -0,0 +1,26 @@
# Pinecone Demo
This is the Dendrite Pinecone demo! It's easy to get started.
To run the homeserver, start at the root of the Dendrite repository and run:
```
go run ./cmd/dendrite-demo-pinecone
```
To connect to the static Pinecone peer used by the mobile demos run:
```
go run ./cmd/dendrite-demo-pinecone -peer wss://pinecone.matrix.org/public
```
The following command line arguments are accepted:
* `-peer tcp://a.b.c.d:e` to specify a static Pinecone peer to connect to - you will need to supply this if you do not have another Pinecone node on your network
* `-port 12345` to specify a port to listen on for client connections
Then point your favourite Matrix client to the homeserver URL`http://localhost:8008` (or whichever `-port` you specified), create an account and log in.
If your peering connection is operational then you should see a `Connected TCP:` line in the log output. If not then try a different peer.
Once logged in, you should be able to open the room directory or join a room by its ID.

4
cmd/dendrite-demo-pinecone/conn/client.go
View file

@ -101,9 +101,7 @@ func CreateFederationClient(
base *base.BaseDendrite, s *pineconeSessions.Sessions,
) *gomatrixserverlib.FederationClient {
return gomatrixserverlib.NewFederationClient(
base.Cfg.Global.ServerName,
base.Cfg.Global.KeyID,
base.Cfg.Global.PrivateKey,
base.Cfg.Global.SigningIdentities(),
gomatrixserverlib.WithTransport(createTransport(s)),
)
}

44
cmd/dendrite-demo-pinecone/main.go
View file

@ -37,6 +37,7 @@ import (
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/keyserver"
@ -51,11 +52,10 @@ import (
pineconeConnections "github.com/matrix-org/pinecone/connections"
pineconeMulticast "github.com/matrix-org/pinecone/multicast"
pineconeRouter "github.com/matrix-org/pinecone/router"
pineconeEvents "github.com/matrix-org/pinecone/router/events"
pineconeSessions "github.com/matrix-org/pinecone/sessions"
"github.com/sirupsen/logrus"
_ "github.com/mattn/go-sqlite3"
)
var (
@ -89,6 +89,7 @@ func main() {
if configFlagSet {
cfg = setup.ParseFlags(true)
sk = cfg.Global.PrivateKey
pk = sk.Public().(ed25519.PublicKey)
} else {
keyfile := filepath.Join(*instanceDir, *instanceName) + ".pem"
if _, err := os.Stat(keyfile); os.IsNotExist(err) {
@ -142,6 +143,9 @@ func main() {
cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName)))
cfg.ClientAPI.RegistrationDisabled = false
cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
cfg.MediaAPI.BasePath = config.Path(*instanceDir)
cfg.SyncAPI.Fulltext.Enabled = true
cfg.SyncAPI.Fulltext.IndexPath = config.Path(*instanceDir)
if err := cfg.Derive(); err != nil {
panic(err)
}
@ -151,9 +155,15 @@ func main() {
cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
base := base.NewBaseDendrite(cfg, "Monolith")
base.ConfigureAdminEndpoints()
defer base.Close() // nolint: errcheck
pineconeEventChannel := make(chan pineconeEvents.Event)
pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk)
pRouter.EnableHopLimiting()
pRouter.EnableWakeupBroadcasts()
pRouter.Subscribe(pineconeEventChannel)
pQUIC := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"})
pMulticast := pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), pRouter)
pManager := pineconeConnections.NewConnectionManager(pRouter, nil)
@ -239,6 +249,8 @@ func main() {
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
httpRouter.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) {
c, err := wsUpgrader.Upgrade(w, r, nil)
if err != nil {
@ -291,5 +303,33 @@ func main() {
logrus.Fatal(http.ListenAndServe(httpBindAddr, httpRouter))
}()
go func(ch <-chan pineconeEvents.Event) {
eLog := logrus.WithField("pinecone", "events")
for event := range ch {
switch e := event.(type) {
case pineconeEvents.PeerAdded:
case pineconeEvents.PeerRemoved:
case pineconeEvents.TreeParentUpdate:
case pineconeEvents.SnakeDescUpdate:
case pineconeEvents.TreeRootAnnUpdate:
case pineconeEvents.SnakeEntryAdded:
case pineconeEvents.SnakeEntryRemoved:
case pineconeEvents.BroadcastReceived:
eLog.Info("Broadcast received from: ", e.PeerID)
req := &api.PerformWakeupServersRequest{
ServerNames: []gomatrixserverlib.ServerName{gomatrixserverlib.ServerName(e.PeerID)},
}
res := &api.PerformWakeupServersResponse{}
if err := fsAPI.PerformWakeupServers(base.Context(), req, res); err != nil {
logrus.WithError(err).Error("Failed to wakeup destination", e.PeerID)
}
case pineconeEvents.BandwidthReport:
default:
}
}
}(pineconeEventChannel)
base.WaitForShutdown()
}

8
cmd/dendrite-demo-pinecone/rooms/rooms.go
View file

@ -58,13 +58,17 @@ func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom {
for _, k := range p.r.Peers() {
list[gomatrixserverlib.ServerName(k.PublicKey)] = struct{}{}
}
return bulkFetchPublicRoomsFromServers(context.Background(), p.fedClient, list)
return bulkFetchPublicRoomsFromServers(
context.Background(), p.fedClient,
gomatrixserverlib.ServerName(p.r.PublicKey().String()), list,
)
}
// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers.
// Returns a list of public rooms.
func bulkFetchPublicRoomsFromServers(
ctx context.Context, fedClient *gomatrixserverlib.FederationClient,
origin gomatrixserverlib.ServerName,
homeservers map[gomatrixserverlib.ServerName]struct{},
) (publicRooms []gomatrixserverlib.PublicRoom) {
limit := 200
@ -82,7 +86,7 @@ func bulkFetchPublicRoomsFromServers(
go func(homeserverDomain gomatrixserverlib.ServerName) {
defer wg.Done()
util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms")
fres, err := fedClient.GetPublicRooms(reqctx, homeserverDomain, int(limit), "", false, "")
fres, err := fedClient.GetPublicRooms(reqctx, origin, homeserverDomain, int(limit), "", false, "")
if err != nil {
util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Warn(
"bulkFetchPublicRoomsFromServers: failed to query hs",

8
cmd/dendrite-demo-yggdrasil/main.go
View file

@ -48,8 +48,6 @@ import (
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/userapi"
"github.com/sirupsen/logrus"
_ "github.com/mattn/go-sqlite3"
)
var (
@ -134,6 +132,9 @@ func main() {
cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName)))
cfg.ClientAPI.RegistrationDisabled = false
cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
cfg.MediaAPI.BasePath = config.Path(*instanceDir)
cfg.SyncAPI.Fulltext.Enabled = true
cfg.SyncAPI.Fulltext.IndexPath = config.Path(*instanceDir)
if err := cfg.Derive(); err != nil {
panic(err)
}
@ -143,6 +144,7 @@ func main() {
cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
base := base.NewBaseDendrite(cfg, "Monolith")
base.ConfigureAdminEndpoints()
defer base.Close() // nolint: errcheck
ygg, err := yggconn.Setup(sk, *instanceName, ".", *instancePeer, *instanceListen)
@ -197,6 +199,8 @@ func main() {
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
embed.Embed(httpRouter, *instancePort, "Yggdrasil Demo")
yggRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()

3
cmd/dendrite-demo-yggdrasil/yggconn/client.go
View file

@ -55,8 +55,7 @@ func (n *Node) CreateFederationClient(
},
)
return gomatrixserverlib.NewFederationClient(
base.Cfg.Global.ServerName, base.Cfg.Global.KeyID,
base.Cfg.Global.PrivateKey,
base.Cfg.Global.SigningIdentities(),
gomatrixserverlib.WithTransport(tr),
)
}

86
cmd/dendrite-demo-yggdrasil/yggconn/node.go
View file

@ -20,6 +20,7 @@ import (
"encoding/hex"
"fmt"
"net"
"regexp"
"strings"
"github.com/matrix-org/gomatrixserverlib"
@ -27,9 +28,9 @@ import (
"github.com/sirupsen/logrus"
ironwoodtypes "github.com/Arceliar/ironwood/types"
yggdrasilconfig "github.com/yggdrasil-network/yggdrasil-go/src/config"
"github.com/yggdrasil-network/yggdrasil-go/src/core"
yggdrasilcore "github.com/yggdrasil-network/yggdrasil-go/src/core"
yggdrasildefaults "github.com/yggdrasil-network/yggdrasil-go/src/defaults"
"github.com/yggdrasil-network/yggdrasil-go/src/multicast"
yggdrasilmulticast "github.com/yggdrasil-network/yggdrasil-go/src/multicast"
gologme "github.com/gologme/log"
@ -37,7 +38,6 @@ import (
type Node struct {
core *yggdrasilcore.Core
config *yggdrasilconfig.NodeConfig
multicast *yggdrasilmulticast.Multicast
log *gologme.Logger
utpSocket *utp.Socket
@ -57,43 +57,52 @@ func (n *Node) DialerContext(ctx context.Context, _, address string) (net.Conn,
func Setup(sk ed25519.PrivateKey, instanceName, storageDirectory, peerURI, listenURI string) (*Node, error) {
n := &Node{
core: &yggdrasilcore.Core{},
config: yggdrasildefaults.GenerateConfig(),
multicast: &yggdrasilmulticast.Multicast{},
log: gologme.New(logrus.StandardLogger().Writer(), "", 0),
incoming: make(chan net.Conn),
log: gologme.New(logrus.StandardLogger().Writer(), "", 0),
incoming: make(chan net.Conn),
}
options := []yggdrasilcore.SetupOption{
yggdrasilcore.AdminListenAddress("none"),
}
if listenURI != "" {
options = append(options, yggdrasilcore.ListenAddress(listenURI))
}
if peerURI != "" {
for _, uri := range strings.Split(peerURI, ",") {
options = append(options, yggdrasilcore.Peer{
URI: uri,
})
}
}
var err error
if n.core, err = yggdrasilcore.New(sk, options...); err != nil {
panic(err)
}
n.log.EnableLevel("error")
n.log.EnableLevel("warn")
n.log.EnableLevel("info")
n.core.SetLogger(n.log)
if n.utpSocket, err = utp.NewSocketFromPacketConnNoClose(n.core); err != nil {
panic(err)
{
var err error
options := []yggdrasilcore.SetupOption{}
if listenURI != "" {
options = append(options, yggdrasilcore.ListenAddress(listenURI))
}
if peerURI != "" {
for _, uri := range strings.Split(peerURI, ",") {
options = append(options, yggdrasilcore.Peer{
URI: uri,
})
}
}
if n.core, err = core.New(sk[:], n.log, options...); err != nil {
panic(err)
}
n.core.SetLogger(n.log)
if n.utpSocket, err = utp.NewSocketFromPacketConnNoClose(n.core); err != nil {
panic(err)
}
}
if err = n.multicast.Init(n.core, n.config, n.log, nil); err != nil {
panic(err)
}
if err = n.multicast.Start(); err != nil {
panic(err)
// Setup the multicast module.
{
var err error
options := []multicast.SetupOption{
multicast.MulticastInterface{
Regex: regexp.MustCompile(".*"),
Beacon: true,
Listen: true,
Port: 0,
Priority: 0,
},
}
if n.multicast, err = multicast.New(n.core, n.log, options...); err != nil {
panic(err)
}
}
n.log.Printf("Public key: %x", n.core.PublicKey())
@ -114,14 +123,7 @@ func (n *Node) DerivedServerName() string {
}
func (n *Node) PrivateKey() ed25519.PrivateKey {
sk := make(ed25519.PrivateKey, ed25519.PrivateKeySize)
sb, err := hex.DecodeString(n.config.PrivateKey)
if err == nil {
copy(sk, sb[:])
} else {
panic(err)
}
return sk
return n.core.PrivateKey()
}
func (n *Node) PublicKey() ed25519.PublicKey {

9
cmd/dendrite-demo-yggdrasil/yggrooms/yggrooms.go
View file

@ -43,13 +43,18 @@ func NewYggdrasilRoomProvider(
}
func (p *YggdrasilRoomProvider) Rooms() []gomatrixserverlib.PublicRoom {
return bulkFetchPublicRoomsFromServers(context.Background(), p.fedClient, p.node.KnownNodes())
return bulkFetchPublicRoomsFromServers(
context.Background(), p.fedClient,
gomatrixserverlib.ServerName(p.node.DerivedServerName()),
p.node.KnownNodes(),
)
}
// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers.
// Returns a list of public rooms.
func bulkFetchPublicRoomsFromServers(
ctx context.Context, fedClient *gomatrixserverlib.FederationClient,
origin gomatrixserverlib.ServerName,
homeservers []gomatrixserverlib.ServerName,
) (publicRooms []gomatrixserverlib.PublicRoom) {
limit := 200
@ -66,7 +71,7 @@ func bulkFetchPublicRoomsFromServers(
go func(homeserverDomain gomatrixserverlib.ServerName) {
defer wg.Done()
util.GetLogger(ctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms")
fres, err := fedClient.GetPublicRooms(ctx, homeserverDomain, int(limit), "", false, "")
fres, err := fedClient.GetPublicRooms(ctx, origin, homeserverDomain, int(limit), "", false, "")
if err != nil {
util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Warn(
"bulkFetchPublicRoomsFromServers: failed to query hs",

2
cmd/dendrite-monolith-server/main.go
View file

@ -30,8 +30,6 @@ import (
"github.com/matrix-org/dendrite/userapi"
uapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/sirupsen/logrus"
_ "github.com/mattn/go-sqlite3"
)
var (

2
cmd/dendrite-polylith-multi/main.go
View file

@ -24,8 +24,6 @@ import (
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/sirupsen/logrus"
_ "github.com/mattn/go-sqlite3"
)
type entrypoint func(base *base.BaseDendrite, cfg *config.Dendrite)

115
cmd/dendrite-upgrade-tests/main.go
View file

@ -7,6 +7,7 @@ import (
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
@ -38,6 +39,7 @@ var (
flagHead = flag.String("head", "", "Location to a dendrite repository to treat as HEAD instead of Github")
flagDockerHost = flag.String("docker-host", "localhost", "The hostname of the docker client. 'localhost' if running locally, 'host.docker.internal' if running in Docker.")
flagDirect = flag.Bool("direct", false, "If a direct upgrade from the defined FROM version to TO should be done")
flagSqlite = flag.Bool("sqlite", false, "Test SQLite instead of PostgreSQL")
alphaNumerics = regexp.MustCompile("[^a-zA-Z0-9]+")
)
@ -49,7 +51,7 @@ const HEAD = "HEAD"
// due to the error:
// When using COPY with more than one source file, the destination must be a directory and end with a /
// We need to run a postgres anyway, so use the dockerfile associated with Complement instead.
const Dockerfile = `FROM golang:1.18-stretch as build
const DockerfilePostgreSQL = `FROM golang:1.18-stretch as build
RUN apt-get update && apt-get install -y postgresql
WORKDIR /build
@ -60,6 +62,7 @@ COPY . .
RUN go build ./cmd/dendrite-monolith-server
RUN go build ./cmd/generate-keys
RUN go build ./cmd/generate-config
RUN go build ./cmd/create-account
RUN ./generate-config --ci > dendrite.yaml
RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
@ -92,6 +95,43 @@ ENV SERVER_NAME=localhost
EXPOSE 8008 8448
CMD /build/run_dendrite.sh `
const DockerfileSQLite = `FROM golang:1.18-stretch as build
RUN apt-get update && apt-get install -y postgresql
WORKDIR /build
# Copy the build context to the repo as this is the right dendrite code. This is different to the
# Complement Dockerfile which wgets a branch.
COPY . .
RUN go build ./cmd/dendrite-monolith-server
RUN go build ./cmd/generate-keys
RUN go build ./cmd/generate-config
RUN go build ./cmd/create-account
RUN ./generate-config --ci > dendrite.yaml
RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
# Make sure the SQLite databases are in a persistent location, we're already mapping
# the postgresql folder so let's just use that for simplicity
RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postgresql\/9.6\/main\/%g" dendrite.yaml
# This entry script starts postgres, waits for it to be up then starts dendrite
RUN echo '\
sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml \n\
PARAMS="--tls-cert server.crt --tls-key server.key --config dendrite.yaml" \n\
./dendrite-monolith-server --really-enable-open-registration ${PARAMS} || ./dendrite-monolith-server ${PARAMS} \n\
' > run_dendrite.sh && chmod +x run_dendrite.sh
ENV SERVER_NAME=localhost
EXPOSE 8008 8448
CMD /build/run_dendrite.sh `
func dockerfile() []byte {
if *flagSqlite {
return []byte(DockerfileSQLite)
}
return []byte(DockerfilePostgreSQL)
}
const dendriteUpgradeTestLabel = "dendrite_upgrade_test"
// downloadArchive downloads an arbitrary github archive of the form:
@ -150,7 +190,7 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir,
if branchOrTagName == HEAD && *flagHead != "" {
log.Printf("%s: Using %s as HEAD", branchOrTagName, *flagHead)
// add top level Dockerfile
err = os.WriteFile(path.Join(*flagHead, "Dockerfile"), []byte(Dockerfile), os.ModePerm)
err = os.WriteFile(path.Join(*flagHead, "Dockerfile"), dockerfile(), os.ModePerm)
if err != nil {
return "", fmt.Errorf("custom HEAD: failed to inject /Dockerfile: %w", err)
}
@ -166,7 +206,7 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir,
// pull an archive, this contains a top-level directory which screws with the build context
// which we need to fix up post download
u := fmt.Sprintf("https://github.com/matrix-org/dendrite/archive/%s.tar.gz", branchOrTagName)
tarball, err = downloadArchive(httpClient, tmpDir, u, []byte(Dockerfile))
tarball, err = downloadArchive(httpClient, tmpDir, u, dockerfile())
if err != nil {
return "", fmt.Errorf("failed to download archive %s: %w", u, err)
}
@ -367,7 +407,8 @@ func runImage(dockerClient *client.Client, volumeName, version, imageID string)
// hit /versions to check it is up
var lastErr error
for i := 0; i < 500; i++ {
res, err := http.Get(versionsURL)
var res *http.Response
res, err = http.Get(versionsURL)
if err != nil {
lastErr = fmt.Errorf("GET %s => error: %s", versionsURL, err)
time.Sleep(50 * time.Millisecond)
@ -381,18 +422,22 @@ func runImage(dockerClient *client.Client, volumeName, version, imageID string)
lastErr = nil
break
}
if lastErr != nil {
logs, err := dockerClient.ContainerLogs(context.Background(), containerID, types.ContainerLogsOptions{
ShowStdout: true,
ShowStderr: true,
})
// ignore errors when cannot get logs, it's just for debugging anyways
if err == nil {
logbody, err := io.ReadAll(logs)
if err == nil {
log.Printf("Container logs:\n\n%s\n\n", string(logbody))
logs, err := dockerClient.ContainerLogs(context.Background(), containerID, types.ContainerLogsOptions{
ShowStdout: true,
ShowStderr: true,
Follow: true,
})
// ignore errors when cannot get logs, it's just for debugging anyways
if err == nil {
go func() {
for {
if body, err := io.ReadAll(logs); err == nil && len(body) > 0 {
log.Printf("%s: %s", version, string(body))
} else {
return
}
}
}
}()
}
return baseURL, containerID, lastErr
}
@ -416,6 +461,46 @@ func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchTo
if err = runTests(csAPIURL, v); err != nil {
return fmt.Errorf("failed to run tests on version %s: %s", v, err)
}
err = testCreateAccount(dockerClient, v, containerID)
if err != nil {
return err
}
return nil
}
// test that create-account is working
func testCreateAccount(dockerClient *client.Client, v string, containerID string) error {
createUser := strings.ToLower("createaccountuser-" + v)
log.Printf("%s: Creating account %s with create-account\n", v, createUser)
respID, err := dockerClient.ContainerExecCreate(context.Background(), containerID, types.ExecConfig{
AttachStderr: true,
AttachStdout: true,
Cmd: []string{
"/build/create-account",
"-username", createUser,
"-password", "someRandomPassword",
},
})
if err != nil {
return fmt.Errorf("failed to ContainerExecCreate: %w", err)
}
response, err := dockerClient.ContainerExecAttach(context.Background(), respID.ID, types.ExecStartCheck{})
if err != nil {
return fmt.Errorf("failed to attach to container: %w", err)
}
defer response.Close()
data, err := ioutil.ReadAll(response.Reader)
if err != nil {
return err
}
if !bytes.Contains(data, []byte("AccessToken")) {
return fmt.Errorf("failed to create-account: %s", string(data))
}
return nil
}

12
cmd/furl/main.go
View file

@ -48,10 +48,15 @@ func main() {
panic("unexpected key block")
}
serverName := gomatrixserverlib.ServerName(*requestFrom)
client := gomatrixserverlib.NewFederationClient(
gomatrixserverlib.ServerName(*requestFrom),
gomatrixserverlib.KeyID(keyBlock.Headers["Key-ID"]),
privateKey,
[]*gomatrixserverlib.SigningIdentity{
{
ServerName: serverName,
KeyID: gomatrixserverlib.KeyID(keyBlock.Headers["Key-ID"]),
PrivateKey: privateKey,
},
},
)
u, err := url.Parse(flag.Arg(0))
@ -79,6 +84,7 @@ func main() {
req := gomatrixserverlib.NewFederationRequest(
method,
serverName,
gomatrixserverlib.ServerName(u.Host),
u.RequestURI(),
)

25
dendrite-sample.monolith.yaml
View file

@ -18,12 +18,17 @@ global:
private_key: matrix_key.pem
# The paths and expiry timestamps (as a UNIX timestamp in millisecond precision)
# to old signing private keys that were formerly in use on this domain. These
# to old signing keys that were formerly in use on this domain name. These
# keys will not be used for federation request or event signing, but will be
# provided to any other homeserver that asks when trying to verify old events.
old_private_keys:
# If the old private key file is available:
# - private_key: old_matrix_key.pem
# expired_at: 1601024554498
# If only the public key (in base64 format) and key ID are known:
# - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM=
# key_id: ed25519:mykeyid
# expired_at: 1601024554498
# How long a remote server can cache our server signing key before requesting it
# again. Increasing this number will reduce the number of requests made by other
@ -37,7 +42,7 @@ global:
# you must configure the "database" block for each component instead.
database:
connection_string: postgresql://username:password@hostname/dendrite?sslmode=disable
max_open_conns: 100
max_open_conns: 90
max_idle_conns: 5
conn_max_lifetime: -1
@ -174,7 +179,13 @@ client_api:
recaptcha_public_key: ""
recaptcha_private_key: ""
recaptcha_bypass_secret: ""
recaptcha_siteverify_api: ""
# To use hcaptcha.com instead of ReCAPTCHA, set the following parameters, otherwise just keep them empty.
# recaptcha_siteverify_api: "https://hcaptcha.com/siteverify"
# recaptcha_api_js_url: "https://js.hcaptcha.com/1/api.js"
# recaptcha_form_field: "h-captcha-response"
# recaptcha_sitekey_class: "h-captcha"
# TURN server information that this homeserver should send to clients.
turn:
@ -305,6 +316,14 @@ user_api:
# The default lifetime is 3600000ms (60 minutes).
# openid_token_lifetime_ms: 3600000
# Users who register on this homeserver will automatically be joined to the rooms listed under "auto_join_rooms" option.
# By default, any room aliases included in this list will be created as a publicly joinable room
# when the first user registers for the homeserver. If the room already exists,
# make certain it is a publicly joinable room, i.e. the join rule of the room must be set to 'public'.
# As Spaces are just rooms under the hood, Space aliases may also be used.
auto_join_rooms:
# - "#main:matrix.org"
# Configuration for Opentracing.
# See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on
# how this works and how to set it up.

23
dendrite-sample.polylith.yaml
View file

@ -18,12 +18,17 @@ global:
private_key: matrix_key.pem
# The paths and expiry timestamps (as a UNIX timestamp in millisecond precision)
# to old signing private keys that were formerly in use on this domain. These
# to old signing keys that were formerly in use on this domain name. These
# keys will not be used for federation request or event signing, but will be
# provided to any other homeserver that asks when trying to verify old events.
old_private_keys:
# If the old private key file is available:
# - private_key: old_matrix_key.pem
# expired_at: 1601024554498
# If only the public key (in base64 format) and key ID are known:
# - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM=
# key_id: ed25519:mykeyid
# expired_at: 1601024554498
# How long a remote server can cache our server signing key before requesting it
# again. Increasing this number will reduce the number of requests made by other
@ -170,7 +175,13 @@ client_api:
recaptcha_public_key: ""
recaptcha_private_key: ""
recaptcha_bypass_secret: ""
recaptcha_siteverify_api: ""
# To use hcaptcha.com instead of ReCAPTCHA, set the following parameters, otherwise just keep them empty.
# recaptcha_siteverify_api: "https://hcaptcha.com/siteverify"
# recaptcha_api_js_url: "https://js.hcaptcha.com/1/api.js"
# recaptcha_form_field: "h-captcha-response"
# recaptcha_sitekey_class: "h-captcha"
# TURN server information that this homeserver should send to clients.
turn:
@ -370,6 +381,14 @@ user_api:
# The default lifetime is 3600000ms (60 minutes).
# openid_token_lifetime_ms: 3600000
# Users who register on this homeserver will automatically be joined to the rooms listed under "auto_join_rooms" option.
# By default, any room aliases included in this list will be created as a publicly joinable room
# when the first user registers for the homeserver. If the room already exists,
# make certain it is a publicly joinable room, i.e. the join rule of the room must be set to 'public'.
# As Spaces are just rooms under the hood, Space aliases may also be used.
auto_join_rooms:
# - "#main:matrix.org"
# Configuration for Opentracing.
# See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on
# how this works and how to set it up.

15
docs/CONTRIBUTING.md
View file

@ -75,7 +75,20 @@ comment. Please avoid doing this if you can.
We also have unit tests which we run via:
```bash
go test --race ./...
DENDRITE_TEST_SKIP_NODB=1 go test --race ./...
```
This only runs SQLite database tests. If you wish to execute Postgres tests as well, you'll either need to
have Postgres installed locally (`createdb` will be used) or have a remote/containerized Postgres instance
available.
To configure the connection to a remote Postgres, you can use the following enviroment variables:
```bash
POSTGRES_USER=postgres
POSTGERS_PASSWORD=yourPostgresPassword
POSTGRES_HOST=localhost
POSTGRES_DB=postgres # the superuser database to use
```
In general, we like submissions that come with tests. Anything that proves that the

4
docs/Gemfile.lock
View file

@ -231,9 +231,9 @@ GEM
jekyll-seo-tag (~> 2.1)
minitest (5.15.0)
multipart-post (2.1.1)
nokogiri (1.13.6-arm64-darwin)
nokogiri (1.13.9-arm64-darwin)
racc (~> 1.4)
nokogiri (1.13.6-x86_64-linux)
nokogiri (1.13.9-x86_64-linux)
racc (~> 1.4)
octokit (4.22.0)
faraday (>= 0.9)

7
docs/administration/1_createusers.md
View file

@ -14,7 +14,7 @@ User accounts can be created on a Dendrite instance in a number of ways.
The `create-account` tool is built in the `bin` folder when building Dendrite with
the `build.sh` script.
It uses the `dendrite.yaml` configuration file to connect to a running Dendrite instance and requires
It uses the `dendrite.yaml` configuration file to connect to a running Dendrite instance and requires
shared secret registration to be enabled as explained below.
An example of using `create-account` to create a **normal account**:
@ -31,11 +31,11 @@ To create a new **admin account**, add the `-admin` flag:
./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin
```
By default `create-account` uses `https://localhost:8448` to connect to Dendrite, this can be overwritten using
By default `create-account` uses `http://localhost:8008` to connect to Dendrite, this can be overwritten using
the `-url` flag:
```bash
./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -url http://localhost:8008
./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -url https://localhost:8448
```
An example of using `create-account` when running in **Docker**, having found the `CONTAINERNAME` from `docker ps`:
@ -43,6 +43,7 @@ An example of using `create-account` when running in **Docker**, having found th
```bash
docker exec -it CONTAINERNAME /usr/bin/create-account -config /path/to/dendrite.yaml -username USERNAME
```
```bash
docker exec -it CONTAINERNAME /usr/bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin
```

5
docs/administration/4_adminapi.md
View file

@ -62,6 +62,11 @@ the full user ID is `@alice:domain.com` then the local part is `alice`.
This endpoint instructs Dendrite to reindex all searchable events (`m.room.message`, `m.room.topic` and `m.room.name`). An empty JSON body will be returned immediately.
Indexing is done in the background, the server logs every 1000 events (or below) when they are being indexed. Once reindexing is done, you'll see something along the lines `Indexed 69586 events in 53.68223182s` in your debug logs.
## POST `/_dendrite/admin/refreshDevices/{userID}`
This endpoint instructs Dendrite to immediately query `/devices/{userID}` on a federated server. An empty JSON body will be returned on success, updating all locally stored user devices/keys. This can be used to possibly resolve E2EE issues, where the remote user can't decrypt messages.
## POST `/_synapse/admin/v1/send_server_notice`
Request body format:

107
docs/caddy/polylith/Caddyfile
View file

@ -1,66 +1,85 @@
# Sample Caddyfile for using Caddy in front of Dendrite.
#
# Customize email address and domain names.
# Optional settings commented out.
#
# BE SURE YOUR DOMAINS ARE POINTED AT YOUR SERVER FIRST.
# Documentation: https://caddyserver.com/docs/
#
# Bonus tip: If your IP address changes, use Caddy's
# dynamic DNS plugin to update your DNS records to
# point to your new IP automatically:
# https://github.com/mholt/caddy-dynamicdns
# Sample Caddyfile for using Caddy in front of Dendrite
#
# Customize email address and domain names
# Optional settings commented out
#
# BE SURE YOUR DOMAINS ARE POINTED AT YOUR SERVER FIRST
# Documentation: <https://caddyserver.com/docs/>
#
# Bonus tip: If your IP address changes, use Caddy's
# dynamic DNS plugin to update your DNS records to
# point to your new IP automatically
# <https://github.com/mholt/caddy-dynamicdns>
#
# Global options block
{
# In case there is a problem with your certificates.
# email example@example.com
# In case there is a problem with your certificates.
# email example@example.com
# Turn off the admin endpoint if you don't need graceful config
# changes and/or are running untrusted code on your machine.
# admin off
# Turn off the admin endpoint if you don't need graceful config
# changes and/or are running untrusted code on your machine.
# admin off
# Enable this if your clients don't send ServerName in TLS handshakes.
# default_sni example.com
# Enable this if your clients don't send ServerName in TLS handshakes.
# default_sni example.com
# Enable debug mode for verbose logging.
# debug
# Enable debug mode for verbose logging.
# debug
# Use Let's Encrypt's staging endpoint for testing.
# acme_ca https://acme-staging-v02.api.letsencrypt.org/directory
# Use Let's Encrypt's staging endpoint for testing.
# acme_ca https://acme-staging-v02.api.letsencrypt.org/directory
# If you're port-forwarding HTTP/HTTPS ports from 80/443 to something
# else, enable these and put the alternate port numbers here.
# http_port 8080
# https_port 8443
# If you're port-forwarding HTTP/HTTPS ports from 80/443 to something
# else, enable these and put the alternate port numbers here.
# http_port 8080
# https_port 8443
}
# The server name of your matrix homeserver. This example shows
# "well-known delegation" from the registered domain to a subdomain,
# "well-known delegation" from the registered domain to a subdomain
# which is only needed if your server_name doesn't match your Matrix
# homeserver URL (i.e. you can show users a vanity domain that looks
# nice and is easy to remember but still have your Matrix server on
# its own subdomain or hosted service).
# its own subdomain or hosted service)
example.com {
header /.well-known/matrix/* Content-Type application/json
header /.well-known/matrix/* Access-Control-Allow-Origin *
respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}`
respond /.well-known/matrix/client `{"m.homeserver": {"base_url": "https://matrix.example.com"}}`
header /.well-known/matrix/*Content-Type application/json
header /.well-known/matrix/* Access-Control-Allow-Origin *
respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}`
respond /.well-known/matrix/client `{"m.homeserver": {"base_url": "https://matrix.example.com"}}`
}
# The actual domain name whereby your Matrix server is accessed.
# The actual domain name whereby your Matrix server is accessed
matrix.example.com {
# Change the end of each reverse_proxy line to the correct
# address for your various services.
@sync_api {
path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$
}
reverse_proxy @sync_api sync_api:8073
# Change the end of each reverse_proxy line to the correct
# address for your various services.
@sync_api {
path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|.*?_?members|context/.*?|relations/.*?|event/.*?))$
}
reverse_proxy @sync_api sync_api:8073
reverse_proxy /_matrix/client* client_api:8071
reverse_proxy /_matrix/federation* federation_api:8071
reverse_proxy /_matrix/key* federation_api:8071
reverse_proxy /_matrix/media* media_api:8071
reverse_proxy /_matrix/client* client_api:8071
reverse_proxy /_matrix/federation* federation_api:8071
reverse_proxy /_matrix/key* federation_api:8071
reverse_proxy /_matrix/media* media_api:8071
}

9
docs/hiawatha/polylith-sample.conf
View file

@ -18,8 +18,15 @@ VirtualHost {
# /_matrix/client/.*/user/{userId}/filter/{filterID}
# /_matrix/client/.*/keys/changes
# /_matrix/client/.*/rooms/{roomId}/messages
# /_matrix/client/.*/rooms/{roomId}/context/{eventID}
# /_matrix/client/.*/rooms/{roomId}/event/{eventID}
# /_matrix/client/.*/rooms/{roomId}/relations/{eventID}
# /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}
# /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}/{eventType}
# /_matrix/client/.*/rooms/{roomId}/members
# /_matrix/client/.*/rooms/{roomId}/joined_members
# to sync_api
ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages) http://localhost:8073 600
ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|.*?_?members|context/.*?|relations/.*?|event/.*?))$ http://localhost:8073 600
ReverseProxy = /_matrix/client http://localhost:8071 600
ReverseProxy = /_matrix/federation http://localhost:8072 600
ReverseProxy = /_matrix/key http://localhost:8072 600

35
docs/installation/11_optimisation.md
View file

@ -11,6 +11,41 @@ permalink: /installation/start/optimisation
Now that you have Dendrite running, the following tweaks will improve the reliability
and performance of your installation.
## PostgreSQL connection limit
A PostgreSQL database engine is configured to allow only a certain number of connections.
This is typically controlled by the `max_connections` and `superuser_reserved_connections`
configuration items in `postgresql.conf`. Once these limits are violated, **PostgreSQL will
immediately stop accepting new connections** until some of the existing connections are closed.
This is a common source of misconfiguration and requires particular care.
If your PostgreSQL `max_connections` is set to `100` and `superuser_reserved_connections` is
set to `3` then you have an effective connection limit of 97 database connections. It is
therefore important to ensure that Dendrite doesn't violate that limit, otherwise database
queries will unexpectedly fail and this will cause problems both within Dendrite and for users.
If you are also running other software that uses the same PostgreSQL database engine, then you
must also take into account that some connections will be already used by your other software
and therefore will not be available to Dendrite. Check the configuration of any other software
using the same database engine for their configured connection limits and adjust your calculations
accordingly.
Dendrite has a `max_open_conns` configuration item in each `database` block to control how many
connections it will open to the database.
**If you are using the `global` database pool** then you only need to configure the
`max_open_conns` setting once in the `global` section.
**If you are defining a `database` config per component** then you will need to ensure that
the **sum total** of all configured `max_open_conns` to a given database server do not exceed
the connection limit. If you configure a total that adds up to more connections than are available
then this will cause database queries to fail.
You may wish to raise the `max_connections` limit on your PostgreSQL server to accommodate
additional connections, in which case you should also update the `max_open_conns` in your
Dendrite configuration accordingly. However be aware that this is only advisable on particularly
powerful servers that can handle the concurrent load of additional queries running at one time.
## File descriptor limit
Most platforms have a limit on how many file descriptors a single process can open. All

6
docs/installation/2_domainname.md
View file

@ -87,6 +87,12 @@ and contain the following JSON document:
For example, this can be done with the following Caddy config:
```
handle /.well-known/matrix/server {
header Content-Type application/json
header Access-Control-Allow-Origin *
respond `{"m.server": "matrix.example.com:8448"}`
}
handle /.well-known/matrix/client {
header Content-Type application/json
header Access-Control-Allow-Origin *

9
docs/nginx/polylith-sample.conf
View file

@ -28,8 +28,15 @@ server {
# /_matrix/client/.*/user/{userId}/filter/{filterID}
# /_matrix/client/.*/keys/changes
# /_matrix/client/.*/rooms/{roomId}/messages
# /_matrix/client/.*/rooms/{roomId}/context/{eventID}
# /_matrix/client/.*/rooms/{roomId}/event/{eventID}
# /_matrix/client/.*/rooms/{roomId}/relations/{eventID}
# /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}
# /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}/{eventType}
# /_matrix/client/.*/rooms/{roomId}/members
# /_matrix/client/.*/rooms/{roomId}/joined_members
# to sync_api
location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$ {
location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|.*?_?members|context/.*?|relations/.*?|event/.*?))$ {
proxy_pass http://sync_api:8073;
}

73
federationapi/api/api.go
View file

@ -21,8 +21,8 @@ type FederationInternalAPI interface {
QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error
LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
MSC2836EventRelationships(ctx context.Context, origin, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
MSC2946Spaces(ctx context.Context, origin, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
// Broadcasts an EDU to all servers in rooms we are joined to. Used in the yggdrasil demos.
PerformBroadcastEDU(
@ -30,6 +30,12 @@ type FederationInternalAPI interface {
request *PerformBroadcastEDURequest,
response *PerformBroadcastEDUResponse,
) error
PerformWakeupServers(
ctx context.Context,
request *PerformWakeupServersRequest,
response *PerformWakeupServersResponse,
) error
}
type ClientFederationAPI interface {
@ -60,18 +66,18 @@ type RoomserverFederationAPI interface {
// containing only the server names (without information for membership events).
// The response will include this server if they are joined to the room.
QueryJoinedHostServerNamesInRoom(ctx context.Context, request *QueryJoinedHostServerNamesInRoomRequest, response *QueryJoinedHostServerNamesInRoomResponse) error
GetEventAuth(ctx context.Context, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
GetEventAuth(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
GetEvent(ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
}
// KeyserverFederationAPI is a subset of gomatrixserverlib.FederationClient functions which the keyserver
// implements as proxy calls, with built-in backoff/retries/etc. Errors returned from functions in
// this interface are of type FederationClientError
type KeyserverFederationAPI interface {
GetUserDevices(ctx context.Context, s gomatrixserverlib.ServerName, userID string) (res gomatrixserverlib.RespUserDevices, err error)
ClaimKeys(ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res gomatrixserverlib.RespClaimKeys, err error)
QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (res gomatrixserverlib.RespQueryKeys, err error)
GetUserDevices(ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string) (res gomatrixserverlib.RespUserDevices, err error)
ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res gomatrixserverlib.RespClaimKeys, err error)
QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (res gomatrixserverlib.RespQueryKeys, err error)
}
// an interface for gmsl.FederationClient - contains functions called by federationapi only.
@ -80,28 +86,28 @@ type FederationClient interface {
SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error)
// Perform operations
LookupRoomAlias(ctx context.Context, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error)
Peek(ctx context.Context, s gomatrixserverlib.ServerName, roomID, peekID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespPeek, err error)
MakeJoin(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error)
SendJoin(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error)
MakeLeave(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string) (res gomatrixserverlib.RespMakeLeave, err error)
SendLeave(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (err error)
SendInviteV2(ctx context.Context, s gomatrixserverlib.ServerName, request gomatrixserverlib.InviteV2Request) (res gomatrixserverlib.RespInviteV2, err error)
LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error)
Peek(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, peekID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespPeek, err error)
MakeJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error)
SendJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error)
MakeLeave(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string) (res gomatrixserverlib.RespMakeLeave, err error)
SendLeave(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (err error)
SendInviteV2(ctx context.Context, origin, s gomatrixserverlib.ServerName, request gomatrixserverlib.InviteV2Request) (res gomatrixserverlib.RespInviteV2, err error)
GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
GetEvent(ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
GetEventAuth(ctx context.Context, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
GetUserDevices(ctx context.Context, s gomatrixserverlib.ServerName, userID string) (gomatrixserverlib.RespUserDevices, error)
ClaimKeys(ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error)
QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error)
Backfill(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string) (res gomatrixserverlib.Transaction, err error)
MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
GetEventAuth(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
GetUserDevices(ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string) (gomatrixserverlib.RespUserDevices, error)
ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error)
QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error)
Backfill(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string) (res gomatrixserverlib.Transaction, err error)
MSC2836EventRelationships(ctx context.Context, origin, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
MSC2946Spaces(ctx context.Context, origin, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
ExchangeThirdPartyInvite(ctx context.Context, s gomatrixserverlib.ServerName, builder gomatrixserverlib.EventBuilder) (err error)
LookupState(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespState, err error)
LookupStateIDs(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error)
LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
ExchangeThirdPartyInvite(ctx context.Context, origin, s gomatrixserverlib.ServerName, builder gomatrixserverlib.EventBuilder) (err error)
LookupState(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespState, err error)
LookupStateIDs(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error)
LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
}
// FederationClientError is returned from FederationClient methods in the event of a problem.
@ -159,6 +165,7 @@ type PerformJoinRequest struct {
// The sorted list of servers to try. Servers will be tried sequentially, after de-duplication.
ServerNames types.ServerNames `json:"server_names"`
Content map[string]interface{} `json:"content"`
Unsigned map[string]interface{} `json:"unsigned"`
}
type PerformJoinResponse struct {
@ -197,8 +204,9 @@ type PerformInviteResponse struct {
// QueryJoinedHostServerNamesInRoomRequest is a request to QueryJoinedHostServerNames
type QueryJoinedHostServerNamesInRoomRequest struct {
RoomID string `json:"room_id"`
ExcludeSelf bool `json:"exclude_self"`
RoomID string `json:"room_id"`
ExcludeSelf bool `json:"exclude_self"`
ExcludeBlacklisted bool `json:"exclude_blacklisted"`
}
// QueryJoinedHostServerNamesInRoomResponse is a response to QueryJoinedHostServerNames
@ -212,6 +220,13 @@ type PerformBroadcastEDURequest struct {
type PerformBroadcastEDUResponse struct {
}
type PerformWakeupServersRequest struct {
ServerNames []gomatrixserverlib.ServerName `json:"server_names"`
}
type PerformWakeupServersResponse struct {
}
type InputPublicKeysRequest struct {
Keys map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult `json:"keys"`
}

65
federationapi/consumers/keychange.go
View file

@ -18,6 +18,11 @@ import (
"context"
"encoding/json"
"github.com/getsentry/sentry-go"
"github.com/matrix-org/gomatrixserverlib"
"github.com/nats-io/nats.go"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/federationapi/queue"
"github.com/matrix-org/dendrite/federationapi/storage"
"github.com/matrix-org/dendrite/federationapi/types"
@ -26,21 +31,18 @@ import (
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/jetstream"
"github.com/matrix-org/dendrite/setup/process"
"github.com/matrix-org/gomatrixserverlib"
"github.com/nats-io/nats.go"
"github.com/sirupsen/logrus"
)
// KeyChangeConsumer consumes events that originate in key server.
type KeyChangeConsumer struct {
ctx context.Context
jetstream nats.JetStreamContext
durable string
db storage.Database
queues *queue.OutgoingQueues
serverName gomatrixserverlib.ServerName
rsAPI roomserverAPI.FederationRoomserverAPI
topic string
ctx context.Context
jetstream nats.JetStreamContext
durable string
db storage.Database
queues *queue.OutgoingQueues
isLocalServerName func(gomatrixserverlib.ServerName) bool
rsAPI roomserverAPI.FederationRoomserverAPI
topic string
}
// NewKeyChangeConsumer creates a new KeyChangeConsumer. Call Start() to begin consuming from key servers.
@ -53,14 +55,14 @@ func NewKeyChangeConsumer(
rsAPI roomserverAPI.FederationRoomserverAPI,
) *KeyChangeConsumer {
return &KeyChangeConsumer{
ctx: process.Context(),
jetstream: js,
durable: cfg.Matrix.JetStream.Prefixed("FederationAPIKeyChangeConsumer"),
topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputKeyChangeEvent),
queues: queues,
db: store,
serverName: cfg.Matrix.ServerName,
rsAPI: rsAPI,
ctx: process.Context(),
jetstream: js,
durable: cfg.Matrix.JetStream.Prefixed("FederationAPIKeyChangeConsumer"),
topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputKeyChangeEvent),
queues: queues,
db: store,
isLocalServerName: cfg.Matrix.IsLocalServerName,
rsAPI: rsAPI,
}
}
@ -78,6 +80,7 @@ func (t *KeyChangeConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) boo
msg := msgs[0] // Guaranteed to exist if onMessage is called
var m api.DeviceMessage
if err := json.Unmarshal(msg.Data, &m); err != nil {
sentry.CaptureException(err)
logrus.WithError(err).Errorf("failed to read device message from key change topic")
return true
}
@ -105,10 +108,11 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
// only send key change events which originated from us
_, originServerName, err := gomatrixserverlib.SplitID('@', m.UserID)
if err != nil {
sentry.CaptureException(err)
logger.WithError(err).Error("Failed to extract domain from key change event")
return true
}
if originServerName != t.serverName {
if !t.isLocalServerName(originServerName) {
return true
}
@ -118,13 +122,15 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
WantMembership: "join",
}, &queryRes)
if err != nil {
sentry.CaptureException(err)
logger.WithError(err).Error("failed to calculate joined rooms for user")
return true
}
// send this key change to all servers who share rooms with this user.
destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true)
destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true)
if err != nil {
sentry.CaptureException(err)
logger.WithError(err).Error("failed to calculate joined hosts for rooms user is in")
return true
}
@ -135,7 +141,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
// Pack the EDU and marshal it
edu := &gomatrixserverlib.EDU{
Type: gomatrixserverlib.MDeviceListUpdate,
Origin: string(t.serverName),
Origin: string(originServerName),
}
event := gomatrixserverlib.DeviceListUpdateEvent{
UserID: m.UserID,
@ -147,12 +153,13 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
Keys: m.KeyJSON,
}
if edu.Content, err = json.Marshal(event); err != nil {
sentry.CaptureException(err)
logger.WithError(err).Error("failed to marshal EDU JSON")
return true
}
logger.Debugf("Sending device list update message to %q", destinations)
err = t.queues.SendEDU(edu, t.serverName, destinations)
err = t.queues.SendEDU(edu, originServerName, destinations)
return err == nil
}
@ -160,10 +167,11 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool {
output := m.CrossSigningKeyUpdate
_, host, err := gomatrixserverlib.SplitID('@', output.UserID)
if err != nil {
sentry.CaptureException(err)
logrus.WithError(err).Errorf("fedsender key change consumer: user ID parse failure")
return true
}
if host != gomatrixserverlib.ServerName(t.serverName) {
if !t.isLocalServerName(host) {
// Ignore any messages that didn't originate locally, otherwise we'll
// end up parroting information we received from other servers.
return true
@ -176,12 +184,14 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool {
WantMembership: "join",
}, &queryRes)
if err != nil {
sentry.CaptureException(err)
logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined rooms for user")
return true
}
// send this key change to all servers who share rooms with this user.
destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true)
destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true)
if err != nil {
sentry.CaptureException(err)
logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined hosts for rooms user is in")
return true
}
@ -193,15 +203,16 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool {
// Pack the EDU and marshal it
edu := &gomatrixserverlib.EDU{
Type: types.MSigningKeyUpdate,
Origin: string(t.serverName),
Origin: string(host),
}
if edu.Content, err = json.Marshal(output); err != nil {
sentry.CaptureException(err)
logger.WithError(err).Error("fedsender key change consumer: failed to marshal output, dropping")
return true
}
logger.Debugf("Sending cross-signing update message to %q", destinations)
err = t.queues.SendEDU(edu, t.serverName, destinations)
err = t.queues.SendEDU(edu, host, destinations)
return err == nil
}

28
federationapi/consumers/presence.go
View file

@ -22,6 +22,7 @@ import (
"github.com/matrix-org/dendrite/federationapi/queue"
"github.com/matrix-org/dendrite/federationapi/storage"
fedTypes "github.com/matrix-org/dendrite/federationapi/types"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/jetstream"
"github.com/matrix-org/dendrite/setup/process"
@ -38,7 +39,8 @@ type OutputPresenceConsumer struct {
durable string
db storage.Database
queues *queue.OutgoingQueues
ServerName gomatrixserverlib.ServerName
isLocalServerName func(gomatrixserverlib.ServerName) bool
rsAPI roomserverAPI.FederationRoomserverAPI
topic string
outboundPresenceEnabled bool
}
@ -50,16 +52,18 @@ func NewOutputPresenceConsumer(
js nats.JetStreamContext,
queues *queue.OutgoingQueues,
store storage.Database,
rsAPI roomserverAPI.FederationRoomserverAPI,
) *OutputPresenceConsumer {
return &OutputPresenceConsumer{
ctx: process.Context(),
jetstream: js,
queues: queues,
db: store,
ServerName: cfg.Matrix.ServerName,
isLocalServerName: cfg.Matrix.IsLocalServerName,
durable: cfg.Matrix.JetStream.Durable("FederationAPIPresenceConsumer"),
topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
outboundPresenceEnabled: cfg.Matrix.Presence.EnableOutbound,
rsAPI: rsAPI,
}
}
@ -85,7 +89,17 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg
log.WithError(err).WithField("user_id", userID).Error("failed to extract domain from receipt sender")
return true
}
if serverName != t.ServerName {
if !t.isLocalServerName(serverName) {
return true
}
var queryRes roomserverAPI.QueryRoomsForUserResponse
err = t.rsAPI.QueryRoomsForUser(t.ctx, &roomserverAPI.QueryRoomsForUserRequest{
UserID: userID,
WantMembership: "join",
}, &queryRes)
if err != nil {
log.WithError(err).Error("failed to calculate joined rooms for user")
return true
}
@ -96,11 +110,13 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg
return true
}
joined, err := t.db.GetAllJoinedHosts(ctx)
// send this presence to all servers who share rooms with this user.
joined, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true)
if err != nil {
log.WithError(err).Error("failed to get joined hosts")
return true
}
if len(joined) == 0 {
return true
}
@ -127,7 +143,7 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg
edu := &gomatrixserverlib.EDU{
Type: gomatrixserverlib.MPresence,
Origin: string(t.ServerName),
Origin: string(serverName),
}
if edu.Content, err = json.Marshal(content); err != nil {
log.WithError(err).Error("failed to marshal EDU JSON")
@ -135,7 +151,7 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg
}
log.Tracef("sending presence EDU to %d servers", len(joined))
if err = t.queues.SendEDU(edu, t.ServerName, joined); err != nil {
if err = t.queues.SendEDU(edu, serverName, joined); err != nil {
log.WithError(err).Error("failed to send EDU")
return false
}

42
federationapi/consumers/receipts.go
View file

@ -34,13 +34,13 @@ import (
// OutputReceiptConsumer consumes events that originate in the clientapi.
type OutputReceiptConsumer struct {
ctx context.Context
jetstream nats.JetStreamContext
durable string
db storage.Database
queues *queue.OutgoingQueues
ServerName gomatrixserverlib.ServerName
topic string
ctx context.Context
jetstream nats.JetStreamContext
durable string
db storage.Database
queues *queue.OutgoingQueues
isLocalServerName func(gomatrixserverlib.ServerName) bool
topic string
}
// NewOutputReceiptConsumer creates a new OutputReceiptConsumer. Call Start() to begin consuming typing events.
@ -52,13 +52,13 @@ func NewOutputReceiptConsumer(
store storage.Database,
) *OutputReceiptConsumer {
return &OutputReceiptConsumer{
ctx: process.Context(),
jetstream: js,
queues: queues,
db: store,
ServerName: cfg.Matrix.ServerName,
durable: cfg.Matrix.JetStream.Durable("FederationAPIReceiptConsumer"),
topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
ctx: process.Context(),
jetstream: js,
queues: queues,
db: store,
isLocalServerName: cfg.Matrix.IsLocalServerName,
durable: cfg.Matrix.JetStream.Durable("FederationAPIReceiptConsumer"),
topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent),
}
}
@ -81,13 +81,21 @@ func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msgs []*nats.Msg)
Type: msg.Header.Get("type"),
}
switch receipt.Type {
case "m.read":
// These are allowed to be sent over federation
case "m.read.private", "m.fully_read":
// These must not be sent over federation
return true
}
// only send receipt events which originated from us
_, receiptServerName, err := gomatrixserverlib.SplitID('@', receipt.UserID)
if err != nil {
log.WithError(err).WithField("user_id", receipt.UserID).Error("failed to extract domain from receipt sender")
return true
}
if receiptServerName != t.ServerName {
if !t.isLocalServerName(receiptServerName) {
return true
}
@ -126,14 +134,14 @@ func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msgs []*nats.Msg)
edu := &gomatrixserverlib.EDU{
Type: gomatrixserverlib.MReceipt,
Origin: string(t.ServerName),
Origin: string(receiptServerName),
}
if edu.Content, err = json.Marshal(content); err != nil {
log.WithError(err).Error("failed to marshal EDU JSON")
return true
}
if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil {
if err := t.queues.SendEDU(edu, receiptServerName, names); err != nil {
log.WithError(err).Error("failed to send EDU")
return false
}

127
federationapi/consumers/roomserver.go
View file

@ -18,6 +18,10 @@ import (
"context"
"encoding/json"
"fmt"
"strconv"
"time"
syncAPITypes "github.com/matrix-org/dendrite/syncapi/types"
"github.com/matrix-org/gomatrixserverlib"
"github.com/nats-io/nats.go"
@ -34,14 +38,16 @@ import (
// OutputRoomEventConsumer consumes events that originated in the room server.
type OutputRoomEventConsumer struct {
ctx context.Context
cfg *config.FederationAPI
rsAPI api.FederationRoomserverAPI
jetstream nats.JetStreamContext
durable string
db storage.Database
queues *queue.OutgoingQueues
topic string
ctx context.Context
cfg *config.FederationAPI
rsAPI api.FederationRoomserverAPI
jetstream nats.JetStreamContext
natsClient *nats.Conn
durable string
db storage.Database
queues *queue.OutgoingQueues
topic string
topicPresence string
}
// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
@ -49,19 +55,22 @@ func NewOutputRoomEventConsumer(
process *process.ProcessContext,
cfg *config.FederationAPI,
js nats.JetStreamContext,
natsClient *nats.Conn,
queues *queue.OutgoingQueues,
store storage.Database,
rsAPI api.FederationRoomserverAPI,
) *OutputRoomEventConsumer {
return &OutputRoomEventConsumer{
ctx: process.Context(),
cfg: cfg,
jetstream: js,
db: store,
queues: queues,
rsAPI: rsAPI,
durable: cfg.Matrix.JetStream.Durable("FederationAPIRoomServerConsumer"),
topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent),
ctx: process.Context(),
cfg: cfg,
jetstream: js,
natsClient: natsClient,
db: store,
queues: queues,
rsAPI: rsAPI,
durable: cfg.Matrix.JetStream.Durable("FederationAPIRoomServerConsumer"),
topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent),
topicPresence: cfg.Matrix.JetStream.Prefixed(jetstream.RequestPresence),
}
}
@ -79,6 +88,13 @@ func (s *OutputRoomEventConsumer) Start() error {
// realises that it cannot update the room state using the deltas.
func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool {
msg := msgs[0] // Guaranteed to exist if onMessage is called
receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType))
// Only handle events we care about
if receivedType != api.OutputTypeNewRoomEvent && receivedType != api.OutputTypeNewInboundPeek {
return true
}
// Parse out the event JSON
var output api.OutputEvent
if err := json.Unmarshal(msg.Data, &output); err != nil {
@ -139,6 +155,7 @@ func (s *OutputRoomEventConsumer) processInboundPeek(orp api.OutputNewInboundPee
// processMessage updates the list of currently joined hosts in the room
// and then sends the event to the hosts that were joined before the event.
func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rewritesState bool) error {
addsStateEvents, missingEventIDs := ore.NeededStateEventIDs()
// Ask the roomserver and add in the rest of the results into the set.
@ -177,6 +194,14 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
return err
}
// If we added new hosts, inform them about our known presence events for this room
if len(addsJoinedHosts) > 0 && ore.Event.Type() == gomatrixserverlib.MRoomMember && ore.Event.StateKey() != nil {
membership, _ := ore.Event.Membership()
if membership == gomatrixserverlib.Join {
s.sendPresence(ore.Event.RoomID(), addsJoinedHosts)
}
}
if oldJoinedHosts == nil {
// This means that there is nothing to update as this is a duplicate
// message.
@ -206,6 +231,76 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
)
}
func (s *OutputRoomEventConsumer) sendPresence(roomID string, addedJoined []types.JoinedHost) {
joined := make([]gomatrixserverlib.ServerName, len(addedJoined))
for _, added := range addedJoined {
joined = append(joined, added.ServerName)
}
// get our locally joined users
var queryRes api.QueryMembershipsForRoomResponse
err := s.rsAPI.QueryMembershipsForRoom(s.ctx, &api.QueryMembershipsForRoomRequest{
JoinedOnly: true,
LocalOnly: true,
RoomID: roomID,
}, &queryRes)
if err != nil {
log.WithError(err).Error("failed to calculate joined rooms for user")
return
}
// send every presence we know about to the remote server
content := types.Presence{}
for _, ev := range queryRes.JoinEvents {
msg := nats.NewMsg(s.topicPresence)
msg.Header.Set(jetstream.UserID, ev.Sender)
var presence *nats.Msg
presence, err = s.natsClient.RequestMsg(msg, time.Second*10)
if err != nil {
log.WithError(err).Errorf("unable to get presence")
continue
}
statusMsg := presence.Header.Get("status_msg")
e := presence.Header.Get("error")
if e != "" {
continue
}
var lastActive int
lastActive, err = strconv.Atoi(presence.Header.Get("last_active_ts"))
if err != nil {
continue
}
p := syncAPITypes.PresenceInternal{LastActiveTS: gomatrixserverlib.Timestamp(lastActive)}
content.Push = append(content.Push, types.PresenceContent{
CurrentlyActive: p.CurrentlyActive(),
LastActiveAgo: p.LastActiveAgo(),
Presence: presence.Header.Get("presence"),
StatusMsg: &statusMsg,
UserID: ev.Sender,
})
}
if len(content.Push) == 0 {
return
}
edu := &gomatrixserverlib.EDU{
Type: gomatrixserverlib.MPresence,
Origin: string(s.cfg.Matrix.ServerName),
}
if edu.Content, err = json.Marshal(content); err != nil {
log.WithError(err).Error("failed to marshal EDU JSON")
return
}
if err := s.queues.SendEDU(edu, s.cfg.Matrix.ServerName, joined); err != nil {
log.WithError(err).Error("failed to send EDU")
}
}
// joinedHostsAtEvent works out a list of matrix servers that were joined to
// the room at the event (including peeking ones)
// It is important to use the state at the event for sending messages because:

51
federationapi/consumers/sendtodevice.go
View file

@ -18,27 +18,29 @@ import (
"context"
"encoding/json"
"github.com/getsentry/sentry-go"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/nats-io/nats.go"
log "github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/federationapi/queue"
"github.com/matrix-org/dendrite/federationapi/storage"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/jetstream"
"github.com/matrix-org/dendrite/setup/process"
syncTypes "github.com/matrix-org/dendrite/syncapi/types"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/nats-io/nats.go"
log "github.com/sirupsen/logrus"
)
// OutputSendToDeviceConsumer consumes events that originate in the clientapi.
type OutputSendToDeviceConsumer struct {
ctx context.Context
jetstream nats.JetStreamContext
durable string
db storage.Database
queues *queue.OutgoingQueues
ServerName gomatrixserverlib.ServerName
topic string
ctx context.Context
jetstream nats.JetStreamContext
durable string
db storage.Database
queues *queue.OutgoingQueues
isLocalServerName func(gomatrixserverlib.ServerName) bool
topic string
}
// NewOutputSendToDeviceConsumer creates a new OutputSendToDeviceConsumer. Call Start() to begin consuming send-to-device events.
@ -50,13 +52,13 @@ func NewOutputSendToDeviceConsumer(
store storage.Database,
) *OutputSendToDeviceConsumer {
return &OutputSendToDeviceConsumer{
ctx: process.Context(),
jetstream: js,
queues: queues,
db: store,
ServerName: cfg.Matrix.ServerName,
durable: cfg.Matrix.JetStream.Durable("FederationAPIESendToDeviceConsumer"),
topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
ctx: process.Context(),
jetstream: js,
queues: queues,
db: store,
isLocalServerName: cfg.Matrix.IsLocalServerName,
durable: cfg.Matrix.JetStream.Durable("FederationAPIESendToDeviceConsumer"),
topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
}
}
@ -76,35 +78,37 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats
sender := msg.Header.Get("sender")
_, originServerName, err := gomatrixserverlib.SplitID('@', sender)
if err != nil {
sentry.CaptureException(err)
log.WithError(err).WithField("user_id", sender).Error("Failed to extract domain from send-to-device sender")
return true
}
if originServerName != t.ServerName {
log.WithField("other_server", originServerName).Info("Suppressing send-to-device: originated elsewhere")
if !t.isLocalServerName(originServerName) {
return true
}
// Extract the send-to-device event from msg.
var ote syncTypes.OutputSendToDeviceEvent
if err = json.Unmarshal(msg.Data, &ote); err != nil {
sentry.CaptureException(err)
log.WithError(err).Errorf("output log: message parse failed (expected send-to-device)")
return true
}
_, destServerName, err := gomatrixserverlib.SplitID('@', ote.UserID)
if err != nil {
sentry.CaptureException(err)
log.WithError(err).WithField("user_id", ote.UserID).Error("Failed to extract domain from send-to-device destination")
return true
}
// The SyncAPI is already handling sendToDevice for the local server
if destServerName == t.ServerName {
if t.isLocalServerName(destServerName) {
return true
}
// Pack the EDU and marshal it
edu := &gomatrixserverlib.EDU{
Type: gomatrixserverlib.MDirectToDevice,
Origin: string(t.ServerName),
Origin: string(originServerName),
}
tdm := gomatrixserverlib.ToDeviceMessage{
Sender: ote.Sender,
@ -117,12 +121,13 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats
},
}
if edu.Content, err = json.Marshal(tdm); err != nil {
sentry.CaptureException(err)
log.WithError(err).Error("failed to marshal EDU JSON")
return true
}
log.Debugf("Sending send-to-device message into %q destination queue", destServerName)
if err := t.queues.SendEDU(edu, t.ServerName, []gomatrixserverlib.ServerName{destServerName}); err != nil {
if err := t.queues.SendEDU(edu, originServerName, []gomatrixserverlib.ServerName{destServerName}); err != nil {
log.WithError(err).Error("failed to send EDU")
return false
}

32
federationapi/consumers/typing.go
View file

@ -31,13 +31,13 @@ import (
// OutputTypingConsumer consumes events that originate in the clientapi.
type OutputTypingConsumer struct {
ctx context.Context
jetstream nats.JetStreamContext
durable string
db storage.Database
queues *queue.OutgoingQueues
ServerName gomatrixserverlib.ServerName
topic string
ctx context.Context
jetstream nats.JetStreamContext
durable string
db storage.Database
queues *queue.OutgoingQueues
isLocalServerName func(gomatrixserverlib.ServerName) bool
topic string
}
// NewOutputTypingConsumer creates a new OutputTypingConsumer. Call Start() to begin consuming typing events.
@ -49,13 +49,13 @@ func NewOutputTypingConsumer(
store storage.Database,
) *OutputTypingConsumer {
return &OutputTypingConsumer{
ctx: process.Context(),
jetstream: js,
queues: queues,
db: store,
ServerName: cfg.Matrix.ServerName,
durable: cfg.Matrix.JetStream.Durable("FederationAPITypingConsumer"),
topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
ctx: process.Context(),
jetstream: js,
queues: queues,
db: store,
isLocalServerName: cfg.Matrix.IsLocalServerName,
durable: cfg.Matrix.JetStream.Durable("FederationAPITypingConsumer"),
topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent),
}
}
@ -87,7 +87,7 @@ func (t *OutputTypingConsumer) onMessage(ctx context.Context, msgs []*nats.Msg)
_ = msg.Ack()
return true
}
if typingServerName != t.ServerName {
if !t.isLocalServerName(typingServerName) {
return true
}
@ -111,7 +111,7 @@ func (t *OutputTypingConsumer) onMessage(ctx context.Context, msgs []*nats.Msg)
log.WithError(err).Error("failed to marshal EDU JSON")
return true
}
if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil {
if err := t.queues.SendEDU(edu, typingServerName, names); err != nil {
log.WithError(err).Error("failed to send EDU")
return false
}

27
federationapi/federationapi.go
View file

@ -69,7 +69,7 @@ func AddPublicRoutes(
TopicPresenceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent),
TopicDeviceListUpdate: cfg.Matrix.JetStream.Prefixed(jetstream.InputDeviceListUpdate),
TopicSigningKeyUpdate: cfg.Matrix.JetStream.Prefixed(jetstream.InputSigningKeyUpdate),
ServerName: cfg.Matrix.ServerName,
Config: cfg,
UserAPI: userAPI,
}
@ -107,7 +107,7 @@ func NewInternalAPI(
) api.FederationInternalAPI {
cfg := &base.Cfg.FederationAPI
federationDB, err := storage.NewDatabase(base, &cfg.Database, base.Caches, base.Cfg.Global.ServerName)
federationDB, err := storage.NewDatabase(base, &cfg.Database, base.Caches, base.Cfg.Global.IsLocalServerName)
if err != nil {
logrus.WithError(err).Panic("failed to connect to federation sender db")
}
@ -116,26 +116,21 @@ func NewInternalAPI(
_ = federationDB.RemoveAllServersFromBlacklist()
}
stats := &statistics.Statistics{
DB: federationDB,
FailuresUntilBlacklist: cfg.FederationMaxRetries,
}
stats := statistics.NewStatistics(federationDB, cfg.FederationMaxRetries+1)
js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
js, nats := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
signingInfo := base.Cfg.Global.SigningIdentities()
queues := queue.NewOutgoingQueues(
federationDB, base.ProcessContext,
cfg.Matrix.DisableFederation,
cfg.Matrix.ServerName, federation, rsAPI, stats,
&queue.SigningInfo{
KeyID: cfg.Matrix.KeyID,
PrivateKey: cfg.Matrix.PrivateKey,
ServerName: cfg.Matrix.ServerName,
},
cfg.Matrix.ServerName, federation, rsAPI, &stats,
signingInfo,
)
rsConsumer := consumers.NewOutputRoomEventConsumer(
base.ProcessContext, cfg, js, queues,
base.ProcessContext, cfg, js, nats, queues,
federationDB, rsAPI,
)
if err = rsConsumer.Start(); err != nil {
@ -167,7 +162,7 @@ func NewInternalAPI(
}
presenceConsumer := consumers.NewOutputPresenceConsumer(
base.ProcessContext, cfg, js, queues, federationDB,
base.ProcessContext, cfg, js, queues, federationDB, rsAPI,
)
if err = presenceConsumer.Start(); err != nil {
logrus.WithError(err).Panic("failed to start presence consumer")
@ -183,5 +178,5 @@ func NewInternalAPI(
}
time.AfterFunc(time.Minute, cleanExpiredEDUs)
return internal.NewFederationInternalAPI(federationDB, cfg, rsAPI, federation, stats, caches, queues, keyRing)
return internal.NewFederationInternalAPI(federationDB, cfg, rsAPI, federation, &stats, caches, queues, keyRing)
}

6
federationapi/federationapi_keys_test.go
View file

@ -87,6 +87,7 @@ func TestMain(m *testing.M) {
cfg.Global.JetStream.StoragePath = config.Path(d)
cfg.Global.KeyID = serverKeyID
cfg.Global.KeyValidityPeriod = s.validity
cfg.FederationAPI.KeyPerspectives = nil
f, err := os.CreateTemp(d, "federation_keys_test*.db")
if err != nil {
return -1
@ -103,7 +104,7 @@ func TestMain(m *testing.M) {
// Create the federation client.
s.fedclient = gomatrixserverlib.NewFederationClient(
s.config.Matrix.ServerName, serverKeyID, testPriv,
s.config.Matrix.SigningIdentities(),
gomatrixserverlib.WithTransport(transport),
)
@ -136,7 +137,7 @@ func (m *MockRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err
}
// Get the keys and JSON-ify them.
keys := routing.LocalKeys(s.config)
keys := routing.LocalKeys(s.config, gomatrixserverlib.ServerName(req.Host))
body, err := json.MarshalIndent(keys.JSON, "", " ")
if err != nil {
return nil, err
@ -207,7 +208,6 @@ func TestRenewalBehaviour(t *testing.T) {
// happy at this point that the key that we already have is from the past
// then repeating a key fetch should cause us to try and renew the key.
// If so, then the new key will end up in our cache.
serverC.renew()
res, err = serverA.api.FetchKeys(

9
federationapi/federationapi_test.go
View file

@ -103,7 +103,7 @@ func (f *fedClient) GetServerKeys(ctx context.Context, matrixServer gomatrixserv
return keys, nil
}
func (f *fedClient) MakeJoin(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error) {
func (f *fedClient) MakeJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error) {
for _, r := range f.allowJoins {
if r.ID == roomID {
res.RoomVersion = r.Version
@ -127,7 +127,7 @@ func (f *fedClient) MakeJoin(ctx context.Context, s gomatrixserverlib.ServerName
}
return
}
func (f *fedClient) SendJoin(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error) {
func (f *fedClient) SendJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error) {
f.fedClientMutex.Lock()
defer f.fedClientMutex.Unlock()
for _, r := range f.allowJoins {
@ -164,6 +164,7 @@ func TestFederationAPIJoinThenKeyUpdate(t *testing.T) {
func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) {
base, close := testrig.CreateBaseDendrite(t, dbType)
base.Cfg.FederationAPI.PreferDirectFetch = true
base.Cfg.FederationAPI.KeyPerspectives = nil
defer close()
jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream)
@ -282,7 +283,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://"))
fedCli := gomatrixserverlib.NewFederationClient(
serverName, cfg.Global.KeyID, cfg.Global.PrivateKey,
cfg.Global.SigningIdentities(),
gomatrixserverlib.WithSkipVerify(true),
)
@ -325,7 +326,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
t.Errorf("failed to create invite v2 request: %s", err)
continue
}
_, err = fedCli.SendInviteV2(context.Background(), serverName, invReq)
_, err = fedCli.SendInviteV2(context.Background(), cfg.Global.ServerName, serverName, invReq)
if err == nil {
t.Errorf("expected an error, got none")
continue

46
federationapi/internal/federationclient.go
View file

@ -11,13 +11,13 @@ import (
// client.
func (a *FederationInternalAPI) GetEventAuth(
ctx context.Context, s gomatrixserverlib.ServerName,
ctx context.Context, origin, s gomatrixserverlib.ServerName,
roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string,
) (res gomatrixserverlib.RespEventAuth, err error) {
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
defer cancel()
ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
return a.federation.GetEventAuth(ctx, s, roomVersion, roomID, eventID)
return a.federation.GetEventAuth(ctx, origin, s, roomVersion, roomID, eventID)
})
if err != nil {
return gomatrixserverlib.RespEventAuth{}, err
@ -26,12 +26,12 @@ func (a *FederationInternalAPI) GetEventAuth(
}
func (a *FederationInternalAPI) GetUserDevices(
ctx context.Context, s gomatrixserverlib.ServerName, userID string,
ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string,
) (gomatrixserverlib.RespUserDevices, error) {
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
defer cancel()
ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
return a.federation.GetUserDevices(ctx, s, userID)
return a.federation.GetUserDevices(ctx, origin, s, userID)
})
if err != nil {
return gomatrixserverlib.RespUserDevices{}, err
@ -40,12 +40,12 @@ func (a *FederationInternalAPI) GetUserDevices(
}
func (a *FederationInternalAPI) ClaimKeys(
ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string,
ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string,
) (gomatrixserverlib.RespClaimKeys, error) {
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
defer cancel()
ires, err := a.doRequestIfNotBackingOffOrBlacklisted(s, func() (interface{}, error) {
return a.federation.ClaimKeys(ctx, s, oneTimeKeys)
ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
return a.federation.ClaimKeys(ctx, origin, s, oneTimeKeys)
})
if err != nil {
return gomatrixserverlib.RespClaimKeys{}, err
@ -54,10 +54,10 @@ func (a *FederationInternalAPI) ClaimKeys(
}
func (a *FederationInternalAPI) QueryKeys(
ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string,
ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string,
) (gomatrixserverlib.RespQueryKeys, error) {
ires, err := a.doRequestIfNotBackingOffOrBlacklisted(s, func() (interface{}, error) {
return a.federation.QueryKeys(ctx, s, keys)
return a.federation.QueryKeys(ctx, origin, s, keys)
})
if err != nil {
return gomatrixserverlib.RespQueryKeys{}, err
@ -66,12 +66,12 @@ func (a *FederationInternalAPI) QueryKeys(
}
func (a *FederationInternalAPI) Backfill(
ctx context.Context, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string,
ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string,
) (res gomatrixserverlib.Transaction, err error) {
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
defer cancel()
ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
return a.federation.Backfill(ctx, s, roomID, limit, eventIDs)
return a.federation.Backfill(ctx, origin, s, roomID, limit, eventIDs)
})
if err != nil {
return gomatrixserverlib.Transaction{}, err
@ -80,12 +80,12 @@ func (a *FederationInternalAPI) Backfill(
}
func (a *FederationInternalAPI) LookupState(
ctx context.Context, s gomatrixserverlib.ServerName, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion,
ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion,
) (res gomatrixserverlib.RespState, err error) {
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
defer cancel()
ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
return a.federation.LookupState(ctx, s, roomID, eventID, roomVersion)
return a.federation.LookupState(ctx, origin, s, roomID, eventID, roomVersion)
})
if err != nil {
return gomatrixserverlib.RespState{}, err
@ -94,12 +94,12 @@ func (a *FederationInternalAPI) LookupState(
}
func (a *FederationInternalAPI) LookupStateIDs(
ctx context.Context, s gomatrixserverlib.ServerName, roomID, eventID string,
ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, eventID string,
) (res gomatrixserverlib.RespStateIDs, err error) {
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
defer cancel()
ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
return a.federation.LookupStateIDs(ctx, s, roomID, eventID)
return a.federation.LookupStateIDs(ctx, origin, s, roomID, eventID)
})
if err != nil {
return gomatrixserverlib.RespStateIDs{}, err
@ -108,13 +108,13 @@ func (a *FederationInternalAPI) LookupStateIDs(
}
func (a *FederationInternalAPI) LookupMissingEvents(
ctx context.Context, s gomatrixserverlib.ServerName, roomID string,
ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string,
missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion,
) (res gomatrixserverlib.RespMissingEvents, err error) {
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
defer cancel()
ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
return a.federation.LookupMissingEvents(ctx, s, roomID, missing, roomVersion)
return a.federation.LookupMissingEvents(ctx, origin, s, roomID, missing, roomVersion)
})
if err != nil {
return gomatrixserverlib.RespMissingEvents{}, err
@ -123,12 +123,12 @@ func (a *FederationInternalAPI) LookupMissingEvents(
}
func (a *FederationInternalAPI) GetEvent(
ctx context.Context, s gomatrixserverlib.ServerName, eventID string,
ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string,
) (res gomatrixserverlib.Transaction, err error) {
ctx, cancel := context.WithTimeout(ctx, time.Second*30)
defer cancel()
ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
return a.federation.GetEvent(ctx, s, eventID)
return a.federation.GetEvent(ctx, origin, s, eventID)
})
if err != nil {
return gomatrixserverlib.Transaction{}, err
@ -151,13 +151,13 @@ func (a *FederationInternalAPI) LookupServerKeys(
}
func (a *FederationInternalAPI) MSC2836EventRelationships(
ctx context.Context, s gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest,
ctx context.Context, origin, s gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest,
roomVersion gomatrixserverlib.RoomVersion,
) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error) {
ctx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel()
ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
return a.federation.MSC2836EventRelationships(ctx, s, r, roomVersion)
return a.federation.MSC2836EventRelationships(ctx, origin, s, r, roomVersion)
})
if err != nil {
return res, err
@ -166,12 +166,12 @@ func (a *FederationInternalAPI) MSC2836EventRelationships(
}
func (a *FederationInternalAPI) MSC2946Spaces(
ctx context.Context, s gomatrixserverlib.ServerName, roomID string, suggestedOnly bool,
ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, suggestedOnly bool,
) (res gomatrixserverlib.MSC2946SpacesResponse, err error) {
ctx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel()
ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) {
return a.federation.MSC2946Spaces(ctx, s, roomID, suggestedOnly)
return a.federation.MSC2946Spaces(ctx, origin, s, roomID, suggestedOnly)
})
if err != nil {
return res, err

2
federationapi/internal/keys.go
View file

@ -99,7 +99,7 @@ func (s *FederationInternalAPI) handleLocalKeys(
results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult,
) {
for req := range requests {
if req.ServerName != s.cfg.Matrix.ServerName {
if !s.cfg.Matrix.IsLocalServerName(req.ServerName) {
continue
}
if req.KeyID == s.cfg.Matrix.KeyID {

77
federationapi/internal/perform.go
View file

@ -7,14 +7,15 @@ import (
"fmt"
"time"
"github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/federationapi/consumers"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/roomserver/version"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/federationapi/consumers"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/roomserver/version"
)
// PerformLeaveRequest implements api.FederationInternalAPI
@ -25,6 +26,7 @@ func (r *FederationInternalAPI) PerformDirectoryLookup(
) (err error) {
dir, err := r.federation.LookupRoomAlias(
ctx,
r.cfg.Matrix.ServerName,
request.ServerName,
request.RoomAlias,
)
@ -76,7 +78,7 @@ func (r *FederationInternalAPI) PerformJoin(
seenSet := make(map[gomatrixserverlib.ServerName]bool)
var uniqueList []gomatrixserverlib.ServerName
for _, srv := range request.ServerNames {
if seenSet[srv] || srv == r.cfg.Matrix.ServerName {
if seenSet[srv] || r.cfg.Matrix.IsLocalServerName(srv) {
continue
}
seenSet[srv] = true
@ -95,6 +97,7 @@ func (r *FederationInternalAPI) PerformJoin(
request.Content,
serverName,
supportedVersions,
request.Unsigned,
); err != nil {
logrus.WithError(err).WithFields(logrus.Fields{
"server_name": serverName,
@ -139,11 +142,18 @@ func (r *FederationInternalAPI) performJoinUsingServer(
content map[string]interface{},
serverName gomatrixserverlib.ServerName,
supportedVersions []gomatrixserverlib.RoomVersion,
unsigned map[string]interface{},
) error {
_, origin, err := r.cfg.Matrix.SplitLocalID('@', userID)
if err != nil {
return err
}
// Try to perform a make_join using the information supplied in the
// request.
respMakeJoin, err := r.federation.MakeJoin(
ctx,
origin,
serverName,
roomID,
userID,
@ -189,7 +199,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
// Build the join event.
event, err := respMakeJoin.JoinEvent.Build(
time.Now(),
r.cfg.Matrix.ServerName,
origin,
r.cfg.Matrix.KeyID,
r.cfg.Matrix.PrivateKey,
respMakeJoin.RoomVersion,
@ -201,6 +211,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
// Try to perform a send_join using the newly built event.
respSendJoin, err := r.federation.SendJoin(
context.Background(),
origin,
serverName,
event,
)
@ -243,7 +254,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
respMakeJoin.RoomVersion,
r.keyRing,
event,
federatedAuthProvider(ctx, r.federation, r.keyRing, serverName),
federatedAuthProvider(ctx, r.federation, r.keyRing, origin, serverName),
)
if err != nil {
return fmt.Errorf("respSendJoin.Check: %w", err)
@ -259,7 +270,7 @@ func (r *FederationInternalAPI) performJoinUsingServer(
if err != nil {
return fmt.Errorf("JoinedHostsFromEvents: failed to get joined hosts: %s", err)
}
logrus.WithField("hosts", joinedHosts).WithField("room", roomID).Info("Joined federated room with hosts")
logrus.WithField("room", roomID).Infof("Joined federated room with %d hosts", len(joinedHosts))
if _, err = r.db.UpdateRoom(context.Background(), roomID, joinedHosts, nil, true); err != nil {
return fmt.Errorf("UpdatedRoom: failed to update room with joined hosts: %s", err)
}
@ -267,9 +278,18 @@ func (r *FederationInternalAPI) performJoinUsingServer(
// If we successfully performed a send_join above then the other
// server now thinks we're a part of the room. Send the newly
// returned state to the roomserver to update our local view.
if unsigned != nil {
event, err = event.SetUnsigned(unsigned)
if err != nil {
// non-fatal, log and continue
logrus.WithError(err).Errorf("Failed to set unsigned content")
}
}
if err = roomserverAPI.SendEventWithState(
context.Background(),
r.rsAPI,
origin,
roomserverAPI.KindNew,
respState,
event.Headered(respMakeJoin.RoomVersion),
@ -416,6 +436,7 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer(
// request.
respPeek, err := r.federation.Peek(
ctx,
r.cfg.Matrix.ServerName,
serverName,
roomID,
peekID,
@ -442,7 +463,7 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer(
// authenticate the state returned (check its auth events etc)
// the equivalent of CheckSendJoinResponse()
authEvents, _, err := respState.Check(ctx, respPeek.RoomVersion, r.keyRing, federatedAuthProvider(ctx, r.federation, r.keyRing, serverName))
authEvents, _, err := respState.Check(ctx, respPeek.RoomVersion, r.keyRing, federatedAuthProvider(ctx, r.federation, r.keyRing, r.cfg.Matrix.ServerName, serverName))
if err != nil {
return fmt.Errorf("error checking state returned from peeking: %w", err)
}
@ -464,7 +485,7 @@ func (r *FederationInternalAPI) performOutboundPeekUsingServer(
// logrus.Warnf("got respPeek %#v", respPeek)
// Send the newly returned state to the roomserver to update our local view.
if err = roomserverAPI.SendEventWithState(
ctx, r.rsAPI,
ctx, r.rsAPI, r.cfg.Matrix.ServerName,
roomserverAPI.KindNew,
&respState,
respPeek.LatestEvent.Headered(respPeek.RoomVersion),
@ -484,6 +505,11 @@ func (r *FederationInternalAPI) PerformLeave(
request *api.PerformLeaveRequest,
response *api.PerformLeaveResponse,
) (err error) {
_, origin, err := r.cfg.Matrix.SplitLocalID('@', request.UserID)
if err != nil {
return err
}
// Deduplicate the server names we were provided.
util.SortAndUnique(request.ServerNames)
@ -494,6 +520,7 @@ func (r *FederationInternalAPI) PerformLeave(
// request.
respMakeLeave, err := r.federation.MakeLeave(
ctx,
origin,
serverName,
request.RoomID,
request.UserID,
@ -535,7 +562,7 @@ func (r *FederationInternalAPI) PerformLeave(
// Build the leave event.
event, err := respMakeLeave.LeaveEvent.Build(
time.Now(),
r.cfg.Matrix.ServerName,
origin,
r.cfg.Matrix.KeyID,
r.cfg.Matrix.PrivateKey,
respMakeLeave.RoomVersion,
@ -548,6 +575,7 @@ func (r *FederationInternalAPI) PerformLeave(
// Try to perform a send_leave using the newly built event.
err = r.federation.SendLeave(
ctx,
origin,
serverName,
event,
)
@ -574,6 +602,11 @@ func (r *FederationInternalAPI) PerformInvite(
request *api.PerformInviteRequest,
response *api.PerformInviteResponse,
) (err error) {
_, origin, err := r.cfg.Matrix.SplitLocalID('@', request.Event.Sender())
if err != nil {
return err
}
if request.Event.StateKey() == nil {
return errors.New("invite must be a state event")
}
@ -596,7 +629,7 @@ func (r *FederationInternalAPI) PerformInvite(
return fmt.Errorf("gomatrixserverlib.NewInviteV2Request: %w", err)
}
inviteRes, err := r.federation.SendInviteV2(ctx, destination, inviteReq)
inviteRes, err := r.federation.SendInviteV2(ctx, origin, destination, inviteReq)
if err != nil {
return fmt.Errorf("r.federation.SendInviteV2: failed to send invite: %w", err)
}
@ -637,9 +670,23 @@ func (r *FederationInternalAPI) PerformBroadcastEDU(
return nil
}
// PerformWakeupServers implements api.FederationInternalAPI
func (r *FederationInternalAPI) PerformWakeupServers(
ctx context.Context,
request *api.PerformWakeupServersRequest,
response *api.PerformWakeupServersResponse,
) (err error) {
r.MarkServersAlive(request.ServerNames)
return nil
}
func (r *FederationInternalAPI) MarkServersAlive(destinations []gomatrixserverlib.ServerName) {
for _, srv := range destinations {
_ = r.db.RemoveServerFromBlacklist(srv)
// Check the statistics cache for the blacklist status to prevent hitting
// the database unnecessarily.
if r.queues.IsServerBlacklisted(srv) {
_ = r.db.RemoveServerFromBlacklist(srv)
}
r.queues.RetryServer(srv)
}
}
@ -697,7 +744,7 @@ func setDefaultRoomVersionFromJoinEvent(joinEvent gomatrixserverlib.EventBuilder
// FederatedAuthProvider is an auth chain provider which fetches events from the server provided
func federatedAuthProvider(
ctx context.Context, federation api.FederationClient,
keyRing gomatrixserverlib.JSONVerifier, server gomatrixserverlib.ServerName,
keyRing gomatrixserverlib.JSONVerifier, origin, server gomatrixserverlib.ServerName,
) gomatrixserverlib.AuthChainProvider {
// A list of events that we have retried, if they were not included in
// the auth events supplied in the send_join.
@ -727,7 +774,7 @@ func federatedAuthProvider(
// Try to retrieve the event from the server that sent us the send
// join response.
tx, txerr := federation.GetEvent(ctx, server, eventID)
tx, txerr := federation.GetEvent(ctx, origin, server, eventID)
if txerr != nil {
return nil, fmt.Errorf("missingAuth r.federation.GetEvent: %w", txerr)
}

2
federationapi/internal/query.go
View file

@ -16,7 +16,7 @@ func (f *FederationInternalAPI) QueryJoinedHostServerNamesInRoom(
request *api.QueryJoinedHostServerNamesInRoomRequest,
response *api.QueryJoinedHostServerNamesInRoomResponse,
) (err error) {
joinedHosts, err := f.db.GetJoinedHostsForRooms(ctx, []string{request.RoomID}, request.ExcludeSelf)
joinedHosts, err := f.db.GetJoinedHostsForRooms(ctx, []string{request.RoomID}, request.ExcludeSelf, request.ExcludeBlacklisted)
if err != nil {
return
}

Some files were not shown because too many files have changed in this diff Show more