mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2026-01-16 18:43:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge upstream Dendrite 0.11 changes, Dendrite and lib tests passing (#1440)

Browse source 
Signed-off-by: Brian Meek <brian@hntlabs.com>
This commit is contained in:
Brian Meek 2023-02-08 16:58:48 -08:00 committed by GitHub
parent 9a85e1b3c1
commit b90114cda3
408 changed files with 21179 additions and 5285 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
.github/codecov.yaml vendored Normal file
View file

@ -0,0 +1,20 @@
flag_management:
default_rules:
carryforward: true
coverage:
status:
project:
default:
target: auto
threshold: 0%
base: auto
flags:
- unittests
patch:
default:
target: 75%
threshold: 0%
base: auto
flags:
- unittests

296
.github/workflows/dendrite.yml vendored
View file

@ -26,22 +26,14 @@ jobs:
uses: actions/setup-go@v3
with:
go-version: 1.19
- uses: actions/cache@v2
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-wasm-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-wasm
cache: true
- name: Install Node
uses: actions/setup-node@v2
with:
node-version: 14
- uses: actions/cache@v2
- uses: actions/cache@v3
with:
path: ~/.npm
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@ -76,10 +68,147 @@ jobs:
# run go test with different go versions
test:
timeout-minutes: 5
timeout-minutes: 10
name: Unit tests (Go ${{ matrix.go }})
runs-on: ubuntu-latest
# Service containers to run with `container-job`
services:
# Label used to access the service container
postgres:
# Docker Hub image
image: postgres:13-alpine
# Provide the password for postgres
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: dendrite
ports:
# Maps tcp port 5432 on service container to the host
- 5432:5432
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
strategy:
fail-fast: false
matrix:
go: ["1.19"]
steps:
- uses: actions/checkout@v3
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- uses: actions/cache@v3
# manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}-unit-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go${{ matrix.go }}-unit-
- name: Set up gotestfmt
uses: gotesttools/gotestfmt-action@v2
with:
# Optional: pass GITHUB_TOKEN to avoid rate limiting.
token: ${{ secrets.GITHUB_TOKEN }}
- run: go test -json -v ./... 2>&1 | gotestfmt
env:
POSTGRES_HOST: localhost
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: dendrite
# build Dendrite for linux with different architectures and go versions
build:
name: Build for Linux
timeout-minutes: 10
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
go: ["1.19"]
goos: ["linux"]
goarch: ["amd64", "386"]
steps:
- uses: actions/checkout@v3
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
restore-keys: |
key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-
- name: Install dependencies x86
if: ${{ matrix.goarch == '386' }}
run: sudo apt update && sudo apt-get install -y gcc-multilib
- env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: 1
CGO_CFLAGS: -fno-stack-protector
run: go build -trimpath -v -o "bin/" ./cmd/...
# build for Windows 64-bit
build_windows:
name: Build for Windows
timeout-minutes: 10
runs-on: ubuntu-latest
strategy:
matrix:
go: ["1.19"]
goos: ["windows"]
goarch: ["amd64"]
steps:
- uses: actions/checkout@v3
- name: Setup Go ${{ matrix.go }}
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
restore-keys: |
key: ${{ runner.os }}-go${{ matrix.go }}${{ matrix.goos }}-${{ matrix.goarch }}-
- name: Install dependencies
run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc
- env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: 1
CC: "/usr/bin/x86_64-w64-mingw32-gcc"
run: go build -trimpath -v -o "bin/" ./cmd/...
# Dummy step to gate other tests on without repeating the whole list
initial-tests-done:
name: Initial tests passed
needs: [lint, test, build, build_windows]
runs-on: ubuntu-latest
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
steps:
- name: Check initial tests passed
uses: re-actors/alls-green@release/v1
with:
jobs: ${{ toJSON(needs) }}
# run go test with different go versions
integration:
timeout-minutes: 20
needs: initial-tests-done
name: Integration tests (Go ${{ matrix.go }})
runs-on: ubuntu-latest
# Service containers to run with `container-job`
services:
# Label used to access the service container
postgres:
@ -119,95 +248,19 @@ jobs:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}-test-${{ hashFiles('**/go.sum') }}
key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go${{ matrix.go }}-test-
- run: go test -json -v ./... 2>&1 | gotestfmt
${{ runner.os }}-go${{ matrix.go }}-test-race-
- run: go test -race -json -v -coverpkg=./... -coverprofile=cover.out $(go list ./... | grep -v /cmd/dendrite*) 2>&1 | gotestfmt
env:
POSTGRES_HOST: localhost
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: dendrite
# build Dendrite for linux with different architectures and go versions
build:
name: Build for Linux
timeout-minutes: 10
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
go: [ "1.19"]
goos: ["linux"]
goarch: ["amd64", "386"]
steps:
- uses: actions/checkout@v3
- name: Setup go
uses: actions/setup-go@v3
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
go-version: ${{ matrix.go }}
- name: Install dependencies x86
if: ${{ matrix.goarch == '386' }}
run: sudo apt update && sudo apt-get install -y gcc-multilib
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goarch }}-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goarch }}-
- env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: 1
CGO_CFLAGS: -fno-stack-protector
run: go build -trimpath -v -o "bin/" ./cmd/...
# build for Windows 64-bit
build_windows:
name: Build for Windows
timeout-minutes: 10
runs-on: ubuntu-latest
strategy:
matrix:
go: ["1.19"]
goos: ["windows"]
goarch: ["amd64"]
steps:
- uses: actions/checkout@v3
- name: Setup Go ${{ matrix.go }}
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- name: Install dependencies
run: sudo apt update && sudo apt install -y gcc-mingw-w64-x86-64 # install required gcc
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }}-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go${{ matrix.go }}-${{ matrix.goos }}
- env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
CGO_ENABLED: 1
CC: "/usr/bin/x86_64-w64-mingw32-gcc"
run: go build -trimpath -v -o "bin/" ./cmd/...
# Dummy step to gate other tests on without repeating the whole list
initial-tests-done:
name: Initial tests passed
needs: [lint, test, build, build_windows]
runs-on: ubuntu-latest
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
steps:
- name: Check initial tests passed
uses: re-actors/alls-green@release/v1
with:
jobs: ${{ toJSON(needs) }}
flags: unittests
# run database upgrade tests
upgrade_test:
@ -221,18 +274,13 @@ jobs:
uses: actions/setup-go@v3
with:
go-version: "1.19"
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-upgrade
cache: true
- name: Build upgrade-tests
run: go build ./cmd/dendrite-upgrade-tests
- name: Test upgrade
- name: Test upgrade (PostgreSQL)
run: ./dendrite-upgrade-tests --head .
- name: Test upgrade (SQLite)
run: ./dendrite-upgrade-tests --sqlite --head .
# run database upgrade tests, skipping over one version
upgrade_test_direct:
@ -246,17 +294,12 @@ jobs:
uses: actions/setup-go@v3
with:
go-version: "1.19"
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-upgrade-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-upgrade
cache: true
- name: Build upgrade-tests
run: go build ./cmd/dendrite-upgrade-tests
- name: Test upgrade
- name: Test upgrade (PostgreSQL)
run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
- name: Test upgrade (SQLite)
run: ./dendrite-upgrade-tests -direct -from HEAD-2 --head .
# run Sytest in different variations
@ -288,9 +331,11 @@ jobs:
postgres: postgres
api: full-http
container:
image: matrixdotorg/sytest-dendrite:latest
image: matrixdotorg/sytest-dendrite
volumes:
- ${{ github.workspace }}:/src
- /root/.cache/go-build:/github/home/.cache/go-build
- /root/.cache/go-mod:/gopath/pkg/mod
env:
POSTGRES: ${{ matrix.postgres && 1}}
API: ${{ matrix.api && 1 }}
@ -298,6 +343,14 @@ jobs:
CGO_ENABLED: ${{ matrix.cgo && 1 }}
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
/gopath/pkg/mod
key: ${{ runner.os }}-go-sytest-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-sytest-
- name: Run Sytest
run: /bootstrap.sh dendrite
working-directory: /src
@ -332,12 +385,14 @@ jobs:
matrix:
include:
- label: SQLite native
cgo: 0
- label: SQLite Cgo
cgo: 1
- label: SQLite native, full HTTP APIs
api: full-http
cgo: 0
- label: SQLite Cgo, full HTTP APIs
api: full-http
@ -345,10 +400,12 @@ jobs:
- label: PostgreSQL
postgres: Postgres
cgo: 0
- label: PostgreSQL, full HTTP APIs
postgres: Postgres
api: full-http
cgo: 0
steps:
# Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
# See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
@ -356,14 +413,12 @@ jobs:
run: |
echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
echo "~/go/bin" >> $GITHUB_PATH
- name: "Install Complement Dependencies"
# We don't need to install Go because it is included on the Ubuntu 20.04 image:
# See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
run: |
sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
- name: Run actions/checkout@v3 for dendrite
uses: actions/checkout@v3
with:
@ -389,12 +444,10 @@ jobs:
if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
continue
fi
(wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
done
# Build initial Dendrite image
- run: docker build -t complement-dendrite -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
- run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
working-directory: dendrite
env:
DOCKER_BUILDKIT: 1
@ -406,9 +459,9 @@ jobs:
shell: bash
name: Run Complement Tests
env:
COMPLEMENT_BASE_IMAGE: complement-dendrite:latest
API: ${{ matrix.api && 1 }}
CGO_ENABLED: ${{ matrix.cgo && 1 }}
COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }}
COMPLEMENT_DENDRITE_API: ${{ matrix.api && 1 }}
COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_
working-directory: complement
integration-tests-done:
@ -420,6 +473,7 @@ jobs:
upgrade_test_direct,
sytest,
complement,
integration,
]
runs-on: ubuntu-latest
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
@ -434,7 +488,7 @@ jobs:
permissions:
packages: write
contents: read
security-events: write # To upload Trivy sarif files
security-events: write # To upload Trivy sarif files
if: github.repository == 'matrix-org/dendrite' && github.ref_name == 'main'
needs: [integration-tests-done]
uses: matrix-org/dendrite/.github/workflows/docker.yml@main

56
.github/workflows/docker.yml vendored
View file

@ -32,7 +32,7 @@ jobs:
if: github.event_name == 'release' # Only for GitHub releases
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
@ -68,18 +68,6 @@ jobs:
${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
format: "sarif"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
- name: Build release monolith image
if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_monolith_release
@ -100,6 +88,18 @@ jobs:
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }}
format: "sarif"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
polylith:
name: Polylith image
runs-on: ubuntu-latest
@ -114,7 +114,7 @@ jobs:
if: github.event_name == 'release' # Only for GitHub releases
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
@ -150,18 +150,6 @@ jobs:
${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
format: "sarif"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
- name: Build release polylith image
if: github.event_name == 'release' # Only for GitHub releases
id: docker_build_polylith_release
@ -180,6 +168,18 @@ jobs:
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest
ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }}
format: "sarif"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: "trivy-results.sarif"
demo-pinecone:
name: Pinecone demo image
runs-on: ubuntu-latest
@ -193,7 +193,7 @@ jobs:
if: github.event_name == 'release' # Only for GitHub releases
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
@ -260,7 +260,7 @@ jobs:
if: github.event_name == 'release' # Only for GitHub releases
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV
echo "BUILD=$(git rev-parse --short HEAD || \"\")" >> $GITHUB_ENV
BRANCH=$(git symbolic-ref --short HEAD | tr -d \/)
[ ${BRANCH} == "main" ] && BRANCH=""
echo "BRANCH=${BRANCH}" >> $GITHUB_ENV

52
.github/workflows/gh-pages.yml vendored Normal file
View file

@ -0,0 +1,52 @@
# Sample workflow for building and deploying a Jekyll site to GitHub Pages
name: Deploy GitHub Pages dependencies preinstalled
on:
# Runs on pushes targeting the default branch
push:
branches: ["main"]
paths:
- 'docs/**' # only execute if we have docs changes
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
permissions:
contents: read
pages: write
id-token: write
# Allow one concurrent deployment
concurrency:
group: "pages"
cancel-in-progress: true
jobs:
# Build job
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup Pages
uses: actions/configure-pages@v2
- name: Build with Jekyll
uses: actions/jekyll-build-pages@v1
with:
source: ./docs
destination: ./_site
- name: Upload artifact
uses: actions/upload-pages-artifact@v1
# Deployment job
deploy:
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
runs-on: ubuntu-latest
needs: build
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v1

39
.github/workflows/helm.yml vendored Normal file
View file

@ -0,0 +1,39 @@
name: Release Charts
on:
push:
branches:
- main
paths:
- 'helm/**' # only execute if we have helm chart changes
jobs:
release:
# depending on default permission settings for your org (contents being read-only or read-write for workloads), you will have to add permissions
# see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
permissions:
contents: write
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Configure Git
run: |
git config user.name "$GITHUB_ACTOR"
git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
- name: Install Helm
uses: azure/setup-helm@v3
with:
version: v3.10.0
- name: Run chart-releaser
uses: helm/chart-releaser-action@v1.4.1
env:
CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
with:
config: helm/cr.yaml
charts_dir: helm/

90
.github/workflows/k8s.yml vendored Normal file
View file

@ -0,0 +1,90 @@
name: k8s
on:
push:
branches: ["main"]
paths:
- 'helm/**' # only execute if we have helm chart changes
pull_request:
branches: ["main"]
paths:
- 'helm/**'
jobs:
lint:
name: Lint Helm chart
runs-on: ubuntu-latest
outputs:
changed: ${{ steps.list-changed.outputs.changed }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: azure/setup-helm@v3
with:
version: v3.10.0
- uses: actions/setup-python@v4
with:
python-version: 3.11
check-latest: true
- uses: helm/chart-testing-action@v2.3.1
- name: Get changed status
id: list-changed
run: |
changed=$(ct list-changed --config helm/ct.yaml --target-branch ${{ github.event.repository.default_branch }})
if [[ -n "$changed" ]]; then
echo "::set-output name=changed::true"
fi
- name: Run lint
run: ct lint --config helm/ct.yaml
# only bother to run if lint step reports a change to the helm chart
install:
needs:
- lint
if: ${{ needs.lint.outputs.changed == 'true' }}
name: Install Helm charts
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ inputs.checkoutCommit }}
- name: Install Kubernetes tools
uses: yokawasa/action-setup-kube-tools@v0.8.2
with:
setup-tools: |
helmv3
helm: "3.10.3"
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Set up chart-testing
uses: helm/chart-testing-action@v2.3.1
- name: Create k3d cluster
uses: nolar/setup-k3d-k3s@v1
with:
version: v1.21
- name: Remove node taints
run: |
kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true
- name: Run chart-testing (install)
run: ct install --config helm/ct.yaml
# Install the chart using helm directly and test with create-account
- name: Install chart
run: |
helm install --values helm/dendrite/ci/ct-postgres-sharedsecret-values.yaml dendrite helm/dendrite
- name: Wait for Postgres and Dendrite to be up
run: |
kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=postgresql || kubectl get pods -A
kubectl wait --for=condition=ready --timeout=90s pod -l app.kubernetes.io/name=dendrite || kubectl get pods -A
kubectl get pods -A
kubectl get services
kubectl get ingress
- name: Run create account
run: |
podName=$(kubectl get pods -l app.kubernetes.io/name=dendrite -o name)
kubectl exec "${podName}" -- /usr/bin/create-account -username alice -password somerandompassword

289
.github/workflows/schedules.yaml vendored
View file

@ -10,83 +10,27 @@ concurrency:
cancel-in-progress: true
jobs:
# run go test with different go versions
test:
timeout-minutes: 20
name: Unit tests (Go ${{ matrix.go }})
runs-on: ubuntu-latest
# Service containers to run with `container-job`
services:
# Label used to access the service container
postgres:
# Docker Hub image
image: postgres:13-alpine
# Provide the password for postgres
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: dendrite
ports:
# Maps tcp port 5432 on service container to the host
- 5432:5432
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
strategy:
fail-fast: false
matrix:
go: ["1.18", "1.19"]
steps:
- uses: actions/checkout@v3
- name: Setup go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go${{ matrix.go }}-test-race-
- run: go test -race ./...
env:
POSTGRES_HOST: localhost
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: dendrite
# Dummy step to gate other tests on without repeating the whole list
initial-tests-done:
name: Initial tests passed
needs: [test]
runs-on: ubuntu-latest
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
steps:
- name: Check initial tests passed
uses: re-actors/alls-green@release/v1
with:
jobs: ${{ toJSON(needs) }}
# run Sytest in different variations
sytest:
timeout-minutes: 60
needs: initial-tests-done
name: "Sytest (${{ matrix.label }})"
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- label: SQLite
- label: SQLite native
- label: SQLite, full HTTP APIs
- label: SQLite Cgo
cgo: 1
- label: SQLite native, full HTTP APIs
api: full-http
- label: SQLite Cgo, full HTTP APIs
api: full-http
cgo: 1
- label: PostgreSQL
postgres: postgres
@ -97,13 +41,24 @@ jobs:
image: matrixdotorg/sytest-dendrite:latest
volumes:
- ${{ github.workspace }}:/src
- /root/.cache/go-build:/github/home/.cache/go-build
- /root/.cache/go-mod:/gopath/pkg/mod
env:
POSTGRES: ${{ matrix.postgres && 1}}
API: ${{ matrix.api && 1 }}
SYTEST_BRANCH: ${{ github.head_ref }}
RACE_DETECTION: 1
COVER: 1
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- uses: actions/cache@v3
with:
path: |
~/.cache/go-build
/gopath/pkg/mod
key: ${{ runner.os }}-go-sytest-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-sytest-
- name: Run Sytest
run: /bootstrap.sh dendrite
working-directory: /src
@ -126,3 +81,205 @@ jobs:
path: |
/logs/results.tap
/logs/**/*.log*
sytest-coverage:
timeout-minutes: 5
name: "Sytest Coverage"
runs-on: ubuntu-latest
needs: sytest # only run once Sytest is done
if: ${{ always() }}
steps:
- uses: actions/checkout@v3
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: '>=1.19.0'
cache: true
- name: Download all artifacts
uses: actions/download-artifact@v3
- name: Install gocovmerge
run: go install github.com/wadey/gocovmerge@latest
- name: Run gocovmerge
run: |
find -name 'integrationcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > sytest.cov
go tool cover -func=sytest.cov
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
files: ./sytest.cov
flags: sytest
fail_ci_if_error: true
# run Complement
complement:
name: "Complement (${{ matrix.label }})"
timeout-minutes: 60
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- label: SQLite native
cgo: 0
- label: SQLite Cgo
cgo: 1
- label: SQLite native, full HTTP APIs
api: full-http
cgo: 0
- label: SQLite Cgo, full HTTP APIs
api: full-http
cgo: 1
- label: PostgreSQL
postgres: Postgres
cgo: 0
- label: PostgreSQL, full HTTP APIs
postgres: Postgres
api: full-http
cgo: 0
steps:
# Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env to run Complement.
# See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
- name: "Set Go Version"
run: |
echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
echo "~/go/bin" >> $GITHUB_PATH
- name: "Install Complement Dependencies"
# We don't need to install Go because it is included on the Ubuntu 20.04 image:
# See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
run: |
sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
- name: Run actions/checkout@v3 for dendrite
uses: actions/checkout@v3
with:
path: dendrite
# Attempt to check out the same branch of Complement as the PR. If it
# doesn't exist, fallback to main.
- name: Checkout complement
shell: bash
run: |
mkdir -p complement
# Attempt to use the version of complement which best matches the current
# build. Depending on whether this is a PR or release, etc. we need to
# use different fallbacks.
#
# 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
# for pull requests, otherwise GITHUB_REF).
# 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
# (GITHUB_BASE_REF for pull requests).
# 3. Use the default complement branch ("master").
for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
# Skip empty branch names and merge commits.
if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
continue
fi
(wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
done
# Build initial Dendrite image
- run: docker build --build-arg=CGO=${{ matrix.cgo }} -t complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }} -f build/scripts/Complement${{ matrix.postgres }}.Dockerfile .
working-directory: dendrite
env:
DOCKER_BUILDKIT: 1
- name: Create post test script
run: |
cat <<EOF > /tmp/posttest.sh
#!/bin/bash
mkdir -p /tmp/Complement/logs/\$2/\$1/
docker cp \$1:/dendrite/complementcover.log /tmp/Complement/logs/\$2/\$1/
EOF
chmod +x /tmp/posttest.sh
# Run Complement
- run: |
set -o pipefail &&
go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
shell: bash
name: Run Complement Tests
env:
COMPLEMENT_BASE_IMAGE: complement-dendrite:${{ matrix.postgres }}${{ matrix.api }}${{ matrix.cgo }}
COMPLEMENT_DENDRITE_API: ${{ matrix.api && 1 }}
COMPLEMENT_SHARE_ENV_PREFIX: COMPLEMENT_DENDRITE_
COMPLEMENT_DENDRITE_COVER: 1
COMPLEMENT_POST_TEST_SCRIPT: /tmp/posttest.sh
working-directory: complement
- name: Upload Complement logs
uses: actions/upload-artifact@v2
if: ${{ always() }}
with:
name: Complement Logs - (Dendrite, ${{ join(matrix.*, ', ') }})
path: |
/tmp/Complement/**/complementcover.log
complement-coverage:
timeout-minutes: 5
name: "Complement Coverage"
runs-on: ubuntu-latest
needs: complement # only run once Complement is done
if: ${{ always() }}
steps:
- uses: actions/checkout@v3
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: '>=1.19.0'
cache: true
- name: Download all artifacts
uses: actions/download-artifact@v3
- name: Install gocovmerge
run: go install github.com/wadey/gocovmerge@latest
- name: Run gocovmerge
run: |
find -name 'complementcover.log' -printf '"%p"\n' | xargs gocovmerge | grep -Ev 'relayapi|setup/mscs|api_trace' > complement.cov
go tool cover -func=complement.cov
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
files: ./complement.cov
flags: complement
fail_ci_if_error: true
element_web:
timeout-minutes: 120
runs-on: ubuntu-latest
steps:
- uses: tecolicom/actions-use-apt-tools@v1
with:
# Our test suite includes some screenshot tests with unusual diacritics, which are
# supposed to be covered by STIXGeneral.
tools: fonts-stix
- uses: actions/checkout@v2
with:
repository: matrix-org/matrix-react-sdk
- uses: actions/setup-node@v3
with:
cache: 'yarn'
- name: Fetch layered build
run: scripts/ci/layered.sh
- name: Copy config
run: cp element.io/develop/config.json config.json
working-directory: ./element-web
- name: Build
env:
CI_PACKAGE: true
run: yarn build
working-directory: ./element-web
- name: Edit Test Config
run: |
sed -i '/HOMESERVER/c\ HOMESERVER: "dendrite",' cypress.config.ts
- name: "Run cypress tests"
uses: cypress-io/github-action@v4.1.1
with:
browser: chrome
start: npx serve -p 8080 ./element-web/webapp
wait-on: 'http://localhost:8080'
env:
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
TMPDIR: ${{ runner.temp }}

4
.gitignore vendored
View file

@ -57,6 +57,7 @@ zion-appservice.yaml
# Database files
*.db
*.db-journal
# Log files
*.log*
@ -78,4 +79,5 @@ media_store/
# Debug
**/__debug_bin
.env
.env
build/*

57
CHANGES.md
View file

@ -1,5 +1,62 @@
# Changelog
## Dendrite 0.11.0 (2023-01-20)
The last three missing federation API Sytests have been fixed - bringing us to 100% server-server Synapse parity, with client-server parity at 93% 🎉
### Features
* Added `/_dendrite/admin/purgeRoom/{roomID}` to clean up the database
* The default room version was updated to 10 (contributed by [FSG-Cat](https://github.com/FSG-Cat))
### Fixes
* An oversight in the `create-config` binary, which now correctly sets the media path if specified (contributed by [BieHDC](https://github.com/BieHDC))
* The Helm chart now uses the `$.Chart.AppVersion` as the default image version to pull, with the possibility to override it (contributed by [genofire](https://github.com/genofire))
## Dendrite 0.10.9 (2023-01-17)
### Features
* Stale device lists are now cleaned up on startup, removing entries for users the server doesn't share a room with anymore
* Dendrite now has its own Helm chart
* Guest access is now handled correctly (disallow joins, kick guests on revocation of guest access, as well as over federation)
### Fixes
* Push rules have seen several tweaks and fixes, which should, for example, fix notifications for `m.read_receipts`
* Outgoing presence will now correctly be sent to newly joined hosts
* Fixes the `/_dendrite/admin/resetPassword/{userID}` admin endpoint to use the correct variable
* Federated backfilling for medium/large rooms has been fixed
* `/login` causing wrong device list updates has been resolved
* `/sync` should now return the correct room summary heroes
* The default config options for `recaptcha_sitekey_class` and `recaptcha_form_field` are now set correctly
* `/messages` now omits empty `state` to be more spec compliant (contributed by [handlerug](https://github.com/handlerug))
* `/sync` has been optimised to only query state events for history visibility if they are really needed
## Dendrite 0.10.8 (2022-11-29)
### Features
* The built-in NATS Server has been updated to version 2.9.8
* A number of under-the-hood changes have been merged for future virtual hosting support in Dendrite (running multiple domain names on the same Dendrite deployment)
### Fixes
* Event auth handling of invites has been refactored, which should fix some edge cases being handled incorrectly
* Fix a bug when returning an empty protocol list, which could cause Element to display "The homeserver may be too old to support third party networks" when opening the public room directory
* The sync API will no longer filter out the user's own membership when using lazy-loading
* Dendrite will now correctly detect JetStream consumers being deleted, stopping the consumer goroutine as needed
* A panic in the federation API where the server list could go out of bounds has been fixed
* Blacklisted servers will now be excluded when querying joined servers, which improves CPU usage and performs less unnecessary outbound requests
* A database writer will now be used to assign state key NIDs when requesting NIDs that may not exist yet
* Dendrite will now correctly move local aliases for an upgraded room when the room is upgraded remotely
* Dendrite will now correctly move account data for an upgraded room when the room is upgraded remotely
* Missing state key NIDs will now be allocated on request rather than returning an error
* Guest access is now correctly denied on a number of endpoints
* Presence information will now be correctly sent for new private chats
* A number of unspecced fields have been removed from outbound `/send` transactions
## Dendrite 0.10.7 (2022-11-04)
### Features

32
Dockerfile
View file

@ -24,16 +24,15 @@ RUN --mount=target=. \
go build -v -ldflags="${FLAGS}" -trimpath -o /out/ ./cmd/...
#
# The dendrite base image; mainly creates a user and switches to it
# The dendrite base image
#
FROM alpine:latest AS dendrite-base
RUN apk --update --no-cache add curl
LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
LABEL org.opencontainers.image.licenses="Apache-2.0"
LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/"
LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C."
RUN addgroup dendrite && adduser dendrite -G dendrite -u 1337 -D
USER dendrite
#
# Builds the polylith image and only contains the polylith binary
@ -65,30 +64,3 @@ WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]
EXPOSE 8008 8448
#
# Builds the Complement image, used for integration tests
#
FROM base AS complement
LABEL org.opencontainers.image.title="Dendrite (Complement)"
RUN apk add --no-cache sqlite openssl ca-certificates
COPY --from=build /out/generate-config /usr/bin/generate-config
COPY --from=build /out/generate-keys /usr/bin/generate-keys
COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server
WORKDIR /dendrite
RUN /usr/bin/generate-keys --private-key matrix_key.pem && \
mkdir /ca && \
openssl genrsa -out /ca/ca.key 2048 && \
openssl req -new -x509 -key /ca/ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out /ca/ca.crt
ENV SERVER_NAME=localhost
ENV API=0
EXPOSE 8008 8448
# At runtime, generate TLS cert based on the CA now mounted at /ca
# At runtime, replace the SERVER_NAME with what we are told
CMD /usr/bin/generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /ca/ca.crt --tls-authority-key /ca/ca.key && \
/usr/bin/generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
cp /ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
/usr/bin/dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}

11
appservice/appservice.go
View file

@ -24,6 +24,8 @@ import (
"github.com/gorilla/mux"
"github.com/sirupsen/logrus"
"github.com/matrix-org/gomatrixserverlib"
appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/appservice/consumers"
"github.com/matrix-org/dendrite/appservice/inthttp"
@ -35,8 +37,8 @@ import (
)
// AddInternalRoutes registers HTTP handlers for internal API calls
func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInternalAPI) {
inthttp.AddRoutes(queryAPI, router)
func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInternalAPI, enableMetrics bool) {
inthttp.AddRoutes(queryAPI, router, enableMetrics)
}
// NewInternalAPI returns a concerete implementation of the internal API. Callers
@ -74,7 +76,7 @@ func NewInternalAPI(
// events to be sent out.
for _, appservice := range base.Cfg.Derived.ApplicationServices {
// Create bot account for this AS if it doesn't already exist
if err := generateAppServiceAccount(userAPI, appservice); err != nil {
if err := generateAppServiceAccount(userAPI, appservice, base.Cfg.Global.ServerName); err != nil {
logrus.WithFields(logrus.Fields{
"appservice": appservice.ID,
}).WithError(err).Panicf("failed to generate bot account for appservice")
@ -101,11 +103,13 @@ func NewInternalAPI(
func generateAppServiceAccount(
userAPI userapi.AppserviceUserAPI,
as config.ApplicationService,
serverName gomatrixserverlib.ServerName,
) error {
var accRes userapi.PerformAccountCreationResponse
err := userAPI.PerformAccountCreation(context.Background(), &userapi.PerformAccountCreationRequest{
AccountType: userapi.AccountTypeAppService,
Localpart: as.SenderLocalpart,
ServerName: serverName,
AppServiceID: as.ID,
OnConflict: userapi.ConflictUpdate,
}, &accRes)
@ -115,6 +119,7 @@ func generateAppServiceAccount(
var devRes userapi.PerformDeviceCreationResponse
err = userAPI.PerformDeviceCreation(context.Background(), &userapi.PerformDeviceCreationRequest{
Localpart: as.SenderLocalpart,
ServerName: serverName,
AccessToken: as.ASToken,
DeviceID: &as.SenderLocalpart,
DeviceDisplayName: &as.SenderLocalpart,

223
appservice/appservice_test.go Normal file
View file

@ -0,0 +1,223 @@
package appservice_test
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"reflect"
"regexp"
"strings"
"testing"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/appservice/inthttp"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/userapi"
"github.com/matrix-org/dendrite/test/testrig"
)
func TestAppserviceInternalAPI(t *testing.T) {
// Set expected results
existingProtocol := "irc"
wantLocationResponse := []api.ASLocationResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
wantUserResponse := []api.ASUserResponse{{Protocol: existingProtocol, Fields: []byte("{}")}}
wantProtocolResponse := api.ASProtocolResponse{Instances: []api.ProtocolInstance{{Fields: []byte("{}")}}}
wantProtocolResult := map[string]api.ASProtocolResponse{
existingProtocol: wantProtocolResponse,
}
// create a dummy AS url, handling some cases
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.Contains(r.URL.Path, "location"):
// Check if we've got an existing protocol, if so, return a proper response.
if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
if err := json.NewEncoder(w).Encode(wantLocationResponse); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
}
if err := json.NewEncoder(w).Encode([]api.ASLocationResponse{}); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
case strings.Contains(r.URL.Path, "user"):
if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
if err := json.NewEncoder(w).Encode(wantUserResponse); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
}
if err := json.NewEncoder(w).Encode([]api.UserResponse{}); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
case strings.Contains(r.URL.Path, "protocol"):
if r.URL.Path[len(r.URL.Path)-len(existingProtocol):] == existingProtocol {
if err := json.NewEncoder(w).Encode(wantProtocolResponse); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
}
if err := json.NewEncoder(w).Encode(nil); err != nil {
t.Fatalf("failed to encode response: %s", err)
}
return
default:
t.Logf("hit location: %s", r.URL.Path)
}
}))
// The test cases to run
runCases := func(t *testing.T, testAPI api.AppServiceInternalAPI) {
t.Run("UserIDExists", func(t *testing.T) {
testUserIDExists(t, testAPI, "@as-testing:test", true)
testUserIDExists(t, testAPI, "@as1-testing:test", false)
})
t.Run("AliasExists", func(t *testing.T) {
testAliasExists(t, testAPI, "@asroom-testing:test", true)
testAliasExists(t, testAPI, "@asroom1-testing:test", false)
})
t.Run("Locations", func(t *testing.T) {
testLocations(t, testAPI, existingProtocol, wantLocationResponse)
testLocations(t, testAPI, "abc", nil)
})
t.Run("User", func(t *testing.T) {
testUser(t, testAPI, existingProtocol, wantUserResponse)
testUser(t, testAPI, "abc", nil)
})
t.Run("Protocols", func(t *testing.T) {
testProtocol(t, testAPI, existingProtocol, wantProtocolResult)
testProtocol(t, testAPI, existingProtocol, wantProtocolResult) // tests the cache
testProtocol(t, testAPI, "", wantProtocolResult) // tests getting all protocols
testProtocol(t, testAPI, "abc", nil)
})
}
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
base, closeBase := testrig.CreateBaseDendrite(t, dbType)
defer closeBase()
// Create a dummy application service
base.Cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{
{
ID: "someID",
URL: srv.URL,
ASToken: "",
HSToken: "",
SenderLocalpart: "senderLocalPart",
NamespaceMap: map[string][]config.ApplicationServiceNamespace{
"users": {{RegexpObject: regexp.MustCompile("as-.*")}},
"aliases": {{RegexpObject: regexp.MustCompile("asroom-.*")}},
},
Protocols: []string{existingProtocol},
},
}
// Create required internal APIs
rsAPI := roomserver.NewInternalAPI(base)
usrAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, nil, rsAPI, nil)
asAPI := appservice.NewInternalAPI(base, usrAPI, rsAPI)
// Finally execute the tests
t.Run("HTTP API", func(t *testing.T) {
router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter()
appservice.AddInternalRoutes(router, asAPI, base.EnableMetrics)
apiURL, cancel := test.ListenAndServe(t, router, false)
defer cancel()
asHTTPApi, err := inthttp.NewAppserviceClient(apiURL, &http.Client{})
if err != nil {
t.Fatalf("failed to create HTTP client: %s", err)
}
runCases(t, asHTTPApi)
})
t.Run("Monolith", func(t *testing.T) {
runCases(t, asAPI)
})
})
}
func testUserIDExists(t *testing.T, asAPI api.AppServiceInternalAPI, userID string, wantExists bool) {
ctx := context.Background()
userResp := &api.UserIDExistsResponse{}
if err := asAPI.UserIDExists(ctx, &api.UserIDExistsRequest{
UserID: userID,
}, userResp); err != nil {
t.Errorf("failed to get userID: %s", err)
}
if userResp.UserIDExists != wantExists {
t.Errorf("unexpected result for UserIDExists(%s): %v, expected %v", userID, userResp.UserIDExists, wantExists)
}
}
func testAliasExists(t *testing.T, asAPI api.AppServiceInternalAPI, alias string, wantExists bool) {
ctx := context.Background()
aliasResp := &api.RoomAliasExistsResponse{}
if err := asAPI.RoomAliasExists(ctx, &api.RoomAliasExistsRequest{
Alias: alias,
}, aliasResp); err != nil {
t.Errorf("failed to get alias: %s", err)
}
if aliasResp.AliasExists != wantExists {
t.Errorf("unexpected result for RoomAliasExists(%s): %v, expected %v", alias, aliasResp.AliasExists, wantExists)
}
}
func testLocations(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult []api.ASLocationResponse) {
ctx := context.Background()
locationResp := &api.LocationResponse{}
if err := asAPI.Locations(ctx, &api.LocationRequest{
Protocol: proto,
}, locationResp); err != nil {
t.Errorf("failed to get locations: %s", err)
}
if !reflect.DeepEqual(locationResp.Locations, wantResult) {
t.Errorf("unexpected result for Locations(%s): %+v, expected %+v", proto, locationResp.Locations, wantResult)
}
}
func testUser(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult []api.ASUserResponse) {
ctx := context.Background()
userResp := &api.UserResponse{}
if err := asAPI.User(ctx, &api.UserRequest{
Protocol: proto,
}, userResp); err != nil {
t.Errorf("failed to get user: %s", err)
}
if !reflect.DeepEqual(userResp.Users, wantResult) {
t.Errorf("unexpected result for User(%s): %+v, expected %+v", proto, userResp.Users, wantResult)
}
}
func testProtocol(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, wantResult map[string]api.ASProtocolResponse) {
ctx := context.Background()
protoResp := &api.ProtocolResponse{}
if err := asAPI.Protocols(ctx, &api.ProtocolRequest{
Protocol: proto,
}, protoResp); err != nil {
t.Errorf("failed to get Protocols: %s", err)
}
if !reflect.DeepEqual(protoResp.Protocols, wantResult) {
t.Errorf("unexpected result for Protocols(%s): %+v, expected %+v", proto, protoResp.Protocols[proto], wantResult)
}
}

13
appservice/inthttp/server.go
View file

@ -8,29 +8,30 @@ import (
)
// AddRoutes adds the AppServiceQueryAPI handlers to the http.ServeMux.
func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router) {
func AddRoutes(a api.AppServiceInternalAPI, internalAPIMux *mux.Router, enableMetrics bool) {
internalAPIMux.Handle(
AppServiceRoomAliasExistsPath,
httputil.MakeInternalRPCAPI("AppserviceRoomAliasExists", a.RoomAliasExists),
httputil.MakeInternalRPCAPI("AppserviceRoomAliasExists", enableMetrics, a.RoomAliasExists),
)
internalAPIMux.Handle(
AppServiceUserIDExistsPath,
httputil.MakeInternalRPCAPI("AppserviceUserIDExists", a.UserIDExists),
httputil.MakeInternalRPCAPI("AppserviceUserIDExists", enableMetrics, a.UserIDExists),
)
internalAPIMux.Handle(
AppServiceProtocolsPath,
httputil.MakeInternalRPCAPI("AppserviceProtocols", a.Protocols),
httputil.MakeInternalRPCAPI("AppserviceProtocols", enableMetrics, a.Protocols),
)
internalAPIMux.Handle(
AppServiceLocationsPath,
httputil.MakeInternalRPCAPI("AppserviceLocations", a.Locations),
httputil.MakeInternalRPCAPI("AppserviceLocations", enableMetrics, a.Locations),
)
internalAPIMux.Handle(
AppServiceUserPath,
httputil.MakeInternalRPCAPI("AppserviceUser", a.User),
httputil.MakeInternalRPCAPI("AppserviceUser", enableMetrics, a.User),
)
}

10
are-we-synapse-yet.list
View file

@ -936,4 +936,12 @@ fst Room state after a rejected message event is the same as before
fst Room state after a rejected state event is the same as before
fpb Federation publicRoom Name/topic keys are correct
fed New federated private chats get full presence information (SYN-115) (10 subtests)
dvk Rejects invalid device keys
dvk Rejects invalid device keys
rmv User can create and send/receive messages in a room with version 10
rmv local user can join room with version 10
rmv User can invite local user to room with version 10
rmv remote user can join room with version 10
rmv User can invite remote user to room with version 10
rmv Remote user can backfill in a room with version 10
rmv Can reject invites over federation for rooms with version 10
rmv Can receive redactions from regular users over federation in room version 10

6
build/dendritejs-pinecone/main.go
View file

@ -180,14 +180,14 @@ func startup() {
base := base.NewBaseDendrite(cfg, "Monolith")
defer base.Close() // nolint: errcheck
rsAPI := roomserver.NewInternalAPI(base)
federation := conn.CreateFederationClient(base, pSessions)
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation, rsAPI)
serverKeyAPI := &signing.YggdrasilKeys{}
keyRing := serverKeyAPI.KeyRing()
rsAPI := roomserver.NewInternalAPI(base)
userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
keyAPI.SetUserAPI(userAPI)

21
build/docker/Dockerfile.demo-pinecone
View file

@ -1,12 +1,4 @@
FROM docker.io/golang:1.19-alpine AS base
<<<<<<<< HEAD:servers/dendrite/build/docker/Dockerfile.demo-yggdrasil
========
#
# Needs to be separate from the main Dockerfile for OpenShift,
# as --target is not supported there.
#
>>>>>>>> 5a5e4a4ba659aafc62d142561dbd221c6b8020ec:servers/dendrite/build/docker/Dockerfile.demo-pinecone
#
# Needs to be separate from the main Dockerfile for OpenShift,
@ -20,20 +12,13 @@ WORKDIR /build
COPY . /build
RUN mkdir -p bin
<<<<<<<< HEAD:servers/dendrite/build/docker/Dockerfile.demo-yggdrasil
RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-yggdrasil
========
RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-pinecone
>>>>>>>> 5a5e4a4ba659aafc62d142561dbd221c6b8020ec:servers/dendrite/build/docker/Dockerfile.demo-pinecone
RUN go build -trimpath -o bin/ ./cmd/create-account
RUN go build -trimpath -o bin/ ./cmd/generate-keys
FROM alpine:latest
<<<<<<<< HEAD:servers/dendrite/build/docker/Dockerfile.demo-yggdrasil
LABEL org.opencontainers.image.title="Dendrite (Yggdrasil demo)"
========
RUN apk --update --no-cache add curl
LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)"
>>>>>>>> 5a5e4a4ba659aafc62d142561dbd221c6b8020ec:servers/dendrite/build/docker/Dockerfile.demo-pinecone
LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go"
LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite"
LABEL org.opencontainers.image.licenses="Apache-2.0"
@ -43,8 +28,4 @@ COPY --from=base /build/bin/* /usr/bin/
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
<<<<<<<< HEAD:servers/dendrite/build/docker/Dockerfile.demo-yggdrasil
ENTRYPOINT ["/usr/bin/dendrite-demo-yggdrasil"]
========
ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"]
>>>>>>>> 5a5e4a4ba659aafc62d142561dbd221c6b8020ec:servers/dendrite/build/docker/Dockerfile.demo-pinecone

2
build/gobind-pinecone/build.sh Normal file → Executable file
View file

@ -7,7 +7,7 @@ do
case "$option"
in
a) gomobile bind -v -target android -trimpath -ldflags="-s -w" github.com/matrix-org/dendrite/build/gobind-pinecone ;;
i) gomobile bind -v -target ios -trimpath -ldflags="" github.com/matrix-org/dendrite/build/gobind-pinecone ;;
i) gomobile bind -v -target ios -trimpath -ldflags="" -o ~/DendriteBindings/Gobind.xcframework . ;;
*) echo "No target specified, specify -a or -i"; exit 1 ;;
esac
done

442
build/gobind-pinecone/monolith.go
View file

@ -18,48 +18,25 @@ import (
"context"
"crypto/ed25519"
"crypto/rand"
"crypto/tls"
"encoding/hex"
"fmt"
"io"
"net"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"time"
"go.uber.org/atomic"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/clientapi/userutil"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conduit"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/monolith"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/relay"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/process"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/userapi"
"github.com/matrix-org/dendrite/federationapi/api"
userapiAPI "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/pinecone/types"
"github.com/sirupsen/logrus"
"golang.org/x/net/http2"
"golang.org/x/net/http2/h2c"
pineconeConnections "github.com/matrix-org/pinecone/connections"
pineconeMulticast "github.com/matrix-org/pinecone/multicast"
pineconeRouter "github.com/matrix-org/pinecone/router"
pineconeSessions "github.com/matrix-org/pinecone/sessions"
"github.com/matrix-org/pinecone/types"
_ "golang.org/x/mobile/bind"
)
@ -69,36 +46,37 @@ const (
PeerTypeMulticast = pineconeRouter.PeerTypeMulticast
PeerTypeBluetooth = pineconeRouter.PeerTypeBluetooth
PeerTypeBonjour = pineconeRouter.PeerTypeBonjour
MaxFrameSize = types.MaxFrameSize
)
// Re-export Conduit in this package for bindings.
type Conduit struct {
conduit.Conduit
}
type DendriteMonolith struct {
logger logrus.Logger
PineconeRouter *pineconeRouter.Router
PineconeMulticast *pineconeMulticast.Multicast
PineconeQUIC *pineconeSessions.Sessions
PineconeManager *pineconeConnections.ConnectionManager
StorageDirectory string
CacheDirectory string
listener net.Listener
httpServer *http.Server
processContext *process.ProcessContext
userAPI userapiAPI.UserInternalAPI
logger logrus.Logger
p2pMonolith monolith.P2PMonolith
StorageDirectory string
CacheDirectory string
listener net.Listener
}
func (m *DendriteMonolith) PublicKey() string {
return m.PineconeRouter.PublicKey().String()
return m.p2pMonolith.Router.PublicKey().String()
}
func (m *DendriteMonolith) BaseURL() string {
return fmt.Sprintf("http://%s", m.listener.Addr().String())
return fmt.Sprintf("http://%s", m.p2pMonolith.Addr())
}
func (m *DendriteMonolith) PeerCount(peertype int) int {
return m.PineconeRouter.PeerCount(peertype)
return m.p2pMonolith.Router.PeerCount(peertype)
}
func (m *DendriteMonolith) SessionCount() int {
return len(m.PineconeQUIC.Protocol("matrix").Sessions())
return len(m.p2pMonolith.Sessions.Protocol(monolith.SessionProtocol).Sessions())
}
type InterfaceInfo struct {
@ -140,55 +118,156 @@ func (m *DendriteMonolith) RegisterNetworkCallback(intfCallback InterfaceRetriev
}
return intfs
}
m.PineconeMulticast.RegisterNetworkCallback(callback)
m.p2pMonolith.Multicast.RegisterNetworkCallback(callback)
}
func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) {
if enabled {
m.PineconeMulticast.Start()
m.p2pMonolith.Multicast.Start()
} else {
m.PineconeMulticast.Stop()
m.p2pMonolith.Multicast.Stop()
m.DisconnectType(int(pineconeRouter.PeerTypeMulticast))
}
}
func (m *DendriteMonolith) SetStaticPeer(uri string) {
m.PineconeManager.RemovePeers()
m.p2pMonolith.ConnManager.RemovePeers()
for _, uri := range strings.Split(uri, ",") {
m.PineconeManager.AddPeer(strings.TrimSpace(uri))
m.p2pMonolith.ConnManager.AddPeer(strings.TrimSpace(uri))
}
}
func getServerKeyFromString(nodeID string) (gomatrixserverlib.ServerName, error) {
var nodeKey gomatrixserverlib.ServerName
if userID, err := gomatrixserverlib.NewUserID(nodeID, false); err == nil {
hexKey, decodeErr := hex.DecodeString(string(userID.Domain()))
if decodeErr != nil || len(hexKey) != ed25519.PublicKeySize {
return "", fmt.Errorf("UserID domain is not a valid ed25519 public key: %v", userID.Domain())
} else {
nodeKey = userID.Domain()
}
} else {
hexKey, decodeErr := hex.DecodeString(nodeID)
if decodeErr != nil || len(hexKey) != ed25519.PublicKeySize {
return "", fmt.Errorf("Relay server uri is not a valid ed25519 public key: %v", nodeID)
} else {
nodeKey = gomatrixserverlib.ServerName(nodeID)
}
}
return nodeKey, nil
}
func (m *DendriteMonolith) SetRelayServers(nodeID string, uris string) {
relays := []gomatrixserverlib.ServerName{}
for _, uri := range strings.Split(uris, ",") {
uri = strings.TrimSpace(uri)
if len(uri) == 0 {
continue
}
nodeKey, err := getServerKeyFromString(uri)
if err != nil {
logrus.Errorf(err.Error())
continue
}
relays = append(relays, nodeKey)
}
nodeKey, err := getServerKeyFromString(nodeID)
if err != nil {
logrus.Errorf(err.Error())
return
}
if string(nodeKey) == m.PublicKey() {
logrus.Infof("Setting own relay servers to: %v", relays)
m.p2pMonolith.RelayRetriever.SetRelayServers(relays)
} else {
relay.UpdateNodeRelayServers(
gomatrixserverlib.ServerName(nodeKey),
relays,
m.p2pMonolith.BaseDendrite.Context(),
m.p2pMonolith.GetFederationAPI(),
)
}
}
func (m *DendriteMonolith) GetRelayServers(nodeID string) string {
nodeKey, err := getServerKeyFromString(nodeID)
if err != nil {
logrus.Errorf(err.Error())
return ""
}
relaysString := ""
if string(nodeKey) == m.PublicKey() {
relays := m.p2pMonolith.RelayRetriever.GetRelayServers()
for i, relay := range relays {
if i != 0 {
// Append a comma to the previous entry if there is one.
relaysString += ","
}
relaysString += string(relay)
}
} else {
request := api.P2PQueryRelayServersRequest{Server: gomatrixserverlib.ServerName(nodeKey)}
response := api.P2PQueryRelayServersResponse{}
err := m.p2pMonolith.GetFederationAPI().P2PQueryRelayServers(m.p2pMonolith.BaseDendrite.Context(), &request, &response)
if err != nil {
logrus.Warnf("Failed obtaining list of this node's relay servers: %s", err.Error())
return ""
}
for i, relay := range response.RelayServers {
if i != 0 {
// Append a comma to the previous entry if there is one.
relaysString += ","
}
relaysString += string(relay)
}
}
return relaysString
}
func (m *DendriteMonolith) RelayingEnabled() bool {
return m.p2pMonolith.GetRelayAPI().RelayingEnabled()
}
func (m *DendriteMonolith) SetRelayingEnabled(enabled bool) {
m.p2pMonolith.GetRelayAPI().SetRelayingEnabled(enabled)
}
func (m *DendriteMonolith) DisconnectType(peertype int) {
for _, p := range m.PineconeRouter.Peers() {
for _, p := range m.p2pMonolith.Router.Peers() {
if int(peertype) == p.PeerType {
m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil)
m.p2pMonolith.Router.Disconnect(types.SwitchPortID(p.Port), nil)
}
}
}
func (m *DendriteMonolith) DisconnectZone(zone string) {
for _, p := range m.PineconeRouter.Peers() {
for _, p := range m.p2pMonolith.Router.Peers() {
if zone == p.Zone {
m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil)
m.p2pMonolith.Router.Disconnect(types.SwitchPortID(p.Port), nil)
}
}
}
func (m *DendriteMonolith) DisconnectPort(port int) {
m.PineconeRouter.Disconnect(types.SwitchPortID(port), nil)
m.p2pMonolith.Router.Disconnect(types.SwitchPortID(port), nil)
}
func (m *DendriteMonolith) Conduit(zone string, peertype int) (*Conduit, error) {
l, r := net.Pipe()
conduit := &Conduit{conn: r, port: 0}
newConduit := Conduit{conduit.NewConduit(r, 0)}
go func() {
conduit.portMutex.Lock()
defer conduit.portMutex.Unlock()
logrus.Errorf("Attempting authenticated connect")
var port types.SwitchPortID
var err error
if conduit.port, err = m.PineconeRouter.Connect(
if port, err = m.p2pMonolith.Router.Connect(
l,
pineconeRouter.ConnectionZone(zone),
pineconeRouter.ConnectionPeerType(peertype),
@ -196,16 +275,17 @@ func (m *DendriteMonolith) Conduit(zone string, peertype int) (*Conduit, error)
logrus.Errorf("Authenticated connect failed: %s", err)
_ = l.Close()
_ = r.Close()
_ = conduit.Close()
_ = newConduit.Close()
return
}
logrus.Infof("Authenticated connect succeeded (port %d)", conduit.port)
newConduit.SetPort(port)
logrus.Infof("Authenticated connect succeeded (port %d)", newConduit.Port())
}()
return conduit, nil
return &newConduit, nil
}
func (m *DendriteMonolith) RegisterUser(localpart, password string) (string, error) {
pubkey := m.PineconeRouter.PublicKey()
pubkey := m.p2pMonolith.Router.PublicKey()
userID := userutil.MakeUserID(
localpart,
gomatrixserverlib.ServerName(hex.EncodeToString(pubkey[:])),
@ -216,7 +296,7 @@ func (m *DendriteMonolith) RegisterUser(localpart, password string) (string, err
Password: password,
}
userRes := &userapiAPI.PerformAccountCreationResponse{}
if err := m.userAPI.PerformAccountCreation(context.Background(), userReq, userRes); err != nil {
if err := m.p2pMonolith.GetUserAPI().PerformAccountCreation(context.Background(), userReq, userRes); err != nil {
return userID, fmt.Errorf("userAPI.PerformAccountCreation: %w", err)
}
return userID, nil
@ -234,7 +314,7 @@ func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, e
AccessToken: hex.EncodeToString(accessTokenBytes[:n]),
}
loginRes := &userapiAPI.PerformDeviceCreationResponse{}
if err := m.userAPI.PerformDeviceCreation(context.Background(), loginReq, loginRes); err != nil {
if err := m.p2pMonolith.GetUserAPI().PerformDeviceCreation(context.Background(), loginReq, loginRes); err != nil {
return "", fmt.Errorf("userAPI.PerformDeviceCreation: %w", err)
}
if !loginRes.DeviceCreated {
@ -243,51 +323,10 @@ func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, e
return loginRes.Device.AccessToken, nil
}
// nolint:gocyclo
func (m *DendriteMonolith) Start() {
var sk ed25519.PrivateKey
var pk ed25519.PublicKey
keyfile := filepath.Join(m.StorageDirectory, "p2p.pem")
if _, err := os.Stat(keyfile); os.IsNotExist(err) {
oldkeyfile := filepath.Join(m.StorageDirectory, "p2p.key")
if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
if err = test.NewMatrixKey(keyfile); err != nil {
panic("failed to generate a new PEM key: " + err.Error())
}
if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
panic("failed to load PEM key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
} else {
if sk, err = os.ReadFile(oldkeyfile); err != nil {
panic("failed to read the old private key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
if err = test.SaveMatrixKey(keyfile, sk); err != nil {
panic("failed to convert the private key to PEM format: " + err.Error())
}
}
} else {
if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
panic("failed to load PEM key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
}
pk = sk.Public().(ed25519.PublicKey)
var err error
m.listener, err = net.Listen("tcp", "localhost:65432")
if err != nil {
panic(err)
}
oldKeyfile := filepath.Join(m.StorageDirectory, "p2p.key")
sk, pk := monolith.GetOrCreateKey(keyfile, oldKeyfile)
m.logger = logrus.Logger{
Out: BindLogger{},
@ -295,190 +334,25 @@ func (m *DendriteMonolith) Start() {
m.logger.SetOutput(BindLogger{})
logrus.SetOutput(BindLogger{})
m.PineconeRouter = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk)
m.PineconeQUIC = pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), m.PineconeRouter, []string{"matrix"})
m.PineconeMulticast = pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), m.PineconeRouter)
m.PineconeManager = pineconeConnections.NewConnectionManager(m.PineconeRouter, nil)
m.p2pMonolith = monolith.P2PMonolith{}
m.p2pMonolith.SetupPinecone(sk)
prefix := hex.EncodeToString(pk)
cfg := &config.Dendrite{}
cfg.Defaults(config.DefaultOpts{
Generate: true,
Monolithic: true,
})
cfg := monolith.GenerateDefaultConfig(sk, m.StorageDirectory, m.CacheDirectory, prefix)
cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
cfg.Global.PrivateKey = sk
cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
cfg.Global.JetStream.InMemory = false
cfg.Global.JetStream.StoragePath = config.Path(filepath.Join(m.CacheDirectory, prefix))
cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(m.StorageDirectory, prefix)))
cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(m.StorageDirectory, prefix)))
cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(m.StorageDirectory, prefix)))
cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(m.StorageDirectory, prefix)))
cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(m.StorageDirectory, prefix)))
cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", filepath.Join(m.StorageDirectory, prefix)))
cfg.MediaAPI.BasePath = config.Path(filepath.Join(m.CacheDirectory, "media"))
cfg.MediaAPI.AbsBasePath = config.Path(filepath.Join(m.CacheDirectory, "media"))
cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
cfg.ClientAPI.RegistrationDisabled = false
cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
cfg.SyncAPI.Fulltext.Enabled = true
cfg.SyncAPI.Fulltext.IndexPath = config.Path(filepath.Join(m.CacheDirectory, "search"))
if err = cfg.Derive(); err != nil {
panic(err)
}
// NOTE : disabled for now since there is a 64 bit alignment panic on 32 bit systems
// This isn't actually fixed: https://github.com/blevesearch/zapx/pull/147
cfg.SyncAPI.Fulltext.Enabled = false
base := base.NewBaseDendrite(cfg, "Monolith")
defer base.Close() // nolint: errcheck
federation := conn.CreateFederationClient(base, m.PineconeQUIC)
serverKeyAPI := &signing.YggdrasilKeys{}
keyRing := serverKeyAPI.KeyRing()
rsAPI := roomserver.NewInternalAPI(base)
fsAPI := federationapi.NewInternalAPI(
base, federation, rsAPI, base.Caches, keyRing, true,
)
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
m.userAPI = userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
keyAPI.SetUserAPI(m.userAPI)
asAPI := appservice.NewInternalAPI(base, m.userAPI, rsAPI)
// The underlying roomserver implementation needs to be able to call the fedsender.
// This is different to rsAPI which can be the http client which doesn't need this dependency
rsAPI.SetFederationAPI(fsAPI, keyRing)
userProvider := users.NewPineconeUserProvider(m.PineconeRouter, m.PineconeQUIC, m.userAPI, federation)
roomProvider := rooms.NewPineconeRoomProvider(m.PineconeRouter, m.PineconeQUIC, fsAPI, federation)
monolith := setup.Monolith{
Config: base.Cfg,
Client: conn.CreateClient(base, m.PineconeQUIC),
FedClient: federation,
KeyRing: keyRing,
AppserviceAPI: asAPI,
FederationAPI: fsAPI,
RoomserverAPI: rsAPI,
UserAPI: m.userAPI,
KeyAPI: keyAPI,
ExtPublicRoomsProvider: roomProvider,
ExtUserDirectoryProvider: userProvider,
}
monolith.AddAllPublicRoutes(base)
httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
httpRouter.HandleFunc("/pinecone", m.PineconeRouter.ManholeHandler)
pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()
pMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles)
pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
pHTTP := m.PineconeQUIC.Protocol("matrix").HTTP()
pHTTP.Mux().Handle(users.PublicURL, pMux)
pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux)
pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux)
// Build both ends of a HTTP multiplex.
h2s := &http2.Server{}
m.httpServer = &http.Server{
Addr: ":0",
TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 30 * time.Second,
BaseContext: func(_ net.Listener) context.Context {
return context.Background()
},
Handler: h2c.NewHandler(pMux, h2s),
}
m.processContext = base.ProcessContext
go func() {
m.logger.Info("Listening on ", cfg.Global.ServerName)
switch m.httpServer.Serve(m.PineconeQUIC.Protocol("matrix")) {
case net.ErrClosed, http.ErrServerClosed:
m.logger.Info("Stopped listening on ", cfg.Global.ServerName)
default:
m.logger.Fatal(err)
}
}()
go func() {
logrus.Info("Listening on ", m.listener.Addr())
switch http.Serve(m.listener, httpRouter) {
case net.ErrClosed, http.ErrServerClosed:
m.logger.Info("Stopped listening on ", cfg.Global.ServerName)
default:
m.logger.Fatal(err)
}
}()
enableRelaying := false
enableMetrics := false
enableWebsockets := false
m.p2pMonolith.SetupDendrite(cfg, 65432, enableRelaying, enableMetrics, enableWebsockets)
m.p2pMonolith.StartMonolith()
}
func (m *DendriteMonolith) Stop() {
m.processContext.ShutdownDendrite()
_ = m.listener.Close()
m.PineconeMulticast.Stop()
_ = m.PineconeQUIC.Close()
_ = m.PineconeRouter.Close()
m.processContext.WaitForComponentsToFinish()
}
const MaxFrameSize = types.MaxFrameSize
type Conduit struct {
closed atomic.Bool
conn net.Conn
port types.SwitchPortID
portMutex sync.Mutex
}
func (c *Conduit) Port() int {
c.portMutex.Lock()
defer c.portMutex.Unlock()
return int(c.port)
}
func (c *Conduit) Read(b []byte) (int, error) {
if c.closed.Load() {
return 0, io.EOF
}
return c.conn.Read(b)
}
func (c *Conduit) ReadCopy() ([]byte, error) {
if c.closed.Load() {
return nil, io.EOF
}
var buf [65535 * 2]byte
n, err := c.conn.Read(buf[:])
if err != nil {
return nil, err
}
return buf[:n], nil
}
func (c *Conduit) Write(b []byte) (int, error) {
if c.closed.Load() {
return 0, io.EOF
}
return c.conn.Write(b)
}
func (c *Conduit) Close() error {
if c.closed.Load() {
return io.ErrClosedPipe
}
c.closed.Store(true)
return c.conn.Close()
m.p2pMonolith.Stop()
}

152
build/gobind-pinecone/monolith_test.go Normal file
View file

@ -0,0 +1,152 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gobind
import (
"strings"
"testing"
"github.com/matrix-org/gomatrixserverlib"
)
func TestMonolithStarts(t *testing.T) {
monolith := DendriteMonolith{}
monolith.Start()
monolith.PublicKey()
monolith.Stop()
}
func TestMonolithSetRelayServers(t *testing.T) {
testCases := []struct {
name string
nodeID string
relays string
expectedRelays string
expectSelf bool
}{
{
name: "assorted valid, invalid, empty & self keys",
nodeID: "@valid:abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
relays: "@valid:123456123456abcdef123456abcdef123456abcdef123456abcdef123456abcd,@invalid:notakey,,",
expectedRelays: "123456123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
expectSelf: true,
},
{
name: "invalid node key",
nodeID: "@invalid:notakey",
relays: "@valid:123456123456abcdef123456abcdef123456abcdef123456abcdef123456abcd,@invalid:notakey,,",
expectedRelays: "",
expectSelf: false,
},
{
name: "node is self",
nodeID: "self",
relays: "@valid:123456123456abcdef123456abcdef123456abcdef123456abcdef123456abcd,@invalid:notakey,,",
expectedRelays: "123456123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
expectSelf: false,
},
}
for _, tc := range testCases {
monolith := DendriteMonolith{}
monolith.Start()
inputRelays := tc.relays
expectedRelays := tc.expectedRelays
if tc.expectSelf {
inputRelays += "," + monolith.PublicKey()
expectedRelays += "," + monolith.PublicKey()
}
nodeID := tc.nodeID
if nodeID == "self" {
nodeID = monolith.PublicKey()
}
monolith.SetRelayServers(nodeID, inputRelays)
relays := monolith.GetRelayServers(nodeID)
monolith.Stop()
if !containSameKeys(strings.Split(relays, ","), strings.Split(expectedRelays, ",")) {
t.Fatalf("%s: expected %s got %s", tc.name, expectedRelays, relays)
}
}
}
func containSameKeys(expected []string, actual []string) bool {
if len(expected) != len(actual) {
return false
}
for _, expectedKey := range expected {
hasMatch := false
for _, actualKey := range actual {
if actualKey == expectedKey {
hasMatch = true
}
}
if !hasMatch {
return false
}
}
return true
}
func TestParseServerKey(t *testing.T) {
testCases := []struct {
name string
serverKey string
expectedErr bool
expectedKey gomatrixserverlib.ServerName
}{
{
name: "valid userid as key",
serverKey: "@valid:abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
expectedErr: false,
expectedKey: "abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
},
{
name: "valid key",
serverKey: "abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
expectedErr: false,
expectedKey: "abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcd",
},
{
name: "invalid userid key",
serverKey: "@invalid:notakey",
expectedErr: true,
expectedKey: "",
},
{
name: "invalid key",
serverKey: "@invalid:notakey",
expectedErr: true,
expectedKey: "",
},
}
for _, tc := range testCases {
key, err := getServerKeyFromString(tc.serverKey)
if tc.expectedErr && err == nil {
t.Fatalf("%s: expected an error", tc.name)
} else if !tc.expectedErr && err != nil {
t.Fatalf("%s: didn't expect an error: %s", tc.name, err.Error())
}
if tc.expectedKey != key {
t.Fatalf("%s: keys not equal. expected: %s got: %s", tc.name, tc.expectedKey, key)
}
}
}

5
build/gobind-yggdrasil/monolith.go
View file

@ -150,6 +150,7 @@ func (m *DendriteMonolith) Start() {
}
base := base.NewBaseDendrite(cfg, "Monolith")
base.ConfigureAdminEndpoints()
m.processContext = base.ProcessContext
defer base.Close() // nolint: errcheck
@ -164,7 +165,7 @@ func (m *DendriteMonolith) Start() {
base, federation, rsAPI, base.Caches, keyRing, true,
)
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation, rsAPI)
userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, base.PushGatewayHTTPClient())
keyAPI.SetUserAPI(userAPI)
@ -196,6 +197,8 @@ func (m *DendriteMonolith) Start() {
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
yggRouter := mux.NewRouter()
yggRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)

14
build/scripts/Complement.Dockerfile
View file

@ -1,6 +1,6 @@
#syntax=docker/dockerfile:1.2
FROM golang:1.18-stretch as build
FROM golang:1.19-buster as build
RUN apt-get update && apt-get install -y sqlite3
WORKDIR /build
@ -10,18 +10,22 @@ RUN mkdir /dendrite
# Utilise Docker caching when downloading dependencies, this stops us needlessly
# downloading dependencies every time.
ARG CGO
RUN --mount=target=. \
--mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
go build -o /dendrite ./cmd/generate-config && \
go build -o /dendrite ./cmd/generate-keys && \
go build -o /dendrite ./cmd/dendrite-monolith-server
CGO_ENABLED=${CGO} go build --race -o /dendrite ./cmd/generate-config && \
CGO_ENABLED=${CGO} go build --race -o /dendrite ./cmd/generate-keys && \
CGO_ENABLED=${CGO} go build --race -o /dendrite ./cmd/dendrite-monolith-server && \
CGO_ENABLED=${CGO} go test --race -c -cover -covermode=atomic -o /dendrite/dendrite-monolith-server-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite-monolith-server && \
cp build/scripts/complement-cmd.sh /complement-cmd.sh
WORKDIR /dendrite
RUN ./generate-keys --private-key matrix_key.pem
ENV SERVER_NAME=localhost
ENV API=0
ENV COVER=0
EXPOSE 8008 8448
# At runtime, generate TLS cert based on the CA now mounted at /ca
@ -29,4 +33,4 @@ EXPOSE 8008 8448
CMD ./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key && \
./generate-config -server $SERVER_NAME --ci > dendrite.yaml && \
cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
exec /complement-cmd.sh

5
build/scripts/ComplementLocal.Dockerfile
View file

@ -12,18 +12,20 @@ FROM golang:1.18-stretch
RUN apt-get update && apt-get install -y sqlite3
ENV SERVER_NAME=localhost
ENV COVER=0
EXPOSE 8008 8448
WORKDIR /runtime
# This script compiles Dendrite for us.
RUN echo '\
#!/bin/bash -eux \n\
if test -f "/runtime/dendrite-monolith-server"; then \n\
if test -f "/runtime/dendrite-monolith-server" && test -f "/runtime/dendrite-monolith-server-cover"; then \n\
echo "Skipping compilation; binaries exist" \n\
exit 0 \n\
fi \n\
cd /dendrite \n\
go build -v -o /runtime /dendrite/cmd/dendrite-monolith-server \n\
go test -c -cover -covermode=atomic -o /runtime/dendrite-monolith-server-cover -coverpkg "github.com/matrix-org/..." /dendrite/cmd/dendrite-monolith-server \n\
' > compile.sh && chmod +x compile.sh
# This script runs Dendrite for us. Must be run in the /runtime directory.
@ -33,6 +35,7 @@ RUN echo '\
./generate-keys -keysize 1024 --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /complement/ca/ca.crt --tls-authority-key /complement/ca/ca.key \n\
./generate-config -server $SERVER_NAME --ci > dendrite.yaml \n\
cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates \n\
[ ${COVER} -eq 1 ] && exec ./dendrite-monolith-server-cover --test.coverprofile=integrationcover.log --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml \n\
' > run.sh && chmod +x run.sh

14
build/scripts/ComplementPostgres.Dockerfile
View file

@ -28,18 +28,22 @@ RUN mkdir /dendrite
# Utilise Docker caching when downloading dependencies, this stops us needlessly
# downloading dependencies every time.
ARG CGO
RUN --mount=target=. \
--mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
go build --race -o /dendrite ./cmd/generate-config && \
go build --race -o /dendrite ./cmd/generate-keys && \
go build --race -o /dendrite ./cmd/dendrite-monolith-server
CGO_ENABLED=${CGO} go build --race -o /dendrite ./cmd/generate-config && \
CGO_ENABLED=${CGO} go build --race -o /dendrite ./cmd/generate-keys && \
CGO_ENABLED=${CGO} go build --race -o /dendrite ./cmd/dendrite-monolith-server && \
CGO_ENABLED=${CGO} go test --race -c -cover -covermode=atomic -o /dendrite/dendrite-monolith-server-cover -coverpkg "github.com/matrix-org/..." ./cmd/dendrite-monolith-server && \
cp build/scripts/complement-cmd.sh /complement-cmd.sh
WORKDIR /dendrite
RUN ./generate-keys --private-key matrix_key.pem
ENV SERVER_NAME=localhost
ENV API=0
ENV COVER=0
EXPOSE 8008 8448
@ -50,4 +54,6 @@ CMD /build/run_postgres.sh && ./generate-keys --keysize 1024 --server $SERVER_NA
# Bump max_open_conns up here in the global database config
sed -i 's/max_open_conns:.*$/max_open_conns: 1990/g' dendrite.yaml && \
cp /complement/ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \
exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
<<<<<<< HEAD
exec ./dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0}
exec /complement-cmd.sh

22
build/scripts/complement-cmd.sh Executable file
View file

@ -0,0 +1,22 @@
#!/bin/bash -e
# This script is intended to be used inside a docker container for Complement
if [[ "${COVER}" -eq 1 ]]; then
echo "Running with coverage"
exec /dendrite/dendrite-monolith-server-cover \
--really-enable-open-registration \
--tls-cert server.crt \
--tls-key server.key \
--config dendrite.yaml \
-api=${API:-0} \
--test.coverprofile=complementcover.log
else
echo "Not running with coverage"
exec /dendrite/dendrite-monolith-server \
--really-enable-open-registration \
--tls-cert server.crt \
--tls-key server.key \
--config dendrite.yaml \
-api=${API:-0}
fi

240
clientapi/admin_test.go Normal file
View file

@ -0,0 +1,240 @@
package clientapi
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/syncapi"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/tidwall/gjson"
rsapi "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig"
"github.com/matrix-org/dendrite/userapi"
uapi "github.com/matrix-org/dendrite/userapi/api"
)
type clientRoomserverAPI struct {
rsapi.ClientRoomserverAPI
}
func TestAdminResetPassword(t *testing.T) {
aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
bob := test.NewUser(t, test.WithAccountType(uapi.AccountTypeUser))
vhUser := &test.User{ID: "@vhuser:vh1"}
ctx := context.Background()
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
base, baseClose := testrig.CreateBaseDendrite(t, dbType)
defer baseClose()
// add a vhost
base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{
SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"},
})
rsAPI := roomserver.NewInternalAPI(base)
// Needed for changing the password/login
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
keyAPI.SetUserAPI(userAPI)
// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, nil)
// Create the users in the userapi and login
accessTokens := map[*test.User]string{
aliceAdmin: "",
bob: "",
vhUser: "",
}
for u := range accessTokens {
localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
userRes := &uapi.PerformAccountCreationResponse{}
password := util.RandomString(8)
if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
AccountType: u.AccountType,
Localpart: localpart,
ServerName: serverName,
Password: password,
}, userRes); err != nil {
t.Errorf("failed to create account: %s", err)
}
req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
"type": authtypes.LoginTypePassword,
"identifier": map[string]interface{}{
"type": "m.id.user",
"user": u.ID,
},
"password": password,
}))
rec := httptest.NewRecorder()
base.PublicClientAPIMux.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("failed to login: %s", rec.Body.String())
}
accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String()
}
testCases := []struct {
name string
requestingUser *test.User
userID string
requestOpt test.HTTPRequestOpt
wantOK bool
withHeader bool
}{
{name: "Missing auth", requestingUser: bob, wantOK: false, userID: bob.ID},
{name: "Bob is denied access", requestingUser: bob, wantOK: false, withHeader: true, userID: bob.ID},
{name: "Alice is allowed access", requestingUser: aliceAdmin, wantOK: true, withHeader: true, userID: bob.ID, requestOpt: test.WithJSONBody(t, map[string]interface{}{
"password": util.RandomString(8),
})},
{name: "missing userID does not call function", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: ""}, // this 404s
{name: "rejects empty password", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: bob.ID, requestOpt: test.WithJSONBody(t, map[string]interface{}{
"password": "",
})},
{name: "rejects unknown server name", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: "@doesnotexist:localhost", requestOpt: test.WithJSONBody(t, map[string]interface{}{})},
{name: "rejects unknown user", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: "@doesnotexist:test", requestOpt: test.WithJSONBody(t, map[string]interface{}{})},
{name: "allows changing password for different vhost", requestingUser: aliceAdmin, wantOK: true, withHeader: true, userID: vhUser.ID, requestOpt: test.WithJSONBody(t, map[string]interface{}{
"password": util.RandomString(8),
})},
{name: "rejects existing user, missing body", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: bob.ID},
{name: "rejects invalid userID", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: "!notauserid:test", requestOpt: test.WithJSONBody(t, map[string]interface{}{})},
{name: "rejects invalid json", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: bob.ID, requestOpt: test.WithJSONBody(t, `{invalidJSON}`)},
{name: "rejects too weak password", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: bob.ID, requestOpt: test.WithJSONBody(t, map[string]interface{}{
"password": util.RandomString(6),
})},
{name: "rejects too long password", requestingUser: aliceAdmin, wantOK: false, withHeader: true, userID: bob.ID, requestOpt: test.WithJSONBody(t, map[string]interface{}{
"password": util.RandomString(513),
})},
}
for _, tc := range testCases {
tc := tc // ensure we don't accidentally only test the last test case
t.Run(tc.name, func(t *testing.T) {
req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/resetPassword/"+tc.userID)
if tc.requestOpt != nil {
req = test.NewRequest(t, http.MethodPost, "/_dendrite/admin/resetPassword/"+tc.userID, tc.requestOpt)
}
if tc.withHeader {
req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser])
}
rec := httptest.NewRecorder()
base.DendriteAdminMux.ServeHTTP(rec, req)
t.Logf("%s", rec.Body.String())
if tc.wantOK && rec.Code != http.StatusOK {
t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
}
})
}
})
}
func TestPurgeRoom(t *testing.T) {
aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
bob := test.NewUser(t)
room := test.NewRoom(t, aliceAdmin, test.RoomPreset(test.PresetTrustedPrivateChat))
// Invite Bob
room.CreateAndInsert(t, aliceAdmin, gomatrixserverlib.MRoomMember, map[string]interface{}{
"membership": "invite",
}, test.WithStateKey(bob.ID))
ctx := context.Background()
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
base, baseClose := testrig.CreateBaseDendrite(t, dbType)
defer baseClose()
fedClient := base.CreateFederationClient()
rsAPI := roomserver.NewInternalAPI(base)
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fedClient, rsAPI)
userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
// this starts the JetStream consumers
syncapi.AddPublicRoutes(base, userAPI, rsAPI, &clientRoomserverAPI{}, keyAPI)
federationapi.NewInternalAPI(base, fedClient, rsAPI, base.Caches, nil, true)
rsAPI.SetFederationAPI(nil, nil)
keyAPI.SetUserAPI(userAPI)
// Create the room
if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
t.Fatalf("failed to send events: %v", err)
}
// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
AddPublicRoutes(base, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, nil)
// Create the users in the userapi and login
accessTokens := map[*test.User]string{
aliceAdmin: "",
}
for u := range accessTokens {
localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
userRes := &uapi.PerformAccountCreationResponse{}
password := util.RandomString(8)
if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
AccountType: u.AccountType,
Localpart: localpart,
ServerName: serverName,
Password: password,
}, userRes); err != nil {
t.Errorf("failed to create account: %s", err)
}
req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
"type": authtypes.LoginTypePassword,
"identifier": map[string]interface{}{
"type": "m.id.user",
"user": u.ID,
},
"password": password,
}))
rec := httptest.NewRecorder()
base.PublicClientAPIMux.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("failed to login: %s", rec.Body.String())
}
accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String()
}
testCases := []struct {
name string
roomID string
wantOK bool
}{
{name: "Can purge existing room", wantOK: true, roomID: room.ID},
{name: "Can not purge non-existent room", wantOK: false, roomID: "!doesnotexist:localhost"},
{name: "rejects invalid room ID", wantOK: false, roomID: "@doesnotexist:localhost"},
}
for _, tc := range testCases {
tc := tc // ensure we don't accidentally only test the last test case
t.Run(tc.name, func(t *testing.T) {
req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/purgeRoom/"+tc.roomID)
req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin])
rec := httptest.NewRecorder()
base.DendriteAdminMux.ServeHTTP(rec, req)
t.Logf("%s", rec.Body.String())
if tc.wantOK && rec.Code != http.StatusOK {
t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
}
})
}
})
}

20
clientapi/auth/login_publickey_ethereum.go
View file

@ -25,6 +25,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/userutil"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/sirupsen/logrus"
"github.com/spruceid/siwe-go"
)
@ -70,22 +71,27 @@ func (pk LoginPublicKeyEthereum) AccountExists(ctx context.Context) (string, *js
localPart, _, err := userutil.ParseUsernameParam(pk.UserId, pk.config.Matrix)
if err != nil {
// userId does not exist
return "", jsonerror.Forbidden("the address is incorrect, or the account does not exist.")
logrus.WithError(err).Error("the address is incorrect, userId does not exist", pk.UserId)
return "", jsonerror.Forbidden("the address is incorrect, userId does not exist")
}
if !pk.IsValidUserId(localPart) {
logrus.Warn("the username is not valid", pk.UserId, localPart)
return "", jsonerror.InvalidUsername("the username is not valid.")
}
res := userapi.QueryAccountAvailabilityResponse{}
if err := pk.userAPI.QueryAccountAvailability(ctx, &userapi.QueryAccountAvailabilityRequest{
Localpart: localPart,
Localpart: localPart,
ServerName: pk.config.Matrix.ServerName,
}, &res); err != nil {
return "", jsonerror.Unknown("failed to check availability: " + err.Error())
logrus.WithError(err).Error("failed to check availability")
return "", jsonerror.Unknown("failed to check availability")
}
if localPart == "" || res.Available {
return "", jsonerror.Forbidden("the address is incorrect, account does not exist")
logrus.Warn("the address is incorrect, or the account does not exist", pk.UserId, localPart, res)
return "", jsonerror.Forbidden("the address is incorrect, or the account does not exist")
}
return localPart, nil
@ -116,10 +122,12 @@ func (pk LoginPublicKeyEthereum) ValidateLoginResponse() (bool, *jsonerror.Matri
return false, jsonerror.InvalidParam("auth.message")
}
serverName := pk.config.Matrix.ServerName
// Check signature to verify message was not tempered
_, err = message.Verify(pk.Signature, (*string)(&pk.config.Matrix.ServerName), nil, nil)
_, err = message.Verify(pk.Signature, (*string)(&serverName), nil, nil)
if err != nil {
return false, jsonerror.InvalidSignature(err.Error())
return false, jsonerror.InvalidSignature(fmt.Sprintf("%s signature:%+v server_name:%+v messsage_domain:%+v", err.Error(), pk.Signature, serverName, message.GetDomain()))
}
// Error if the user ID does not match the signed message.

6
clientapi/auth/login_publickey_ethereum_test.go
View file

@ -27,6 +27,7 @@ import (
"github.com/matrix-org/dendrite/setup/config"
testutil "github.com/matrix-org/dendrite/test"
uapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/stretchr/testify/assert"
)
@ -37,8 +38,9 @@ type loginContext struct {
func createLoginContext(_ *testing.T) *loginContext {
cfg := &config.ClientAPI{
Matrix: &config.Global{
ServerName: testutil.TestServerName,
Matrix: &config.Global{SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: gomatrixserverlib.ServerName("localhost"),
},
},
Derived: &config.Derived{},
PasswordAuthenticationDisabled: true,

14
clientapi/auth/login_test.go
View file

@ -22,8 +22,10 @@ import (
"testing"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/userutil"
"github.com/matrix-org/dendrite/setup/config"
uapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
@ -46,7 +48,7 @@ func TestLoginFromJSONReader(t *testing.T) {
"password": "herpassword",
"device_id": "adevice"
}`,
WantUsername: "alice",
WantUsername: "@alice:example.com",
WantDeviceID: "adevice",
},
{
@ -73,7 +75,9 @@ func TestLoginFromJSONReader(t *testing.T) {
var userAPI fakeUserInternalAPI
cfg := &config.ClientAPI{
Matrix: &config.Global{
ServerName: serverName,
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: serverName,
},
},
}
login, cleanup, err := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, &userAPI, &userInteractive, cfg)
@ -158,7 +162,9 @@ func TestBadLoginFromJSONReader(t *testing.T) {
var userAPI fakeUserInternalAPI
cfg := &config.ClientAPI{
Matrix: &config.Global{
ServerName: serverName,
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: serverName,
},
},
}
_, cleanup, errRes := LoginFromJSONReader(ctx, strings.NewReader(tst.Body), &userAPI, &userAPI, &userAPI, &userInteractive, cfg)
@ -185,7 +191,7 @@ func (ua *fakeUserInternalAPI) QueryAccountByPassword(ctx context.Context, req *
return nil
}
res.Exists = true
res.Account = &uapi.Account{}
res.Account = &uapi.Account{UserID: userutil.MakeUserID(req.Localpart, req.ServerName)}
return nil
}

26
clientapi/auth/password.go
View file

@ -61,7 +61,7 @@ func (t *LoginTypePassword) LoginFromJSON(ctx context.Context, reqBytes []byte)
func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) {
r := req.(*PasswordRequest)
username := strings.ToLower(r.Username())
username := r.Username()
if username == "" {
return nil, &util.JSONResponse{
Code: http.StatusUnauthorized,
@ -74,32 +74,45 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
JSON: jsonerror.BadJSON("A password must be supplied."),
}
}
localpart, _, err := userutil.ParseUsernameParam(username, t.Config.Matrix)
localpart, domain, err := userutil.ParseUsernameParam(username, t.Config.Matrix)
if err != nil {
return nil, &util.JSONResponse{
Code: http.StatusUnauthorized,
JSON: jsonerror.InvalidUsername(err.Error()),
}
}
if !t.Config.Matrix.IsLocalServerName(domain) {
return nil, &util.JSONResponse{
Code: http.StatusUnauthorized,
JSON: jsonerror.InvalidUsername("The server name is not known."),
}
}
// Squash username to all lowercase letters
res := &api.QueryAccountByPasswordResponse{}
err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{Localpart: strings.ToLower(localpart), PlaintextPassword: r.Password}, res)
err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
Localpart: strings.ToLower(localpart),
ServerName: domain,
PlaintextPassword: r.Password,
}, res)
if err != nil {
return nil, &util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: jsonerror.Unknown("unable to fetch account by password"),
JSON: jsonerror.Unknown("Unable to fetch account by password."),
}
}
// If we couldn't find the user by the lower cased localpart, try the provided
// localpart as is.
if !res.Exists {
err = t.GetAccountByPassword(ctx, &api.QueryAccountByPasswordRequest{
Localpart: localpart,
ServerName: domain,
PlaintextPassword: r.Password,
}, res)
if err != nil {
return nil, &util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: jsonerror.Unknown("unable to fetch account by password"),
JSON: jsonerror.Unknown("Unable to fetch account by password."),
}
}
// Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows
@ -111,6 +124,9 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login,
}
}
}
// Set the user, so login.Username() can do the right thing
r.Identifier.User = res.Account.UserID
r.User = res.Account.UserID
return &r.Login, nil
}

4
clientapi/auth/user_interactive_test.go
View file

@ -50,7 +50,9 @@ func (d *fakeAccountDatabase) QueryAccountByPassword(ctx context.Context, req *a
func setup() *UserInteractive {
cfg := &config.ClientAPI{
Matrix: &config.Global{
ServerName: serverName,
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: serverName,
},
},
}
accountApi := fakeAccountDatabase{}

3
clientapi/authorization/authorization.go
View file

@ -11,7 +11,7 @@ import (
)
func NewRoomserverAuthorization(cfg *config.ClientAPI, roomQueryAPI roomserver.QueryEventsAPI) authorization.Authorization {
// Load authorization manager for Zion
if flag.Lookup("test.v") == nil {
// normal run
// Load authorization manager for Zion
@ -26,5 +26,4 @@ func NewRoomserverAuthorization(cfg *config.ClientAPI, roomQueryAPI roomserver.Q
// run under go test
return &authorization.DefaultAuthorization{}
}
}

8
clientapi/clientapi.go
View file

@ -15,6 +15,8 @@
package clientapi
import (
"github.com/matrix-org/gomatrixserverlib"
appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/clientapi/api"
"github.com/matrix-org/dendrite/clientapi/producers"
@ -26,7 +28,6 @@ import (
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/jetstream"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
)
// AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component.
@ -57,10 +58,7 @@ func AddPublicRoutes(
}
routing.Setup(
base.PublicClientAPIMux,
base.PublicWellKnownAPIMux,
base.SynapseAdminMux,
base.DendriteAdminMux,
base,
cfg, rsAPI, asAPI,
userAPI, userDirectoryProvider, federation,
syncProducer, transactionsCache, fsAPI, keyAPI,

71
clientapi/routing/admin.go
View file

@ -1,12 +1,14 @@
package routing
import (
"context"
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/nats-io/nats.go"
@ -97,22 +99,77 @@ func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi
}
}
func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
func AdminPurgeRoom(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
localpart, ok := vars["localpart"]
roomID, ok := vars["roomID"]
if !ok {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.MissingArgument("Expecting user localpart."),
JSON: jsonerror.MissingArgument("Expecting room ID."),
}
}
res := &roomserverAPI.PerformAdminPurgeRoomResponse{}
if err := rsAPI.PerformAdminPurgeRoom(
context.Background(),
&roomserverAPI.PerformAdminPurgeRoomRequest{
RoomID: roomID,
},
res,
); err != nil {
return util.ErrorResponse(err)
}
if err := res.Error; err != nil {
return err.JSONResponse()
}
return util.JSONResponse{
Code: 200,
JSON: res,
}
}
func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, userAPI userapi.ClientUserAPI) util.JSONResponse {
if req.Body == nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.Unknown("Missing request body"),
}
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
var localpart string
userID := vars["userID"]
localpart, serverName, err := cfg.Matrix.SplitLocalID('@', userID)
if err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.InvalidArgumentValue(err.Error()),
}
}
accAvailableResp := &userapi.QueryAccountAvailabilityResponse{}
if err = userAPI.QueryAccountAvailability(req.Context(), &userapi.QueryAccountAvailabilityRequest{
Localpart: localpart,
ServerName: serverName,
}, accAvailableResp); err != nil {
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: jsonerror.InternalAPIError(req.Context(), err),
}
}
if accAvailableResp.Available {
return util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.Unknown("User does not exist"),
}
}
request := struct {
Password string `json:"password"`
}{}
if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
if err = json.NewDecoder(req.Body).Decode(&request); err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.Unknown("Failed to decode request body: " + err.Error()),
@ -124,8 +181,14 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap
JSON: jsonerror.MissingArgument("Expecting non-empty password."),
}
}
if err = internal.ValidatePassword(request.Password); err != nil {
return *internal.PasswordResponse(err)
}
updateReq := &userapi.PerformPasswordUpdateRequest{
Localpart: localpart,
ServerName: serverName,
Password: request.Password,
LogoutDevices: true,
}

115
clientapi/routing/auth_fallback.go
View file

@ -15,11 +15,11 @@
package routing
import (
"fmt"
"html/template"
"net/http"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/util"
)
@ -101,14 +101,28 @@ func serveTemplate(w http.ResponseWriter, templateHTML string, data map[string]s
func AuthFallback(
w http.ResponseWriter, req *http.Request, authType string,
cfg *config.ClientAPI,
) *util.JSONResponse {
sessionID := req.URL.Query().Get("session")
) {
// We currently only support "m.login.recaptcha", so fail early if that's not requested
if authType == authtypes.LoginTypeRecaptcha {
if !cfg.RecaptchaEnabled {
writeHTTPMessage(w, req,
"Recaptcha login is disabled on this Homeserver",
http.StatusBadRequest,
)
return
}
} else {
writeHTTPMessage(w, req, fmt.Sprintf("Unknown authtype %q", authType), http.StatusNotImplemented)
return
}
sessionID := req.URL.Query().Get("session")
if sessionID == "" {
return writeHTTPMessage(w, req,
writeHTTPMessage(w, req,
"Session ID not provided",
http.StatusBadRequest,
)
return
}
serveRecaptcha := func() {
@ -130,70 +144,44 @@ func AuthFallback(
if req.Method == http.MethodGet {
// Handle Recaptcha
if authType == authtypes.LoginTypeRecaptcha {
if err := checkRecaptchaEnabled(cfg, w, req); err != nil {
return err
}
serveRecaptcha()
return nil
}
return &util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound("Unknown auth stage type"),
}
serveRecaptcha()
return
} else if req.Method == http.MethodPost {
// Handle Recaptcha
if authType == authtypes.LoginTypeRecaptcha {
if err := checkRecaptchaEnabled(cfg, w, req); err != nil {
return err
}
clientIP := req.RemoteAddr
err := req.ParseForm()
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("req.ParseForm failed")
res := jsonerror.InternalServerError()
return &res
}
response := req.Form.Get(cfg.RecaptchaFormField)
if err := validateRecaptcha(cfg, response, clientIP); err != nil {
util.GetLogger(req.Context()).Error(err)
return err
}
// Success. Add recaptcha as a completed login flow
sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeRecaptcha)
serveSuccess()
return nil
clientIP := req.RemoteAddr
err := req.ParseForm()
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("req.ParseForm failed")
w.WriteHeader(http.StatusBadRequest)
serveRecaptcha()
return
}
return &util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound("Unknown auth stage type"),
response := req.Form.Get(cfg.RecaptchaFormField)
err = validateRecaptcha(cfg, response, clientIP)
switch err {
case ErrMissingResponse:
w.WriteHeader(http.StatusBadRequest)
serveRecaptcha() // serve the initial page again, instead of nothing
return
case ErrInvalidCaptcha:
w.WriteHeader(http.StatusUnauthorized)
serveRecaptcha()
return
case nil:
default: // something else failed
util.GetLogger(req.Context()).WithError(err).Error("failed to validate recaptcha")
serveRecaptcha()
return
}
}
return &util.JSONResponse{
Code: http.StatusMethodNotAllowed,
JSON: jsonerror.NotFound("Bad method"),
}
}
// checkRecaptchaEnabled creates an error response if recaptcha is not usable on homeserver.
func checkRecaptchaEnabled(
cfg *config.ClientAPI,
w http.ResponseWriter,
req *http.Request,
) *util.JSONResponse {
if !cfg.RecaptchaEnabled {
return writeHTTPMessage(w, req,
"Recaptcha login is disabled on this Homeserver",
http.StatusBadRequest,
)
// Success. Add recaptcha as a completed login flow
sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypeRecaptcha)
serveSuccess()
return
}
return nil
writeHTTPMessage(w, req, "Bad method", http.StatusMethodNotAllowed)
}
// writeHTTPMessage writes the given header and message to the HTTP response writer.
@ -201,13 +189,10 @@ func checkRecaptchaEnabled(
func writeHTTPMessage(
w http.ResponseWriter, req *http.Request,
message string, header int,
) *util.JSONResponse {
) {
w.WriteHeader(header)
_, err := w.Write([]byte(message))
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("w.Write failed")
res := jsonerror.InternalServerError()
return &res
}
return nil
}

149
clientapi/routing/auth_fallback_test.go Normal file
View file

@ -0,0 +1,149 @@
package routing
import (
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test/testrig"
)
func Test_AuthFallback(t *testing.T) {
base, _, _ := testrig.Base(nil)
defer base.Close()
for _, useHCaptcha := range []bool{false, true} {
for _, recaptchaEnabled := range []bool{false, true} {
for _, wantErr := range []bool{false, true} {
t.Run(fmt.Sprintf("useHCaptcha(%v) - recaptchaEnabled(%v) - wantErr(%v)", useHCaptcha, recaptchaEnabled, wantErr), func(t *testing.T) {
// Set the defaults for each test
base.Cfg.ClientAPI.Defaults(config.DefaultOpts{Generate: true, Monolithic: true})
base.Cfg.ClientAPI.RecaptchaEnabled = recaptchaEnabled
base.Cfg.ClientAPI.RecaptchaPublicKey = "pub"
base.Cfg.ClientAPI.RecaptchaPrivateKey = "priv"
if useHCaptcha {
base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = "https://hcaptcha.com/siteverify"
base.Cfg.ClientAPI.RecaptchaApiJsUrl = "https://js.hcaptcha.com/1/api.js"
base.Cfg.ClientAPI.RecaptchaFormField = "h-captcha-response"
base.Cfg.ClientAPI.RecaptchaSitekeyClass = "h-captcha"
}
cfgErrs := &config.ConfigErrors{}
base.Cfg.ClientAPI.Verify(cfgErrs, true)
if len(*cfgErrs) > 0 {
t.Fatalf("(hCaptcha=%v) unexpected config errors: %s", useHCaptcha, cfgErrs.Error())
}
req := httptest.NewRequest(http.MethodGet, "/?session=1337", nil)
rec := httptest.NewRecorder()
AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
if !recaptchaEnabled {
if rec.Code != http.StatusBadRequest {
t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusBadRequest)
}
if rec.Body.String() != "Recaptcha login is disabled on this Homeserver" {
t.Fatalf("unexpected response body: %s", rec.Body.String())
}
} else {
if !strings.Contains(rec.Body.String(), base.Cfg.ClientAPI.RecaptchaSitekeyClass) {
t.Fatalf("body does not contain %s: %s", base.Cfg.ClientAPI.RecaptchaSitekeyClass, rec.Body.String())
}
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if wantErr {
_, _ = w.Write([]byte(`{"success":false}`))
return
}
_, _ = w.Write([]byte(`{"success":true}`))
}))
defer srv.Close() // nolint: errcheck
base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL
// check the result after sending the captcha
req = httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
req.Form = url.Values{}
req.Form.Add(base.Cfg.ClientAPI.RecaptchaFormField, "someRandomValue")
rec = httptest.NewRecorder()
AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
if recaptchaEnabled {
if !wantErr {
if rec.Code != http.StatusOK {
t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusOK)
}
if rec.Body.String() != successTemplate {
t.Fatalf("unexpected response: %s, want %s", rec.Body.String(), successTemplate)
}
} else {
if rec.Code != http.StatusUnauthorized {
t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusUnauthorized)
}
wantString := "Authentication"
if !strings.Contains(rec.Body.String(), wantString) {
t.Fatalf("expected response to contain '%s', but didn't: %s", wantString, rec.Body.String())
}
}
} else {
if rec.Code != http.StatusBadRequest {
t.Fatalf("unexpected response code: %d, want %d", rec.Code, http.StatusBadRequest)
}
if rec.Body.String() != "Recaptcha login is disabled on this Homeserver" {
t.Fatalf("unexpected response: %s, want %s", rec.Body.String(), "successTemplate")
}
}
})
}
}
}
t.Run("unknown fallbacks are handled correctly", func(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
rec := httptest.NewRecorder()
AuthFallback(rec, req, "DoesNotExist", &base.Cfg.ClientAPI)
if rec.Code != http.StatusNotImplemented {
t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusNotImplemented)
}
})
t.Run("unknown methods are handled correctly", func(t *testing.T) {
req := httptest.NewRequest(http.MethodDelete, "/?session=1337", nil)
rec := httptest.NewRecorder()
AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
if rec.Code != http.StatusMethodNotAllowed {
t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusMethodNotAllowed)
}
})
t.Run("missing session parameter is handled correctly", func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "/", nil)
rec := httptest.NewRecorder()
AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
if rec.Code != http.StatusBadRequest {
t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
}
})
t.Run("missing session parameter is handled correctly", func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "/", nil)
rec := httptest.NewRecorder()
AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
if rec.Code != http.StatusBadRequest {
t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
}
})
t.Run("missing 'response' is handled correctly", func(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/?session=1337", nil)
rec := httptest.NewRecorder()
AuthFallback(rec, req, authtypes.LoginTypeRecaptcha, &base.Cfg.ClientAPI)
if rec.Code != http.StatusBadRequest {
t.Fatalf("unexpected http status: %d, want %d", rec.Code, http.StatusBadRequest)
}
})
}

2
clientapi/routing/createroom.go
View file

@ -477,7 +477,7 @@ func createRoom(
SendAsServer: roomserverAPI.DoNotSendToOtherServers,
})
}
if err = roomserverAPI.SendInputRoomEvents(ctx, rsAPI, inputs, false); err != nil {
if err = roomserverAPI.SendInputRoomEvents(ctx, rsAPI, device.UserDomain(), inputs, false); err != nil {
util.GetLogger(ctx).WithError(err).Error("roomserverAPI.SendInputRoomEvents failed")
return jsonerror.InternalServerError()
}

2
clientapi/routing/directory.go
View file

@ -77,7 +77,7 @@ func DirectoryRoom(
// If we don't know it locally, do a federation query.
// But don't send the query to ourselves.
if !cfg.Matrix.IsLocalServerName(domain) {
fedRes, fedErr := federation.LookupRoomAlias(req.Context(), domain, roomAlias)
fedRes, fedErr := federation.LookupRoomAlias(req.Context(), cfg.Matrix.ServerName, domain, roomAlias)
if fedErr != nil {
// TODO: Return 502 if the remote server errored.
// TODO: Return 504 if the remote server timed out.

2
clientapi/routing/directory_public.go
View file

@ -74,7 +74,7 @@ func GetPostPublicRooms(
serverName := gomatrixserverlib.ServerName(request.Server)
if serverName != "" && !cfg.Matrix.IsLocalServerName(serverName) {
res, err := federation.GetPublicRoomsFiltered(
req.Context(), serverName,
req.Context(), cfg.Matrix.ServerName, serverName,
int(request.Limit), request.Since,
request.Filter.SearchTerms, false,
"",

10
clientapi/routing/joinroom.go
View file

@ -37,6 +37,7 @@ func JoinRoomByIDOrAlias(
joinReq := roomserverAPI.PerformJoinRequest{
RoomIDOrAlias: roomIDOrAlias,
UserID: device.UserID,
IsGuest: device.AccountType == api.AccountTypeGuest,
Content: map[string]interface{}{},
}
joinRes := roomserverAPI.PerformJoinResponse{}
@ -84,7 +85,14 @@ func JoinRoomByIDOrAlias(
if err := rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes); err != nil {
done <- jsonerror.InternalAPIError(req.Context(), err)
} else if joinRes.Error != nil {
done <- joinRes.Error.JSONResponse()
if joinRes.Error.Code == roomserverAPI.PerformErrorNotAllowed && device.AccountType == api.AccountTypeGuest {
done <- util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.GuestAccessForbidden(joinRes.Error.Msg),
}
} else {
done <- joinRes.Error.JSONResponse()
}
} else {
done <- util.JSONResponse{
Code: http.StatusOK,

158
clientapi/routing/joinroom_test.go Normal file
View file

@ -0,0 +1,158 @@
package routing
import (
"bytes"
"context"
"net/http"
"testing"
"time"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig"
"github.com/matrix-org/dendrite/userapi"
uapi "github.com/matrix-org/dendrite/userapi/api"
)
func TestJoinRoomByIDOrAlias(t *testing.T) {
alice := test.NewUser(t)
bob := test.NewUser(t)
charlie := test.NewUser(t, test.WithAccountType(uapi.AccountTypeGuest))
ctx := context.Background()
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
base, baseClose := testrig.CreateBaseDendrite(t, dbType)
defer baseClose()
rsAPI := roomserver.NewInternalAPI(base)
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc
// Create the users in the userapi
for _, u := range []*test.User{alice, bob, charlie} {
localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
userRes := &uapi.PerformAccountCreationResponse{}
if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
AccountType: u.AccountType,
Localpart: localpart,
ServerName: serverName,
Password: "someRandomPassword",
}, userRes); err != nil {
t.Errorf("failed to create account: %s", err)
}
}
aliceDev := &uapi.Device{UserID: alice.ID}
bobDev := &uapi.Device{UserID: bob.ID}
charlieDev := &uapi.Device{UserID: charlie.ID, AccountType: uapi.AccountTypeGuest}
// create a room with disabled guest access and invite Bob
resp := createRoom(ctx, createRoomRequest{
Name: "testing",
IsDirect: true,
Topic: "testing",
Visibility: "public",
Preset: presetPublicChat,
RoomAliasName: "alias",
Invite: []string{bob.ID},
GuestCanJoin: false,
}, aliceDev, &base.Cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
crResp, ok := resp.JSON.(createRoomResponse)
if !ok {
t.Fatalf("response is not a createRoomResponse: %+v", resp)
}
// create a room with guest access enabled and invite Charlie
resp = createRoom(ctx, createRoomRequest{
Name: "testing",
IsDirect: true,
Topic: "testing",
Visibility: "public",
Preset: presetPublicChat,
Invite: []string{charlie.ID},
GuestCanJoin: true,
}, aliceDev, &base.Cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now())
crRespWithGuestAccess, ok := resp.JSON.(createRoomResponse)
if !ok {
t.Fatalf("response is not a createRoomResponse: %+v", resp)
}
// Dummy request
body := &bytes.Buffer{}
req, err := http.NewRequest(http.MethodPost, "/?server_name=test", body)
if err != nil {
t.Fatal(err)
}
testCases := []struct {
name string
device *uapi.Device
roomID string
wantHTTP200 bool
}{
{
name: "User can join successfully by alias",
device: bobDev,
roomID: crResp.RoomAlias,
wantHTTP200: true,
},
{
name: "User can join successfully by roomID",
device: bobDev,
roomID: crResp.RoomID,
wantHTTP200: true,
},
{
name: "join is forbidden if user is guest",
device: charlieDev,
roomID: crResp.RoomID,
},
{
name: "room does not exist",
device: aliceDev,
roomID: "!doesnotexist:test",
},
{
name: "user from different server",
device: &uapi.Device{UserID: "@wrong:server"},
roomID: crResp.RoomAlias,
},
{
name: "user doesn't exist locally",
device: &uapi.Device{UserID: "@doesnotexist:test"},
roomID: crResp.RoomAlias,
},
{
name: "invalid room ID",
device: aliceDev,
roomID: "invalidRoomID",
},
{
name: "roomAlias does not exist",
device: aliceDev,
roomID: "#doesnotexist:test",
},
{
name: "room with guest_access event",
device: charlieDev,
roomID: crRespWithGuestAccess.RoomID,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
joinResp := JoinRoomByIDOrAlias(req, tc.device, rsAPI, userAPI, tc.roomID)
if tc.wantHTTP200 && !joinResp.Is2xx() {
t.Fatalf("expected join room to succeed, but didn't: %+v", joinResp)
}
})
}
})
}

10
clientapi/routing/leaveroom.go
View file

@ -18,6 +18,7 @@ import (
"net/http"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/httputil"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/util"
@ -44,6 +45,15 @@ func LeaveRoomByID(
JSON: jsonerror.LeaveServerNoticeError(),
}
}
switch e := err.(type) {
case httputil.InternalAPIError:
if e.Message == jsonerror.LeaveServerNoticeError().Error() {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.LeaveServerNoticeError(),
}
}
}
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.Unknown(err.Error()),

10
clientapi/routing/login.go
View file

@ -23,15 +23,13 @@ import (
"github.com/matrix-org/dendrite/clientapi/userutil"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
type loginResponse struct {
UserID string `json:"user_id"`
AccessToken string `json:"access_token"`
HomeServer gomatrixserverlib.ServerName `json:"home_server"`
DeviceID string `json:"device_id"`
UserID string `json:"user_id"`
AccessToken string `json:"access_token"`
DeviceID string `json:"device_id"`
}
type flows struct {
@ -114,6 +112,7 @@ func completeAuth(
DeviceID: login.DeviceID,
AccessToken: token,
Localpart: localpart,
ServerName: serverName,
IPAddr: ipAddr,
UserAgent: userAgent,
}, &performRes)
@ -129,7 +128,6 @@ func completeAuth(
JSON: loginResponse{
UserID: performRes.Device.UserID,
AccessToken: performRes.Device.AccessToken,
HomeServer: serverName,
DeviceID: performRes.Device.ID,
},
}

152
clientapi/routing/login_test.go Normal file
View file

@ -0,0 +1,152 @@
package routing
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig"
"github.com/matrix-org/dendrite/userapi"
uapi "github.com/matrix-org/dendrite/userapi/api"
)
func TestLogin(t *testing.T) {
aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
bobUser := &test.User{ID: "@bob:test", AccountType: uapi.AccountTypeUser}
charlie := &test.User{ID: "@Charlie:test", AccountType: uapi.AccountTypeUser}
vhUser := &test.User{ID: "@vhuser:vh1"}
ctx := context.Background()
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
base, baseClose := testrig.CreateBaseDendrite(t, dbType)
defer baseClose()
base.Cfg.ClientAPI.RateLimiting.Enabled = false
// add a vhost
base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{
SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"},
})
rsAPI := roomserver.NewInternalAPI(base)
// Needed for /login
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
keyAPI.SetUserAPI(userAPI)
// We mostly need the userAPI for this test, so nil for other APIs/caches etc.
Setup(base, &base.Cfg.ClientAPI, nil, nil, userAPI, nil, nil, nil, nil, nil, keyAPI, nil, &base.Cfg.MSCs, nil)
// Create password
password := util.RandomString(8)
// create the users
for _, u := range []*test.User{aliceAdmin, bobUser, vhUser, charlie} {
localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
userRes := &uapi.PerformAccountCreationResponse{}
if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
AccountType: u.AccountType,
Localpart: localpart,
ServerName: serverName,
Password: password,
}, userRes); err != nil {
t.Errorf("failed to create account: %s", err)
}
if !userRes.AccountCreated {
t.Fatalf("account not created")
}
}
testCases := []struct {
name string
userID string
wantOK bool
}{
{
name: "aliceAdmin can login",
userID: aliceAdmin.ID,
wantOK: true,
},
{
name: "bobUser can login",
userID: bobUser.ID,
wantOK: true,
},
{
name: "vhuser can login",
userID: vhUser.ID,
wantOK: true,
},
{
name: "bob with uppercase can login",
userID: "@Bob:test",
wantOK: true,
},
{
name: "Charlie can login (existing uppercase)",
userID: charlie.ID,
wantOK: true,
},
{
name: "Charlie can not login with lowercase userID",
userID: strings.ToLower(charlie.ID),
wantOK: false,
},
}
ctx := context.Background()
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
"type": authtypes.LoginTypePassword,
"identifier": map[string]interface{}{
"type": "m.id.user",
"user": tc.userID,
},
"password": password,
}))
rec := httptest.NewRecorder()
base.PublicClientAPIMux.ServeHTTP(rec, req)
if tc.wantOK && rec.Code != http.StatusOK {
t.Fatalf("failed to login: %s", rec.Body.String())
}
t.Logf("Response: %s", rec.Body.String())
// get the response
resp := loginResponse{}
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
// everything OK
if !tc.wantOK && resp.AccessToken == "" {
return
}
if tc.wantOK && resp.AccessToken == "" {
t.Fatalf("expected accessToken after successful login but got none: %+v", resp)
}
devicesResp := &uapi.QueryDevicesResponse{}
if err := userAPI.QueryDevices(ctx, &uapi.QueryDevicesRequest{UserID: resp.UserID}, devicesResp); err != nil {
t.Fatal(err)
}
for _, dev := range devicesResp.Devices {
// We expect the userID on the device to be the same as resp.UserID
if dev.UserID != resp.UserID {
t.Fatalf("unexpected userID on device: %s", dev.UserID)
}
}
})
}
})
}

8
clientapi/routing/membership.go
View file

@ -110,6 +110,7 @@ func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, devic
ctx, rsAPI,
roomserverAPI.KindNew,
[]*gomatrixserverlib.HeaderedEvent{event.Event.Headered(roomVer)},
device.UserDomain(),
serverName,
serverName,
nil,
@ -322,7 +323,12 @@ func buildMembershipEvent(
return nil, err
}
return eventutil.QueryAndBuildEvent(ctx, &builder, cfg.Matrix, evTime, rsAPI, nil)
identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
if err != nil {
return nil, err
}
return eventutil.QueryAndBuildEvent(ctx, &builder, cfg.Matrix, identity, evTime, rsAPI, nil)
}
// loadProfile lookups the profile of a given user from the database and returns

11
clientapi/routing/notification.go
View file

@ -40,16 +40,17 @@ func GetNotifications(
}
var queryRes userapi.QueryNotificationsResponse
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("SplitID failed")
return jsonerror.InternalServerError()
}
err = userAPI.QueryNotifications(req.Context(), &userapi.QueryNotificationsRequest{
Localpart: localpart,
From: req.URL.Query().Get("from"),
Limit: int(limit),
Only: req.URL.Query().Get("only"),
Localpart: localpart,
ServerName: domain,
From: req.URL.Query().Get("from"),
Limit: int(limit),
Only: req.URL.Query().Get("only"),
}, &queryRes)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("QueryNotifications failed")

17
clientapi/routing/password.go
View file

@ -7,6 +7,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
@ -81,12 +82,12 @@ func Password(
sessions.addCompletedSessionStage(sessionID, authtypes.LoginTypePassword)
// Check the new password strength.
if resErr = validatePassword(r.NewPassword); resErr != nil {
return *resErr
if err := internal.ValidatePassword(r.NewPassword); err != nil {
return *internal.PasswordResponse(err)
}
// Get the local part.
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
return jsonerror.InternalServerError()
@ -94,8 +95,9 @@ func Password(
// Ask the user API to perform the password change.
passwordReq := &api.PerformPasswordUpdateRequest{
Localpart: localpart,
Password: r.NewPassword,
Localpart: localpart,
ServerName: domain,
Password: r.NewPassword,
}
passwordRes := &api.PerformPasswordUpdateResponse{}
if err := userAPI.PerformPasswordUpdate(req.Context(), passwordReq, passwordRes); err != nil {
@ -122,8 +124,9 @@ func Password(
}
pushersReq := &api.PerformPusherDeletionRequest{
Localpart: localpart,
SessionID: device.SessionID,
Localpart: localpart,
ServerName: domain,
SessionID: device.SessionID,
}
if err := userAPI.PerformPusherDeletion(req.Context(), pushersReq, &struct{}{}); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("PerformPusherDeletion failed")

14
clientapi/routing/profile.go
View file

@ -284,7 +284,7 @@ func updateProfile(
}
events, err := buildMembershipEvents(
ctx, res.RoomIDs, *profile, userID, cfg, evTime, rsAPI,
ctx, device, res.RoomIDs, *profile, userID, cfg, evTime, rsAPI,
)
switch e := err.(type) {
case nil:
@ -298,7 +298,7 @@ func updateProfile(
return jsonerror.InternalServerError(), e
}
if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, domain, domain, nil, true); err != nil {
if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, device.UserDomain(), domain, domain, nil, true); err != nil {
util.GetLogger(ctx).WithError(err).Error("SendEvents failed")
return jsonerror.InternalServerError(), err
}
@ -321,7 +321,7 @@ func getProfile(
}
if !cfg.Matrix.IsLocalServerName(domain) {
profile, fedErr := federation.LookupProfile(ctx, domain, userID, "")
profile, fedErr := federation.LookupProfile(ctx, cfg.Matrix.ServerName, domain, userID, "")
if fedErr != nil {
if x, ok := fedErr.(gomatrix.HTTPError); ok {
if x.Code == http.StatusNotFound {
@ -349,6 +349,7 @@ func getProfile(
func buildMembershipEvents(
ctx context.Context,
device *userapi.Device,
roomIDs []string,
newProfile authtypes.Profile, userID string, cfg *config.ClientAPI,
evTime time.Time, rsAPI api.ClientRoomserverAPI,
@ -380,7 +381,12 @@ func buildMembershipEvents(
return nil, err
}
event, err := eventutil.QueryAndBuildEvent(ctx, &builder, cfg.Matrix, evTime, rsAPI, nil)
identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
if err != nil {
return nil, err
}
event, err := eventutil.QueryAndBuildEvent(ctx, &builder, cfg.Matrix, identity, evTime, rsAPI, nil)
if err != nil {
return nil, err
}

8
clientapi/routing/pusher.go
View file

@ -31,13 +31,14 @@ func GetPushers(
userAPI userapi.ClientUserAPI,
) util.JSONResponse {
var queryRes userapi.QueryPushersResponse
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("SplitID failed")
return jsonerror.InternalServerError()
}
err = userAPI.QueryPushers(req.Context(), &userapi.QueryPushersRequest{
Localpart: localpart,
Localpart: localpart,
ServerName: domain,
}, &queryRes)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("QueryPushers failed")
@ -59,7 +60,7 @@ func SetPusher(
req *http.Request, device *userapi.Device,
userAPI userapi.ClientUserAPI,
) util.JSONResponse {
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("SplitID failed")
return jsonerror.InternalServerError()
@ -93,6 +94,7 @@ func SetPusher(
}
body.Localpart = localpart
body.ServerName = domain
body.SessionID = device.SessionID
err = userAPI.PerformPusherSet(req.Context(), &body, &struct{}{})
if err != nil {

9
clientapi/routing/redaction.go
View file

@ -123,8 +123,13 @@ func SendRedaction(
return jsonerror.InternalServerError()
}
identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
if err != nil {
return jsonerror.InternalServerError()
}
var queryRes roomserverAPI.QueryLatestEventsAndStateResponse
e, err := eventutil.QueryAndBuildEvent(req.Context(), &builder, cfg.Matrix, time.Now(), rsAPI, &queryRes)
e, err := eventutil.QueryAndBuildEvent(req.Context(), &builder, cfg.Matrix, identity, time.Now(), rsAPI, &queryRes)
if err == eventutil.ErrRoomNoExists {
return util.JSONResponse{
Code: http.StatusNotFound,
@ -132,7 +137,7 @@ func SendRedaction(
}
}
domain := device.UserDomain()
if err = roomserverAPI.SendEvents(context.Background(), rsAPI, roomserverAPI.KindNew, []*gomatrixserverlib.HeaderedEvent{e}, domain, domain, nil, false); err != nil {
if err = roomserverAPI.SendEvents(context.Background(), rsAPI, roomserverAPI.KindNew, []*gomatrixserverlib.HeaderedEvent{e}, device.UserDomain(), domain, domain, nil, false); err != nil {
util.GetLogger(req.Context()).WithError(err).Errorf("failed to SendEvents")
return jsonerror.InternalServerError()
}

283
clientapi/routing/register.go
View file

@ -18,18 +18,21 @@ package routing
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/matrix-org/dendrite/internal"
internalHTTPUtil "github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/tidwall/gjson"
"github.com/matrix-org/dendrite/internal/eventutil"
@ -60,12 +63,7 @@ var (
)
)
const (
minPasswordLength = 8 // http://matrix.org/docs/spec/client_server/r0.2.0.html#password-based
maxPasswordLength = 512 // https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
maxUsernameLength = 254 // http://matrix.org/speculator/spec/HEAD/intro.html#user-identifiers TODO account for domain
sessionIDLength = 24
)
const sessionIDLength = 24
// sessionsDict keeps track of completed auth stages for each session.
// It shouldn't be passed by value because it contains a mutex.
@ -207,8 +205,7 @@ func (d *sessionsDict) getDeviceToDelete(sessionID string) (string, bool) {
}
var (
sessions = newSessionsDict()
validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
sessions = newSessionsDict()
)
// registerRequest represents the submitted registration request.
@ -219,9 +216,10 @@ var (
// previous parameters with the ones supplied. This mean you cannot "build up" request params.
type registerRequest struct {
// registration parameters
Password string `json:"password"`
Username string `json:"username"`
Admin bool `json:"admin"`
Password string `json:"password"`
Username string `json:"username"`
ServerName gomatrixserverlib.ServerName `json:"-"`
Admin bool `json:"admin"`
// user-interactive auth params
Auth authDict `json:"auth"`
@ -279,10 +277,9 @@ func newUserInteractiveResponse(
// http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
type registerResponse struct {
UserID string `json:"user_id"`
AccessToken string `json:"access_token,omitempty"`
HomeServer gomatrixserverlib.ServerName `json:"home_server"`
DeviceID string `json:"device_id,omitempty"`
UserID string `json:"user_id"`
AccessToken string `json:"access_token,omitempty"`
DeviceID string `json:"device_id,omitempty"`
}
// recaptchaResponse represents the HTTP response from a Google Recaptcha server
@ -293,83 +290,28 @@ type recaptchaResponse struct {
ErrorCodes []int `json:"error-codes"`
}
// validateUsername returns an error response if the username is invalid
func validateUsername(localpart string, domain gomatrixserverlib.ServerName) *util.JSONResponse {
// https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
if id := fmt.Sprintf("@%s:%s", localpart, domain); len(id) > maxUsernameLength {
return &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON(fmt.Sprintf("%q exceeds the maximum length of %d characters", id, maxUsernameLength)),
}
} else if !validUsernameRegex.MatchString(localpart) {
return &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./='"),
}
} else if localpart[0] == '_' { // Regex checks its not a zero length string
return &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.InvalidUsername("Username cannot start with a '_'"),
}
}
return nil
}
// validateApplicationServiceUsername returns an error response if the username is invalid for an application service
func validateApplicationServiceUsername(localpart string, domain gomatrixserverlib.ServerName) *util.JSONResponse {
if id := fmt.Sprintf("@%s:%s", localpart, domain); len(id) > maxUsernameLength {
return &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON(fmt.Sprintf("%q exceeds the maximum length of %d characters", id, maxUsernameLength)),
}
} else if !validUsernameRegex.MatchString(localpart) {
return &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./='"),
}
}
return nil
}
// validatePassword returns an error response if the password is invalid
func validatePassword(password string) *util.JSONResponse {
// https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
if len(password) > maxPasswordLength {
return &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON(fmt.Sprintf("'password' >%d characters", maxPasswordLength)),
}
} else if len(password) > 0 && len(password) < minPasswordLength {
return &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.WeakPassword(fmt.Sprintf("password too weak: min %d chars", minPasswordLength)),
}
}
return nil
}
var (
ErrInvalidCaptcha = errors.New("invalid captcha response")
ErrMissingResponse = errors.New("captcha response is required")
ErrCaptchaDisabled = errors.New("captcha registration is disabled")
)
// validateRecaptcha returns an error response if the captcha response is invalid
func validateRecaptcha(
cfg *config.ClientAPI,
response string,
clientip string,
) *util.JSONResponse {
) error {
ip, _, _ := net.SplitHostPort(clientip)
if !cfg.RecaptchaEnabled {
return &util.JSONResponse{
Code: http.StatusConflict,
JSON: jsonerror.Unknown("Captcha registration is disabled"),
}
return ErrCaptchaDisabled
}
if response == "" {
return &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON("Captcha response is required"),
}
return ErrMissingResponse
}
// Make a POST request to Google's API to check the captcha response
// Make a POST request to the captcha provider API to check the captcha response
resp, err := http.PostForm(cfg.RecaptchaSiteVerifyAPI,
url.Values{
"secret": {cfg.RecaptchaPrivateKey},
@ -379,10 +321,7 @@ func validateRecaptcha(
)
if err != nil {
return &util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: jsonerror.BadJSON("Error in requesting validation of captcha response"),
}
return err
}
// Close the request once we're finishing reading from it
@ -392,25 +331,16 @@ func validateRecaptcha(
var r recaptchaResponse
body, err := io.ReadAll(resp.Body)
if err != nil {
return &util.JSONResponse{
Code: http.StatusGatewayTimeout,
JSON: jsonerror.Unknown("Error in contacting captcha server" + err.Error()),
}
return err
}
err = json.Unmarshal(body, &r)
if err != nil {
return &util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: jsonerror.BadJSON("Error in unmarshaling captcha server's response: " + err.Error()),
}
return err
}
// Check that we received a "success"
if !r.Success {
return &util.JSONResponse{
Code: http.StatusUnauthorized,
JSON: jsonerror.BadJSON("Invalid captcha response. Please try again."),
}
return ErrInvalidCaptcha
}
return nil
}
@ -542,8 +472,8 @@ func validateApplicationService(
}
// Check username application service is trying to register is valid
if err := validateApplicationServiceUsername(username, cfg.Matrix.ServerName); err != nil {
return "", err
if err := internal.ValidateApplicationServiceUsername(username, cfg.Matrix.ServerName); err != nil {
return "", internal.UsernameResponse(err)
}
// No errors, registration valid
@ -567,6 +497,12 @@ func Register(
}
var r registerRequest
host := gomatrixserverlib.ServerName(req.Host)
if v := cfg.Matrix.VirtualHostForHTTPHost(host); v != nil {
r.ServerName = v.ServerName
} else {
r.ServerName = cfg.Matrix.ServerName
}
sessionID := gjson.GetBytes(reqBody, "auth.session").String()
if sessionID == "" {
// Generate a new, random session ID
@ -576,6 +512,7 @@ func Register(
// Some of these might end up being overwritten if the
// values are specified again in the request body.
r.Username = data.Username
r.ServerName = data.ServerName
r.Password = data.Password
r.DeviceID = data.DeviceID
r.InitialDisplayName = data.InitialDisplayName
@ -587,7 +524,6 @@ func Register(
JSON: response,
}
}
}
if resErr := httputil.UnmarshalJSON(reqBody, &r); resErr != nil {
return *resErr
@ -597,7 +533,7 @@ func Register(
}
// Don't allow numeric usernames less than MAX_INT64.
if _, err := strconv.ParseInt(r.Username, 10, 64); err == nil {
if _, err = strconv.ParseInt(r.Username, 10, 64); err == nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.InvalidUsername("Numeric user IDs are reserved"),
@ -605,12 +541,15 @@ func Register(
}
// Auto generate a numeric username if r.Username is empty
if r.Username == "" {
res := &userapi.QueryNumericLocalpartResponse{}
if err := userAPI.QueryNumericLocalpart(req.Context(), res); err != nil {
nreq := &userapi.QueryNumericLocalpartRequest{
ServerName: r.ServerName,
}
nres := &userapi.QueryNumericLocalpartResponse{}
if err = userAPI.QueryNumericLocalpart(req.Context(), nreq, nres); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryNumericLocalpart failed")
return jsonerror.InternalServerError()
}
r.Username = strconv.FormatInt(res.ID, 10)
r.Username = strconv.FormatInt(nres.ID, 10)
}
// Is this an appservice registration? It will be if the access
@ -623,8 +562,8 @@ func Register(
case r.Type == authtypes.LoginTypeApplicationService && accessTokenErr == nil:
// Spec-compliant case (the access_token is specified and the login type
// is correctly set, so it's an appservice registration)
if resErr := validateApplicationServiceUsername(r.Username, cfg.Matrix.ServerName); resErr != nil {
return *resErr
if err = internal.ValidateApplicationServiceUsername(r.Username, r.ServerName); err != nil {
return *internal.UsernameResponse(err)
}
case accessTokenErr == nil:
// Non-spec-compliant case (the access_token is specified but the login
@ -640,12 +579,12 @@ func Register(
default:
// Spec-compliant case (neither the access_token nor the login type are
// specified, so it's a normal user registration)
if resErr := validateUsername(r.Username, cfg.Matrix.ServerName); resErr != nil {
return *resErr
if err = internal.ValidateUsername(r.Username, r.ServerName); err != nil {
return *internal.UsernameResponse(err)
}
}
if resErr := validatePassword(r.Password); resErr != nil {
return *resErr
if err = internal.ValidatePassword(r.Password); err != nil {
return *internal.PasswordResponse(err)
}
logger := util.GetLogger(req.Context())
@ -664,16 +603,25 @@ func handleGuestRegistration(
cfg *config.ClientAPI,
userAPI userapi.ClientUserAPI,
) util.JSONResponse {
if cfg.RegistrationDisabled || cfg.GuestsDisabled {
registrationEnabled := !cfg.RegistrationDisabled
guestsEnabled := !cfg.GuestsDisabled
if v := cfg.Matrix.VirtualHost(r.ServerName); v != nil {
registrationEnabled, guestsEnabled = v.RegistrationAllowed()
}
if !registrationEnabled || !guestsEnabled {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Guest registration is disabled"),
JSON: jsonerror.Forbidden(
fmt.Sprintf("Guest registration is disabled on %q", r.ServerName),
),
}
}
var res userapi.PerformAccountCreationResponse
err := userAPI.PerformAccountCreation(req.Context(), &userapi.PerformAccountCreationRequest{
AccountType: userapi.AccountTypeGuest,
ServerName: r.ServerName,
}, &res)
if err != nil {
return util.JSONResponse{
@ -697,6 +645,7 @@ func handleGuestRegistration(
var devRes userapi.PerformDeviceCreationResponse
err = userAPI.PerformDeviceCreation(req.Context(), &userapi.PerformDeviceCreationRequest{
Localpart: res.Account.Localpart,
ServerName: res.Account.ServerName,
DeviceDisplayName: r.InitialDisplayName,
AccessToken: token,
IPAddr: req.RemoteAddr,
@ -713,7 +662,6 @@ func handleGuestRegistration(
JSON: registerResponse{
UserID: devRes.Device.UserID,
AccessToken: devRes.Device.AccessToken,
HomeServer: res.Account.ServerName,
DeviceID: devRes.Device.ID,
},
}
@ -750,10 +698,16 @@ func handleRegistrationFlow(
)
}
if cfg.RegistrationDisabled && r.Auth.Type != authtypes.LoginTypeSharedSecret {
registrationEnabled := !cfg.RegistrationDisabled
if v := cfg.Matrix.VirtualHost(r.ServerName); v != nil {
registrationEnabled, _ = v.RegistrationAllowed()
}
if !registrationEnabled && r.Auth.Type != authtypes.LoginTypeSharedSecret {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Registration is disabled"),
JSON: jsonerror.Forbidden(
fmt.Sprintf("Registration is disabled on %q", r.ServerName),
),
}
}
@ -772,9 +726,18 @@ func handleRegistrationFlow(
switch r.Auth.Type {
case authtypes.LoginTypeRecaptcha:
// Check given captcha response
resErr := validateRecaptcha(cfg, r.Auth.Response, req.RemoteAddr)
if resErr != nil {
return *resErr
err := validateRecaptcha(cfg, r.Auth.Response, req.RemoteAddr)
switch err {
case ErrCaptchaDisabled:
return util.JSONResponse{Code: http.StatusForbidden, JSON: jsonerror.Unknown(err.Error())}
case ErrMissingResponse:
return util.JSONResponse{Code: http.StatusBadRequest, JSON: jsonerror.BadJSON(err.Error())}
case ErrInvalidCaptcha:
return util.JSONResponse{Code: http.StatusUnauthorized, JSON: jsonerror.BadJSON(err.Error())}
case nil:
default:
util.GetLogger(req.Context()).WithError(err).Error("failed to validate recaptcha")
return util.JSONResponse{Code: http.StatusInternalServerError, JSON: jsonerror.InternalServerError()}
}
// Add Recaptcha to the list of completed registration stages
@ -851,8 +814,9 @@ func handleApplicationServiceRegistration(
// Don't need to worry about appending to registration stages as
// application service registration is entirely separate.
return completeRegistration(
req.Context(), userAPI, r.Username, "", appserviceID, req.RemoteAddr, req.UserAgent(), r.Auth.Session,
r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeAppService,
req.Context(), userAPI, r.Username, r.ServerName, "", "", appserviceID, req.RemoteAddr,
req.UserAgent(), r.Auth.Session, r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
userapi.AccountTypeAppService,
)
}
@ -870,8 +834,9 @@ func checkAndCompleteFlow(
if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
// This flow was completed, registration can continue
return completeRegistration(
req.Context(), userAPI, r.Username, r.Password, "", req.RemoteAddr, req.UserAgent(), sessionID,
r.InhibitLogin, r.InitialDisplayName, r.DeviceID, userapi.AccountTypeUser,
req.Context(), userAPI, r.Username, r.ServerName, "", r.Password, "", req.RemoteAddr,
req.UserAgent(), sessionID, r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
userapi.AccountTypeUser,
)
}
sessions.addParams(sessionID, r)
@ -893,9 +858,10 @@ func checkAndCompleteFlow(
func completeRegistration(
ctx context.Context,
userAPI userapi.ClientUserAPI,
username, password, appserviceID, ipAddr, userAgent, sessionID string,
username string, serverName gomatrixserverlib.ServerName, displayName string,
password, appserviceID, ipAddr, userAgent, sessionID string,
inhibitLogin eventutil.WeakBoolean,
displayName, deviceID *string,
deviceDisplayName, deviceID *string,
accType userapi.AccountType,
) util.JSONResponse {
if username == "" {
@ -915,6 +881,7 @@ func completeRegistration(
err := userAPI.PerformAccountCreation(ctx, &userapi.PerformAccountCreationRequest{
AppServiceID: appserviceID,
Localpart: username,
ServerName: serverName,
Password: password,
AccountType: accType,
OnConflict: userapi.ConflictAbort,
@ -926,6 +893,16 @@ func completeRegistration(
JSON: jsonerror.UserInUse("Desired user ID is already taken."),
}
}
switch e := err.(type) {
case internalHTTPUtil.InternalAPIError:
conflictErr := &userapi.ErrorConflict{Message: sqlutil.ErrUserExists.Error()}
if e.Message == conflictErr.Error() {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.UserInUse("Desired user ID is already taken."),
}
}
}
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: jsonerror.Unknown("failed to create account: " + err.Error()),
@ -941,8 +918,7 @@ func completeRegistration(
return util.JSONResponse{
Code: http.StatusOK,
JSON: registerResponse{
UserID: userutil.MakeUserID(username, accRes.Account.ServerName),
HomeServer: accRes.Account.ServerName,
UserID: userutil.MakeUserID(username, accRes.Account.ServerName),
},
}
}
@ -955,11 +931,28 @@ func completeRegistration(
}
}
if displayName != "" {
nameReq := userapi.PerformUpdateDisplayNameRequest{
Localpart: username,
ServerName: serverName,
DisplayName: displayName,
}
var nameRes userapi.PerformUpdateDisplayNameResponse
err = userAPI.SetDisplayName(ctx, &nameReq, &nameRes)
if err != nil {
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: jsonerror.Unknown("failed to set display name: " + err.Error()),
}
}
}
var devRes userapi.PerformDeviceCreationResponse
err = userAPI.PerformDeviceCreation(ctx, &userapi.PerformDeviceCreationRequest{
Localpart: username,
ServerName: serverName,
AccessToken: token,
DeviceDisplayName: displayName,
DeviceDisplayName: deviceDisplayName,
DeviceID: deviceID,
IPAddr: ipAddr,
UserAgent: userAgent,
@ -974,7 +967,6 @@ func completeRegistration(
result := registerResponse{
UserID: devRes.Device.UserID,
AccessToken: devRes.Device.AccessToken,
HomeServer: accRes.Account.ServerName,
DeviceID: devRes.Device.ID,
}
sessions.addCompletedRegistration(sessionID, result)
@ -1051,13 +1043,31 @@ func RegisterAvailable(
// Squash username to all lowercase letters
username = strings.ToLower(username)
domain := cfg.Matrix.ServerName
host := gomatrixserverlib.ServerName(req.Host)
if v := cfg.Matrix.VirtualHostForHTTPHost(host); v != nil {
domain = v.ServerName
}
if u, l, err := cfg.Matrix.SplitLocalID('@', username); err == nil {
username, domain = u, l
}
for _, v := range cfg.Matrix.VirtualHosts {
if v.ServerName == domain && !v.AllowRegistration {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden(
fmt.Sprintf("Registration is not allowed on %q", string(v.ServerName)),
),
}
}
}
if err := validateUsername(username, cfg.Matrix.ServerName); err != nil {
return *err
if err := internal.ValidateUsername(username, domain); err != nil {
return *internal.UsernameResponse(err)
}
// Check if this username is reserved by an application service
userID := userutil.MakeUserID(username, cfg.Matrix.ServerName)
userID := userutil.MakeUserID(username, domain)
for _, appservice := range cfg.Derived.ApplicationServices {
if appservice.OwnsNamespaceCoveringUserId(userID) {
return util.JSONResponse{
@ -1069,7 +1079,8 @@ func RegisterAvailable(
res := &userapi.QueryAccountAvailabilityResponse{}
err := registerAPI.QueryAccountAvailability(req.Context(), &userapi.QueryAccountAvailabilityRequest{
Localpart: username,
Localpart: username,
ServerName: domain,
}, res)
if err != nil {
return util.JSONResponse{
@ -1114,11 +1125,11 @@ func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.Clien
// downcase capitals
ssrr.User = strings.ToLower(ssrr.User)
if resErr := validateUsername(ssrr.User, cfg.Matrix.ServerName); resErr != nil {
return *resErr
if err = internal.ValidateUsername(ssrr.User, cfg.Matrix.ServerName); err != nil {
return *internal.UsernameResponse(err)
}
if resErr := validatePassword(ssrr.Password); resErr != nil {
return *resErr
if err = internal.ValidatePassword(ssrr.Password); err != nil {
return *internal.PasswordResponse(err)
}
deviceID := "shared_secret_registration"
@ -1126,5 +1137,5 @@ func handleSharedSecretRegistration(cfg *config.ClientAPI, userAPI userapi.Clien
if ssrr.Admin {
accType = userapi.AccountTypeAdmin
}
return completeRegistration(req.Context(), userAPI, ssrr.User, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType)
return completeRegistration(req.Context(), userAPI, ssrr.User, cfg.Matrix.ServerName, ssrr.DisplayName, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), "", false, &ssrr.User, &deviceID, accType)
}

8
clientapi/routing/register_publickey_test.go
View file

@ -29,6 +29,7 @@ import (
"github.com/matrix-org/dendrite/setup/config"
testutil "github.com/matrix-org/dendrite/test"
uapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/stretchr/testify/assert"
)
@ -42,9 +43,9 @@ type registerContext struct {
func createRegisterContext(_ *testing.T) *registerContext {
cfg := &config.ClientAPI{
Matrix: &config.Global{
ServerName: testutil.TestServerName,
},
Matrix: &config.Global{SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: gomatrixserverlib.ServerName("localhost"),
}},
Derived: &config.Derived{},
PasswordAuthenticationDisabled: true,
PublicKeyAuthentication: config.PublicKeyAuthentication{
@ -266,6 +267,7 @@ func TestRegisterEthereum(t *testing.T) {
// Asserts
assert := assert.New(t)
assert.NotNil(response, "response actual: nil, expected: not nil")
fmt.Println(response.JSON)
registerRes := response.JSON.(registerResponse)
assert.Truef(
registerRes.UserID == wallet.Eip155UserId,

13
clientapi/routing/register_secret.go
View file

@ -18,12 +18,13 @@ import (
)
type SharedSecretRegistrationRequest struct {
User string `json:"username"`
Password string `json:"password"`
Nonce string `json:"nonce"`
MacBytes []byte
MacStr string `json:"mac"`
Admin bool `json:"admin"`
User string `json:"username"`
Password string `json:"password"`
Nonce string `json:"nonce"`
MacBytes []byte
MacStr string `json:"mac"`
Admin bool `json:"admin"`
DisplayName string `json:"displayname,omitempty"`
}
func NewSharedSecretRegistrationRequest(reader io.ReadCloser) (*SharedSecretRegistrationRequest, error) {

2
clientapi/routing/register_secret_test.go
View file

@ -10,7 +10,7 @@ import (
func TestSharedSecretRegister(t *testing.T) {
// these values have come from a local synapse instance to ensure compatibility
jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice"}`)
jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`)
sharedSecret := "dendritetest"
req, err := NewSharedSecretRegistrationRequest(io.NopCloser(bytes.NewBuffer(jsonStr)))

399
clientapi/routing/register_test.go
View file

@ -15,12 +15,31 @@
package routing
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"reflect"
"regexp"
"strings"
"testing"
"time"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig"
"github.com/matrix-org/dendrite/userapi"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/util"
"github.com/patrickmn/go-cache"
"github.com/stretchr/testify/assert"
)
var (
@ -264,3 +283,383 @@ func TestSessionCleanUp(t *testing.T) {
}
})
}
func Test_register(t *testing.T) {
testCases := []struct {
name string
kind string
password string
username string
loginType string
forceEmpty bool
registrationDisabled bool
guestsDisabled bool
enableRecaptcha bool
captchaBody string
wantResponse util.JSONResponse
}{
{
name: "disallow guests",
kind: "guest",
guestsDisabled: true,
wantResponse: util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden(`Guest registration is disabled on "test"`),
},
},
{
name: "allow guests",
kind: "guest",
},
{
name: "unknown login type",
loginType: "im.not.known",
wantResponse: util.JSONResponse{
Code: http.StatusNotImplemented,
JSON: jsonerror.Unknown("unknown/unimplemented auth type"),
},
},
{
name: "disabled registration",
registrationDisabled: true,
wantResponse: util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden(`Registration is disabled on "test"`),
},
},
{
name: "successful registration, numeric ID",
username: "",
password: "someRandomPassword",
forceEmpty: true,
},
{
name: "successful registration",
username: "success",
},
{
name: "failing registration - user already exists",
username: "success",
wantResponse: util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.UserInUse("Desired user ID is already taken."),
},
},
{
name: "successful registration uppercase username",
username: "LOWERCASED", // this is going to be lower-cased
},
{
name: "invalid username",
username: "#totalyNotValid",
wantResponse: *internal.UsernameResponse(internal.ErrUsernameInvalid),
},
{
name: "numeric username is forbidden",
username: "1337",
wantResponse: util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.InvalidUsername("Numeric user IDs are reserved"),
},
},
{
name: "disabled recaptcha login",
loginType: authtypes.LoginTypeRecaptcha,
wantResponse: util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Unknown(ErrCaptchaDisabled.Error()),
},
},
{
name: "enabled recaptcha, no response defined",
enableRecaptcha: true,
loginType: authtypes.LoginTypeRecaptcha,
wantResponse: util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON(ErrMissingResponse.Error()),
},
},
{
name: "invalid captcha response",
enableRecaptcha: true,
loginType: authtypes.LoginTypeRecaptcha,
captchaBody: `notvalid`,
wantResponse: util.JSONResponse{
Code: http.StatusUnauthorized,
JSON: jsonerror.BadJSON(ErrInvalidCaptcha.Error()),
},
},
{
name: "valid captcha response",
enableRecaptcha: true,
loginType: authtypes.LoginTypeRecaptcha,
captchaBody: `success`,
},
{
name: "captcha invalid from remote",
enableRecaptcha: true,
loginType: authtypes.LoginTypeRecaptcha,
captchaBody: `i should fail for other reasons`,
wantResponse: util.JSONResponse{Code: http.StatusInternalServerError, JSON: jsonerror.InternalServerError()},
},
}
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
base, baseClose := testrig.CreateBaseDendrite(t, dbType)
defer baseClose()
rsAPI := roomserver.NewInternalAPI(base)
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
keyAPI.SetUserAPI(userAPI)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
if tc.enableRecaptcha {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
t.Fatal(err)
}
response := r.Form.Get("response")
// Respond with valid JSON or no JSON at all to test happy/error cases
switch response {
case "success":
json.NewEncoder(w).Encode(recaptchaResponse{Success: true})
case "notvalid":
json.NewEncoder(w).Encode(recaptchaResponse{Success: false})
default:
}
}))
defer srv.Close()
base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL
}
if err := base.Cfg.Derive(); err != nil {
t.Fatalf("failed to derive config: %s", err)
}
base.Cfg.ClientAPI.RecaptchaEnabled = tc.enableRecaptcha
base.Cfg.ClientAPI.RegistrationDisabled = tc.registrationDisabled
base.Cfg.ClientAPI.GuestsDisabled = tc.guestsDisabled
if tc.kind == "" {
tc.kind = "user"
}
if tc.password == "" && !tc.forceEmpty {
tc.password = "someRandomPassword"
}
if tc.username == "" && !tc.forceEmpty {
tc.username = "valid"
}
if tc.loginType == "" {
tc.loginType = "m.login.dummy"
}
reg := registerRequest{
Password: tc.password,
Username: tc.username,
}
body := &bytes.Buffer{}
err := json.NewEncoder(body).Encode(reg)
if err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/?kind=%s", tc.kind), body)
resp := Register(req, userAPI, &base.Cfg.ClientAPI)
t.Logf("Resp: %+v", resp)
// The first request should return a userInteractiveResponse
switch r := resp.JSON.(type) {
case UserInteractiveResponse:
// Check that the flows are the ones we configured
if !reflect.DeepEqual(r.Flows, base.Cfg.Derived.Registration.Flows) {
t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, base.Cfg.Derived.Registration.Flows)
}
case *jsonerror.MatrixError:
if !reflect.DeepEqual(tc.wantResponse, resp) {
t.Fatalf("(%s), unexpected response: %+v, want: %+v", tc.name, resp, tc.wantResponse)
}
return
case registerResponse:
// this should only be possible on guest user registration, never for normal users
if tc.kind != "guest" {
t.Fatalf("got register response on first request: %+v", r)
}
// assert we've got a UserID, AccessToken and DeviceID
if r.UserID == "" {
t.Fatalf("missing userID in response")
}
if r.AccessToken == "" {
t.Fatalf("missing accessToken in response")
}
if r.DeviceID == "" {
t.Fatalf("missing deviceID in response")
}
return
default:
t.Logf("Got response: %T", resp.JSON)
}
// If we reached this, we should have received a UIA response
uia, ok := resp.JSON.(UserInteractiveResponse)
if !ok {
t.Fatalf("did not receive a userInteractiveResponse: %T", resp.JSON)
}
t.Logf("%+v", uia)
// Register the user
reg.Auth = authDict{
Type: authtypes.LoginType(tc.loginType),
Session: uia.Session,
}
if tc.captchaBody != "" {
reg.Auth.Response = tc.captchaBody
}
dummy := "dummy"
reg.DeviceID = &dummy
reg.InitialDisplayName = &dummy
reg.Type = authtypes.LoginType(tc.loginType)
err = json.NewEncoder(body).Encode(reg)
if err != nil {
t.Fatal(err)
}
req = httptest.NewRequest(http.MethodPost, "/", body)
resp = Register(req, userAPI, &base.Cfg.ClientAPI)
switch resp.JSON.(type) {
case *jsonerror.MatrixError:
if !reflect.DeepEqual(tc.wantResponse, resp) {
t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
}
return
case util.JSONResponse:
if !reflect.DeepEqual(tc.wantResponse, resp) {
t.Fatalf("unexpected response: %+v, want: %+v", resp, tc.wantResponse)
}
return
}
rr, ok := resp.JSON.(registerResponse)
if !ok {
t.Fatalf("expected a registerresponse, got %T", resp.JSON)
}
// validate the response
if tc.forceEmpty {
// when not supplying a username, one will be generated. Given this _SHOULD_ be
// the second user, set the username accordingly
reg.Username = "2"
}
wantUserID := strings.ToLower(fmt.Sprintf("@%s:%s", reg.Username, "test"))
if wantUserID != rr.UserID {
t.Fatalf("unexpected userID: %s, want %s", rr.UserID, wantUserID)
}
if rr.DeviceID != *reg.DeviceID {
t.Fatalf("unexpected deviceID: %s, want %s", rr.DeviceID, *reg.DeviceID)
}
if rr.AccessToken == "" {
t.Fatalf("missing accessToken in response")
}
})
}
})
}
func TestRegisterUserWithDisplayName(t *testing.T) {
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
base, baseClose := testrig.CreateBaseDendrite(t, dbType)
defer baseClose()
base.Cfg.Global.ServerName = "server"
rsAPI := roomserver.NewInternalAPI(base)
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
keyAPI.SetUserAPI(userAPI)
deviceName, deviceID := "deviceName", "deviceID"
expectedDisplayName := "DisplayName"
response := completeRegistration(
base.Context(),
userAPI,
"user",
"server",
expectedDisplayName,
"password",
"",
"localhost",
"user agent",
"session",
false,
&deviceName,
&deviceID,
api.AccountTypeAdmin,
)
assert.Equal(t, http.StatusOK, response.Code)
req := api.QueryProfileRequest{UserID: "@user:server"}
var res api.QueryProfileResponse
err := userAPI.QueryProfile(base.Context(), &req, &res)
assert.NoError(t, err)
assert.Equal(t, expectedDisplayName, res.DisplayName)
})
}
func TestRegisterAdminUsingSharedSecret(t *testing.T) {
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
base, baseClose := testrig.CreateBaseDendrite(t, dbType)
defer baseClose()
base.Cfg.Global.ServerName = "server"
sharedSecret := "dendritetest"
base.Cfg.ClientAPI.RegistrationSharedSecret = sharedSecret
rsAPI := roomserver.NewInternalAPI(base)
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, nil, rsAPI)
userAPI := userapi.NewInternalAPI(base, &base.Cfg.UserAPI, nil, keyAPI, rsAPI, nil)
keyAPI.SetUserAPI(userAPI)
expectedDisplayName := "rabbit"
jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`)
req, err := NewSharedSecretRegistrationRequest(io.NopCloser(bytes.NewBuffer(jsonStr)))
assert.NoError(t, err)
if err != nil {
t.Fatalf("failed to read request: %s", err)
}
r := NewSharedSecretRegistration(sharedSecret)
// force the nonce to be known
r.nonces.Set(req.Nonce, true, cache.DefaultExpiration)
_, err = r.IsValidMacLogin(req.Nonce, req.User, req.Password, req.Admin, req.MacBytes)
assert.NoError(t, err)
body := &bytes.Buffer{}
err = json.NewEncoder(body).Encode(req)
assert.NoError(t, err)
ssrr := httptest.NewRequest(http.MethodPost, "/", body)
response := handleSharedSecretRegistration(
&base.Cfg.ClientAPI,
userAPI,
r,
ssrr,
)
assert.Equal(t, http.StatusOK, response.Code)
profilReq := api.QueryProfileRequest{UserID: "@alice:server"}
var profileRes api.QueryProfileResponse
err = userAPI.QueryProfile(base.Context(), &profilReq, &profileRes)
assert.NoError(t, err)
assert.Equal(t, expectedDisplayName, profileRes.DisplayName)
})
}

119
clientapi/routing/routing.go
View file

@ -18,8 +18,10 @@ import (
"context"
"net/http"
"strings"
"sync"
"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/nats-io/nats.go"
@ -53,7 +55,7 @@ var ReleaseVersion string
// applied:
// nolint: gocyclo
func Setup(
publicAPIMux, wkMux, synapseAdminRouter, dendriteAdminRouter *mux.Router,
base *base.BaseDendrite,
cfg *config.ClientAPI,
rsAPI roomserverAPI.ClientRoomserverAPI,
asAPI appserviceAPI.AppServiceInternalAPI,
@ -72,7 +74,14 @@ func Setup(
"ReleaseVersion": ReleaseVersion,
}).Info("Started clientAPI router with ReleaseVersion")
prometheus.MustRegister(amtRegUsers, sendEventDuration)
publicAPIMux := base.PublicClientAPIMux
wkMux := base.PublicWellKnownAPIMux
synapseAdminRouter := base.SynapseAdminMux
dendriteAdminRouter := base.DendriteAdminMux
if base.EnableMetrics {
prometheus.MustRegister(amtRegUsers, sendEventDuration)
}
rateLimits := httputil.NewRateLimits(&cfg.RateLimiting)
userInteractiveAuth := auth.NewUserInteractive(userAPI, userAPI, cfg)
@ -168,7 +177,13 @@ func Setup(
}),
).Methods(http.MethodGet, http.MethodOptions)
dendriteAdminRouter.Handle("/admin/resetPassword/{localpart}",
dendriteAdminRouter.Handle("/admin/purgeRoom/{roomID}",
httputil.MakeAdminAPI("admin_purge_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return AdminPurgeRoom(req, cfg, device, rsAPI)
}),
).Methods(http.MethodPost, http.MethodOptions)
dendriteAdminRouter.Handle("/admin/resetPassword/{userID}",
httputil.MakeAdminAPI("admin_reset_password", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return AdminResetPassword(req, cfg, device, userAPI)
}),
@ -195,18 +210,24 @@ func Setup(
// server notifications
if cfg.Matrix.ServerNotices.Enabled {
logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice")
serverNotificationSender, err := getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
if err != nil {
logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
}
var serverNotificationSender *userapi.Device
var err error
notificationSenderOnce := &sync.Once{}
synapseAdminRouter.Handle("/admin/v1/send_server_notice/{txnID}",
httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
notificationSenderOnce.Do(func() {
serverNotificationSender, err = getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
if err != nil {
logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
}
})
// not specced, but ensure we're rate limiting requests to this endpoint
if r := rateLimits.Limit(req, device); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
var vars map[string]string
vars, err = httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
@ -222,6 +243,12 @@ func Setup(
synapseAdminRouter.Handle("/admin/v1/send_server_notice",
httputil.MakeAuthAPI("send_server_notice", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
notificationSenderOnce.Do(func() {
serverNotificationSender, err = getSenderDevice(context.Background(), rsAPI, userAPI, cfg)
if err != nil {
logrus.WithError(err).Fatal("unable to get account for sending sending server notices")
}
})
// not specced, but ensure we're rate limiting requests to this endpoint
if r := rateLimits.Limit(req, device); r != nil {
return *r
@ -261,13 +288,13 @@ func Setup(
return util.ErrorResponse(err)
}
isAllowed, err := authorization.IsAllowed(authz.AuthorizationArgs{
isAllowed, _ := authorization.IsAllowed(authz.AuthorizationArgs{
RoomId: vars["roomIDOrAlias"],
UserId: device.UserID,
Permission: authz.PermissionRead,
})
if !isAllowed || err != nil {
if !isAllowed {
return util.JSONResponse{
Code: http.StatusUnauthorized,
JSON: jsonerror.Forbidden("Unauthorised"),
@ -277,7 +304,7 @@ func Setup(
return JoinRoomByIDOrAlias(
req, device, rsAPI, userAPI, vars["roomIDOrAlias"],
)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPost, http.MethodOptions)
if mscCfg.Enabled("msc2753") {
@ -299,7 +326,7 @@ func Setup(
v3mux.Handle("/joined_rooms",
httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return GetJoinedRooms(req, device, rsAPI)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/join",
httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@ -313,7 +340,7 @@ func Setup(
return JoinRoomByIDOrAlias(
req, device, rsAPI, userAPI, vars["roomID"],
)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/leave",
httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@ -327,7 +354,7 @@ func Setup(
return LeaveRoomByID(
req, device, rsAPI, vars["roomID"],
)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/unpeek",
httputil.MakeAuthAPI("unpeek", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@ -414,7 +441,7 @@ func Setup(
return util.ErrorResponse(err)
}
return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, rsAPI, nil)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}",
httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@ -439,7 +466,7 @@ func Setup(
txnID := vars["txnID"]
return SendEvent(req, device, vars["roomID"], vars["eventType"], &txnID,
nil, cfg, rsAPI, transactionsCache)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPut, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@ -448,7 +475,7 @@ func Setup(
return util.ErrorResponse(err)
}
return OnIncomingStateRequest(req.Context(), device, rsAPI, vars["roomID"])
})).Methods(http.MethodGet, http.MethodOptions)
}, httputil.WithAllowGuests())).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/aliases", httputil.MakeAuthAPI("aliases", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -467,7 +494,7 @@ func Setup(
eventType := strings.TrimSuffix(vars["type"], "/")
eventFormat := req.URL.Query().Get("format") == "event"
return OnIncomingStateTypeRequest(req.Context(), device, rsAPI, vars["roomID"], eventType, "", eventFormat)
})).Methods(http.MethodGet, http.MethodOptions)
}, httputil.WithAllowGuests())).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
@ -476,7 +503,7 @@ func Setup(
}
eventFormat := req.URL.Query().Get("format") == "event"
return OnIncomingStateTypeRequest(req.Context(), device, rsAPI, vars["roomID"], vars["type"], vars["stateKey"], eventFormat)
})).Methods(http.MethodGet, http.MethodOptions)
}, httputil.WithAllowGuests())).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}",
httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@ -487,7 +514,7 @@ func Setup(
emptyString := ""
eventType := strings.TrimSuffix(vars["eventType"], "/")
return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, rsAPI, nil)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPut, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}",
@ -498,7 +525,7 @@ func Setup(
}
stateKey := vars["stateKey"]
return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, rsAPI, nil)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPut, http.MethodOptions)
v3mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse {
@ -641,7 +668,7 @@ func Setup(
}
txnID := vars["txnID"]
return SendToDevice(req, device, syncProducer, transactionsCache, vars["eventType"], &txnID)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPut, http.MethodOptions)
// This is only here because sytest refers to /unstable for this endpoint
@ -655,7 +682,7 @@ func Setup(
}
txnID := vars["txnID"]
return SendToDevice(req, device, syncProducer, transactionsCache, vars["eventType"], &txnID)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPut, http.MethodOptions)
v3mux.Handle("/account/whoami",
@ -664,7 +691,7 @@ func Setup(
return *r
}
return Whoami(req, device)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/account/password",
@ -697,9 +724,9 @@ func Setup(
).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
v3mux.Handle("/auth/{authType}/fallback/web",
httputil.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse {
httputil.MakeHTMLAPI("auth_fallback", base.EnableMetrics, func(w http.ResponseWriter, req *http.Request) {
vars := mux.Vars(req)
return AuthFallback(w, req, vars["authType"], cfg)
AuthFallback(w, req, vars["authType"], cfg)
}),
).Methods(http.MethodGet, http.MethodPost, http.MethodOptions)
@ -896,7 +923,7 @@ func Setup(
return util.ErrorResponse(err)
}
return SetDisplayName(req, userAPI, device, vars["userID"], cfg, rsAPI)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPut, http.MethodOptions)
// Browsers use the OPTIONS HTTP method to check if the CORS policy allows
// PUT requests, so we need to allow this method
@ -937,7 +964,7 @@ func Setup(
v3mux.Handle("/thirdparty/protocols",
httputil.MakeAuthAPI("thirdparty_protocols", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return Protocols(req, asAPI, device, "")
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/thirdparty/protocol/{protocolID}",
@ -947,7 +974,7 @@ func Setup(
return util.ErrorResponse(err)
}
return Protocols(req, asAPI, device, vars["protocolID"])
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/thirdparty/user/{protocolID}",
@ -957,13 +984,13 @@ func Setup(
return util.ErrorResponse(err)
}
return User(req, asAPI, device, vars["protocolID"], req.URL.Query())
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/thirdparty/user",
httputil.MakeAuthAPI("thirdparty_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return User(req, asAPI, device, "", req.URL.Query())
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/thirdparty/location/{protocolID}",
@ -973,13 +1000,13 @@ func Setup(
return util.ErrorResponse(err)
}
return Location(req, asAPI, device, vars["protocolID"], req.URL.Query())
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/thirdparty/location",
httputil.MakeAuthAPI("thirdparty_location", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return Location(req, asAPI, device, "", req.URL.Query())
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/initialSync",
@ -1120,7 +1147,7 @@ func Setup(
v3mux.Handle("/devices",
httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return GetDevicesByLocalpart(req, userAPI, device)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/devices/{deviceID}",
@ -1130,7 +1157,7 @@ func Setup(
return util.ErrorResponse(err)
}
return GetDeviceByID(req, userAPI, device, vars["deviceID"])
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/devices/{deviceID}",
@ -1140,7 +1167,7 @@ func Setup(
return util.ErrorResponse(err)
}
return UpdateDeviceByID(req, userAPI, device, vars["deviceID"])
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPut, http.MethodOptions)
v3mux.Handle("/devices/{deviceID}",
@ -1182,21 +1209,21 @@ func Setup(
// Stub implementations for sytest
v3mux.Handle("/events",
httputil.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse {
httputil.MakeAuthAPI("events", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{
"chunk": []interface{}{},
"start": "",
"end": "",
}}
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/initialSync",
httputil.MakeExternalAPI("initial_sync", func(req *http.Request) util.JSONResponse {
httputil.MakeAuthAPI("initial_sync", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{
"end": "",
}}
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/user/{userId}/rooms/{roomId}/tags",
@ -1235,7 +1262,7 @@ func Setup(
return *r
}
return GetCapabilities(req, rsAPI)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
// Key Backup Versions (Metadata)
@ -1416,7 +1443,7 @@ func Setup(
postDeviceSigningSignatures := httputil.MakeAuthAPI("post_device_signing_signatures", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return UploadCrossSigningDeviceSignatures(req, keyAPI, device)
})
}, httputil.WithAllowGuests())
v3mux.Handle("/keys/device_signing/upload", postDeviceSigningKeys).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/keys/signatures/upload", postDeviceSigningSignatures).Methods(http.MethodPost, http.MethodOptions)
@ -1428,22 +1455,22 @@ func Setup(
v3mux.Handle("/keys/upload/{deviceID}",
httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return UploadKeys(req, keyAPI, device)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/keys/upload",
httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return UploadKeys(req, keyAPI, device)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/keys/query",
httputil.MakeAuthAPI("keys_query", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return QueryKeys(req, keyAPI, device)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/keys/claim",
httputil.MakeAuthAPI("keys_claim", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return ClaimKeys(req, keyAPI)
}),
}, httputil.WithAllowGuests()),
).Methods(http.MethodPost, http.MethodOptions)
v3mux.Handle("/rooms/{roomId}/receipt/{receiptType}/{eventId}",
httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {

9
clientapi/routing/sendevent.go
View file

@ -186,6 +186,7 @@ func SendEvent(
[]*gomatrixserverlib.HeaderedEvent{
e.Headered(verRes.RoomVersion),
},
device.UserDomain(),
domain,
domain,
txnAndSessionID,
@ -275,8 +276,14 @@ func generateSendEvent(
return nil, &resErr
}
identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
if err != nil {
resErr := jsonerror.InternalServerError()
return nil, &resErr
}
var queryRes api.QueryLatestEventsAndStateResponse
e, err := eventutil.QueryAndBuildEvent(ctx, &builder, cfg.Matrix, evTime, rsAPI, &queryRes)
e, err := eventutil.QueryAndBuildEvent(ctx, &builder, cfg.Matrix, identity, evTime, rsAPI, &queryRes)
if err == eventutil.ErrRoomNoExists {
return nil, &util.JSONResponse{
Code: http.StatusNotFound,

9
clientapi/routing/server_notices.go
View file

@ -231,6 +231,7 @@ func SendServerNotice(
[]*gomatrixserverlib.HeaderedEvent{
e.Headered(roomVersion),
},
device.UserDomain(),
cfgClient.Matrix.ServerName,
cfgClient.Matrix.ServerName,
txnAndSessionID,
@ -286,6 +287,7 @@ func getSenderDevice(
err := userAPI.PerformAccountCreation(ctx, &userapi.PerformAccountCreationRequest{
AccountType: userapi.AccountTypeUser,
Localpart: cfg.Matrix.ServerNotices.LocalPart,
ServerName: cfg.Matrix.ServerName,
OnConflict: userapi.ConflictUpdate,
}, &accRes)
if err != nil {
@ -295,8 +297,9 @@ func getSenderDevice(
// Set the avatarurl for the user
avatarRes := &userapi.PerformSetAvatarURLResponse{}
if err = userAPI.SetAvatarURL(ctx, &userapi.PerformSetAvatarURLRequest{
Localpart: cfg.Matrix.ServerNotices.LocalPart,
AvatarURL: cfg.Matrix.ServerNotices.AvatarURL,
Localpart: cfg.Matrix.ServerNotices.LocalPart,
ServerName: cfg.Matrix.ServerName,
AvatarURL: cfg.Matrix.ServerNotices.AvatarURL,
}, avatarRes); err != nil {
util.GetLogger(ctx).WithError(err).Error("userAPI.SetAvatarURL failed")
return nil, err
@ -308,6 +311,7 @@ func getSenderDevice(
displayNameRes := &userapi.PerformUpdateDisplayNameResponse{}
if err = userAPI.SetDisplayName(ctx, &userapi.PerformUpdateDisplayNameRequest{
Localpart: cfg.Matrix.ServerNotices.LocalPart,
ServerName: cfg.Matrix.ServerName,
DisplayName: cfg.Matrix.ServerNotices.DisplayName,
}, displayNameRes); err != nil {
util.GetLogger(ctx).WithError(err).Error("userAPI.SetDisplayName failed")
@ -353,6 +357,7 @@ func getSenderDevice(
var devRes userapi.PerformDeviceCreationResponse
err = userAPI.PerformDeviceCreation(ctx, &userapi.PerformDeviceCreationRequest{
Localpart: cfg.Matrix.ServerNotices.LocalPart,
ServerName: cfg.Matrix.ServerName,
DeviceDisplayName: &cfg.Matrix.ServerNotices.LocalPart,
AccessToken: token,
NoDeviceListUpdate: true,

10
clientapi/routing/thirdparty.go
View file

@ -36,9 +36,15 @@ func Protocols(req *http.Request, asAPI appserviceAPI.AppServiceInternalAPI, dev
return jsonerror.InternalServerError()
}
if !resp.Exists {
if protocol != "" {
return util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound("The protocol is unknown."),
}
}
return util.JSONResponse{
Code: http.StatusNotFound,
JSON: jsonerror.NotFound("The protocol is unknown."),
Code: http.StatusOK,
JSON: struct{}{},
}
}
if protocol != "" {

14
clientapi/routing/threepid.go
View file

@ -136,16 +136,17 @@ func CheckAndSave3PIDAssociation(
}
// Save the association in the database
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
return jsonerror.InternalServerError()
}
if err = threePIDAPI.PerformSaveThreePIDAssociation(req.Context(), &api.PerformSaveThreePIDAssociationRequest{
ThreePID: address,
Localpart: localpart,
Medium: medium,
ThreePID: address,
Localpart: localpart,
ServerName: domain,
Medium: medium,
}, &struct{}{}); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("threePIDAPI.PerformSaveThreePIDAssociation failed")
return jsonerror.InternalServerError()
@ -161,7 +162,7 @@ func CheckAndSave3PIDAssociation(
func GetAssociated3PIDs(
req *http.Request, threepidAPI api.ClientUserAPI, device *api.Device,
) util.JSONResponse {
localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
localpart, domain, err := gomatrixserverlib.SplitID('@', device.UserID)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
return jsonerror.InternalServerError()
@ -169,7 +170,8 @@ func GetAssociated3PIDs(
res := &api.QueryThreePIDsForLocalpartResponse{}
err = threepidAPI.QueryThreePIDsForLocalpart(req.Context(), &api.QueryThreePIDsForLocalpartRequest{
Localpart: localpart,
Localpart: localpart,
ServerName: domain,
}, res)
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("threepidAPI.QueryThreePIDsForLocalpart failed")

2
clientapi/routing/userdirectory.go
View file

@ -106,7 +106,7 @@ knownUsersLoop:
continue
}
// TODO: We should probably cache/store this
fedProfile, fedErr := federation.LookupProfile(ctx, serverName, userID, "")
fedProfile, fedErr := federation.LookupProfile(ctx, localServerName, serverName, userID, "")
if fedErr != nil {
if x, ok := fedErr.(gomatrix.HTTPError); ok {
if x.Code == http.StatusNotFound {

8
clientapi/threepid/invites.go
View file

@ -359,8 +359,13 @@ func emit3PIDInviteEvent(
return err
}
identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
if err != nil {
return err
}
queryRes := api.QueryLatestEventsAndStateResponse{}
event, err := eventutil.QueryAndBuildEvent(ctx, builder, cfg.Matrix, evTime, rsAPI, &queryRes)
event, err := eventutil.QueryAndBuildEvent(ctx, builder, cfg.Matrix, identity, evTime, rsAPI, &queryRes)
if err != nil {
return err
}
@ -371,6 +376,7 @@ func emit3PIDInviteEvent(
[]*gomatrixserverlib.HeaderedEvent{
event.Headered(queryRes.RoomVersion),
},
device.UserDomain(),
cfg.Matrix.ServerName,
cfg.Matrix.ServerName,
nil,

16
clientapi/userutil/userutil_test.go
View file

@ -30,7 +30,9 @@ var (
// TestGoodUserID checks that correct localpart is returned for a valid user ID.
func TestGoodUserID(t *testing.T) {
cfg := &config.Global{
ServerName: serverName,
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: serverName,
},
}
lp, _, err := ParseUsernameParam(goodUserID, cfg)
@ -47,7 +49,9 @@ func TestGoodUserID(t *testing.T) {
// TestWithLocalpartOnly checks that localpart is returned when usernameParam contains only localpart.
func TestWithLocalpartOnly(t *testing.T) {
cfg := &config.Global{
ServerName: serverName,
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: serverName,
},
}
lp, _, err := ParseUsernameParam(localpart, cfg)
@ -64,7 +68,9 @@ func TestWithLocalpartOnly(t *testing.T) {
// TestIncorrectDomain checks for error when there's server name mismatch.
func TestIncorrectDomain(t *testing.T) {
cfg := &config.Global{
ServerName: invalidServerName,
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: invalidServerName,
},
}
_, _, err := ParseUsernameParam(goodUserID, cfg)
@ -77,7 +83,9 @@ func TestIncorrectDomain(t *testing.T) {
// TestBadUserID checks that ParseUsernameParam fails for invalid user ID
func TestBadUserID(t *testing.T) {
cfg := &config.Global{
ServerName: serverName,
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: serverName,
},
}
_, _, err := ParseUsernameParam(badUserID, cfg)

34
cmd/create-account/main.go
View file

@ -25,10 +25,10 @@ import (
"io"
"net/http"
"os"
"regexp"
"strings"
"time"
"github.com/matrix-org/dendrite/internal"
"github.com/tidwall/gjson"
"github.com/sirupsen/logrus"
@ -58,15 +58,14 @@ Arguments:
`
var (
username = flag.String("username", "", "The username of the account to register (specify the localpart only, e.g. 'alice' for '@alice:domain.com')")
password = flag.String("password", "", "The password to associate with the account")
pwdFile = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)")
pwdStdin = flag.Bool("passwordstdin", false, "Reads the password from stdin")
isAdmin = flag.Bool("admin", false, "Create an admin account")
resetPassword = flag.Bool("reset-password", false, "Deprecated")
serverURL = flag.String("url", "http://localhost:8008", "The URL to connect to.")
validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
timeout = flag.Duration("timeout", time.Second*30, "Timeout for the http client when connecting to the server")
username = flag.String("username", "", "The username of the account to register (specify the localpart only, e.g. 'alice' for '@alice:domain.com')")
password = flag.String("password", "", "The password to associate with the account")
pwdFile = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)")
pwdStdin = flag.Bool("passwordstdin", false, "Reads the password from stdin")
isAdmin = flag.Bool("admin", false, "Create an admin account")
resetPassword = flag.Bool("reset-password", false, "Deprecated")
serverURL = flag.String("url", "http://localhost:8008", "The URL to connect to.")
timeout = flag.Duration("timeout", time.Second*30, "Timeout for the http client when connecting to the server")
)
var cl = http.Client{
@ -95,20 +94,21 @@ func main() {
os.Exit(1)
}
if !validUsernameRegex.MatchString(*username) {
logrus.Warn("Username can only contain characters a-z, 0-9, or '_-./='")
if err := internal.ValidateUsername(*username, cfg.Global.ServerName); err != nil {
logrus.WithError(err).Error("Specified username is invalid")
os.Exit(1)
}
if len(fmt.Sprintf("@%s:%s", *username, cfg.Global.ServerName)) > 255 {
logrus.Fatalf("Username can not be longer than 255 characters: %s", fmt.Sprintf("@%s:%s", *username, cfg.Global.ServerName))
}
pass, err := getPassword(*password, *pwdFile, *pwdStdin, os.Stdin)
if err != nil {
logrus.Fatalln(err)
}
if err = internal.ValidatePassword(pass); err != nil {
logrus.WithError(err).Error("Specified password is invalid")
os.Exit(1)
}
cl.Timeout = *timeout
accessToken, err := sharedSecretRegister(cfg.ClientAPI.RegistrationSharedSecret, *serverURL, *username, pass, *isAdmin)
@ -177,7 +177,7 @@ func sharedSecretRegister(sharedSecret, serverURL, localpart, password string, a
defer regResp.Body.Close() // nolint: errcheck
if regResp.StatusCode < 200 || regResp.StatusCode >= 300 {
body, _ = io.ReadAll(regResp.Body)
return "", fmt.Errorf(gjson.GetBytes(body, "error").Str)
return "", fmt.Errorf("got HTTP %d error from server: %s", regResp.StatusCode, string(body))
}
r, err := io.ReadAll(regResp.Body)
if err != nil {

39
cmd/dendrite-demo-pinecone/README.md
View file

@ -24,3 +24,42 @@ Then point your favourite Matrix client to the homeserver URL`http://localhost:
If your peering connection is operational then you should see a `Connected TCP:` line in the log output. If not then try a different peer.
Once logged in, you should be able to open the room directory or join a room by its ID.
## Store & Forward Relays
To test out the store & forward relay functionality, you need a minimum of 3 instances.
One instance will act as the relay, and the other two instances will be the users trying to communicate.
Then you can send messages between the two nodes and watch as the relay is used if the receiving node is offline.
### Launching the Nodes
Relay Server:
```
go run cmd/dendrite-demo-pinecone/main.go -dir relay/ -listen "[::]:49000"
```
Node 1:
```
go run cmd/dendrite-demo-pinecone/main.go -dir node-1/ -peer "[::]:49000" -port 8007
```
Node 2:
```
go run cmd/dendrite-demo-pinecone/main.go -dir node-2/ -peer "[::]:49000" -port 8009
```
### Database Setup
At the moment, the database must be manually configured.
For both `Node 1` and `Node 2` add the following entries to their respective `relay_server` table in the federationapi database:
```
server_name: {node_1_public_key}, relay_server_name: {relay_public_key}
server_name: {node_2_public_key}, relay_server_name: {relay_public_key}
```
After editing the database you will need to relaunch the nodes for the changes to be picked up by dendrite.
### Testing
Now you can run two separate instances of element and connect them to `Node 1` and `Node 2`.
You can shutdown one of the nodes and continue sending messages. If you wait long enough, the message will be sent to the relay server. (you can see this in the log output of the relay server)

84
cmd/dendrite-demo-pinecone/conduit/conduit.go Normal file
View file

@ -0,0 +1,84 @@
// Copyright 2023 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package conduit
import (
"io"
"net"
"sync"
"github.com/matrix-org/pinecone/types"
"go.uber.org/atomic"
)
type Conduit struct {
closed atomic.Bool
conn net.Conn
portMutex sync.Mutex
port types.SwitchPortID
}
func NewConduit(conn net.Conn, port int) Conduit {
return Conduit{
conn: conn,
port: types.SwitchPortID(port),
}
}
func (c *Conduit) Port() int {
c.portMutex.Lock()
defer c.portMutex.Unlock()
return int(c.port)
}
func (c *Conduit) SetPort(port types.SwitchPortID) {
c.portMutex.Lock()
defer c.portMutex.Unlock()
c.port = port
}
func (c *Conduit) Read(b []byte) (int, error) {
if c.closed.Load() {
return 0, io.EOF
}
return c.conn.Read(b)
}
func (c *Conduit) ReadCopy() ([]byte, error) {
if c.closed.Load() {
return nil, io.EOF
}
var buf [65535 * 2]byte
n, err := c.conn.Read(buf[:])
if err != nil {
return nil, err
}
return buf[:n], nil
}
func (c *Conduit) Write(b []byte) (int, error) {
if c.closed.Load() {
return 0, io.EOF
}
return c.conn.Write(b)
}
func (c *Conduit) Close() error {
if c.closed.Load() {
return io.ErrClosedPipe
}
c.closed.Store(true)
return c.conn.Close()
}

121
cmd/dendrite-demo-pinecone/conduit/conduit_test.go Normal file
View file

@ -0,0 +1,121 @@
// Copyright 2023 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package conduit
import (
"fmt"
"net"
"testing"
"github.com/stretchr/testify/assert"
)
var TestBuf = []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
type TestNetConn struct {
net.Conn
shouldFail bool
}
func (t *TestNetConn) Read(b []byte) (int, error) {
if t.shouldFail {
return 0, fmt.Errorf("Failed")
} else {
n := copy(b, TestBuf)
return n, nil
}
}
func (t *TestNetConn) Write(b []byte) (int, error) {
if t.shouldFail {
return 0, fmt.Errorf("Failed")
} else {
return len(b), nil
}
}
func (t *TestNetConn) Close() error {
if t.shouldFail {
return fmt.Errorf("Failed")
} else {
return nil
}
}
func TestConduitStoresPort(t *testing.T) {
conduit := Conduit{port: 7}
assert.Equal(t, 7, conduit.Port())
}
func TestConduitRead(t *testing.T) {
conduit := Conduit{conn: &TestNetConn{}}
b := make([]byte, len(TestBuf))
bytes, err := conduit.Read(b)
assert.NoError(t, err)
assert.Equal(t, len(TestBuf), bytes)
assert.Equal(t, TestBuf, b)
}
func TestConduitReadCopy(t *testing.T) {
conduit := Conduit{conn: &TestNetConn{}}
result, err := conduit.ReadCopy()
assert.NoError(t, err)
assert.Equal(t, TestBuf, result)
}
func TestConduitWrite(t *testing.T) {
conduit := Conduit{conn: &TestNetConn{}}
bytes, err := conduit.Write(TestBuf)
assert.NoError(t, err)
assert.Equal(t, len(TestBuf), bytes)
}
func TestConduitClose(t *testing.T) {
conduit := Conduit{conn: &TestNetConn{}}
err := conduit.Close()
assert.NoError(t, err)
assert.True(t, conduit.closed.Load())
}
func TestConduitReadClosed(t *testing.T) {
conduit := Conduit{conn: &TestNetConn{}}
err := conduit.Close()
assert.NoError(t, err)
b := make([]byte, len(TestBuf))
_, err = conduit.Read(b)
assert.Error(t, err)
}
func TestConduitReadCopyClosed(t *testing.T) {
conduit := Conduit{conn: &TestNetConn{}}
err := conduit.Close()
assert.NoError(t, err)
_, err = conduit.ReadCopy()
assert.Error(t, err)
}
func TestConduitWriteClosed(t *testing.T) {
conduit := Conduit{conn: &TestNetConn{}}
err := conduit.Close()
assert.NoError(t, err)
_, err = conduit.Write(TestBuf)
assert.Error(t, err)
}
func TestConduitReadCopyFails(t *testing.T) {
conduit := Conduit{conn: &TestNetConn{shouldFail: true}}
_, err := conduit.ReadCopy()
assert.Error(t, err)
}

4
cmd/dendrite-demo-pinecone/conn/client.go
View file

@ -101,9 +101,7 @@ func CreateFederationClient(
base *base.BaseDendrite, s *pineconeSessions.Sessions,
) *gomatrixserverlib.FederationClient {
return gomatrixserverlib.NewFederationClient(
base.Cfg.Global.ServerName,
base.Cfg.Global.KeyID,
base.Cfg.Global.PrivateKey,
base.Cfg.Global.SigningIdentities(),
gomatrixserverlib.WithTransport(createTransport(s)),
)
}

222
cmd/dendrite-demo-pinecone/main.go
View file

@ -15,56 +15,35 @@
package main
import (
"context"
"crypto/ed25519"
"crypto/tls"
"encoding/hex"
"flag"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/gorilla/mux"
"github.com/gorilla/websocket"
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/embed"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/monolith"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/userapi"
"github.com/matrix-org/gomatrixserverlib"
pineconeConnections "github.com/matrix-org/pinecone/connections"
pineconeMulticast "github.com/matrix-org/pinecone/multicast"
pineconeRouter "github.com/matrix-org/pinecone/router"
pineconeSessions "github.com/matrix-org/pinecone/sessions"
"github.com/sirupsen/logrus"
pineconeRouter "github.com/matrix-org/pinecone/router"
)
var (
instanceName = flag.String("name", "dendrite-p2p-pinecone", "the name of this P2P demo instance")
instancePort = flag.Int("port", 8008, "the port that the client API will listen on")
instancePeer = flag.String("peer", "", "the static Pinecone peers to connect to, comma separated-list")
instanceListen = flag.String("listen", ":0", "the port Pinecone peers can connect to")
instanceDir = flag.String("dir", ".", "the directory to store the databases in (if --config not specified)")
instanceName = flag.String("name", "dendrite-p2p-pinecone", "the name of this P2P demo instance")
instancePort = flag.Int("port", 8008, "the port that the client API will listen on")
instancePeer = flag.String("peer", "", "the static Pinecone peers to connect to, comma separated-list")
instanceListen = flag.String("listen", ":0", "the port Pinecone peers can connect to")
instanceDir = flag.String("dir", ".", "the directory to store the databases in (if --config not specified)")
instanceRelayingEnabled = flag.Bool("relay", false, "whether to enable store & forward relaying for other nodes")
)
// nolint:gocyclo
func main() {
flag.Parse()
internal.SetupPprof()
@ -81,7 +60,7 @@ func main() {
}
}
cfg := &config.Dendrite{}
var cfg *config.Dendrite
// use custom config if config flag is set
if configFlagSet {
@ -90,82 +69,30 @@ func main() {
pk = sk.Public().(ed25519.PublicKey)
} else {
keyfile := filepath.Join(*instanceDir, *instanceName) + ".pem"
if _, err := os.Stat(keyfile); os.IsNotExist(err) {
oldkeyfile := *instanceName + ".key"
if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) {
if err = test.NewMatrixKey(keyfile); err != nil {
panic("failed to generate a new PEM key: " + err.Error())
}
if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
panic("failed to load PEM key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
} else {
if sk, err = os.ReadFile(oldkeyfile); err != nil {
panic("failed to read the old private key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
if err := test.SaveMatrixKey(keyfile, sk); err != nil {
panic("failed to convert the private key to PEM format: " + err.Error())
}
}
} else {
var err error
if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
panic("failed to load PEM key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
}
pk = sk.Public().(ed25519.PublicKey)
cfg.Defaults(config.DefaultOpts{
Generate: true,
Monolithic: true,
})
cfg.Global.PrivateKey = sk
cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", filepath.Join(*instanceDir, *instanceName)))
cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(*instanceDir, *instanceName)))
cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(*instanceDir, *instanceName)))
cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(*instanceDir, *instanceName)))
cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(*instanceDir, *instanceName)))
cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(*instanceDir, *instanceName)))
cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", filepath.Join(*instanceDir, *instanceName)))
cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName)))
cfg.ClientAPI.RegistrationDisabled = false
cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
cfg.MediaAPI.BasePath = config.Path(*instanceDir)
cfg.SyncAPI.Fulltext.Enabled = true
cfg.SyncAPI.Fulltext.IndexPath = config.Path(*instanceDir)
if err := cfg.Derive(); err != nil {
panic(err)
}
oldKeyfile := *instanceName + ".key"
sk, pk = monolith.GetOrCreateKey(keyfile, oldKeyfile)
cfg = monolith.GenerateDefaultConfig(sk, *instanceDir, *instanceDir, *instanceName)
}
cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk))
cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
base := base.NewBaseDendrite(cfg, "Monolith")
defer base.Close() // nolint: errcheck
p2pMonolith := monolith.P2PMonolith{}
p2pMonolith.SetupPinecone(sk)
p2pMonolith.Multicast.Start()
pRouter := pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk)
pQUIC := pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), pRouter, []string{"matrix"})
pMulticast := pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), pRouter)
pManager := pineconeConnections.NewConnectionManager(pRouter, nil)
pMulticast.Start()
if instancePeer != nil && *instancePeer != "" {
for _, peer := range strings.Split(*instancePeer, ",") {
pManager.AddPeer(strings.Trim(peer, " \t\r\n"))
p2pMonolith.ConnManager.AddPeer(strings.Trim(peer, " \t\r\n"))
}
}
enableMetrics := true
enableWebsockets := true
p2pMonolith.SetupDendrite(cfg, *instancePort, *instanceRelayingEnabled, enableMetrics, enableWebsockets)
p2pMonolith.StartMonolith()
p2pMonolith.WaitForShutdown()
go func() {
listener, err := net.Listen("tcp", *instanceListen)
if err != nil {
@ -181,7 +108,7 @@ func main() {
continue
}
port, err := pRouter.Connect(
port, err := p2pMonolith.Router.Connect(
conn,
pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote),
)
@ -193,105 +120,4 @@ func main() {
fmt.Println("Inbound connection", conn.RemoteAddr(), "is connected to port", port)
}
}()
federation := conn.CreateFederationClient(base, pQUIC)
serverKeyAPI := &signing.YggdrasilKeys{}
keyRing := serverKeyAPI.KeyRing()
rsComponent := roomserver.NewInternalAPI(base)
rsAPI := rsComponent
fsAPI := federationapi.NewInternalAPI(
base, federation, rsAPI, base.Caches, keyRing, true,
)
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
keyAPI.SetUserAPI(userAPI)
asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
rsComponent.SetFederationAPI(fsAPI, keyRing)
userProvider := users.NewPineconeUserProvider(pRouter, pQUIC, userAPI, federation)
roomProvider := rooms.NewPineconeRoomProvider(pRouter, pQUIC, fsAPI, federation)
monolith := setup.Monolith{
Config: base.Cfg,
Client: conn.CreateClient(base, pQUIC),
FedClient: federation,
KeyRing: keyRing,
AppserviceAPI: asAPI,
FederationAPI: fsAPI,
RoomserverAPI: rsAPI,
UserAPI: userAPI,
KeyAPI: keyAPI,
ExtPublicRoomsProvider: roomProvider,
ExtUserDirectoryProvider: userProvider,
}
monolith.AddAllPublicRoutes(base)
wsUpgrader := websocket.Upgrader{
CheckOrigin: func(_ *http.Request) bool {
return true
},
}
httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
httpRouter.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) {
c, err := wsUpgrader.Upgrade(w, r, nil)
if err != nil {
logrus.WithError(err).Error("Failed to upgrade WebSocket connection")
return
}
conn := conn.WrapWebSocketConn(c)
if _, err = pRouter.Connect(
conn,
pineconeRouter.ConnectionZone("websocket"),
pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote),
); err != nil {
logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch")
}
})
httpRouter.HandleFunc("/pinecone", pRouter.ManholeHandler)
embed.Embed(httpRouter, *instancePort, "Pinecone Demo")
pMux := mux.NewRouter().SkipClean(true).UseEncodedPath()
pMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles)
pMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(base.PublicFederationAPIMux)
pMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
pHTTP := pQUIC.Protocol("matrix").HTTP()
pHTTP.Mux().Handle(users.PublicURL, pMux)
pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, pMux)
pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, pMux)
// Build both ends of a HTTP multiplex.
httpServer := &http.Server{
Addr: ":0",
TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 60 * time.Second,
BaseContext: func(_ net.Listener) context.Context {
return context.Background()
},
Handler: pMux,
}
go func() {
pubkey := pRouter.PublicKey()
logrus.Info("Listening on ", hex.EncodeToString(pubkey[:]))
logrus.Fatal(httpServer.Serve(pQUIC.Protocol("matrix")))
}()
go func() {
httpBindAddr := fmt.Sprintf(":%d", *instancePort)
logrus.Info("Listening on ", httpBindAddr)
logrus.Fatal(http.ListenAndServe(httpBindAddr, httpRouter))
}()
base.WaitForShutdown()
}

63
cmd/dendrite-demo-pinecone/monolith/keys.go Normal file
View file

@ -0,0 +1,63 @@
// Copyright 2023 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package monolith
import (
"crypto/ed25519"
"os"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/test"
)
func GetOrCreateKey(keyfile string, oldKeyfile string) (ed25519.PrivateKey, ed25519.PublicKey) {
var sk ed25519.PrivateKey
var pk ed25519.PublicKey
if _, err := os.Stat(keyfile); os.IsNotExist(err) {
if _, err = os.Stat(oldKeyfile); os.IsNotExist(err) {
if err = test.NewMatrixKey(keyfile); err != nil {
panic("failed to generate a new PEM key: " + err.Error())
}
if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
panic("failed to load PEM key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
} else {
if sk, err = os.ReadFile(oldKeyfile); err != nil {
panic("failed to read the old private key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
if err = test.SaveMatrixKey(keyfile, sk); err != nil {
panic("failed to convert the private key to PEM format: " + err.Error())
}
}
} else {
if _, sk, err = config.LoadMatrixKey(keyfile, os.ReadFile); err != nil {
panic("failed to load PEM key: " + err.Error())
}
if len(sk) != ed25519.PrivateKeySize {
panic("the private key is not long enough")
}
}
pk = sk.Public().(ed25519.PublicKey)
return sk, pk
}

359
cmd/dendrite-demo-pinecone/monolith/monolith.go Normal file
View file

@ -0,0 +1,359 @@
// Copyright 2023 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package monolith
import (
"context"
"crypto/ed25519"
"crypto/tls"
"encoding/hex"
"fmt"
"net"
"net/http"
"path/filepath"
"time"
"github.com/gorilla/mux"
"github.com/gorilla/websocket"
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/conn"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/embed"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/relay"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/rooms"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-pinecone/users"
"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing"
"github.com/matrix-org/dendrite/federationapi"
federationAPI "github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/federationapi/producers"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/relayapi"
relayAPI "github.com/matrix-org/dendrite/relayapi/api"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/jetstream"
"github.com/matrix-org/dendrite/userapi"
userAPI "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
pineconeConnections "github.com/matrix-org/pinecone/connections"
pineconeMulticast "github.com/matrix-org/pinecone/multicast"
pineconeRouter "github.com/matrix-org/pinecone/router"
pineconeEvents "github.com/matrix-org/pinecone/router/events"
pineconeSessions "github.com/matrix-org/pinecone/sessions"
)
const SessionProtocol = "matrix"
type P2PMonolith struct {
BaseDendrite *base.BaseDendrite
Sessions *pineconeSessions.Sessions
Multicast *pineconeMulticast.Multicast
ConnManager *pineconeConnections.ConnectionManager
Router *pineconeRouter.Router
EventChannel chan pineconeEvents.Event
RelayRetriever relay.RelayServerRetriever
dendrite setup.Monolith
port int
httpMux *mux.Router
pineconeMux *mux.Router
listener net.Listener
httpListenAddr string
}
func GenerateDefaultConfig(sk ed25519.PrivateKey, storageDir string, cacheDir string, dbPrefix string) *config.Dendrite {
cfg := config.Dendrite{}
cfg.Defaults(config.DefaultOpts{
Generate: true,
Monolithic: true,
})
cfg.Global.PrivateKey = sk
cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", filepath.Join(cacheDir, dbPrefix)))
cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(storageDir, dbPrefix)))
cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(storageDir, dbPrefix)))
cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(storageDir, dbPrefix)))
cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(storageDir, dbPrefix)))
cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(storageDir, dbPrefix)))
cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", filepath.Join(storageDir, dbPrefix)))
cfg.RelayAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-relayapi.db", filepath.Join(storageDir, dbPrefix)))
cfg.MSCs.MSCs = []string{"msc2836", "msc2946"}
cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(storageDir, dbPrefix)))
cfg.ClientAPI.RegistrationDisabled = false
cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true
cfg.MediaAPI.BasePath = config.Path(filepath.Join(cacheDir, "media"))
cfg.MediaAPI.AbsBasePath = config.Path(filepath.Join(cacheDir, "media"))
cfg.SyncAPI.Fulltext.Enabled = true
cfg.SyncAPI.Fulltext.IndexPath = config.Path(filepath.Join(cacheDir, "search"))
if err := cfg.Derive(); err != nil {
panic(err)
}
return &cfg
}
func (p *P2PMonolith) SetupPinecone(sk ed25519.PrivateKey) {
p.EventChannel = make(chan pineconeEvents.Event)
p.Router = pineconeRouter.NewRouter(logrus.WithField("pinecone", "router"), sk)
p.Router.EnableHopLimiting()
p.Router.EnableWakeupBroadcasts()
p.Router.Subscribe(p.EventChannel)
p.Sessions = pineconeSessions.NewSessions(logrus.WithField("pinecone", "sessions"), p.Router, []string{SessionProtocol})
p.Multicast = pineconeMulticast.NewMulticast(logrus.WithField("pinecone", "multicast"), p.Router)
p.ConnManager = pineconeConnections.NewConnectionManager(p.Router, nil)
}
func (p *P2PMonolith) SetupDendrite(cfg *config.Dendrite, port int, enableRelaying bool, enableMetrics bool, enableWebsockets bool) {
if enableMetrics {
p.BaseDendrite = base.NewBaseDendrite(cfg, "Monolith")
} else {
p.BaseDendrite = base.NewBaseDendrite(cfg, "Monolith", base.DisableMetrics)
}
p.port = port
p.BaseDendrite.ConfigureAdminEndpoints()
federation := conn.CreateFederationClient(p.BaseDendrite, p.Sessions)
serverKeyAPI := &signing.YggdrasilKeys{}
keyRing := serverKeyAPI.KeyRing()
rsComponent := roomserver.NewInternalAPI(p.BaseDendrite)
rsAPI := rsComponent
fsAPI := federationapi.NewInternalAPI(
p.BaseDendrite, federation, rsAPI, p.BaseDendrite.Caches, keyRing, true,
)
keyAPI := keyserver.NewInternalAPI(p.BaseDendrite, &p.BaseDendrite.Cfg.KeyServer, fsAPI, rsComponent)
userAPI := userapi.NewInternalAPI(p.BaseDendrite, &cfg.UserAPI, nil, keyAPI, rsAPI, p.BaseDendrite.PushGatewayHTTPClient())
keyAPI.SetUserAPI(userAPI)
asAPI := appservice.NewInternalAPI(p.BaseDendrite, userAPI, rsAPI)
rsComponent.SetFederationAPI(fsAPI, keyRing)
userProvider := users.NewPineconeUserProvider(p.Router, p.Sessions, userAPI, federation)
roomProvider := rooms.NewPineconeRoomProvider(p.Router, p.Sessions, fsAPI, federation)
js, _ := p.BaseDendrite.NATS.Prepare(p.BaseDendrite.ProcessContext, &p.BaseDendrite.Cfg.Global.JetStream)
producer := &producers.SyncAPIProducer{
JetStream: js,
TopicReceiptEvent: p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.OutputReceiptEvent),
TopicSendToDeviceEvent: p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent),
TopicTypingEvent: p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.OutputTypingEvent),
TopicPresenceEvent: p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.OutputPresenceEvent),
TopicDeviceListUpdate: p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.InputDeviceListUpdate),
TopicSigningKeyUpdate: p.BaseDendrite.Cfg.Global.JetStream.Prefixed(jetstream.InputSigningKeyUpdate),
Config: &p.BaseDendrite.Cfg.FederationAPI,
UserAPI: userAPI,
}
relayAPI := relayapi.NewRelayInternalAPI(p.BaseDendrite, federation, rsAPI, keyRing, producer, enableRelaying)
logrus.Infof("Relaying enabled: %v", relayAPI.RelayingEnabled())
p.dendrite = setup.Monolith{
Config: p.BaseDendrite.Cfg,
Client: conn.CreateClient(p.BaseDendrite, p.Sessions),
FedClient: federation,
KeyRing: keyRing,
AppserviceAPI: asAPI,
FederationAPI: fsAPI,
RoomserverAPI: rsAPI,
UserAPI: userAPI,
KeyAPI: keyAPI,
RelayAPI: relayAPI,
ExtPublicRoomsProvider: roomProvider,
ExtUserDirectoryProvider: userProvider,
}
p.dendrite.AddAllPublicRoutes(p.BaseDendrite)
p.setupHttpServers(userProvider, enableWebsockets)
}
func (p *P2PMonolith) GetFederationAPI() federationAPI.FederationInternalAPI {
return p.dendrite.FederationAPI
}
func (p *P2PMonolith) GetRelayAPI() relayAPI.RelayInternalAPI {
return p.dendrite.RelayAPI
}
func (p *P2PMonolith) GetUserAPI() userAPI.UserInternalAPI {
return p.dendrite.UserAPI
}
func (p *P2PMonolith) StartMonolith() {
p.startHTTPServers()
p.startEventHandler()
}
func (p *P2PMonolith) Stop() {
_ = p.BaseDendrite.Close()
p.WaitForShutdown()
}
func (p *P2PMonolith) WaitForShutdown() {
p.BaseDendrite.WaitForShutdown()
p.closeAllResources()
}
func (p *P2PMonolith) closeAllResources() {
if p.listener != nil {
_ = p.listener.Close()
}
if p.Multicast != nil {
p.Multicast.Stop()
}
if p.Sessions != nil {
_ = p.Sessions.Close()
}
if p.Router != nil {
_ = p.Router.Close()
}
}
func (p *P2PMonolith) Addr() string {
return p.httpListenAddr
}
func (p *P2PMonolith) setupHttpServers(userProvider *users.PineconeUserProvider, enableWebsockets bool) {
p.httpMux = mux.NewRouter().SkipClean(true).UseEncodedPath()
p.httpMux.PathPrefix(httputil.InternalPathPrefix).Handler(p.BaseDendrite.InternalAPIMux)
p.httpMux.PathPrefix(httputil.PublicClientPathPrefix).Handler(p.BaseDendrite.PublicClientAPIMux)
p.httpMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(p.BaseDendrite.PublicMediaAPIMux)
p.httpMux.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(p.BaseDendrite.DendriteAdminMux)
p.httpMux.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(p.BaseDendrite.SynapseAdminMux)
if enableWebsockets {
wsUpgrader := websocket.Upgrader{
CheckOrigin: func(_ *http.Request) bool {
return true
},
}
p.httpMux.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) {
c, err := wsUpgrader.Upgrade(w, r, nil)
if err != nil {
logrus.WithError(err).Error("Failed to upgrade WebSocket connection")
return
}
conn := conn.WrapWebSocketConn(c)
if _, err = p.Router.Connect(
conn,
pineconeRouter.ConnectionZone("websocket"),
pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote),
); err != nil {
logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch")
}
})
}
p.httpMux.HandleFunc("/pinecone", p.Router.ManholeHandler)
if enableWebsockets {
embed.Embed(p.httpMux, p.port, "Pinecone Demo")
}
p.pineconeMux = mux.NewRouter().SkipClean(true).UseEncodedPath()
p.pineconeMux.PathPrefix(users.PublicURL).HandlerFunc(userProvider.FederatedUserProfiles)
p.pineconeMux.PathPrefix(httputil.PublicFederationPathPrefix).Handler(p.BaseDendrite.PublicFederationAPIMux)
p.pineconeMux.PathPrefix(httputil.PublicMediaPathPrefix).Handler(p.BaseDendrite.PublicMediaAPIMux)
pHTTP := p.Sessions.Protocol(SessionProtocol).HTTP()
pHTTP.Mux().Handle(users.PublicURL, p.pineconeMux)
pHTTP.Mux().Handle(httputil.PublicFederationPathPrefix, p.pineconeMux)
pHTTP.Mux().Handle(httputil.PublicMediaPathPrefix, p.pineconeMux)
}
func (p *P2PMonolith) startHTTPServers() {
go func() {
// Build both ends of a HTTP multiplex.
httpServer := &http.Server{
Addr: ":0",
TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){},
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 30 * time.Second,
BaseContext: func(_ net.Listener) context.Context {
return context.Background()
},
Handler: p.pineconeMux,
}
pubkey := p.Router.PublicKey()
pubkeyString := hex.EncodeToString(pubkey[:])
logrus.Info("Listening on ", pubkeyString)
switch httpServer.Serve(p.Sessions.Protocol(SessionProtocol)) {
case net.ErrClosed, http.ErrServerClosed:
logrus.Info("Stopped listening on ", pubkeyString)
default:
logrus.Error("Stopped listening on ", pubkeyString)
}
}()
p.httpListenAddr = fmt.Sprintf(":%d", p.port)
go func() {
logrus.Info("Listening on ", p.httpListenAddr)
switch http.ListenAndServe(p.httpListenAddr, p.httpMux) {
case net.ErrClosed, http.ErrServerClosed:
logrus.Info("Stopped listening on ", p.httpListenAddr)
default:
logrus.Error("Stopped listening on ", p.httpListenAddr)
}
}()
}
func (p *P2PMonolith) startEventHandler() {
stopRelayServerSync := make(chan bool)
eLog := logrus.WithField("pinecone", "events")
p.RelayRetriever = relay.NewRelayServerRetriever(
context.Background(),
gomatrixserverlib.ServerName(p.Router.PublicKey().String()),
p.dendrite.FederationAPI,
p.dendrite.RelayAPI,
stopRelayServerSync,
)
p.RelayRetriever.InitializeRelayServers(eLog)
go func(ch <-chan pineconeEvents.Event) {
for event := range ch {
switch e := event.(type) {
case pineconeEvents.PeerAdded:
p.RelayRetriever.StartSync()
case pineconeEvents.PeerRemoved:
if p.RelayRetriever.IsRunning() && p.Router.TotalPeerCount() == 0 {
stopRelayServerSync <- true
}
case pineconeEvents.BroadcastReceived:
// eLog.Info("Broadcast received from: ", e.PeerID)
req := &federationAPI.PerformWakeupServersRequest{
ServerNames: []gomatrixserverlib.ServerName{gomatrixserverlib.ServerName(e.PeerID)},
}
res := &federationAPI.PerformWakeupServersResponse{}
if err := p.dendrite.FederationAPI.PerformWakeupServers(p.BaseDendrite.Context(), req, res); err != nil {
eLog.WithError(err).Error("Failed to wakeup destination", e.PeerID)
}
default:
}
}
}(p.EventChannel)
}

237
cmd/dendrite-demo-pinecone/relay/retriever.go Normal file
View file

@ -0,0 +1,237 @@
// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package relay
import (
"context"
"sync"
"time"
federationAPI "github.com/matrix-org/dendrite/federationapi/api"
relayServerAPI "github.com/matrix-org/dendrite/relayapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
"go.uber.org/atomic"
)
const (
relayServerRetryInterval = time.Second * 30
)
type RelayServerRetriever struct {
ctx context.Context
serverName gomatrixserverlib.ServerName
federationAPI federationAPI.FederationInternalAPI
relayAPI relayServerAPI.RelayInternalAPI
relayServersQueried map[gomatrixserverlib.ServerName]bool
queriedServersMutex sync.Mutex
running atomic.Bool
quit <-chan bool
}
func NewRelayServerRetriever(
ctx context.Context,
serverName gomatrixserverlib.ServerName,
federationAPI federationAPI.FederationInternalAPI,
relayAPI relayServerAPI.RelayInternalAPI,
quit <-chan bool,
) RelayServerRetriever {
return RelayServerRetriever{
ctx: ctx,
serverName: serverName,
federationAPI: federationAPI,
relayAPI: relayAPI,
relayServersQueried: make(map[gomatrixserverlib.ServerName]bool),
running: *atomic.NewBool(false),
quit: quit,
}
}
func (r *RelayServerRetriever) InitializeRelayServers(eLog *logrus.Entry) {
request := federationAPI.P2PQueryRelayServersRequest{Server: gomatrixserverlib.ServerName(r.serverName)}
response := federationAPI.P2PQueryRelayServersResponse{}
err := r.federationAPI.P2PQueryRelayServers(r.ctx, &request, &response)
if err != nil {
eLog.Warnf("Failed obtaining list of this node's relay servers: %s", err.Error())
}
r.queriedServersMutex.Lock()
defer r.queriedServersMutex.Unlock()
for _, server := range response.RelayServers {
r.relayServersQueried[server] = false
}
eLog.Infof("Registered relay servers: %v", response.RelayServers)
}
func (r *RelayServerRetriever) SetRelayServers(servers []gomatrixserverlib.ServerName) {
UpdateNodeRelayServers(r.serverName, servers, r.ctx, r.federationAPI)
// Replace list of servers to sync with and mark them all as unsynced.
r.queriedServersMutex.Lock()
defer r.queriedServersMutex.Unlock()
r.relayServersQueried = make(map[gomatrixserverlib.ServerName]bool)
for _, server := range servers {
r.relayServersQueried[server] = false
}
r.StartSync()
}
func (r *RelayServerRetriever) GetRelayServers() []gomatrixserverlib.ServerName {
r.queriedServersMutex.Lock()
defer r.queriedServersMutex.Unlock()
relayServers := []gomatrixserverlib.ServerName{}
for server := range r.relayServersQueried {
relayServers = append(relayServers, server)
}
return relayServers
}
func (r *RelayServerRetriever) GetQueriedServerStatus() map[gomatrixserverlib.ServerName]bool {
r.queriedServersMutex.Lock()
defer r.queriedServersMutex.Unlock()
result := map[gomatrixserverlib.ServerName]bool{}
for server, queried := range r.relayServersQueried {
result[server] = queried
}
return result
}
func (r *RelayServerRetriever) StartSync() {
if !r.running.Load() {
logrus.Info("Starting relay server sync")
go r.SyncRelayServers(r.quit)
}
}
func (r *RelayServerRetriever) IsRunning() bool {
return r.running.Load()
}
func (r *RelayServerRetriever) SyncRelayServers(stop <-chan bool) {
defer r.running.Store(false)
t := time.NewTimer(relayServerRetryInterval)
for {
relayServersToQuery := []gomatrixserverlib.ServerName{}
func() {
r.queriedServersMutex.Lock()
defer r.queriedServersMutex.Unlock()
for server, complete := range r.relayServersQueried {
if !complete {
relayServersToQuery = append(relayServersToQuery, server)
}
}
}()
if len(relayServersToQuery) == 0 {
// All relay servers have been synced.
logrus.Info("Finished syncing with all known relays")
return
}
r.queryRelayServers(relayServersToQuery)
t.Reset(relayServerRetryInterval)
select {
case <-stop:
if !t.Stop() {
<-t.C
}
return
case <-t.C:
}
}
}
func (r *RelayServerRetriever) queryRelayServers(relayServers []gomatrixserverlib.ServerName) {
logrus.Info("Querying relay servers for any available transactions")
for _, server := range relayServers {
userID, err := gomatrixserverlib.NewUserID("@user:"+string(r.serverName), false)
if err != nil {
return
}
logrus.Infof("Syncing with relay: %s", string(server))
err = r.relayAPI.PerformRelayServerSync(context.Background(), *userID, server)
if err == nil {
func() {
r.queriedServersMutex.Lock()
defer r.queriedServersMutex.Unlock()
r.relayServersQueried[server] = true
}()
// TODO : What happens if your relay receives new messages after this point?
// Should you continue to check with them, or should they try and contact you?
// They could send a "new_async_events" message your way maybe?
// Then you could mark them as needing to be queried again.
// What if you miss this message?
// Maybe you should try querying them again after a certain period of time as a backup?
} else {
logrus.Errorf("Failed querying relay server: %s", err.Error())
}
}
}
func UpdateNodeRelayServers(
node gomatrixserverlib.ServerName,
relays []gomatrixserverlib.ServerName,
ctx context.Context,
fedAPI federationAPI.FederationInternalAPI,
) {
// Get the current relay list
request := federationAPI.P2PQueryRelayServersRequest{Server: node}
response := federationAPI.P2PQueryRelayServersResponse{}
err := fedAPI.P2PQueryRelayServers(ctx, &request, &response)
if err != nil {
logrus.Warnf("Failed obtaining list of relay servers for %s: %s", node, err.Error())
}
// Remove old, non-matching relays
var serversToRemove []gomatrixserverlib.ServerName
for _, existingServer := range response.RelayServers {
shouldRemove := true
for _, newServer := range relays {
if newServer == existingServer {
shouldRemove = false
break
}
}
if shouldRemove {
serversToRemove = append(serversToRemove, existingServer)
}
}
removeRequest := federationAPI.P2PRemoveRelayServersRequest{
Server: node,
RelayServers: serversToRemove,
}
removeResponse := federationAPI.P2PRemoveRelayServersResponse{}
err = fedAPI.P2PRemoveRelayServers(ctx, &removeRequest, &removeResponse)
if err != nil {
logrus.Warnf("Failed removing old relay servers for %s: %s", node, err.Error())
}
// Add new relays
addRequest := federationAPI.P2PAddRelayServersRequest{
Server: node,
RelayServers: relays,
}
addResponse := federationAPI.P2PAddRelayServersResponse{}
err = fedAPI.P2PAddRelayServers(ctx, &addRequest, &addResponse)
if err != nil {
logrus.Warnf("Failed adding relay servers for %s: %s", node, err.Error())
}
}

99
cmd/dendrite-demo-pinecone/relay/retriever_test.go Normal file
View file

@ -0,0 +1,99 @@
// Copyright 2023 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package relay
import (
"context"
"testing"
"time"
federationAPI "github.com/matrix-org/dendrite/federationapi/api"
relayServerAPI "github.com/matrix-org/dendrite/relayapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"gotest.tools/v3/poll"
)
var testRelayServers = []gomatrixserverlib.ServerName{"relay1", "relay2"}
type FakeFedAPI struct {
federationAPI.FederationInternalAPI
}
func (f *FakeFedAPI) P2PQueryRelayServers(
ctx context.Context,
req *federationAPI.P2PQueryRelayServersRequest,
res *federationAPI.P2PQueryRelayServersResponse,
) error {
res.RelayServers = testRelayServers
return nil
}
type FakeRelayAPI struct {
relayServerAPI.RelayInternalAPI
}
func (r *FakeRelayAPI) PerformRelayServerSync(
ctx context.Context,
userID gomatrixserverlib.UserID,
relayServer gomatrixserverlib.ServerName,
) error {
return nil
}
func TestRelayRetrieverInitialization(t *testing.T) {
retriever := NewRelayServerRetriever(
context.Background(),
"server",
&FakeFedAPI{},
&FakeRelayAPI{},
make(<-chan bool),
)
retriever.InitializeRelayServers(logrus.WithField("test", "relay"))
relayServers := retriever.GetQueriedServerStatus()
assert.Equal(t, 2, len(relayServers))
}
func TestRelayRetrieverSync(t *testing.T) {
retriever := NewRelayServerRetriever(
context.Background(),
"server",
&FakeFedAPI{},
&FakeRelayAPI{},
make(<-chan bool),
)
retriever.InitializeRelayServers(logrus.WithField("test", "relay"))
relayServers := retriever.GetQueriedServerStatus()
assert.Equal(t, 2, len(relayServers))
stopRelayServerSync := make(chan bool)
go retriever.SyncRelayServers(stopRelayServerSync)
check := func(log poll.LogT) poll.Result {
relayServers := retriever.GetQueriedServerStatus()
for _, queried := range relayServers {
if !queried {
return poll.Continue("waiting for all servers to be queried")
}
}
stopRelayServerSync <- true
return poll.Success()
}
poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond))
}

8
cmd/dendrite-demo-pinecone/rooms/rooms.go
View file

@ -58,13 +58,17 @@ func (p *PineconeRoomProvider) Rooms() []gomatrixserverlib.PublicRoom {
for _, k := range p.r.Peers() {
list[gomatrixserverlib.ServerName(k.PublicKey)] = struct{}{}
}
return bulkFetchPublicRoomsFromServers(context.Background(), p.fedClient, list)
return bulkFetchPublicRoomsFromServers(
context.Background(), p.fedClient,
gomatrixserverlib.ServerName(p.r.PublicKey().String()), list,
)
}
// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers.
// Returns a list of public rooms.
func bulkFetchPublicRoomsFromServers(
ctx context.Context, fedClient *gomatrixserverlib.FederationClient,
origin gomatrixserverlib.ServerName,
homeservers map[gomatrixserverlib.ServerName]struct{},
) (publicRooms []gomatrixserverlib.PublicRoom) {
limit := 200
@ -82,7 +86,7 @@ func bulkFetchPublicRoomsFromServers(
go func(homeserverDomain gomatrixserverlib.ServerName) {
defer wg.Done()
util.GetLogger(reqctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms")
fres, err := fedClient.GetPublicRooms(reqctx, homeserverDomain, int(limit), "", false, "")
fres, err := fedClient.GetPublicRooms(reqctx, origin, homeserverDomain, int(limit), "", false, "")
if err != nil {
util.GetLogger(reqctx).WithError(err).WithField("hs", homeserverDomain).Warn(
"bulkFetchPublicRoomsFromServers: failed to query hs",

8
cmd/dendrite-demo-yggdrasil/main.go
View file

@ -144,6 +144,7 @@ func main() {
cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
base := base.NewBaseDendrite(cfg, "Monolith")
base.ConfigureAdminEndpoints()
defer base.Close() // nolint: errcheck
ygg, err := yggconn.Setup(sk, *instanceName, ".", *instancePeer, *instanceListen)
@ -156,11 +157,12 @@ func main() {
serverKeyAPI := &signing.YggdrasilKeys{}
keyRing := serverKeyAPI.KeyRing()
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation)
rsComponent := roomserver.NewInternalAPI(
base,
)
keyAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, federation, rsComponent)
rsAPI := rsComponent
userAPI := userapi.NewInternalAPI(base, &cfg.UserAPI, nil, keyAPI, rsAPI, base.PushGatewayHTTPClient())
@ -198,6 +200,8 @@ func main() {
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
httpRouter.PathPrefix(httputil.PublicClientPathPrefix).Handler(base.PublicClientAPIMux)
httpRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(base.PublicMediaAPIMux)
httpRouter.PathPrefix(httputil.DendriteAdminPathPrefix).Handler(base.DendriteAdminMux)
httpRouter.PathPrefix(httputil.SynapseAdminPathPrefix).Handler(base.SynapseAdminMux)
embed.Embed(httpRouter, *instancePort, "Yggdrasil Demo")
yggRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()

3
cmd/dendrite-demo-yggdrasil/yggconn/client.go
View file

@ -55,8 +55,7 @@ func (n *Node) CreateFederationClient(
},
)
return gomatrixserverlib.NewFederationClient(
base.Cfg.Global.ServerName, base.Cfg.Global.KeyID,
base.Cfg.Global.PrivateKey,
base.Cfg.Global.SigningIdentities(),
gomatrixserverlib.WithTransport(tr),
)
}

9
cmd/dendrite-demo-yggdrasil/yggrooms/yggrooms.go
View file

@ -43,13 +43,18 @@ func NewYggdrasilRoomProvider(
}
func (p *YggdrasilRoomProvider) Rooms() []gomatrixserverlib.PublicRoom {
return bulkFetchPublicRoomsFromServers(context.Background(), p.fedClient, p.node.KnownNodes())
return bulkFetchPublicRoomsFromServers(
context.Background(), p.fedClient,
gomatrixserverlib.ServerName(p.node.DerivedServerName()),
p.node.KnownNodes(),
)
}
// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers.
// Returns a list of public rooms.
func bulkFetchPublicRoomsFromServers(
ctx context.Context, fedClient *gomatrixserverlib.FederationClient,
origin gomatrixserverlib.ServerName,
homeservers []gomatrixserverlib.ServerName,
) (publicRooms []gomatrixserverlib.PublicRoom) {
limit := 200
@ -66,7 +71,7 @@ func bulkFetchPublicRoomsFromServers(
go func(homeserverDomain gomatrixserverlib.ServerName) {
defer wg.Done()
util.GetLogger(ctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms")
fres, err := fedClient.GetPublicRooms(ctx, homeserverDomain, int(limit), "", false, "")
fres, err := fedClient.GetPublicRooms(ctx, origin, homeserverDomain, int(limit), "", false, "")
if err != nil {
util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Warn(
"bulkFetchPublicRoomsFromServers: failed to query hs",

17
cmd/dendrite-monolith-server/main.go
View file

@ -18,6 +18,8 @@ import (
"flag"
"os"
"github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/keyserver"
@ -29,7 +31,6 @@ import (
"github.com/matrix-org/dendrite/setup/mscs"
"github.com/matrix-org/dendrite/userapi"
uapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/sirupsen/logrus"
)
var (
@ -75,7 +76,7 @@ func main() {
// call functions directly on the impl unless running in HTTP mode
rsAPI := rsImpl
if base.UseHTTPAPIs {
roomserver.AddInternalRoutes(base.InternalAPIMux, rsImpl)
roomserver.AddInternalRoutes(base.InternalAPIMux, rsImpl, base.EnableMetrics)
rsAPI = base.RoomserverHTTPClient()
}
if traceInternal {
@ -89,15 +90,15 @@ func main() {
)
fsImplAPI := fsAPI
if base.UseHTTPAPIs {
federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI)
federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI, base.EnableMetrics)
fsAPI = base.FederationAPIHTTPClient()
}
keyRing := fsAPI.KeyRing()
keyImpl := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
keyImpl := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI)
keyAPI := keyImpl
if base.UseHTTPAPIs {
keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI)
keyserver.AddInternalRoutes(base.InternalAPIMux, keyAPI, base.EnableMetrics)
keyAPI = base.KeyServerHTTPClient()
}
@ -105,10 +106,12 @@ func main() {
userImpl := userapi.NewInternalAPI(base, &cfg.UserAPI, cfg.Derived.ApplicationServices, keyAPI, rsAPI, pgClient)
userAPI := userImpl
if base.UseHTTPAPIs {
userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
userapi.AddInternalRoutes(base.InternalAPIMux, userAPI, base.EnableMetrics)
userAPI = base.UserAPIClient()
}
if traceInternal {
logrus.Warnf("The traceInternal option is enabled")
userAPI = &uapi.UserInternalAPITrace{
Impl: userAPI,
}
@ -119,7 +122,7 @@ func main() {
// before the listeners are up.
asAPI := appservice.NewInternalAPI(base, userImpl, rsAPI)
if base.UseHTTPAPIs {
appservice.AddInternalRoutes(base.InternalAPIMux, asAPI)
appservice.AddInternalRoutes(base.InternalAPIMux, asAPI, base.EnableMetrics)
asAPI = base.AppserviceHTTPClient()
}

2
cmd/dendrite-polylith-multi/personalities/appservice.go
View file

@ -26,7 +26,7 @@ func Appservice(base *base.BaseDendrite, cfg *config.Dendrite) {
rsAPI := base.RoomserverHTTPClient()
intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
appservice.AddInternalRoutes(base.InternalAPIMux, intAPI)
appservice.AddInternalRoutes(base.InternalAPIMux, intAPI, base.EnableMetrics)
base.SetupAndServeHTTP(
base.Cfg.AppServiceAPI.InternalAPI.Listen, // internal listener

2
cmd/dendrite-polylith-multi/personalities/federationapi.go
View file

@ -34,7 +34,7 @@ func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
rsAPI, fsAPI, keyAPI, nil,
)
federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI)
federationapi.AddInternalRoutes(base.InternalAPIMux, fsAPI, base.EnableMetrics)
base.SetupAndServeHTTP(
base.Cfg.FederationAPI.InternalAPI.Listen,

5
cmd/dendrite-polylith-multi/personalities/keyserver.go
View file

@ -22,10 +22,11 @@ import (
func KeyServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
fsAPI := base.FederationAPIHTTPClient()
intAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI)
rsAPI := base.RoomserverHTTPClient()
intAPI := keyserver.NewInternalAPI(base, &base.Cfg.KeyServer, fsAPI, rsAPI)
intAPI.SetUserAPI(base.UserAPIClient())
keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
keyserver.AddInternalRoutes(base.InternalAPIMux, intAPI, base.EnableMetrics)
base.SetupAndServeHTTP(
base.Cfg.KeyServer.InternalAPI.Listen, // internal listener

2
cmd/dendrite-polylith-multi/personalities/roomserver.go
View file

@ -26,7 +26,7 @@ func RoomServer(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
rsAPI := roomserver.NewInternalAPI(base)
rsAPI.SetFederationAPI(fsAPI, fsAPI.KeyRing())
rsAPI.SetAppserviceAPI(asAPI)
roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI)
roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI, base.EnableMetrics)
base.SetupAndServeHTTP(
base.Cfg.RoomServer.InternalAPI.Listen, // internal listener

2
cmd/dendrite-polylith-multi/personalities/userapi.go
View file

@ -27,7 +27,7 @@ func UserAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) {
base.PushGatewayHTTPClient(),
)
userapi.AddInternalRoutes(base.InternalAPIMux, userAPI)
userapi.AddInternalRoutes(base.InternalAPIMux, userAPI, base.EnableMetrics)
base.SetupAndServeHTTP(
base.Cfg.UserAPI.InternalAPI.Listen, // internal listener

115
cmd/dendrite-upgrade-tests/main.go
View file

@ -38,6 +38,7 @@ var (
flagHead = flag.String("head", "", "Location to a dendrite repository to treat as HEAD instead of Github")
flagDockerHost = flag.String("docker-host", "localhost", "The hostname of the docker client. 'localhost' if running locally, 'host.docker.internal' if running in Docker.")
flagDirect = flag.Bool("direct", false, "If a direct upgrade from the defined FROM version to TO should be done")
flagSqlite = flag.Bool("sqlite", false, "Test SQLite instead of PostgreSQL")
alphaNumerics = regexp.MustCompile("[^a-zA-Z0-9]+")
)
@ -49,7 +50,9 @@ const HEAD = "HEAD"
// due to the error:
// When using COPY with more than one source file, the destination must be a directory and end with a /
// We need to run a postgres anyway, so use the dockerfile associated with Complement instead.
const Dockerfile = `FROM golang:1.19-buster as build
const DockerfilePostgreSQL = `FROM golang:1.19-buster as build
RUN apt-get update && apt-get install -y postgresql
WORKDIR /build
@ -60,6 +63,7 @@ COPY . .
RUN go build --race ./cmd/dendrite-monolith-server
RUN go build --race ./cmd/generate-keys
RUN go build --race ./cmd/generate-config
RUN go build --race ./cmd/create-account
RUN ./generate-config --ci > dendrite.yaml
RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
@ -92,6 +96,43 @@ ENV SERVER_NAME=localhost
EXPOSE 8008 8448
CMD /build/run_dendrite.sh `
const DockerfileSQLite = `FROM golang:1.19-buster as build
RUN apt-get update && apt-get install -y postgresql
WORKDIR /build
# Copy the build context to the repo as this is the right dendrite code. This is different to the
# Complement Dockerfile which wgets a branch.
COPY . .
RUN go build --race ./cmd/dendrite-monolith-server
RUN go build --race ./cmd/generate-keys
RUN go build --race ./cmd/generate-config
RUN go build --race ./cmd/create-account
RUN ./generate-config --ci > dendrite.yaml
RUN ./generate-keys --private-key matrix_key.pem --tls-cert server.crt --tls-key server.key
# Make sure the SQLite databases are in a persistent location, we're already mapping
# the postgresql folder so let's just use that for simplicity
RUN sed -i "s%connection_string:.file:%connection_string: file:\/var\/lib\/postgresql\/9.6\/main\/%g" dendrite.yaml
# This entry script starts postgres, waits for it to be up then starts dendrite
RUN echo '\
sed -i "s/server_name: localhost/server_name: ${SERVER_NAME}/g" dendrite.yaml \n\
PARAMS="--tls-cert server.crt --tls-key server.key --config dendrite.yaml" \n\
./dendrite-monolith-server --really-enable-open-registration ${PARAMS} || ./dendrite-monolith-server ${PARAMS} \n\
' > run_dendrite.sh && chmod +x run_dendrite.sh
ENV SERVER_NAME=localhost
EXPOSE 8008 8448
CMD /build/run_dendrite.sh `
func dockerfile() []byte {
if *flagSqlite {
return []byte(DockerfileSQLite)
}
return []byte(DockerfilePostgreSQL)
}
const dendriteUpgradeTestLabel = "dendrite_upgrade_test"
// downloadArchive downloads an arbitrary github archive of the form:
@ -150,7 +191,7 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir,
if branchOrTagName == HEAD && *flagHead != "" {
log.Printf("%s: Using %s as HEAD", branchOrTagName, *flagHead)
// add top level Dockerfile
err = os.WriteFile(path.Join(*flagHead, "Dockerfile"), []byte(Dockerfile), os.ModePerm)
err = os.WriteFile(path.Join(*flagHead, "Dockerfile"), dockerfile(), os.ModePerm)
if err != nil {
return "", fmt.Errorf("custom HEAD: failed to inject /Dockerfile: %w", err)
}
@ -166,7 +207,7 @@ func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir,
// pull an archive, this contains a top-level directory which screws with the build context
// which we need to fix up post download
u := fmt.Sprintf("https://github.com/matrix-org/dendrite/archive/%s.tar.gz", branchOrTagName)
tarball, err = downloadArchive(httpClient, tmpDir, u, []byte(Dockerfile))
tarball, err = downloadArchive(httpClient, tmpDir, u, dockerfile())
if err != nil {
return "", fmt.Errorf("failed to download archive %s: %w", u, err)
}
@ -367,7 +408,8 @@ func runImage(dockerClient *client.Client, volumeName, version, imageID string)
// hit /versions to check it is up
var lastErr error
for i := 0; i < 500; i++ {
res, err := http.Get(versionsURL)
var res *http.Response
res, err = http.Get(versionsURL)
if err != nil {
lastErr = fmt.Errorf("GET %s => error: %s", versionsURL, err)
time.Sleep(50 * time.Millisecond)
@ -381,18 +423,22 @@ func runImage(dockerClient *client.Client, volumeName, version, imageID string)
lastErr = nil
break
}
if lastErr != nil {
logs, err := dockerClient.ContainerLogs(context.Background(), containerID, types.ContainerLogsOptions{
ShowStdout: true,
ShowStderr: true,
})
// ignore errors when cannot get logs, it's just for debugging anyways
if err == nil {
logbody, err := io.ReadAll(logs)
if err == nil {
log.Printf("Container logs:\n\n%s\n\n", string(logbody))
logs, err := dockerClient.ContainerLogs(context.Background(), containerID, types.ContainerLogsOptions{
ShowStdout: true,
ShowStderr: true,
Follow: true,
})
// ignore errors when cannot get logs, it's just for debugging anyways
if err == nil {
go func() {
for {
if body, err := io.ReadAll(logs); err == nil && len(body) > 0 {
log.Printf("%s: %s", version, string(body))
} else {
return
}
}
}
}()
}
return baseURL, containerID, lastErr
}
@ -419,6 +465,45 @@ func loadAndRunTests(dockerClient *client.Client, volumeName, v string, branchTo
// Sleep to let the database sync before returning and destroying the dendrite container
time.Sleep(5 * time.Second)
err = testCreateAccount(dockerClient, v, containerID)
if err != nil {
return err
}
return nil
}
// test that create-account is working
func testCreateAccount(dockerClient *client.Client, v string, containerID string) error {
createUser := strings.ToLower("createaccountuser-" + v)
log.Printf("%s: Creating account %s with create-account\n", v, createUser)
respID, err := dockerClient.ContainerExecCreate(context.Background(), containerID, types.ExecConfig{
AttachStderr: true,
AttachStdout: true,
Cmd: []string{
"/build/create-account",
"-username", createUser,
"-password", "someRandomPassword",
},
})
if err != nil {
return fmt.Errorf("failed to ContainerExecCreate: %w", err)
}
response, err := dockerClient.ContainerExecAttach(context.Background(), respID.ID, types.ExecStartCheck{})
if err != nil {
return fmt.Errorf("failed to attach to container: %w", err)
}
defer response.Close()
data, err := io.ReadAll(response.Reader)
if err != nil {
return err
}
if !bytes.Contains(data, []byte("AccessToken")) {
return fmt.Errorf("failed to create-account: %s", string(data))
}
return nil
}

12
cmd/furl/main.go
View file

@ -48,10 +48,15 @@ func main() {
panic("unexpected key block")
}
serverName := gomatrixserverlib.ServerName(*requestFrom)
client := gomatrixserverlib.NewFederationClient(
gomatrixserverlib.ServerName(*requestFrom),
gomatrixserverlib.KeyID(keyBlock.Headers["Key-ID"]),
privateKey,
[]*gomatrixserverlib.SigningIdentity{
{
ServerName: serverName,
KeyID: gomatrixserverlib.KeyID(keyBlock.Headers["Key-ID"]),
PrivateKey: privateKey,
},
},
)
u, err := url.Parse(flag.Arg(0))
@ -79,6 +84,7 @@ func main() {
req := gomatrixserverlib.NewFederationRequest(
method,
serverName,
gomatrixserverlib.ServerName(u.Host),
u.RequestURI(),
)

4
cmd/generate-config/main.go
View file

@ -54,6 +54,9 @@ func main() {
} else {
cfg.Global.DatabaseOptions.ConnectionString = uri
}
cfg.MediaAPI.BasePath = config.Path(filepath.Join(*dirPath, "media"))
cfg.Global.JetStream.StoragePath = config.Path(*dirPath)
cfg.SyncAPI.Fulltext.IndexPath = config.Path(filepath.Join(*dirPath, "searchindex"))
cfg.Logging = []config.LogrusHook{
{
Type: "file",
@ -67,6 +70,7 @@ func main() {
cfg.AppServiceAPI.DisableTLSValidation = true
cfg.ClientAPI.RateLimiting.Enabled = false
cfg.FederationAPI.DisableTLSValidation = false
cfg.FederationAPI.DisableHTTPKeepalives = true
// don't hit matrix.org when running tests!!!
cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{}
cfg.MediaAPI.BasePath = config.Path(filepath.Join(*dirPath, "media"))

58
docs/FAQ.md
View file

@ -6,6 +6,12 @@ permalink: /faq
# FAQ
## Why does Dendrite exist?
Dendrite aims to provide a matrix compatible server that has low resource usage compared to [Synapse](https://github.com/matrix-org/synapse).
It also aims to provide more flexibility when scaling either up or down.
Dendrite's code is also very easy to hack on which makes it suitable for experimenting with new matrix features such as peer-to-peer.
## Is Dendrite stable?
Mostly, although there are still bugs and missing features. If you are a confident power user and you are happy to spend some time debugging things when they go wrong, then please try out Dendrite. If you are a community, organisation or business that demands stability and uptime, then Dendrite is not for you yet - please install Synapse instead.
@ -34,6 +40,10 @@ No, Dendrite has a very different database schema to Synapse and the two are not
Monolith deployments are always preferred where possible, and at this time, are far better tested than polylith deployments are. The only reason to consider a polylith deployment is if you wish to run different Dendrite components on separate physical machines, but this is an advanced configuration which we don't
recommend.
## Can I configure which port Dendrite listens on?
Yes, use the cli flag `-http-bind-address`.
## I've installed Dendrite but federation isn't working
Check the [Federation Tester](https://federationtester.matrix.org). You need at least:
@ -42,6 +52,10 @@ Check the [Federation Tester](https://federationtester.matrix.org). You need at
* A valid TLS certificate for that DNS name
* Either DNS SRV records or well-known files
## Whenever I try to connect from Element it says unable to connect to homeserver
Check that your dendrite instance is running. Otherwise this is most likely due to a reverse proxy misconfiguration.
## Does Dendrite work with my favourite client?
It should do, although we are aware of some minor issues:
@ -49,6 +63,10 @@ It should do, although we are aware of some minor issues:
* **Element Android**: registration does not work, but logging in with an existing account does
* **Hydrogen**: occasionally sync can fail due to gaps in the `since` parameter, but clearing the cache fixes this
## Is there a public instance of Dendrite I can try out?
Use [dendrite.matrix.org](https://dendrite.matrix.org) which we officially support.
## Does Dendrite support Space Summaries?
Yes, [Space Summaries](https://github.com/matrix-org/matrix-spec-proposals/pull/2946) were merged into the Matrix Spec as of 2022-01-17 however, they are still treated as an MSC (Matrix Specification Change) in Dendrite. In order to enable Space Summaries in Dendrite, you must add the MSC to the MSC configuration section in the configuration YAML. If the MSC is not enabled, a user will typically see a perpetual loading icon on the summary page. See below for a demonstration of how to add to the Dendrite configuration:
@ -84,14 +102,46 @@ Remember to add the config file(s) to the `app_service_api` section of the confi
Yes, you can do this by disabling federation - set `disable_federation` to `true` in the `global` section of the Dendrite configuration file.
## How can I migrate a room in order to change the internal ID?
This can be done by performing a room upgrade. Use the command `/upgraderoom <version>` in Element to do this.
## How do I reset somebody's password on my server?
Use the admin endpoint [resetpassword](https://matrix-org.github.io/dendrite/administration/adminapi#post-_dendriteadminresetpassworduserid)
## Should I use PostgreSQL or SQLite for my databases?
Please use PostgreSQL wherever possible, especially if you are planning to run a homeserver that caters to more than a couple of users.
## What data needs to be kept if transferring/backing up Dendrite?
The list of files that need to be stored is:
- matrix-key.pem
- dendrite.yaml
- the postgres or sqlite DB
- the media store
- the search index (although this can be regenerated)
Note that this list may change / be out of date. We don't officially maintain instructions for migrations like this.
## How can I prepare enough storage for media caches?
This might be what you want: [matrix-media-repo](https://github.com/turt2live/matrix-media-repo)
We don't officially support this or any other dedicated media storage solutions.
## Is there an upgrade guide for Dendrite?
Run a newer docker image. We don't officially support deployments other than Docker.
Most of the time you should be able to just
- stop
- replace binary
- start
## Dendrite is using a lot of CPU
Generally speaking, you should expect to see some CPU spikes, particularly if you are joining or participating in large rooms. However, constant/sustained high CPU usage is not expected - if you are experiencing that, please join `#dendrite-dev:matrix.org` and let us know what you were doing when the
CPU usage shot up, or file a GitHub issue. If you can take a [CPU profile](PROFILING.md) then that would
CPU usage shot up, or file a GitHub issue. If you can take a [CPU profile](development/PROFILING.md) then that would
be a huge help too, as that will help us to understand where the CPU time is going.
## Dendrite is using a lot of RAM
@ -99,9 +149,13 @@ be a huge help too, as that will help us to understand where the CPU time is goi
As above with CPU usage, some memory spikes are expected if Dendrite is doing particularly heavy work
at a given instant. However, if it is using more RAM than you expect for a long time, that's probably
not expected. Join `#dendrite-dev:matrix.org` and let us know what you were doing when the memory usage
ballooned, or file a GitHub issue if you can. If you can take a [memory profile](PROFILING.md) then that
ballooned, or file a GitHub issue if you can. If you can take a [memory profile](development/PROFILING.md) then that
would be a huge help too, as that will help us to understand where the memory usage is happening.
## Do I need to generate the self-signed certificate if I'm going to use a reverse proxy?
No, if you already have a proper certificate from some provider, like Let's Encrypt, and use that on your reverse proxy, and the reverse proxy does TLS termination, then you’re good and can use HTTP to the dendrite process.
## Dendrite is running out of PostgreSQL database connections
You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode!

18
docs/Gemfile.lock
View file

@ -1,7 +1,7 @@
GEM
remote: https://rubygems.org/
specs:
activesupport (6.0.5)
activesupport (6.0.6.1)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 0.7, < 2)
minitest (~> 5.1)
@ -14,8 +14,8 @@ GEM
execjs
coffee-script-source (1.11.1)
colorator (1.1.0)
commonmarker (0.23.6)
concurrent-ruby (1.1.10)
commonmarker (0.23.7)
concurrent-ruby (1.2.0)
dnsruby (1.61.9)
simpleidn (~> 0.1)
em-websocket (0.5.3)
@ -229,11 +229,11 @@ GEM
jekyll (>= 3.5, < 5.0)
jekyll-feed (~> 0.9)
jekyll-seo-tag (~> 2.1)
minitest (5.15.0)
minitest (5.17.0)
multipart-post (2.1.1)
nokogiri (1.13.9-arm64-darwin)
nokogiri (1.13.10-arm64-darwin)
racc (~> 1.4)
nokogiri (1.13.9-x86_64-linux)
nokogiri (1.13.10-x86_64-linux)
racc (~> 1.4)
octokit (4.22.0)
faraday (>= 0.9)
@ -241,7 +241,7 @@ GEM
pathutil (0.16.2)
forwardable-extended (~> 2.6)
public_suffix (4.0.7)
racc (1.6.0)
racc (1.6.1)
rb-fsevent (0.11.1)
rb-inotify (0.10.1)
ffi (~> 1.0)
@ -265,13 +265,13 @@ GEM
thread_safe (0.3.6)
typhoeus (1.4.0)
ethon (>= 0.9.0)
tzinfo (1.2.10)
tzinfo (1.2.11)
thread_safe (~> 0.1)
unf (0.1.4)
unf_ext
unf_ext (0.0.8.1)
unicode-display_width (1.8.0)
zeitwerk (2.5.4)
zeitwerk (2.6.6)
PLATFORMS
arm64-darwin-21

7
docs/administration/4_adminapi.md
View file

@ -44,7 +44,9 @@ This endpoint will instruct Dendrite to part the given local `userID` in the URL
all rooms which they are currently joined. A JSON body will be returned containing
the room IDs of all affected rooms.
## POST `/_dendrite/admin/resetPassword/{localpart}`
## POST `/_dendrite/admin/resetPassword/{userID}`
Reset the password of a local user.
Request body format:
@ -54,9 +56,6 @@ Request body format:
}
```
Reset the password of a local user. The `localpart` is the username only, i.e. if
the full user ID is `@alice:domain.com` then the local part is `alice`.
## GET `/_dendrite/admin/fulltext/reindex`
This endpoint instructs Dendrite to reindex all searchable events (`m.room.message`, `m.room.topic` and `m.room.name`). An empty JSON body will be returned immediately.

41
docs/CONTRIBUTING.md → docs/development/CONTRIBUTING.md
View file

@ -9,6 +9,28 @@ permalink: /development/contributing
Everyone is welcome to contribute to Dendrite! We aim to make it as easy as
possible to get started.
## Contribution types
We are a small team maintaining a large project. As a result, we cannot merge every feature, even if it
is bug-free and useful, because we then commit to maintaining it indefinitely. We will always accept:
- bug fixes
- security fixes (please responsibly disclose via security@matrix.org *before* creating pull requests)
We will accept the following with caveats:
- documentation fixes, provided they do not add additional instructions which can end up going out-of-date,
e.g example configs, shell commands.
- performance fixes, provided they do not add significantly more maintenance burden.
- additional functionality on existing features, provided the functionality is small and maintainable.
- additional functionality that, in its absence, would impact the ecosystem e.g spam and abuse mitigations
- test-only changes, provided they help improve coverage or test tricky code.
The following items are at risk of not being accepted:
- Configuration or CLI changes, particularly ones which increase the overall configuration surface.
The following items are unlikely to be accepted into a main Dendrite release for now:
- New MSC implementations.
- New features which are not in the specification.
## Sign off
We require that everyone who contributes to the project signs off their contributions
@ -35,7 +57,7 @@ to do so for future contributions.
## Getting up and running
See the [Installation](installation) section for information on how to build an
See the [Installation](../installation) section for information on how to build an
instance of Dendrite. You will likely need this in order to test your changes.
## Code style
@ -75,7 +97,20 @@ comment. Please avoid doing this if you can.
We also have unit tests which we run via:
```bash
go test --race ./...
DENDRITE_TEST_SKIP_NODB=1 go test --race ./...
```
This only runs SQLite database tests. If you wish to execute Postgres tests as well, you'll either need to
have Postgres installed locally (`createdb` will be used) or have a remote/containerized Postgres instance
available.
To configure the connection to a remote Postgres, you can use the following enviroment variables:
```bash
POSTGRES_USER=postgres
POSTGERS_PASSWORD=yourPostgresPassword
POSTGRES_HOST=localhost
POSTGRES_DB=postgres # the superuser database to use
```
In general, we like submissions that come with tests. Anything that proves that the
@ -116,7 +151,7 @@ significant amount of CPU and RAM.
Once the code builds, run [Sytest](https://github.com/matrix-org/sytest)
according to the guide in
[docs/sytest.md](https://github.com/matrix-org/dendrite/blob/main/docs/sytest.md#using-a-sytest-docker-image)
[docs/development/sytest.md](https://github.com/matrix-org/dendrite/blob/main/docs/development/sytest.md#using-a-sytest-docker-image)
so you can see whether something is being broken and whether there are newly
passing tests.

0
docs/PROFILING.md → docs/development/PROFILING.md
View file

0
docs/coverage.md → docs/development/coverage.md
View file

0
docs/sytest.md → docs/development/sytest.md
View file

0
docs/tracing/opentracing.md → docs/development/tracing/opentracing.md
View file

0
docs/tracing/setup.md → docs/development/tracing/setup.md
View file

2
docs/installation/2_domainname.md
View file

@ -90,7 +90,7 @@ For example, this can be done with the following Caddy config:
handle /.well-known/matrix/server {
header Content-Type application/json
header Access-Control-Allow-Origin *
respond `"m.server": "matrix.example.com:8448"`
respond `{"m.server": "matrix.example.com:8448"}`
}
handle /.well-known/matrix/client {

125
federationapi/api/api.go
View file

@ -18,11 +18,12 @@ type FederationInternalAPI interface {
gomatrixserverlib.KeyDatabase
ClientFederationAPI
RoomserverFederationAPI
P2PFederationAPI
QueryServerKeys(ctx context.Context, request *QueryServerKeysRequest, response *QueryServerKeysResponse) error
LookupServerKeys(ctx context.Context, s gomatrixserverlib.ServerName, keyRequests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) ([]gomatrixserverlib.ServerKeys, error)
MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
MSC2836EventRelationships(ctx context.Context, origin, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
MSC2946Spaces(ctx context.Context, origin, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
// Broadcasts an EDU to all servers in rooms we are joined to. Used in the yggdrasil demos.
PerformBroadcastEDU(
@ -30,6 +31,11 @@ type FederationInternalAPI interface {
request *PerformBroadcastEDURequest,
response *PerformBroadcastEDUResponse,
) error
PerformWakeupServers(
ctx context.Context,
request *PerformWakeupServersRequest,
response *PerformWakeupServersResponse,
) error
}
type ClientFederationAPI interface {
@ -60,48 +66,77 @@ type RoomserverFederationAPI interface {
// containing only the server names (without information for membership events).
// The response will include this server if they are joined to the room.
QueryJoinedHostServerNamesInRoom(ctx context.Context, request *QueryJoinedHostServerNamesInRoomRequest, response *QueryJoinedHostServerNamesInRoomResponse) error
GetEventAuth(ctx context.Context, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
GetEventAuth(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
GetEvent(ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
}
type P2PFederationAPI interface {
// Get the relay servers associated for the given server.
P2PQueryRelayServers(
ctx context.Context,
request *P2PQueryRelayServersRequest,
response *P2PQueryRelayServersResponse,
) error
// Add relay server associations to the given server.
P2PAddRelayServers(
ctx context.Context,
request *P2PAddRelayServersRequest,
response *P2PAddRelayServersResponse,
) error
// Remove relay server associations from the given server.
P2PRemoveRelayServers(
ctx context.Context,
request *P2PRemoveRelayServersRequest,
response *P2PRemoveRelayServersResponse,
) error
}
// KeyserverFederationAPI is a subset of gomatrixserverlib.FederationClient functions which the keyserver
// implements as proxy calls, with built-in backoff/retries/etc. Errors returned from functions in
// this interface are of type FederationClientError
type KeyserverFederationAPI interface {
GetUserDevices(ctx context.Context, s gomatrixserverlib.ServerName, userID string) (res gomatrixserverlib.RespUserDevices, err error)
ClaimKeys(ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res gomatrixserverlib.RespClaimKeys, err error)
QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (res gomatrixserverlib.RespQueryKeys, err error)
GetUserDevices(ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string) (res gomatrixserverlib.RespUserDevices, err error)
ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (res gomatrixserverlib.RespClaimKeys, err error)
QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (res gomatrixserverlib.RespQueryKeys, err error)
}
// an interface for gmsl.FederationClient - contains functions called by federationapi only.
type FederationClient interface {
P2PFederationClient
gomatrixserverlib.KeyClient
SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error)
// Perform operations
LookupRoomAlias(ctx context.Context, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error)
Peek(ctx context.Context, s gomatrixserverlib.ServerName, roomID, peekID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespPeek, err error)
MakeJoin(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error)
SendJoin(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error)
MakeLeave(ctx context.Context, s gomatrixserverlib.ServerName, roomID, userID string) (res gomatrixserverlib.RespMakeLeave, err error)
SendLeave(ctx context.Context, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (err error)
SendInviteV2(ctx context.Context, s gomatrixserverlib.ServerName, request gomatrixserverlib.InviteV2Request) (res gomatrixserverlib.RespInviteV2, err error)
LookupRoomAlias(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomAlias string) (res gomatrixserverlib.RespDirectory, err error)
Peek(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, peekID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespPeek, err error)
MakeJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string, roomVersions []gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMakeJoin, err error)
SendJoin(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (res gomatrixserverlib.RespSendJoin, err error)
MakeLeave(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID, userID string) (res gomatrixserverlib.RespMakeLeave, err error)
SendLeave(ctx context.Context, origin, s gomatrixserverlib.ServerName, event *gomatrixserverlib.Event) (err error)
SendInviteV2(ctx context.Context, origin, s gomatrixserverlib.ServerName, request gomatrixserverlib.InviteV2Request) (res gomatrixserverlib.RespInviteV2, err error)
GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
GetEvent(ctx context.Context, origin, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error)
GetEventAuth(ctx context.Context, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
GetUserDevices(ctx context.Context, s gomatrixserverlib.ServerName, userID string) (gomatrixserverlib.RespUserDevices, error)
ClaimKeys(ctx context.Context, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error)
QueryKeys(ctx context.Context, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error)
Backfill(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string) (res gomatrixserverlib.Transaction, err error)
MSC2836EventRelationships(ctx context.Context, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
MSC2946Spaces(ctx context.Context, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
GetEventAuth(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) (res gomatrixserverlib.RespEventAuth, err error)
GetUserDevices(ctx context.Context, origin, s gomatrixserverlib.ServerName, userID string) (gomatrixserverlib.RespUserDevices, error)
ClaimKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, oneTimeKeys map[string]map[string]string) (gomatrixserverlib.RespClaimKeys, error)
QueryKeys(ctx context.Context, origin, s gomatrixserverlib.ServerName, keys map[string][]string) (gomatrixserverlib.RespQueryKeys, error)
Backfill(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, limit int, eventIDs []string) (res gomatrixserverlib.Transaction, err error)
MSC2836EventRelationships(ctx context.Context, origin, dst gomatrixserverlib.ServerName, r gomatrixserverlib.MSC2836EventRelationshipsRequest, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.MSC2836EventRelationshipsResponse, err error)
MSC2946Spaces(ctx context.Context, origin, dst gomatrixserverlib.ServerName, roomID string, suggestedOnly bool) (res gomatrixserverlib.MSC2946SpacesResponse, err error)
ExchangeThirdPartyInvite(ctx context.Context, s gomatrixserverlib.ServerName, builder gomatrixserverlib.EventBuilder) (err error)
LookupState(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespState, err error)
LookupStateIDs(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error)
LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
ExchangeThirdPartyInvite(ctx context.Context, origin, s gomatrixserverlib.ServerName, builder gomatrixserverlib.EventBuilder) (err error)
LookupState(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespState, err error)
LookupStateIDs(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error)
LookupMissingEvents(ctx context.Context, origin, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error)
}
type P2PFederationClient interface {
P2PSendTransactionToRelay(ctx context.Context, u gomatrixserverlib.UserID, t gomatrixserverlib.Transaction, forwardingServer gomatrixserverlib.ServerName) (res gomatrixserverlib.EmptyResp, err error)
P2PGetTransactionFromRelay(ctx context.Context, u gomatrixserverlib.UserID, prev gomatrixserverlib.RelayEntry, relayServer gomatrixserverlib.ServerName) (res gomatrixserverlib.RespGetRelayTransaction, err error)
}
// FederationClientError is returned from FederationClient methods in the event of a problem.
@ -198,8 +233,9 @@ type PerformInviteResponse struct {
// QueryJoinedHostServerNamesInRoomRequest is a request to QueryJoinedHostServerNames
type QueryJoinedHostServerNamesInRoomRequest struct {
RoomID string `json:"room_id"`
ExcludeSelf bool `json:"exclude_self"`
RoomID string `json:"room_id"`
ExcludeSelf bool `json:"exclude_self"`
ExcludeBlacklisted bool `json:"exclude_blacklisted"`
}
// QueryJoinedHostServerNamesInRoomResponse is a response to QueryJoinedHostServerNames
@ -213,9 +249,40 @@ type PerformBroadcastEDURequest struct {
type PerformBroadcastEDUResponse struct {
}
type PerformWakeupServersRequest struct {
ServerNames []gomatrixserverlib.ServerName `json:"server_names"`
}
type PerformWakeupServersResponse struct {
}
type InputPublicKeysRequest struct {
Keys map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult `json:"keys"`
}
type InputPublicKeysResponse struct {
}
type P2PQueryRelayServersRequest struct {
Server gomatrixserverlib.ServerName
}
type P2PQueryRelayServersResponse struct {
RelayServers []gomatrixserverlib.ServerName
}
type P2PAddRelayServersRequest struct {
Server gomatrixserverlib.ServerName
RelayServers []gomatrixserverlib.ServerName
}
type P2PAddRelayServersResponse struct {
}
type P2PRemoveRelayServersRequest struct {
Server gomatrixserverlib.ServerName
RelayServers []gomatrixserverlib.ServerName
}
type P2PRemoveRelayServersResponse struct {
}

4
federationapi/consumers/keychange.go
View file

@ -128,7 +128,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool {
}
// send this key change to all servers who share rooms with this user.
destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true)
destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true)
if err != nil {
sentry.CaptureException(err)
logger.WithError(err).Error("failed to calculate joined hosts for rooms user is in")
@ -189,7 +189,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool {
return true
}
// send this key change to all servers who share rooms with this user.
destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true)
destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true)
if err != nil {
sentry.CaptureException(err)
logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined hosts for rooms user is in")

2
federationapi/consumers/presence.go
View file

@ -111,7 +111,7 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg
}
// send this presence to all servers who share rooms with this user.
joined, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true)
joined, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true, true)
if err != nil {
log.WithError(err).Error("failed to get joined hosts")
return true

135
federationapi/consumers/roomserver.go
View file

@ -18,9 +18,14 @@ import (
"context"
"encoding/json"
"fmt"
"strconv"
"time"
syncAPITypes "github.com/matrix-org/dendrite/syncapi/types"
"github.com/matrix-org/gomatrixserverlib"
"github.com/nats-io/nats.go"
"github.com/sirupsen/logrus"
log "github.com/sirupsen/logrus"
"github.com/matrix-org/dendrite/federationapi/queue"
@ -34,14 +39,16 @@ import (
// OutputRoomEventConsumer consumes events that originated in the room server.
type OutputRoomEventConsumer struct {
ctx context.Context
cfg *config.FederationAPI
rsAPI api.FederationRoomserverAPI
jetstream nats.JetStreamContext
durable string
db storage.Database
queues *queue.OutgoingQueues
topic string
ctx context.Context
cfg *config.FederationAPI
rsAPI api.FederationRoomserverAPI
jetstream nats.JetStreamContext
natsClient *nats.Conn
durable string
db storage.Database
queues *queue.OutgoingQueues
topic string
topicPresence string
}
// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
@ -49,19 +56,22 @@ func NewOutputRoomEventConsumer(
process *process.ProcessContext,
cfg *config.FederationAPI,
js nats.JetStreamContext,
natsClient *nats.Conn,
queues *queue.OutgoingQueues,
store storage.Database,
rsAPI api.FederationRoomserverAPI,
) *OutputRoomEventConsumer {
return &OutputRoomEventConsumer{
ctx: process.Context(),
cfg: cfg,
jetstream: js,
db: store,
queues: queues,
rsAPI: rsAPI,
durable: cfg.Matrix.JetStream.Durable("FederationAPIRoomServerConsumer"),
topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent),
ctx: process.Context(),
cfg: cfg,
jetstream: js,
natsClient: natsClient,
db: store,
queues: queues,
rsAPI: rsAPI,
durable: cfg.Matrix.JetStream.Durable("FederationAPIRoomServerConsumer"),
topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputRoomEvent),
topicPresence: cfg.Matrix.JetStream.Prefixed(jetstream.RequestPresence),
}
}
@ -81,8 +91,10 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms
msg := msgs[0] // Guaranteed to exist if onMessage is called
receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType))
// Only handle events we care about
if receivedType != api.OutputTypeNewRoomEvent && receivedType != api.OutputTypeNewInboundPeek {
// Only handle events we care about, avoids unneeded unmarshalling
switch receivedType {
case api.OutputTypeNewRoomEvent, api.OutputTypeNewInboundPeek, api.OutputTypePurgeRoom:
default:
return true
}
@ -117,6 +129,14 @@ func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Ms
return false
}
case api.OutputTypePurgeRoom:
log.WithField("room_id", output.PurgeRoom.RoomID).Warn("Purging room from federation API")
if err := s.db.PurgeRoom(ctx, output.PurgeRoom.RoomID); err != nil {
logrus.WithField("room_id", output.PurgeRoom.RoomID).WithError(err).Error("Failed to purge room from federation API")
} else {
logrus.WithField("room_id", output.PurgeRoom.RoomID).Warn("Room purged from federation API")
}
default:
log.WithField("type", output.Type).Debug(
"roomserver output log: ignoring unknown output type",
@ -146,6 +166,7 @@ func (s *OutputRoomEventConsumer) processInboundPeek(orp api.OutputNewInboundPee
// processMessage updates the list of currently joined hosts in the room
// and then sends the event to the hosts that were joined before the event.
func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rewritesState bool) error {
addsStateEvents, missingEventIDs := ore.NeededStateEventIDs()
// Ask the roomserver and add in the rest of the results into the set.
@ -184,6 +205,14 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
return err
}
// If we added new hosts, inform them about our known presence events for this room
if s.cfg.Matrix.Presence.EnableOutbound && len(addsJoinedHosts) > 0 && ore.Event.Type() == gomatrixserverlib.MRoomMember && ore.Event.StateKey() != nil {
membership, _ := ore.Event.Membership()
if membership == gomatrixserverlib.Join {
s.sendPresence(ore.Event.RoomID(), addsJoinedHosts)
}
}
if oldJoinedHosts == nil {
// This means that there is nothing to update as this is a duplicate
// message.
@ -213,6 +242,76 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
)
}
func (s *OutputRoomEventConsumer) sendPresence(roomID string, addedJoined []types.JoinedHost) {
joined := make([]gomatrixserverlib.ServerName, 0, len(addedJoined))
for _, added := range addedJoined {
joined = append(joined, added.ServerName)
}
// get our locally joined users
var queryRes api.QueryMembershipsForRoomResponse
err := s.rsAPI.QueryMembershipsForRoom(s.ctx, &api.QueryMembershipsForRoomRequest{
JoinedOnly: true,
LocalOnly: true,
RoomID: roomID,
}, &queryRes)
if err != nil {
log.WithError(err).Error("failed to calculate joined rooms for user")
return
}
// send every presence we know about to the remote server
content := types.Presence{}
for _, ev := range queryRes.JoinEvents {
msg := nats.NewMsg(s.topicPresence)
msg.Header.Set(jetstream.UserID, ev.Sender)
var presence *nats.Msg
presence, err = s.natsClient.RequestMsg(msg, time.Second*10)
if err != nil {
log.WithError(err).Errorf("unable to get presence")
continue
}
statusMsg := presence.Header.Get("status_msg")
e := presence.Header.Get("error")
if e != "" {
continue
}
var lastActive int
lastActive, err = strconv.Atoi(presence.Header.Get("last_active_ts"))
if err != nil {
continue
}
p := syncAPITypes.PresenceInternal{LastActiveTS: gomatrixserverlib.Timestamp(lastActive)}
content.Push = append(content.Push, types.PresenceContent{
CurrentlyActive: p.CurrentlyActive(),
LastActiveAgo: p.LastActiveAgo(),
Presence: presence.Header.Get("presence"),
StatusMsg: &statusMsg,
UserID: ev.Sender,
})
}
if len(content.Push) == 0 {
return
}
edu := &gomatrixserverlib.EDU{
Type: gomatrixserverlib.MPresence,
Origin: string(s.cfg.Matrix.ServerName),
}
if edu.Content, err = json.Marshal(content); err != nil {
log.WithError(err).Error("failed to marshal EDU JSON")
return
}
if err := s.queues.SendEDU(edu, s.cfg.Matrix.ServerName, joined); err != nil {
log.WithError(err).Error("failed to send EDU")
}
}
// joinedHostsAtEvent works out a list of matrix servers that were joined to
// the room at the event (including peeking ones)
// It is important to use the state at the event for sending messages because:

26
federationapi/federationapi.go
View file

@ -43,8 +43,8 @@ import (
// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions
// on the given input API.
func AddInternalRoutes(router *mux.Router, intAPI api.FederationInternalAPI) {
inthttp.AddRoutes(intAPI, router)
func AddInternalRoutes(router *mux.Router, intAPI api.FederationInternalAPI, enableMetrics bool) {
inthttp.AddRoutes(intAPI, router, enableMetrics)
}
// AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component.
@ -85,10 +85,7 @@ func AddPublicRoutes(
}
routing.Setup(
base.PublicFederationAPIMux,
base.PublicKeyAPIMux,
base.PublicWellKnownAPIMux,
cfg,
base,
rsAPI, f, keyRing,
federation, userAPI, keyAPI, mscCfg,
servers, producer,
@ -116,23 +113,24 @@ func NewInternalAPI(
_ = federationDB.RemoveAllServersFromBlacklist()
}
stats := statistics.NewStatistics(federationDB, cfg.FederationMaxRetries+1)
stats := statistics.NewStatistics(
federationDB,
cfg.FederationMaxRetries+1,
cfg.P2PFederationRetriesUntilAssumedOffline+1)
js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
js, nats := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream)
signingInfo := base.Cfg.Global.SigningIdentities()
queues := queue.NewOutgoingQueues(
federationDB, base.ProcessContext,
cfg.Matrix.DisableFederation,
cfg.Matrix.ServerName, federation, rsAPI, &stats,
&queue.SigningInfo{
KeyID: cfg.Matrix.KeyID,
PrivateKey: cfg.Matrix.PrivateKey,
ServerName: cfg.Matrix.ServerName,
},
signingInfo,
)
rsConsumer := consumers.NewOutputRoomEventConsumer(
base.ProcessContext, cfg, js, queues,
base.ProcessContext, cfg, js, nats, queues,
federationDB, rsAPI,
)
if err = rsConsumer.Start(); err != nil {

Some files were not shown because too many files have changed in this diff Show more