mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2025-12-06 14:33:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:matrix-org/dendrite into update_postgres

Browse source
This commit is contained in:
Till Faelligen 2024-02-29 08:29:22 +01:00
parent 2f8bf7ed96 66865597e2
commit fe80a2ad71
No known key found for this signature in database
GPG key ID: 3DF82D8AB9211D4E
77 changed files with 1698 additions and 434 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/codecov.yaml vendored
View file

@ -7,7 +7,7 @@ coverage:
project:
default:
target: auto
threshold: 0%
threshold: 0.1%
base: auto
flags:
- unittests

71
.github/workflows/dendrite.yml vendored
View file

@ -28,10 +28,10 @@ jobs:
runs-on: ubuntu-latest
if: ${{ false }} # disable for now
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v3
uses: actions/setup-go@v4
with:
go-version: "stable"
cache: true
@ -41,7 +41,7 @@ jobs:
with:
node-version: 14
- uses: actions/cache@v3
- uses: actions/cache@v4
with:
path: ~/.npm
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@ -66,11 +66,11 @@ jobs:
name: Linting
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install libolm
run: sudo apt-get install libolm-dev libolm3
- name: Install Go
uses: actions/setup-go@v3
uses: actions/setup-go@v4
with:
go-version: "stable"
- name: golangci-lint
@ -102,14 +102,14 @@ jobs:
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install libolm
run: sudo apt-get install libolm-dev libolm3
- name: Setup go
uses: actions/setup-go@v3
uses: actions/setup-go@v4
with:
go-version: "stable"
- uses: actions/cache@v3
- uses: actions/cache@v4
# manually set up caches, as they otherwise clash with different steps using setup-go with cache=true
with:
path: |
@ -141,12 +141,12 @@ jobs:
goos: ["linux"]
goarch: ["amd64", "386"]
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Setup go
uses: actions/setup-go@v3
uses: actions/setup-go@v4
with:
go-version: "stable"
- uses: actions/cache@v3
- uses: actions/cache@v4
with:
path: |
~/.cache/go-build
@ -174,12 +174,12 @@ jobs:
goos: ["windows"]
goarch: ["amd64"]
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v3
uses: actions/setup-go@v4
with:
go-version: "stable"
- uses: actions/cache@v3
- uses: actions/cache@v4
with:
path: |
~/.cache/go-build
@ -235,11 +235,11 @@ jobs:
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install libolm
run: sudo apt-get install libolm-dev libolm3
- name: Setup go
uses: actions/setup-go@v3
uses: actions/setup-go@v4
with:
go-version: "stable"
- name: Set up gotestfmt
@ -247,7 +247,7 @@ jobs:
with:
# Optional: pass GITHUB_TOKEN to avoid rate limiting.
token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/cache@v3
- uses: actions/cache@v4
with:
path: |
~/.cache/go-build
@ -262,10 +262,11 @@ jobs:
POSTGRES_PASSWORD: postgres
POSTGRES_DB: dendrite
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
uses: codecov/codecov-action@v4
with:
flags: unittests
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
# run database upgrade tests
upgrade_test:
@ -274,12 +275,20 @@ jobs:
needs: initial-tests-done
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Setup go
uses: actions/setup-go@v3
uses: actions/setup-go@v4
with:
go-version: "stable"
cache: true
- uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-upgrade-test-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-upgrade-test-
- name: Docker version
run: docker version
- name: Build upgrade-tests
@ -296,12 +305,20 @@ jobs:
needs: initial-tests-done
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Setup go
uses: actions/setup-go@v3
uses: actions/setup-go@v4
with:
go-version: "stable"
cache: true
- uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-upgrade-direct-test-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-upgrade-direct-test-
- name: Docker version
run: docker version
- name: Build upgrade-tests
@ -340,8 +357,8 @@ jobs:
SYTEST_BRANCH: ${{ github.head_ref }}
CGO_ENABLED: ${{ matrix.cgo && 1 }}
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v4
- uses: actions/cache@v4
with:
path: |
~/.cache/go-build
@ -364,7 +381,7 @@ jobs:
run: /src/are-we-synapse-yet.py /logs/results.tap -v
continue-on-error: true # not fatal
- name: Upload Sytest logs
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v4
if: ${{ always() }}
with:
name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
@ -404,8 +421,8 @@ jobs:
run: |
sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
- name: Run actions/checkout@v3 for dendrite
uses: actions/checkout@v3
- name: Run actions/checkout@v4 for dendrite
uses: actions/checkout@v4
with:
path: dendrite

30
.github/workflows/docker.yml vendored
View file

@ -27,22 +27,22 @@ jobs:
security-events: write # To upload Trivy sarif files
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Get release tag & build flags
if: github.event_name == 'release' # Only for GitHub releases
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
@ -98,22 +98,22 @@ jobs:
packages: write
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Get release tag & build flags
if: github.event_name == 'release' # Only for GitHub releases
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
@ -159,22 +159,22 @@ jobs:
packages: write
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Get release tag & build flags
if: github.event_name == 'release' # Only for GitHub releases
run: |
echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Login to GitHub Containers
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}

2
.github/workflows/gh-pages.yml vendored
View file

@ -28,7 +28,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Setup Pages
uses: actions/configure-pages@v2
- name: Build with Jekyll

2
.github/workflows/helm.yml vendored
View file

@ -17,7 +17,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v4
with:
fetch-depth: 0

6
.github/workflows/k8s.yml vendored
View file

@ -17,7 +17,7 @@ jobs:
outputs:
changed: ${{ steps.list-changed.outputs.changed }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: azure/setup-helm@v3
@ -48,7 +48,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ inputs.checkoutCommit }}
@ -66,7 +66,7 @@ jobs:
- name: Create k3d cluster
uses: nolar/setup-k3d-k3s@v1
with:
version: v1.21
version: v1.28
- name: Remove node taints
run: |
kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true

66
.github/workflows/schedules.yaml vendored
View file

@ -10,8 +10,26 @@ concurrency:
cancel-in-progress: true
jobs:
check_date: # https://stackoverflow.com/questions/63014786/how-to-schedule-a-github-actions-nightly-build-but-run-it-only-when-there-where
runs-on: ubuntu-latest
name: Check latest commit
outputs:
should_run: ${{ steps.should_run.outputs.should_run }}
steps:
- uses: actions/checkout@v4
- name: print latest_commit
run: echo ${{ github.sha }}
- id: should_run
continue-on-error: true
name: check latest commit is less than a day
if: ${{ github.event_name == 'schedule' }}
run: test -z $(git rev-list --after="24 hours" ${{ github.sha }}) && echo "::set-output name=should_run::false"
# run Sytest in different variations
sytest:
needs: check_date
if: ${{ needs.check_date.outputs.should_run != 'false' }}
timeout-minutes: 60
name: "Sytest (${{ matrix.label }})"
runs-on: ubuntu-latest
@ -38,8 +56,8 @@ jobs:
RACE_DETECTION: 1
COVER: 1
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v4
- uses: actions/cache@v4
with:
path: |
~/.cache/go-build
@ -62,7 +80,7 @@ jobs:
run: /src/are-we-synapse-yet.py /logs/results.tap -v
continue-on-error: true # not fatal
- name: Upload Sytest logs
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v4
if: ${{ always() }}
with:
name: Sytest Logs - ${{ job.status }} - (Dendrite ${{ join(matrix.*, ' ') }})
@ -75,31 +93,34 @@ jobs:
timeout-minutes: 5
name: "Sytest Coverage"
runs-on: ubuntu-latest
needs: sytest # only run once Sytest is done
if: ${{ always() }}
needs: [ sytest, check_date ] # only run once Sytest is done and there was a commit
if: ${{ always() && needs.check_date.outputs.should_run != 'false' }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v3
uses: actions/setup-go@v4
with:
go-version: 'stable'
cache: true
- name: Download all artifacts
uses: actions/download-artifact@v3
uses: actions/download-artifact@v4
- name: Collect coverage
run: |
go tool covdata textfmt -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o sytest.cov
grep -Ev 'relayapi|setup/mscs|api_trace' sytest.cov > final.cov
go tool covdata func -i="$(find Sytest* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
uses: codecov/codecov-action@v4
with:
files: ./final.cov
flags: sytest
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
# run Complement
complement:
needs: check_date
if: ${{ needs.check_date.outputs.should_run != 'false' }}
name: "Complement (${{ matrix.label }})"
timeout-minutes: 60
runs-on: ubuntu-latest
@ -129,8 +150,8 @@ jobs:
run: |
sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
- name: Run actions/checkout@v3 for dendrite
uses: actions/checkout@v3
- name: Run actions/checkout@v4 for dendrite
uses: actions/checkout@v4
with:
path: dendrite
@ -174,7 +195,7 @@ jobs:
# Run Complement
- run: |
set -o pipefail &&
go test -v -json -tags dendrite_blacklist ./tests/... 2>&1 | gotestfmt
go test -v -json -tags dendrite_blacklist ./tests ./tests/csapi 2>&1 | gotestfmt -hide all
shell: bash
name: Run Complement Tests
env:
@ -185,7 +206,7 @@ jobs:
working-directory: complement
- name: Upload Complement logs
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v4
if: ${{ always() }}
with:
name: Complement Logs - (Dendrite ${{ join(matrix.*, ' ') }})
@ -196,30 +217,32 @@ jobs:
timeout-minutes: 5
name: "Complement Coverage"
runs-on: ubuntu-latest
needs: complement # only run once Complement is done
if: ${{ always() }}
needs: [ complement, check_date ] # only run once Complements is done and there was a commit
if: ${{ always() && needs.check_date.outputs.should_run != 'false' }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v3
uses: actions/setup-go@v4
with:
go-version: 'stable'
cache: true
- name: Download all artifacts
uses: actions/download-artifact@v3
uses: actions/download-artifact@v4
- name: Collect coverage
run: |
go tool covdata textfmt -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)" -o complement.cov
grep -Ev 'relayapi|setup/mscs|api_trace' complement.cov > final.cov
go tool covdata func -i="$(find Complement* -name 'covmeta*' -type f -exec dirname {} \; | uniq | paste -s -d ',' -)"
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
uses: codecov/codecov-action@v4
with:
files: ./final.cov
flags: complement
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }} # required
element-web:
if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright
timeout-minutes: 120
runs-on: ubuntu-latest
steps:
@ -228,7 +251,7 @@ jobs:
# Our test suite includes some screenshot tests with unusual diacritics, which are
# supposed to be covered by STIXGeneral.
tools: fonts-stix
- uses: actions/checkout@v2
- uses: actions/checkout@v4
with:
repository: matrix-org/matrix-react-sdk
- uses: actions/setup-node@v3
@ -259,6 +282,7 @@ jobs:
TMPDIR: ${{ runner.temp }}
element-web-pinecone:
if: ${{ false }} # disable for now, as Cypress has been replaced by Playwright
timeout-minutes: 120
runs-on: ubuntu-latest
steps:
@ -267,7 +291,7 @@ jobs:
# Our test suite includes some screenshot tests with unusual diacritics, which are
# supposed to be covered by STIXGeneral.
tools: fonts-stix
- uses: actions/checkout@v2
- uses: actions/checkout@v4
with:
repository: matrix-org/matrix-react-sdk
- uses: actions/setup-node@v3

21
CHANGES.md
View file

@ -1,5 +1,26 @@
# Changelog
## Dendrite 0.13.6 (2024-01-26)
Upgrading to this version is **highly** recommended, as it contains several QoL improvements.
### Fixes
- Use `AckExplicitPolicy` for JetStream consumers, so messages don't pile up in NATS
- A rare panic when assigning a state key NID has been fixed
- A rare panic when checking powerlevels has been fixed
- Notary keys requests for all keys now work correctly
- Spec compliance:
- Return `M_INVALID_PARAM` when querying room aliases
- Handle empty `from` parameter when requesting `/messages`
- Add CORP headers on media endpoints
- Remove `aliases` from `/publicRooms` responses
- Allow `+` in MXIDs (Contributed by [RosstheRoss](https://github.com/RosstheRoss))
- Fixes membership transitions from `knock` to `join` in `knock_restricted` rooms
- Incremental syncs now batch querying events (Contributed by [recht](https://github.com/recht))
- Move `/joined_members` back to the clientAPI/roomserver, which should make bridges happier again
- Backfilling from other servers now only uses at max 100 events instead of potentially thousands
## Dendrite 0.13.5 (2023-12-12)
Upgrading to this version is **highly** recommended, as it fixes several long-standing bugs in

14
appservice/api/query.go
View file

@ -82,9 +82,17 @@ type UserIDExistsResponse struct {
}
const (
ASProtocolPath = "/_matrix/app/unstable/thirdparty/protocol/"
ASUserPath = "/_matrix/app/unstable/thirdparty/user"
ASLocationPath = "/_matrix/app/unstable/thirdparty/location"
ASProtocolLegacyPath = "/_matrix/app/unstable/thirdparty/protocol/"
ASUserLegacyPath = "/_matrix/app/unstable/thirdparty/user"
ASLocationLegacyPath = "/_matrix/app/unstable/thirdparty/location"
ASRoomAliasExistsLegacyPath = "/rooms/"
ASUserExistsLegacyPath = "/users/"
ASProtocolPath = "/_matrix/app/v1/thirdparty/protocol/"
ASUserPath = "/_matrix/app/v1/thirdparty/user"
ASLocationPath = "/_matrix/app/v1/thirdparty/location"
ASRoomAliasExistsPath = "/_matrix/app/v1/rooms/"
ASUserExistsPath = "/_matrix/app/v1/users/"
)
type ProtocolRequest struct {

12
appservice/consumers/roomserver.go
View file

@ -206,13 +206,21 @@ func (s *OutputRoomEventConsumer) sendEvents(
}
// Send the transaction to the appservice.
// https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid
address := fmt.Sprintf("%s/transactions/%s?access_token=%s", state.RequestUrl(), txnID, url.QueryEscape(state.HSToken))
// https://spec.matrix.org/v1.9/application-service-api/#pushing-events
path := "_matrix/app/v1/transactions"
if s.cfg.LegacyPaths {
path = "transactions"
}
address := fmt.Sprintf("%s/%s/%s", state.RequestUrl(), path, txnID)
if s.cfg.LegacyAuth {
address += "?access_token=" + url.QueryEscape(state.HSToken)
}
req, err := http.NewRequestWithContext(ctx, "PUT", address, bytes.NewBuffer(transaction))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", state.HSToken))
resp, err := state.HTTPClient.Do(req)
if err != nil {
return state.backoffAndPause(err)

98
appservice/query/query.go
View file

@ -19,10 +19,10 @@ package query
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"sync"
log "github.com/sirupsen/logrus"
@ -32,9 +32,6 @@ import (
"github.com/matrix-org/dendrite/setup/config"
)
const roomAliasExistsPath = "/rooms/"
const userIDExistsPath = "/users/"
// AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI
type AppServiceQueryAPI struct {
Cfg *config.AppServiceAPI
@ -55,14 +52,23 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
// Determine which application service should handle this request
for _, appservice := range a.Cfg.Derived.ApplicationServices {
if appservice.URL != "" && appservice.IsInterestedInRoomAlias(request.Alias) {
path := api.ASRoomAliasExistsPath
if a.Cfg.LegacyPaths {
path = api.ASRoomAliasExistsLegacyPath
}
// The full path to the rooms API, includes hs token
URL, err := url.Parse(appservice.RequestUrl() + roomAliasExistsPath)
URL, err := url.Parse(appservice.RequestUrl() + path)
if err != nil {
return err
}
URL.Path += request.Alias
apiURL := URL.String() + "?access_token=" + appservice.HSToken
if a.Cfg.LegacyAuth {
q := URL.Query()
q.Set("access_token", appservice.HSToken)
URL.RawQuery = q.Encode()
}
apiURL := URL.String()
// Send a request to each application service. If one responds that it has
// created the room, immediately return.
@ -70,6 +76,7 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
if err != nil {
return err
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", appservice.HSToken))
req = req.WithContext(ctx)
resp, err := appservice.HTTPClient.Do(req)
@ -123,12 +130,21 @@ func (a *AppServiceQueryAPI) UserIDExists(
for _, appservice := range a.Cfg.Derived.ApplicationServices {
if appservice.URL != "" && appservice.IsInterestedInUserID(request.UserID) {
// The full path to the rooms API, includes hs token
URL, err := url.Parse(appservice.RequestUrl() + userIDExistsPath)
path := api.ASUserExistsPath
if a.Cfg.LegacyPaths {
path = api.ASUserExistsLegacyPath
}
URL, err := url.Parse(appservice.RequestUrl() + path)
if err != nil {
return err
}
URL.Path += request.UserID
apiURL := URL.String() + "?access_token=" + appservice.HSToken
if a.Cfg.LegacyAuth {
q := URL.Query()
q.Set("access_token", appservice.HSToken)
URL.RawQuery = q.Encode()
}
apiURL := URL.String()
// Send a request to each application service. If one responds that it has
// created the user, immediately return.
@ -136,6 +152,7 @@ func (a *AppServiceQueryAPI) UserIDExists(
if err != nil {
return err
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", appservice.HSToken))
resp, err := appservice.HTTPClient.Do(req.WithContext(ctx))
if resp != nil {
defer func() {
@ -176,25 +193,22 @@ type thirdpartyResponses interface {
api.ASProtocolResponse | []api.ASUserResponse | []api.ASLocationResponse
}
func requestDo[T thirdpartyResponses](client *http.Client, url string, response *T) (err error) {
origURL := url
// try v1 and unstable appservice endpoints
for _, version := range []string{"v1", "unstable"} {
var resp *http.Response
var body []byte
asURL := strings.Replace(origURL, "unstable", version, 1)
resp, err = client.Get(asURL)
if err != nil {
continue
}
defer resp.Body.Close() // nolint: errcheck
body, err = io.ReadAll(resp.Body)
if err != nil {
continue
}
return json.Unmarshal(body, &response)
func requestDo[T thirdpartyResponses](as *config.ApplicationService, url string, response *T) error {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return err
}
return err
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", as.HSToken))
resp, err := as.HTTPClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close() // nolint: errcheck
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
return json.Unmarshal(body, &response)
}
func (a *AppServiceQueryAPI) Locations(
@ -207,16 +221,22 @@ func (a *AppServiceQueryAPI) Locations(
return err
}
path := api.ASLocationPath
if a.Cfg.LegacyPaths {
path = api.ASLocationLegacyPath
}
for _, as := range a.Cfg.Derived.ApplicationServices {
var asLocations []api.ASLocationResponse
params.Set("access_token", as.HSToken)
if a.Cfg.LegacyAuth {
params.Set("access_token", as.HSToken)
}
url := as.RequestUrl() + api.ASLocationPath
url := as.RequestUrl() + path
if req.Protocol != "" {
url += "/" + req.Protocol
}
if err := requestDo[[]api.ASLocationResponse](as.HTTPClient, url+"?"+params.Encode(), &asLocations); err != nil {
if err := requestDo[[]api.ASLocationResponse](&as, url+"?"+params.Encode(), &asLocations); err != nil {
log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'locations' from application service")
continue
}
@ -242,16 +262,22 @@ func (a *AppServiceQueryAPI) User(
return err
}
path := api.ASUserPath
if a.Cfg.LegacyPaths {
path = api.ASUserLegacyPath
}
for _, as := range a.Cfg.Derived.ApplicationServices {
var asUsers []api.ASUserResponse
params.Set("access_token", as.HSToken)
if a.Cfg.LegacyAuth {
params.Set("access_token", as.HSToken)
}
url := as.RequestUrl() + api.ASUserPath
url := as.RequestUrl() + path
if req.Protocol != "" {
url += "/" + req.Protocol
}
if err := requestDo[[]api.ASUserResponse](as.HTTPClient, url+"?"+params.Encode(), &asUsers); err != nil {
if err := requestDo[[]api.ASUserResponse](&as, url+"?"+params.Encode(), &asUsers); err != nil {
log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'user' from application service")
continue
}
@ -272,6 +298,10 @@ func (a *AppServiceQueryAPI) Protocols(
req *api.ProtocolRequest,
resp *api.ProtocolResponse,
) error {
protocolPath := api.ASProtocolPath
if a.Cfg.LegacyPaths {
protocolPath = api.ASProtocolLegacyPath
}
// get a single protocol response
if req.Protocol != "" {
@ -289,7 +319,7 @@ func (a *AppServiceQueryAPI) Protocols(
response := api.ASProtocolResponse{}
for _, as := range a.Cfg.Derived.ApplicationServices {
var proto api.ASProtocolResponse
if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+req.Protocol, &proto); err != nil {
if err := requestDo[api.ASProtocolResponse](&as, as.RequestUrl()+protocolPath+req.Protocol, &proto); err != nil {
log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service")
continue
}
@ -319,7 +349,7 @@ func (a *AppServiceQueryAPI) Protocols(
for _, as := range a.Cfg.Derived.ApplicationServices {
for _, p := range as.Protocols {
var proto api.ASProtocolResponse
if err := requestDo[api.ASProtocolResponse](as.HTTPClient, as.RequestUrl()+api.ASProtocolPath+p, &proto); err != nil {
if err := requestDo[api.ASProtocolResponse](&as, as.RequestUrl()+protocolPath+p, &proto); err != nil {
log.WithError(err).WithField("application_service", as.ID).Error("unable to get 'protocol' from application service")
continue
}

201
clientapi/clientapi_test.go
View file

@ -958,7 +958,8 @@ func TestCapabilities(t *testing.T) {
cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
// Needed to create accounts
rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
rsAPI.SetFederationAPI(nil, nil)
userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
@ -1005,7 +1006,8 @@ func TestTurnserver(t *testing.T) {
cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
// Needed to create accounts
rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
rsAPI.SetFederationAPI(nil, nil)
userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
//rsAPI.SetUserAPI(userAPI)
@ -1103,7 +1105,8 @@ func Test3PID(t *testing.T) {
cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
// Needed to create accounts
rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, caching.DisableMetrics)
caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
rsAPI.SetFederationAPI(nil, nil)
userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
// We mostly need the rsAPI/userAPI for this test, so nil for other APIs etc.
@ -2151,3 +2154,195 @@ func TestKeyBackup(t *testing.T) {
}
})
}
func TestGetMembership(t *testing.T) {
alice := test.NewUser(t)
bob := test.NewUser(t)
testCases := []struct {
name string
roomID string
user *test.User
additionalEvents func(t *testing.T, room *test.Room)
request func(t *testing.T, room *test.Room, accessToken string) *http.Request
wantOK bool
wantMemberCount int
}{
{
name: "/joined_members - Bob never joined",
user: bob,
request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
"access_token": accessToken,
}))
},
wantOK: false,
},
{
name: "/joined_members - Alice joined",
user: alice,
request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
"access_token": accessToken,
}))
},
wantOK: true,
wantMemberCount: 1,
},
{
name: "/joined_members - Alice leaves, shouldn't be able to see members ",
user: alice,
request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
"access_token": accessToken,
}))
},
additionalEvents: func(t *testing.T, room *test.Room) {
room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
"membership": "leave",
}, test.WithStateKey(alice.ID))
},
wantOK: false,
},
{
name: "/joined_members - Bob joins, Alice sees two members",
user: alice,
request: func(t *testing.T, room *test.Room, accessToken string) *http.Request {
return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
"access_token": accessToken,
}))
},
additionalEvents: func(t *testing.T, room *test.Room) {
room.CreateAndInsert(t, bob, spec.MRoomMember, map[string]interface{}{
"membership": "join",
}, test.WithStateKey(bob.ID))
},
wantOK: true,
wantMemberCount: 2,
},
}
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
cfg, processCtx, close := testrig.CreateConfig(t, dbType)
routers := httputil.NewRouters()
cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
defer close()
natsInstance := jetstream.NATSInstance{}
jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
// Use an actual roomserver for this
rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
rsAPI.SetFederationAPI(nil, nil)
userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
accessTokens := map[*test.User]userDevice{
alice: {},
bob: {},
}
createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
room := test.NewRoom(t, alice)
t.Cleanup(func() {
t.Logf("running cleanup for %s", tc.name)
})
// inject additional events
if tc.additionalEvents != nil {
tc.additionalEvents(t, room)
}
if err := api.SendEvents(context.Background(), rsAPI, api.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
t.Fatalf("failed to send events: %v", err)
}
w := httptest.NewRecorder()
routers.Client.ServeHTTP(w, tc.request(t, room, accessTokens[tc.user].accessToken))
if w.Code != 200 && tc.wantOK {
t.Logf("%s", w.Body.String())
t.Fatalf("got HTTP %d want %d", w.Code, 200)
}
t.Logf("[%s] Resp: %s", tc.name, w.Body.String())
// check we got the expected events
if tc.wantOK {
memberCount := len(gjson.GetBytes(w.Body.Bytes(), "joined").Map())
if memberCount != tc.wantMemberCount {
t.Fatalf("expected %d members, got %d", tc.wantMemberCount, memberCount)
}
}
})
}
})
}
func TestCreateRoomInvite(t *testing.T) {
alice := test.NewUser(t)
bob := test.NewUser(t)
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
cfg, processCtx, close := testrig.CreateConfig(t, dbType)
routers := httputil.NewRouters()
cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
defer close()
natsInstance := jetstream.NATSInstance{}
jsctx, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
defer jetstream.DeleteAllStreams(jsctx, &cfg.Global.JetStream)
// Use an actual roomserver for this
rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
rsAPI.SetFederationAPI(nil, nil)
userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil, caching.DisableMetrics, testIsBlacklistedOrBackingOff)
// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
accessTokens := map[*test.User]userDevice{
alice: {},
}
createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
reqBody := map[string]any{
"invite": []string{bob.ID},
}
body, err := json.Marshal(reqBody)
if err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/createRoom", strings.NewReader(string(body)))
req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
routers.Client.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected room creation to be successful, got HTTP %d instead: %s", w.Code, w.Body.String())
}
roomID := gjson.GetBytes(w.Body.Bytes(), "room_id").Str
validRoomID, _ := spec.NewRoomID(roomID)
// Now ask the roomserver about the membership event of Bob
ev, err := rsAPI.CurrentStateEvent(context.Background(), *validRoomID, spec.MRoomMember, bob.ID)
if err != nil {
t.Fatal(err)
}
if ev == nil {
t.Fatal("Membership event for Bob does not exist")
}
// Validate that there is NO displayname in content
if gjson.GetBytes(ev.Content(), "displayname").Exists() {
t.Fatal("Found displayname in invite")
}
})
}

4
clientapi/routing/directory.go
View file

@ -55,7 +55,7 @@ func DirectoryRoom(
if err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: spec.BadJSON("Room alias must be in the form '#localpart:domain'"),
JSON: spec.InvalidParam("Room alias must be in the form '#localpart:domain'"),
}
}
@ -134,7 +134,7 @@ func SetLocalAlias(
if err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: spec.BadJSON("Room alias must be in the form '#localpart:domain'"),
JSON: spec.InvalidParam("Room alias must be in the form '#localpart:domain'"),
}
}

2
clientapi/routing/keys.go
View file

@ -93,7 +93,6 @@ func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device)
type queryKeysRequest struct {
Timeout int `json:"timeout"`
Token string `json:"token"`
DeviceKeys map[string][]string `json:"device_keys"`
}
@ -119,7 +118,6 @@ func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *api.Device) u
UserID: device.UserID,
UserToDevices: r.DeviceKeys,
Timeout: r.GetTimeout(),
// TODO: Token?
}, &queryRes)
return util.JSONResponse{
Code: 200,

31
clientapi/routing/membership.go
View file

@ -324,19 +324,18 @@ func SendInvite(
}
// We already received the return value, so no need to check for an error here.
response, _ := sendInvite(req.Context(), profileAPI, device, roomID, body.UserID, body.Reason, cfg, rsAPI, asAPI, evTime)
response, _ := sendInvite(req.Context(), device, roomID, body.UserID, body.Reason, cfg, rsAPI, evTime)
return response
}
// sendInvite sends an invitation to a user. Returns a JSONResponse and an error
func sendInvite(
ctx context.Context,
profileAPI userapi.ClientUserAPI,
device *userapi.Device,
roomID, userID, reason string,
cfg *config.ClientAPI,
rsAPI roomserverAPI.ClientRoomserverAPI,
asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time,
evTime time.Time,
) (util.JSONResponse, error) {
validRoomID, err := spec.NewRoomID(roomID)
if err != nil {
@ -359,13 +358,7 @@ func sendInvite(
JSON: spec.InvalidParam("UserID is invalid"),
}, err
}
profile, err := loadProfile(ctx, userID, cfg, profileAPI, asAPI)
if err != nil {
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: spec.InternalServerError{},
}, err
}
identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
if err != nil {
return util.JSONResponse{
@ -375,16 +368,14 @@ func sendInvite(
}
err = rsAPI.PerformInvite(ctx, &api.PerformInviteRequest{
InviteInput: roomserverAPI.InviteInput{
RoomID: *validRoomID,
Inviter: *inviter,
Invitee: *invitee,
DisplayName: profile.DisplayName,
AvatarURL: profile.AvatarURL,
Reason: reason,
IsDirect: false,
KeyID: identity.KeyID,
PrivateKey: identity.PrivateKey,
EventTime: evTime,
RoomID: *validRoomID,
Inviter: *inviter,
Invitee: *invitee,
Reason: reason,
IsDirect: false,
KeyID: identity.KeyID,
PrivateKey: identity.PrivateKey,
EventTime: evTime,
},
InviteRoomState: nil, // ask the roomserver to draw up invite room state for us
SendAsServer: string(device.UserDomain()),

139
clientapi/routing/memberships.go Normal file
View file

@ -0,0 +1,139 @@
// Copyright 2024 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package routing
import (
"encoding/json"
"net/http"
"github.com/matrix-org/dendrite/roomserver/api"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib/spec"
"github.com/matrix-org/util"
)
// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members
type getJoinedMembersResponse struct {
Joined map[string]joinedMember `json:"joined"`
}
type joinedMember struct {
DisplayName string `json:"display_name"`
AvatarURL string `json:"avatar_url"`
}
// The database stores 'displayname' without an underscore.
// Deserialize into this and then change to the actual API response
type databaseJoinedMember struct {
DisplayName string `json:"displayname"`
AvatarURL string `json:"avatar_url"`
}
// GetJoinedMembers implements
//
// GET /rooms/{roomId}/joined_members
func GetJoinedMembers(
req *http.Request, device *userapi.Device, roomID string,
rsAPI api.ClientRoomserverAPI,
) util.JSONResponse {
// Validate the userID
userID, err := spec.NewUserID(device.UserID, true)
if err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: spec.InvalidParam("Device UserID is invalid"),
}
}
// Validate the roomID
validRoomID, err := spec.NewRoomID(roomID)
if err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: spec.InvalidParam("RoomID is invalid"),
}
}
// Get the current memberships for the requesting user to determine
// if they are allowed to query this endpoint.
queryReq := api.QueryMembershipForUserRequest{
RoomID: validRoomID.String(),
UserID: *userID,
}
var queryRes api.QueryMembershipForUserResponse
if queryErr := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); queryErr != nil {
util.GetLogger(req.Context()).WithError(queryErr).Error("rsAPI.QueryMembershipsForRoom failed")
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: spec.InternalServerError{},
}
}
if !queryRes.HasBeenInRoom {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
}
}
if !queryRes.IsInRoom {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
}
}
// Get the current membership events
var membershipsForRoomResp api.QueryMembershipsForRoomResponse
if err = rsAPI.QueryMembershipsForRoom(req.Context(), &api.QueryMembershipsForRoomRequest{
JoinedOnly: true,
RoomID: validRoomID.String(),
}, &membershipsForRoomResp); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryEventsByID failed")
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: spec.InternalServerError{},
}
}
var res getJoinedMembersResponse
res.Joined = make(map[string]joinedMember)
for _, ev := range membershipsForRoomResp.JoinEvents {
var content databaseJoinedMember
if err := json.Unmarshal(ev.Content, &content); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content")
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: spec.InternalServerError{},
}
}
userID, err := rsAPI.QueryUserIDForSender(req.Context(), *validRoomID, spec.SenderID(ev.Sender))
if err != nil || userID == nil {
util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryUserIDForSender failed")
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: spec.InternalServerError{},
}
}
res.Joined[userID.String()] = joinedMember(content)
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: res,
}
}

2
clientapi/routing/register.go
View file

@ -630,6 +630,7 @@ func handleGuestRegistration(
AccessToken: token,
IPAddr: req.RemoteAddr,
UserAgent: req.UserAgent(),
FromRegistration: true,
}, &devRes)
if err != nil {
return util.JSONResponse{
@ -919,6 +920,7 @@ func completeRegistration(
DeviceID: deviceID,
IPAddr: ipAddr,
UserAgent: userAgent,
FromRegistration: true,
}, &devRes)
if err != nil {
return util.JSONResponse{

10
clientapi/routing/routing.go
View file

@ -1513,4 +1513,14 @@ func Setup(
return GetPresence(req, device, natsClient, cfg.Matrix.JetStream.Prefixed(jetstream.RequestPresence), vars["userId"])
}),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/joined_members",
httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return GetJoinedMembers(req, device, vars["roomID"], rsAPI)
}),
).Methods(http.MethodGet, http.MethodOptions)
}

2
clientapi/routing/server_notices.go
View file

@ -215,7 +215,7 @@ func SendServerNotice(
}
if !membershipRes.IsInRoom {
// re-invite the user
res, err := sendInvite(ctx, userAPI, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, asAPI, time.Now())
res, err := sendInvite(ctx, senderDevice, roomID, r.UserID, "Server notice room", cfgClient, rsAPI, time.Now())
if err != nil {
return res
}

7
dendrite-sample.yaml
View file

@ -154,6 +154,13 @@ app_service_api:
# to be sent to an insecure endpoint.
disable_tls_validation: false
# Send the access_token query parameter with appservice requests in addition
# to the Authorization header. This can cause hs_tokens to be saved to logs,
# so it should not be enabled unless absolutely necessary.
legacy_auth: false
# Use the legacy unprefixed paths for appservice requests.
legacy_paths: false
# Appservice configuration files to load into this homeserver.
config_files:
# - /path/to/appservice_registration.yaml

1
docs/FAQ.md
View file

@ -117,6 +117,7 @@ The list of files that need to be stored is:
- matrix-key.pem
- dendrite.yaml
- the postgres or sqlite DB
- the jetstream directory
- the media store
- the search index (although this can be regenerated)

6
docs/Gemfile.lock
View file

@ -231,9 +231,9 @@ GEM
jekyll-seo-tag (~> 2.1)
minitest (5.17.0)
multipart-post (2.1.1)
nokogiri (1.14.3-arm64-darwin)
nokogiri (1.16.2-arm64-darwin)
racc (~> 1.4)
nokogiri (1.14.3-x86_64-linux)
nokogiri (1.16.2-x86_64-linux)
racc (~> 1.4)
octokit (4.22.0)
faraday (>= 0.9)
@ -241,7 +241,7 @@ GEM
pathutil (0.16.2)
forwardable-extended (~> 2.6)
public_suffix (4.0.7)
racc (1.6.2)
racc (1.7.3)
rb-fsevent (0.11.1)
rb-inotify (0.10.1)
ffi (~> 1.0)

5
federationapi/federationapi.go
View file

@ -113,10 +113,7 @@ func NewInternalAPI(
_ = federationDB.RemoveAllServersFromBlacklist()
}
stats := statistics.NewStatistics(
federationDB,
cfg.FederationMaxRetries+1,
cfg.P2PFederationRetriesUntilAssumedOffline+1)
stats := statistics.NewStatistics(federationDB, cfg.FederationMaxRetries+1, cfg.P2PFederationRetriesUntilAssumedOffline+1, cfg.EnableRelays)
js, nats := natsInstance.Prepare(processContext, &cfg.Matrix.JetStream)

129
federationapi/federationapi_test.go
View file

@ -5,11 +5,14 @@ import (
"crypto/ed25519"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"github.com/matrix-org/dendrite/federationapi/routing"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/sqlutil"
@ -17,7 +20,10 @@ import (
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/gomatrixserverlib/fclient"
"github.com/matrix-org/gomatrixserverlib/spec"
"github.com/matrix-org/util"
"github.com/nats-io/nats.go"
"github.com/stretchr/testify/assert"
"github.com/tidwall/gjson"
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/federationapi/api"
@ -362,3 +368,126 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
}
}
}
func TestNotaryServer(t *testing.T) {
testCases := []struct {
name string
httpBody string
pubKeyRequest *gomatrixserverlib.PublicKeyNotaryLookupRequest
validateFunc func(t *testing.T, response util.JSONResponse)
}{
{
name: "empty httpBody",
validateFunc: func(t *testing.T, resp util.JSONResponse) {
assert.Equal(t, http.StatusBadRequest, resp.Code)
nk, ok := resp.JSON.(spec.MatrixError)
assert.True(t, ok)
assert.Equal(t, spec.ErrorBadJSON, nk.ErrCode)
},
},
{
name: "valid but empty httpBody",
httpBody: "{}",
validateFunc: func(t *testing.T, resp util.JSONResponse) {
want := util.JSONResponse{
Code: http.StatusOK,
JSON: routing.NotaryKeysResponse{ServerKeys: []json.RawMessage{}},
}
assert.Equal(t, want, resp)
},
},
{
name: "request all keys using an empty criteria",
httpBody: `{"server_keys":{"servera":{}}}`,
validateFunc: func(t *testing.T, resp util.JSONResponse) {
assert.Equal(t, http.StatusOK, resp.Code)
nk, ok := resp.JSON.(routing.NotaryKeysResponse)
assert.True(t, ok)
assert.Equal(t, "servera", gjson.GetBytes(nk.ServerKeys[0], "server_name").Str)
assert.True(t, gjson.GetBytes(nk.ServerKeys[0], "verify_keys.ed25519:someID").Exists())
},
},
{
name: "request all keys using null as the criteria",
httpBody: `{"server_keys":{"servera":null}}`,
validateFunc: func(t *testing.T, resp util.JSONResponse) {
assert.Equal(t, http.StatusOK, resp.Code)
nk, ok := resp.JSON.(routing.NotaryKeysResponse)
assert.True(t, ok)
assert.Equal(t, "servera", gjson.GetBytes(nk.ServerKeys[0], "server_name").Str)
assert.True(t, gjson.GetBytes(nk.ServerKeys[0], "verify_keys.ed25519:someID").Exists())
},
},
{
name: "request specific key",
httpBody: `{"server_keys":{"servera":{"ed25519:someID":{}}}}`,
validateFunc: func(t *testing.T, resp util.JSONResponse) {
assert.Equal(t, http.StatusOK, resp.Code)
nk, ok := resp.JSON.(routing.NotaryKeysResponse)
assert.True(t, ok)
assert.Equal(t, "servera", gjson.GetBytes(nk.ServerKeys[0], "server_name").Str)
assert.True(t, gjson.GetBytes(nk.ServerKeys[0], "verify_keys.ed25519:someID").Exists())
},
},
{
name: "request multiple servers",
httpBody: `{"server_keys":{"servera":{"ed25519:someID":{}},"serverb":{"ed25519:someID":{}}}}`,
validateFunc: func(t *testing.T, resp util.JSONResponse) {
assert.Equal(t, http.StatusOK, resp.Code)
nk, ok := resp.JSON.(routing.NotaryKeysResponse)
assert.True(t, ok)
wantServers := map[string]struct{}{
"servera": {},
"serverb": {},
}
for _, js := range nk.ServerKeys {
serverName := gjson.GetBytes(js, "server_name").Str
_, ok = wantServers[serverName]
assert.True(t, ok, "unexpected servername: %s", serverName)
delete(wantServers, serverName)
assert.True(t, gjson.GetBytes(js, "verify_keys.ed25519:someID").Exists())
}
if len(wantServers) > 0 {
t.Fatalf("expected response to also contain: %#v", wantServers)
}
},
},
}
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
cfg, processCtx, close := testrig.CreateConfig(t, dbType)
defer close()
cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
natsInstance := jetstream.NATSInstance{}
fc := &fedClient{
keys: map[spec.ServerName]struct {
key ed25519.PrivateKey
keyID gomatrixserverlib.KeyID
}{
"servera": {
key: test.PrivateKeyA,
keyID: "ed25519:someID",
},
"serverb": {
key: test.PrivateKeyB,
keyID: "ed25519:someID",
},
},
}
fedAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, fc, nil, caches, nil, true)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(tc.httpBody))
req.Host = string(cfg.Global.ServerName)
resp := routing.NotaryKeys(req, &cfg.FederationAPI, fedAPI, tc.pubKeyRequest)
// assert that we received the expected response
tc.validateFunc(t, resp)
})
}
})
}

10
federationapi/internal/federationclient_test.go
View file

@ -61,7 +61,7 @@ func TestFederationClientQueryKeys(t *testing.T) {
},
}
fedClient := &testFedClient{}
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
queues := queue.NewOutgoingQueues(
testDB, process.NewProcessContext(),
false,
@ -92,7 +92,7 @@ func TestFederationClientQueryKeysBlacklisted(t *testing.T) {
},
}
fedClient := &testFedClient{}
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
queues := queue.NewOutgoingQueues(
testDB, process.NewProcessContext(),
false,
@ -122,7 +122,7 @@ func TestFederationClientQueryKeysFailure(t *testing.T) {
},
}
fedClient := &testFedClient{shouldFail: true}
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
queues := queue.NewOutgoingQueues(
testDB, process.NewProcessContext(),
false,
@ -152,7 +152,7 @@ func TestFederationClientClaimKeys(t *testing.T) {
},
}
fedClient := &testFedClient{}
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
queues := queue.NewOutgoingQueues(
testDB, process.NewProcessContext(),
false,
@ -183,7 +183,7 @@ func TestFederationClientClaimKeysBlacklisted(t *testing.T) {
},
}
fedClient := &testFedClient{}
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
queues := queue.NewOutgoingQueues(
testDB, process.NewProcessContext(),
false,

10
federationapi/internal/perform_test.go
View file

@ -66,7 +66,7 @@ func TestPerformWakeupServers(t *testing.T) {
},
}
fedClient := &testFedClient{}
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, true)
queues := queue.NewOutgoingQueues(
testDB, process.NewProcessContext(),
false,
@ -112,7 +112,7 @@ func TestQueryRelayServers(t *testing.T) {
},
}
fedClient := &testFedClient{}
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
queues := queue.NewOutgoingQueues(
testDB, process.NewProcessContext(),
false,
@ -153,7 +153,7 @@ func TestRemoveRelayServers(t *testing.T) {
},
}
fedClient := &testFedClient{}
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
queues := queue.NewOutgoingQueues(
testDB, process.NewProcessContext(),
false,
@ -193,7 +193,7 @@ func TestPerformDirectoryLookup(t *testing.T) {
},
}
fedClient := &testFedClient{}
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
queues := queue.NewOutgoingQueues(
testDB, process.NewProcessContext(),
false,
@ -232,7 +232,7 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) {
},
}
fedClient := &testFedClient{}
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
stats := statistics.NewStatistics(testDB, FailuresUntilBlacklist, FailuresUntilAssumedOffline, true)
queues := queue.NewOutgoingQueues(
testDB, process.NewProcessContext(),
false,

9
federationapi/internal/query.go
View file

@ -43,6 +43,15 @@ func (a *FederationInternalAPI) fetchServerKeysFromCache(
ctx context.Context, req *api.QueryServerKeysRequest,
) ([]gomatrixserverlib.ServerKeys, error) {
var results []gomatrixserverlib.ServerKeys
// We got a request for _all_ server keys, return them.
if len(req.KeyIDToCriteria) == 0 {
serverKeysResponses, _ := a.db.GetNotaryKeys(ctx, req.ServerName, []gomatrixserverlib.KeyID{})
if len(serverKeysResponses) == 0 {
return nil, fmt.Errorf("failed to find server key response for server %s", req.ServerName)
}
return serverKeysResponses, nil
}
for keyID, criteria := range req.KeyIDToCriteria {
serverKeysResponses, _ := a.db.GetNotaryKeys(ctx, req.ServerName, []gomatrixserverlib.KeyID{keyID})
if len(serverKeysResponses) == 0 {

2
federationapi/queue/queue_test.go
View file

@ -117,7 +117,7 @@ func testSetup(failuresUntilBlacklist uint32, failuresUntilAssumedOffline uint32
txRelayCount: *atomic.NewUint32(0),
}
stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline)
stats := statistics.NewStatistics(db, failuresUntilBlacklist, failuresUntilAssumedOffline, false)
signingInfo := []*fclient.SigningIdentity{
{
KeyID: "ed21019:auto",

6
federationapi/routing/backfill.go
View file

@ -95,6 +95,12 @@ func Backfill(
}
}
// Enforce a limit of 100 events, as not to hit the DB to hard.
// Synapse has a hard limit of 100 events as well.
if req.Limit > 100 {
req.Limit = 100
}
// Query the Roomserver.
if err = rsAPI.PerformBackfill(httpReq.Context(), &req, &res); err != nil {
util.GetLogger(httpReq.Context()).WithError(err).Error("query.PerformBackfill failed")

9
federationapi/routing/keys.go
View file

@ -197,6 +197,10 @@ func localKeys(cfg *config.FederationAPI, serverName spec.ServerName) (*gomatrix
return &keys, err
}
type NotaryKeysResponse struct {
ServerKeys []json.RawMessage `json:"server_keys"`
}
func NotaryKeys(
httpReq *http.Request, cfg *config.FederationAPI,
fsAPI federationAPI.FederationInternalAPI,
@ -217,10 +221,9 @@ func NotaryKeys(
}
}
var response struct {
ServerKeys []json.RawMessage `json:"server_keys"`
response := NotaryKeysResponse{
ServerKeys: []json.RawMessage{},
}
response.ServerKeys = []json.RawMessage{}
for serverName, kidToCriteria := range req.ServerKeys {
var keyList []gomatrixserverlib.ServerKeys

2
federationapi/routing/routing.go
View file

@ -647,6 +647,8 @@ func MakeFedAPI(
// add the user to Sentry, if enabled
hub := sentry.GetHubFromContext(req.Context())
if hub != nil {
// clone the hub, so we don't send garbage events with e.g. mismatching rooms/event_ids
hub = hub.Clone()
hub.Scope().SetTag("origin", string(fedReq.Origin()))
hub.Scope().SetTag("uri", fedReq.RequestURI())
}

11
federationapi/statistics/statistics.go
View file

@ -34,12 +34,15 @@ type Statistics struct {
// mark the destination as offline. At this point we should attempt
// to send messages to the user's async relay servers if we know them.
FailuresUntilAssumedOffline uint32
enableRelays bool
}
func NewStatistics(
db storage.Database,
failuresUntilBlacklist uint32,
failuresUntilAssumedOffline uint32,
enableRelays bool,
) Statistics {
return Statistics{
DB: db,
@ -47,6 +50,7 @@ func NewStatistics(
FailuresUntilAssumedOffline: failuresUntilAssumedOffline,
backoffTimers: make(map[spec.ServerName]*time.Timer),
servers: make(map[spec.ServerName]*ServerStatistics),
enableRelays: enableRelays,
}
}
@ -73,6 +77,13 @@ func (s *Statistics) ForServer(serverName spec.ServerName) *ServerStatistics {
} else {
server.blacklisted.Store(blacklisted)
}
// Don't bother hitting the database 2 additional times
// if we don't want to use relays.
if !s.enableRelays {
return server
}
assumedOffline, err := s.DB.IsServerAssumedOffline(context.Background(), serverName)
if err != nil {
logrus.WithError(err).Errorf("Failed to get assumed offline entry %q", serverName)

4
federationapi/statistics/statistics_test.go
View file

@ -16,7 +16,7 @@ const (
)
func TestBackoff(t *testing.T) {
stats := NewStatistics(nil, FailuresUntilBlacklist, FailuresUntilAssumedOffline)
stats := NewStatistics(nil, FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
server := ServerStatistics{
statistics: &stats,
serverName: "test.com",
@ -106,7 +106,7 @@ func TestBackoff(t *testing.T) {
}
func TestRelayServersListing(t *testing.T) {
stats := NewStatistics(test.NewInMemoryFederationDatabase(), FailuresUntilBlacklist, FailuresUntilAssumedOffline)
stats := NewStatistics(test.NewInMemoryFederationDatabase(), FailuresUntilBlacklist, FailuresUntilAssumedOffline, false)
server := ServerStatistics{statistics: &stats}
server.AddRelayServers([]spec.ServerName{"relay1", "relay1", "relay2"})
relayServers := server.KnownRelayServers()

4
go.mod
View file

@ -22,7 +22,7 @@ require (
github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
github.com/matrix-org/gomatrixserverlib v0.0.0-20231212115925-41497b7563eb
github.com/matrix-org/gomatrixserverlib v0.0.0-20240109180417-3495e573f2b7
github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7
github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
github.com/mattn/go-sqlite3 v1.14.17
@ -115,7 +115,7 @@ require (
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.10.1 // indirect
github.com/quic-go/qtls-go1-20 v0.3.2 // indirect
github.com/quic-go/quic-go v0.37.4 // indirect
github.com/quic-go/quic-go v0.37.7 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
github.com/rs/zerolog v1.29.1 // indirect

8
go.sum
View file

@ -208,8 +208,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
github.com/matrix-org/gomatrixserverlib v0.0.0-20231212115925-41497b7563eb h1:Nn+Fr96oi7bIfdOwX5A2L6A2MZCM+lqwLe4/+3+nYj8=
github.com/matrix-org/gomatrixserverlib v0.0.0-20231212115925-41497b7563eb/go.mod h1:M8m7seOroO5ePlgxA7AFZymnG90Cnh94rYQyngSrZkk=
github.com/matrix-org/gomatrixserverlib v0.0.0-20240109180417-3495e573f2b7 h1:EaUvK2ay6cxMxeshC1p6QswS9+rQFbUc2YerkRFyVXQ=
github.com/matrix-org/gomatrixserverlib v0.0.0-20240109180417-3495e573f2b7/go.mod h1:HZGsVJ3bUE+DkZtufkH9H0mlsvbhEGK5CpX0Zlavylg=
github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7 h1:6t8kJr8i1/1I5nNttw6nn1ryQJgzVlBmSGgPiiaTdw4=
github.com/matrix-org/pinecone v0.11.1-0.20230810010612-ea4c33717fd7/go.mod h1:ReWMS/LoVnOiRAdq9sNUC2NZnd1mZkMNB52QhpTRWjg=
github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=
@ -286,8 +286,8 @@ github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+Pymzi
github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI=
github.com/quic-go/qtls-go1-20 v0.3.2/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
github.com/quic-go/quic-go v0.37.4 h1:ke8B73yMCWGq9MfrCCAw0Uzdm7GaViC3i39dsIdDlH4=
github.com/quic-go/quic-go v0.37.4/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU=
github.com/quic-go/quic-go v0.37.7 h1:AgKsQLZ1+YCwZd2GYhBUsJDYZwEkA5gENtAjb+MxONU=
github.com/quic-go/quic-go v0.37.7/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=

2
helm/dendrite/Chart.yaml
View file

@ -1,7 +1,7 @@
apiVersion: v2
name: dendrite
version: "0.14.0"
appVersion: "0.13.5"
appVersion: "0.13.6"
description: Dendrite Matrix Homeserver
type: application
icon: https://avatars.githubusercontent.com/u/8418310?s=48&v=4

7
helm/dendrite/README.md
View file

@ -1,7 +1,7 @@
# dendrite
![Version: 0.13.6](https://img.shields.io/badge/Version-0.13.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.5](https://img.shields.io/badge/AppVersion-0.13.5-informational?style=flat-square)
![Version: 0.13.8](https://img.shields.io/badge/Version-0.13.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.13.6](https://img.shields.io/badge/AppVersion-0.13.6-informational?style=flat-square)
Dendrite Matrix Homeserver
Status: **NOT PRODUCTION READY**
@ -48,13 +48,16 @@ Create a folder `appservices` and place your configurations in there. The confi
| signing_key.create | bool | `true` | Create a new signing key, if not exists |
| signing_key.existingSecret | string | `""` | Use an existing secret |
| resources | object | sets some sane default values | Default resource requests/limits. |
| persistence.jetstream | object | `{"capacity":"1Gi","existingClaim":""}` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. # If defined, storageClassName: <storageClass> # If set to "-", storageClassName: "", which disables dynamic provisioning # If undefined (the default) or set to null, no storageClassName spec is # set, choosing the default provisioner. (gp2 on AWS, standard on # GKE, AWS & OpenStack) # storageClass: "" |
| persistence.storageClass | string | `nil` | The storage class to use for volume claims. Used unless specified at the specific component. Defaults to the cluster default storage class. If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. (gp2 on AWS, standard on GKE, AWS & OpenStack) |
| persistence.jetstream.existingClaim | string | `""` | Use an existing volume claim for jetstream |
| persistence.jetstream.capacity | string | `"1Gi"` | PVC Storage Request for the jetstream volume |
| persistence.jetstream.storageClass | string | `nil` | The storage class to use for volume claims. Defaults to persistence.storageClass If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. (gp2 on AWS, standard on GKE, AWS & OpenStack) |
| persistence.media.existingClaim | string | `""` | Use an existing volume claim for media files |
| persistence.media.capacity | string | `"1Gi"` | PVC Storage Request for the media volume |
| persistence.media.storageClass | string | `nil` | The storage class to use for volume claims. Defaults to persistence.storageClass If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. (gp2 on AWS, standard on GKE, AWS & OpenStack) |
| persistence.search.existingClaim | string | `""` | Use an existing volume claim for the fulltext search index |
| persistence.search.capacity | string | `"1Gi"` | PVC Storage Request for the search volume |
| persistence.search.storageClass | string | `nil` | The storage class to use for volume claims. Defaults to persistence.storageClass If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. (gp2 on AWS, standard on GKE, AWS & OpenStack) |
| extraVolumes | list | `[]` | Add additional volumes to the Dendrite Pod |
| extraVolumeMounts | list | `[]` | Configure additional mount points volumes in the Dendrite Pod |
| strategy.type | string | `"RollingUpdate"` | Strategy to use for rolling updates (e.g. Recreate, RollingUpdate) If you are using ReadWriteOnce volumes, you should probably use Recreate |

9
helm/dendrite/templates/ingress.yaml
View file

@ -4,6 +4,7 @@
{{- $wellKnownServerHost := default $serverNameHost (regexFind "^(\\[.+\\])?[^:]*" .Values.dendrite_config.global.well_known_server_name) -}}
{{- $wellKnownClientHost := default $serverNameHost (regexFind "//(\\[.+\\])?[^:/]*" .Values.dendrite_config.global.well_known_client_name | trimAll "/") -}}
{{- $allHosts := list $serverNameHost $wellKnownServerHost $wellKnownClientHost | uniq -}}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
@ -56,7 +57,7 @@ spec:
service:
name: {{ $fullName }}
port:
name: http
number: {{ $.Values.service.port }}
{{- else }}
serviceName: {{ $fullName }}
servicePort: http
@ -72,7 +73,7 @@ spec:
service:
name: {{ $fullName }}
port:
name: http
number: {{ $.Values.service.port }}
{{- else }}
serviceName: {{ $fullName }}
servicePort: http
@ -88,7 +89,7 @@ spec:
service:
name: {{ $fullName }}
port:
name: http
number: {{ $.Values.service.port }}
{{- else }}
serviceName: {{ $fullName }}
servicePort: http
@ -105,7 +106,7 @@ spec:
service:
name: {{ $fullName }}
port:
name: http
number: {{ $.Values.service.port }}
{{- else }}
serviceName: {{ $fullName }}
servicePort: http

2
helm/dendrite/templates/service.yaml
View file

@ -14,4 +14,4 @@ spec:
- name: http
protocol: TCP
port: {{ .Values.service.port }}
targetPort: 8008
targetPort: http

55
helm/dendrite/values.yaml
View file

@ -26,13 +26,13 @@ persistence:
# -- The storage class to use for volume claims.
# Used unless specified at the specific component.
# Defaults to the cluster default storage class.
## If defined, storageClassName: <storageClass>
## If set to "-", storageClassName: "", which disables dynamic provisioning
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner. (gp2 on AWS, standard on
## GKE, AWS & OpenStack)
##
# storageClass: ""
# If defined, storageClassName: <storageClass>
# If set to "-", storageClassName: "", which disables dynamic provisioning
# If undefined (the default) or set to null, no storageClassName spec is
# set, choosing the default provisioner. (gp2 on AWS, standard on
# GKE, AWS & OpenStack)
#
storageClass:
jetstream:
# -- Use an existing volume claim for jetstream
existingClaim: ""
@ -40,13 +40,12 @@ persistence:
capacity: "1Gi"
# -- The storage class to use for volume claims.
# Defaults to persistence.storageClass
## If defined, storageClassName: <storageClass>
## If set to "-", storageClassName: "", which disables dynamic provisioning
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner. (gp2 on AWS, standard on
## GKE, AWS & OpenStack)
##
# storageClass: ""
# If defined, storageClassName: <storageClass>
# If set to "-", storageClassName: "", which disables dynamic provisioning
# If undefined (the default) or set to null, no storageClassName spec is
# set, choosing the default provisioner. (gp2 on AWS, standard on
# GKE, AWS & OpenStack)
storageClass:
media:
# -- Use an existing volume claim for media files
existingClaim: ""
@ -54,13 +53,12 @@ persistence:
capacity: "1Gi"
# -- The storage class to use for volume claims.
# Defaults to persistence.storageClass
## If defined, storageClassName: <storageClass>
## If set to "-", storageClassName: "", which disables dynamic provisioning
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner. (gp2 on AWS, standard on
## GKE, AWS & OpenStack)
##
# storageClass: ""
# If defined, storageClassName: <storageClass>
# If set to "-", storageClassName: "", which disables dynamic provisioning
# If undefined (the default) or set to null, no storageClassName spec is
# set, choosing the default provisioner. (gp2 on AWS, standard on
# GKE, AWS & OpenStack)
storageClass:
search:
# -- Use an existing volume claim for the fulltext search index
existingClaim: ""
@ -68,13 +66,12 @@ persistence:
capacity: "1Gi"
# -- The storage class to use for volume claims.
# Defaults to persistence.storageClass
## If defined, storageClassName: <storageClass>
## If set to "-", storageClassName: "", which disables dynamic provisioning
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner. (gp2 on AWS, standard on
## GKE, AWS & OpenStack)
##
# storageClass: ""
# If defined, storageClassName: <storageClass>
# If set to "-", storageClassName: "", which disables dynamic provisioning
# If undefined (the default) or set to null, no storageClassName spec is
# set, choosing the default provisioner. (gp2 on AWS, standard on
# GKE, AWS & OpenStack)
storageClass:
# -- Add additional volumes to the Dendrite Pod
extraVolumes: []
@ -92,7 +89,7 @@ extraVolumeMounts: []
strategy:
# -- Strategy to use for rolling updates (e.g. Recreate, RollingUpdate)
# If you are using ReadWriteOnce volumes, you should probably use Recreate
type: RollingUpdate
type: Recreate
rollingUpdate:
# -- Maximum number of pods that can be unavailable during the update process
maxUnavailable: 25%

2
internal/httputil/httpapi.go
View file

@ -76,6 +76,8 @@ func MakeAuthAPI(
// add the user to Sentry, if enabled
hub := sentry.GetHubFromContext(req.Context())
if hub != nil {
// clone the hub, so we don't send garbage events with e.g. mismatching rooms/event_ids
hub = hub.Clone()
hub.Scope().SetUser(sentry.User{
Username: device.UserID,
})

4
internal/validate.go
View file

@ -38,9 +38,9 @@ var (
ErrPasswordTooLong = fmt.Errorf("password too long: max %d characters", maxPasswordLength)
ErrPasswordWeak = fmt.Errorf("password too weak: min %d characters", minPasswordLength)
ErrUsernameTooLong = fmt.Errorf("username exceeds the maximum length of %d characters", maxUsernameLength)
ErrUsernameInvalid = errors.New("username can only contain characters a-z, 0-9, or '_-./='")
ErrUsernameInvalid = errors.New("username can only contain characters a-z, 0-9, or '_+-./='")
ErrUsernameUnderscore = errors.New("username cannot start with a '_'")
validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`)
validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-+=./]+$`)
)
// ValidatePassword returns an error if the password is invalid

15
internal/validate_test.go
View file

@ -129,6 +129,11 @@ func Test_validateUsername(t *testing.T) {
localpart: "i_am_allowed=1",
domain: "localhost",
},
{
name: "special characters are allowed 3",
localpart: "+55555555555",
domain: "localhost",
},
{
name: "not all special characters are allowed",
localpart: "notallowed#", // contains #
@ -139,6 +144,16 @@ func Test_validateUsername(t *testing.T) {
JSON: spec.InvalidUsername(ErrUsernameInvalid.Error()),
},
},
{
name: "not all special characters are allowed 2",
localpart: "<notallowed", // contains <
domain: "localhost",
wantErr: ErrUsernameInvalid,
wantJSON: &util.JSONResponse{
Code: http.StatusBadRequest,
JSON: spec.InvalidUsername(ErrUsernameInvalid.Error()),
},
},
{
name: "username containing numbers",
localpart: "hello1337",

2
internal/version.go
View file

@ -18,7 +18,7 @@ var build string
const (
VersionMajor = 0
VersionMinor = 13
VersionPatch = 5
VersionPatch = 6
VersionTag = "" // example: "rc1"
gitRevLen = 7 // 7 matches the displayed characters on github.com

1
mediaapi/routing/routing.go
View file

@ -123,6 +123,7 @@ func makeDownloadAPI(
// Set internal headers returned regardless of the outcome of the request
util.SetCORSHeaders(w)
w.Header().Set("Cross-Origin-Resource-Policy", "cross-origin")
// Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors
w.Header().Set("Content-Type", "application/json")

2
relayapi/routing/routing.go
View file

@ -108,6 +108,8 @@ func MakeRelayAPI(
// add the user to Sentry, if enabled
hub := sentry.GetHubFromContext(req.Context())
if hub != nil {
// clone the hub, so we don't send garbage events with e.g. mismatching rooms/event_ids
hub = hub.Clone()
hub.Scope().SetTag("origin", string(fedReq.Origin()))
hub.Scope().SetTag("uri", fedReq.RequestURI())
}

82
roomserver/acls/acls.go
View file

@ -23,7 +23,7 @@ import (
"strings"
"sync"
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/dendrite/roomserver/storage/tables"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/gomatrixserverlib/spec"
"github.com/sirupsen/logrus"
@ -34,22 +34,28 @@ const MRoomServerACL = "m.room.server_acl"
type ServerACLDatabase interface {
// GetKnownRooms returns a list of all rooms we know about.
GetKnownRooms(ctx context.Context) ([]string, error)
// GetStateEvent returns the state event of a given type for a given room with a given state key
// If no event could be found, returns nil
// If there was an issue during the retrieval, returns an error
GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*types.HeaderedEvent, error)
// GetBulkStateContent returns all state events which match a given room ID and a given state key tuple. Both must be satisfied for a match.
// If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned.
GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error)
}
type ServerACLs struct {
acls map[string]*serverACL // room ID -> ACL
aclsMutex sync.RWMutex // protects the above
acls map[string]*serverACL // room ID -> ACL
aclsMutex sync.RWMutex // protects the above
aclRegexCache map[string]**regexp.Regexp // Cache from "serverName" -> pointer to a regex
aclRegexCacheMutex sync.RWMutex // protects the above
}
func NewServerACLs(db ServerACLDatabase) *ServerACLs {
ctx := context.TODO()
acls := &ServerACLs{
acls: make(map[string]*serverACL),
// Be generous when creating the cache, as in reality
// there are hundreds of servers in an ACL.
aclRegexCache: make(map[string]**regexp.Regexp, 100),
}
// Look up all of the rooms that the current state server knows about.
rooms, err := db.GetKnownRooms(ctx)
if err != nil {
@ -58,16 +64,16 @@ func NewServerACLs(db ServerACLDatabase) *ServerACLs {
// For each room, let's see if we have a server ACL state event. If we
// do then we'll process it into memory so that we have the regexes to
// hand.
for _, room := range rooms {
state, err := db.GetStateEvent(ctx, room, MRoomServerACL, "")
if err != nil {
logrus.WithError(err).Errorf("Failed to get server ACLs for room %q", room)
continue
}
if state != nil {
acls.OnServerACLUpdate(state.PDU)
}
events, err := db.GetBulkStateContent(ctx, rooms, []gomatrixserverlib.StateKeyTuple{{EventType: MRoomServerACL, StateKey: ""}}, false)
if err != nil {
logrus.WithError(err).Errorf("Failed to get server ACLs for all rooms: %q", err)
}
for _, event := range events {
acls.OnServerACLUpdate(event)
}
return acls
}
@ -79,8 +85,8 @@ type ServerACL struct {
type serverACL struct {
ServerACL
allowedRegexes []*regexp.Regexp
deniedRegexes []*regexp.Regexp
allowedRegexes []**regexp.Regexp
deniedRegexes []**regexp.Regexp
}
func compileACLRegex(orig string) (*regexp.Regexp, error) {
@ -90,9 +96,28 @@ func compileACLRegex(orig string) (*regexp.Regexp, error) {
return regexp.Compile(escaped)
}
func (s *ServerACLs) OnServerACLUpdate(state gomatrixserverlib.PDU) {
// cachedCompileACLRegex is a wrapper around compileACLRegex with added caching
func (s *ServerACLs) cachedCompileACLRegex(orig string) (**regexp.Regexp, error) {
s.aclRegexCacheMutex.RLock()
re, ok := s.aclRegexCache[orig]
if ok {
s.aclRegexCacheMutex.RUnlock()
return re, nil
}
s.aclRegexCacheMutex.RUnlock()
compiled, err := compileACLRegex(orig)
if err != nil {
return nil, err
}
s.aclRegexCacheMutex.Lock()
defer s.aclRegexCacheMutex.Unlock()
s.aclRegexCache[orig] = &compiled
return &compiled, nil
}
func (s *ServerACLs) OnServerACLUpdate(strippedEvent tables.StrippedEvent) {
acls := &serverACL{}
if err := json.Unmarshal(state.Content(), &acls.ServerACL); err != nil {
if err := json.Unmarshal([]byte(strippedEvent.ContentValue), &acls.ServerACL); err != nil {
logrus.WithError(err).Errorf("Failed to unmarshal state content for server ACLs")
return
}
@ -101,14 +126,14 @@ func (s *ServerACLs) OnServerACLUpdate(state gomatrixserverlib.PDU) {
// special characters and then replace * and ? with their regex counterparts.
// https://matrix.org/docs/spec/client_server/r0.6.1#m-room-server-acl
for _, orig := range acls.Allowed {
if expr, err := compileACLRegex(orig); err != nil {
if expr, err := s.cachedCompileACLRegex(orig); err != nil {
logrus.WithError(err).Errorf("Failed to compile allowed regex")
} else {
acls.allowedRegexes = append(acls.allowedRegexes, expr)
}
}
for _, orig := range acls.Denied {
if expr, err := compileACLRegex(orig); err != nil {
if expr, err := s.cachedCompileACLRegex(orig); err != nil {
logrus.WithError(err).Errorf("Failed to compile denied regex")
} else {
acls.deniedRegexes = append(acls.deniedRegexes, expr)
@ -118,10 +143,15 @@ func (s *ServerACLs) OnServerACLUpdate(state gomatrixserverlib.PDU) {
"allow_ip_literals": acls.AllowIPLiterals,
"num_allowed": len(acls.allowedRegexes),
"num_denied": len(acls.deniedRegexes),
}).Debugf("Updating server ACLs for %q", state.RoomID())
}).Debugf("Updating server ACLs for %q", strippedEvent.RoomID)
// Clear out Denied and Allowed, now that we have the compiled regexes.
// They are not needed anymore from this point on.
acls.Denied = nil
acls.Allowed = nil
s.aclsMutex.Lock()
defer s.aclsMutex.Unlock()
s.acls[state.RoomID().String()] = acls
s.acls[strippedEvent.RoomID] = acls
}
func (s *ServerACLs) IsServerBannedFromRoom(serverName spec.ServerName, roomID string) bool {
@ -151,14 +181,14 @@ func (s *ServerACLs) IsServerBannedFromRoom(serverName spec.ServerName, roomID s
// Check if the hostname matches one of the denied regexes. If it does then
// the server is banned from the room.
for _, expr := range acls.deniedRegexes {
if expr.MatchString(string(serverName)) {
if (*expr).MatchString(string(serverName)) {
return true
}
}
// Check if the hostname matches one of the allowed regexes. If it does then
// the server is NOT banned from the room.
for _, expr := range acls.allowedRegexes {
if expr.MatchString(string(serverName)) {
if (*expr).MatchString(string(serverName)) {
return false
}
}

56
roomserver/acls/acls_test.go
View file

@ -15,8 +15,14 @@
package acls
import (
"context"
"regexp"
"testing"
"github.com/matrix-org/dendrite/roomserver/storage/tables"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/gomatrixserverlib/spec"
"github.com/stretchr/testify/assert"
)
func TestOpenACLsWithBlacklist(t *testing.T) {
@ -38,8 +44,8 @@ func TestOpenACLsWithBlacklist(t *testing.T) {
ServerACL: ServerACL{
AllowIPLiterals: true,
},
allowedRegexes: []*regexp.Regexp{allowRegex},
deniedRegexes: []*regexp.Regexp{denyRegex},
allowedRegexes: []**regexp.Regexp{&allowRegex},
deniedRegexes: []**regexp.Regexp{&denyRegex},
}
if acls.IsServerBannedFromRoom("1.2.3.4", roomID) {
@ -77,8 +83,8 @@ func TestDefaultACLsWithWhitelist(t *testing.T) {
ServerACL: ServerACL{
AllowIPLiterals: false,
},
allowedRegexes: []*regexp.Regexp{allowRegex},
deniedRegexes: []*regexp.Regexp{},
allowedRegexes: []**regexp.Regexp{&allowRegex},
deniedRegexes: []**regexp.Regexp{},
}
if !acls.IsServerBannedFromRoom("1.2.3.4", roomID) {
@ -103,3 +109,45 @@ func TestDefaultACLsWithWhitelist(t *testing.T) {
t.Fatal("Expected qux.com:4567 to be allowed but wasn't")
}
}
var (
content1 = `{"allow":["*"],"allow_ip_literals":false,"deny":["hello.world", "*.hello.world"]}`
)
type dummyACLDB struct{}
func (d dummyACLDB) GetKnownRooms(ctx context.Context) ([]string, error) {
return []string{"1", "2"}, nil
}
func (d dummyACLDB) GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error) {
return []tables.StrippedEvent{
{
RoomID: "1",
ContentValue: content1,
},
{
RoomID: "2",
ContentValue: content1,
},
}, nil
}
func TestCachedRegex(t *testing.T) {
db := dummyACLDB{}
wantBannedServer := spec.ServerName("hello.world")
acls := NewServerACLs(db)
// Check that hello.world is banned in room 1
banned := acls.IsServerBannedFromRoom(wantBannedServer, "1")
assert.True(t, banned)
// Check that hello.world is banned in room 2
banned = acls.IsServerBannedFromRoom(wantBannedServer, "2")
assert.True(t, banned)
// Check that matrix.hello.world is banned in room 2
banned = acls.IsServerBannedFromRoom("matrix."+wantBannedServer, "2")
assert.True(t, banned)
}

61
roomserver/api/perform.go
View file

@ -8,7 +8,6 @@ import (
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/gomatrixserverlib/spec"
"github.com/matrix-org/util"
)
type PerformCreateRoomRequest struct {
@ -51,16 +50,14 @@ type PerformLeaveResponse struct {
}
type InviteInput struct {
RoomID spec.RoomID
Inviter spec.UserID
Invitee spec.UserID
DisplayName string
AvatarURL string
Reason string
IsDirect bool
KeyID gomatrixserverlib.KeyID
PrivateKey ed25519.PrivateKey
EventTime time.Time
RoomID spec.RoomID
Inviter spec.UserID
Invitee spec.UserID
Reason string
IsDirect bool
KeyID gomatrixserverlib.KeyID
PrivateKey ed25519.PrivateKey
EventTime time.Time
}
type PerformInviteRequest struct {
@ -91,14 +88,44 @@ type PerformBackfillRequest struct {
VirtualHost spec.ServerName `json:"virtual_host"`
}
// PrevEventIDs returns the prev_event IDs of all backwards extremities, de-duplicated in a lexicographically sorted order.
// limitPrevEventIDs is the maximum of eventIDs we
// return when calling PrevEventIDs.
const limitPrevEventIDs = 100
// PrevEventIDs returns the prev_event IDs of either 100 backwards extremities or
// len(r.BackwardsExtremities). Limited to 100, due to Synapse/Dendrite stopping after reaching
// this limit. (which sounds sane)
func (r *PerformBackfillRequest) PrevEventIDs() []string {
var prevEventIDs []string
for _, pes := range r.BackwardsExtremities {
prevEventIDs = append(prevEventIDs, pes...)
var uniqueIDs map[string]struct{}
// Create a unique eventID map of either 100 or len(r.BackwardsExtremities).
// 100 since Synapse/Dendrite stops after reaching 100 events.
if len(r.BackwardsExtremities) > limitPrevEventIDs {
uniqueIDs = make(map[string]struct{}, limitPrevEventIDs)
} else {
uniqueIDs = make(map[string]struct{}, len(r.BackwardsExtremities))
}
prevEventIDs = util.UniqueStrings(prevEventIDs)
return prevEventIDs
outerLoop:
for _, pes := range r.BackwardsExtremities {
for _, evID := range pes {
uniqueIDs[evID] = struct{}{}
// We found enough unique eventIDs.
if len(uniqueIDs) >= limitPrevEventIDs {
break outerLoop
}
}
}
// map -> []string
result := make([]string, len(uniqueIDs))
i := 0
for evID := range uniqueIDs {
result[i] = evID
i++
}
return result
}
// PerformBackfillResponse is a response to PerformBackfill.

81
roomserver/api/perform_test.go Normal file
View file

@ -0,0 +1,81 @@
package api
import (
"fmt"
"math/rand"
"testing"
"github.com/stretchr/testify/assert"
)
func BenchmarkPrevEventIDs(b *testing.B) {
for _, x := range []int64{1, 10, 100, 500, 1000, 2000} {
benchPrevEventIDs(b, int(x))
}
}
func benchPrevEventIDs(b *testing.B, count int) {
bwExtrems := generateBackwardsExtremities(b, count)
backfiller := PerformBackfillRequest{
BackwardsExtremities: bwExtrems,
}
b.Run(fmt.Sprintf("Original%d", count), func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
prevIDs := backfiller.PrevEventIDs()
_ = prevIDs
}
})
}
type testLike interface {
Helper()
}
const randomIDCharsCount = 10
func generateBackwardsExtremities(t testLike, count int) map[string][]string {
t.Helper()
result := make(map[string][]string, count)
for i := 0; i < count; i++ {
eventID := randomEventId(int64(i))
result[eventID] = []string{
randomEventId(int64(i + 1)),
randomEventId(int64(i + 2)),
randomEventId(int64(i + 3)),
}
}
return result
}
const alphanumerics = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
// randomEventId generates a pseudo-random string of length n.
func randomEventId(src int64) string {
randSrc := rand.NewSource(src)
b := make([]byte, randomIDCharsCount)
for i := range b {
b[i] = alphanumerics[randSrc.Int63()%int64(len(alphanumerics))]
}
return string(b)
}
func TestPrevEventIDs(t *testing.T) {
// generate 10 backwards extremities
bwExtrems := generateBackwardsExtremities(t, 10)
backfiller := PerformBackfillRequest{
BackwardsExtremities: bwExtrems,
}
prevIDs := backfiller.PrevEventIDs()
// Given how "generateBackwardsExtremities" works, this
// generates 12 unique event IDs
assert.Equal(t, 12, len(prevIDs))
// generate 200 backwards extremities
backfiller.BackwardsExtremities = generateBackwardsExtremities(t, 200)
prevIDs = backfiller.PrevEventIDs()
// PrevEventIDs returns at max 100 event IDs
assert.Equal(t, 100, len(prevIDs))
}

16
roomserver/internal/input/input.go
View file

@ -108,12 +108,14 @@ type worker struct {
r *Inputer
roomID string
subscription *nats.Subscription
sentryHub *sentry.Hub
}
func (r *Inputer) startWorkerForRoom(roomID string) {
v, loaded := r.workers.LoadOrStore(roomID, &worker{
r: r,
roomID: roomID,
r: r,
roomID: roomID,
sentryHub: sentry.CurrentHub().Clone(),
})
w := v.(*worker)
w.Lock()
@ -265,9 +267,9 @@ func (w *worker) _next() {
// Look up what the next event is that's waiting to be processed.
ctx, cancel := context.WithTimeout(w.r.ProcessContext.Context(), time.Minute)
defer cancel()
if scope := sentry.CurrentHub().Scope(); scope != nil {
w.sentryHub.ConfigureScope(func(scope *sentry.Scope) {
scope.SetTag("room_id", w.roomID)
}
})
msgs, err := w.subscription.Fetch(1, nats.Context(ctx))
switch err {
case nil:
@ -323,9 +325,9 @@ func (w *worker) _next() {
return
}
if scope := sentry.CurrentHub().Scope(); scope != nil {
w.sentryHub.ConfigureScope(func(scope *sentry.Scope) {
scope.SetTag("event_id", inputRoomEvent.Event.EventID())
}
})
// Process the room event. If something goes wrong then we'll tell
// NATS to terminate the message. We'll store the error result as
@ -347,7 +349,7 @@ func (w *worker) _next() {
}).Warn("Roomserver rejected event")
default:
if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) {
sentry.CaptureException(err)
w.sentryHub.CaptureException(err)
}
logrus.WithError(err).WithFields(logrus.Fields{
"room_id": w.roomID,

9
roomserver/internal/input/input_events.go
View file

@ -24,6 +24,7 @@ import (
"fmt"
"time"
"github.com/matrix-org/dendrite/roomserver/storage/tables"
"github.com/tidwall/gjson"
"github.com/matrix-org/gomatrixserverlib"
@ -509,7 +510,13 @@ func (r *Inputer) processRoomEvent(
logrus.WithError(err).Error("failed to get server ACLs")
}
if aclEvent != nil {
r.ACLs.OnServerACLUpdate(aclEvent)
strippedEvent := tables.StrippedEvent{
RoomID: aclEvent.RoomID().String(),
EventType: aclEvent.Type(),
StateKey: *aclEvent.StateKey(),
ContentValue: string(aclEvent.Content()),
}
r.ACLs.OnServerACLUpdate(strippedEvent)
}
}
}

1
roomserver/internal/input/input_latest_events.go
View file

@ -298,6 +298,7 @@ func (u *latestEventsUpdater) latestState() error {
}).Warnf("State reset detected (removing %d events)", removed)
sentry.WithScope(func(scope *sentry.Scope) {
scope.SetLevel("warning")
scope.SetTag("room_id", u.event.RoomID().String())
scope.SetContext("State reset", map[string]interface{}{
"Event ID": u.event.EventID(),
"Old state NID": fmt.Sprintf("%d", u.oldStateNID),

18
roomserver/internal/perform/perform_create_room.go
View file

@ -503,16 +503,14 @@ func (c *Creator) PerformCreateRoom(ctx context.Context, userID spec.UserID, roo
err = c.RSAPI.PerformInvite(ctx, &api.PerformInviteRequest{
InviteInput: api.InviteInput{
RoomID: roomID,
Inviter: userID,
Invitee: *inviteeUserID,
DisplayName: createRequest.UserDisplayName,
AvatarURL: createRequest.UserAvatarURL,
Reason: "",
IsDirect: createRequest.IsDirect,
KeyID: createRequest.KeyID,
PrivateKey: createRequest.PrivateKey,
EventTime: createRequest.EventTime,
RoomID: roomID,
Inviter: userID,
Invitee: *inviteeUserID,
Reason: "",
IsDirect: createRequest.IsDirect,
KeyID: createRequest.KeyID,
PrivateKey: createRequest.PrivateKey,
EventTime: createRequest.EventTime,
},
InviteRoomState: globalStrippedState,
SendAsServer: string(userID.Domain()),

8
roomserver/internal/perform/perform_invite.go
View file

@ -144,11 +144,9 @@ func (r *Inviter) PerformInvite(
}
content := gomatrixserverlib.MemberContent{
Membership: spec.Invite,
DisplayName: req.InviteInput.DisplayName,
AvatarURL: req.InviteInput.AvatarURL,
Reason: req.InviteInput.Reason,
IsDirect: req.InviteInput.IsDirect,
Membership: spec.Invite,
Reason: req.InviteInput.Reason,
IsDirect: req.InviteInput.IsDirect,
}
if err = proto.SetContent(content); err != nil {

9
roomserver/producers/roomevent.go
View file

@ -17,6 +17,7 @@ package producers
import (
"encoding/json"
"github.com/matrix-org/dendrite/roomserver/storage/tables"
"github.com/nats-io/nats.go"
log "github.com/sirupsen/logrus"
"github.com/tidwall/gjson"
@ -75,7 +76,13 @@ func (r *RoomEventProducer) ProduceRoomEvents(roomID string, updates []api.Outpu
if eventType == acls.MRoomServerACL && update.NewRoomEvent.Event.StateKeyEquals("") {
ev := update.NewRoomEvent.Event.PDU
defer r.ACLs.OnServerACLUpdate(ev)
strippedEvent := tables.StrippedEvent{
RoomID: ev.RoomID().String(),
EventType: ev.Type(),
StateKey: *ev.StateKey(),
ContentValue: string(ev.Content()),
}
defer r.ACLs.OnServerACLUpdate(strippedEvent)
}
}
logger.Tracef("Producing to topic '%s'", r.Topic)

17
roomserver/storage/shared/storage.go
View file

@ -889,10 +889,10 @@ func (d *Database) assignRoomNID(
}
// Check if we already have a numeric ID in the database.
roomNID, err := d.RoomsTable.SelectRoomNID(ctx, txn, roomID)
if err == sql.ErrNoRows {
if errors.Is(err, sql.ErrNoRows) {
// We don't have a numeric ID so insert one into the database.
roomNID, err = d.RoomsTable.InsertRoomNID(ctx, txn, roomID, roomVersion)
if err == sql.ErrNoRows {
if errors.Is(err, sql.ErrNoRows) {
// We raced with another insert so run the select again.
roomNID, err = d.RoomsTable.SelectRoomNID(ctx, txn, roomID)
}
@ -914,10 +914,10 @@ func (d *Database) assignEventTypeNID(
}
// Check if we already have a numeric ID in the database.
eventTypeNID, err := d.EventTypesTable.SelectEventTypeNID(ctx, txn, eventType)
if err == sql.ErrNoRows {
if errors.Is(err, sql.ErrNoRows) {
// We don't have a numeric ID so insert one into the database.
eventTypeNID, err = d.EventTypesTable.InsertEventTypeNID(ctx, txn, eventType)
if err == sql.ErrNoRows {
if errors.Is(err, sql.ErrNoRows) {
// We raced with another insert so run the select again.
eventTypeNID, err = d.EventTypesTable.SelectEventTypeNID(ctx, txn, eventType)
}
@ -938,16 +938,19 @@ func (d *EventDatabase) assignStateKeyNID(
}
// Check if we already have a numeric ID in the database.
eventStateKeyNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, txn, eventStateKey)
if err == sql.ErrNoRows {
if errors.Is(err, sql.ErrNoRows) {
// We don't have a numeric ID so insert one into the database.
eventStateKeyNID, err = d.EventStateKeysTable.InsertEventStateKeyNID(ctx, txn, eventStateKey)
if err == sql.ErrNoRows {
if errors.Is(err, sql.ErrNoRows) {
// We raced with another insert so run the select again.
eventStateKeyNID, err = d.EventStateKeysTable.SelectEventStateKeyNID(ctx, txn, eventStateKey)
}
}
if err != nil {
return 0, err
}
d.Cache.StoreEventStateKey(eventStateKeyNID, eventStateKey)
return eventStateKeyNID, err
return eventStateKeyNID, nil
}
func extractRoomVersionFromCreateEvent(event gomatrixserverlib.PDU) (

4
roomserver/storage/tables/interface.go
View file

@ -235,6 +235,10 @@ func ExtractContentValue(ev *types.HeaderedEvent) string {
key = "topic"
case "m.room.guest_access":
key = "guest_access"
case "m.room.server_acl":
// We need the entire content and not only one key, so we can use it
// on startup to generate the ACLs. This is merely a workaround.
return string(content)
}
result := gjson.GetBytes(content, key)
if !result.Exists() {

11
setup/base/base.go
View file

@ -50,6 +50,9 @@ import (
//go:embed static/*.gotmpl
var staticContent embed.FS
//go:embed static/client/login
var loginFallback embed.FS
const HTTPServerTimeout = time.Minute * 5
// CreateClient creates a new client (normally used for media fetch requests).
@ -158,6 +161,14 @@ func SetupAndServeHTTP(
_, _ = w.Write(landingPage.Bytes())
})
// We only need the files beneath the static/client/login folder.
sub, err := fs.Sub(loginFallback, "static/client/login")
if err != nil {
logrus.Panicf("unable to read embedded files, this should never happen: %s", err)
}
// Serve a static page for login fallback
routers.Static.PathPrefix("/client/login/").Handler(http.StripPrefix("/_matrix/static/client/login/", http.FileServer(http.FS(sub))))
var clientHandler http.Handler
clientHandler = routers.Client
if cfg.Global.Sentry.Enabled {

47
setup/base/static/client/login/index.html Normal file
View file

@ -0,0 +1,47 @@
<!doctype html>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title> Login </title>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="stylesheet" href="style.css">
<script src="js/jquery-3.4.1.min.js"></script>
<script src="js/login.js"></script>
</head>
<body onload="matrixLogin.onLoad()">
<div id="container">
<h1 id="title"></h1>
<span id="feedback"></span>
<div id="loading">
<img src="spinner.gif" />
</div>
<div id="sso_flow" class="login_flow" style="display: none;">
Single-sign on:
<form id="sso_form" action="/_matrix/client/v3/login/sso/redirect" method="get">
<input id="sso_redirect_url" type="hidden" name="redirectUrl" value=""/>
<input type="submit" value="Log in"/>
</form>
</div>
<div id="password_flow" class="login_flow" style="display: none;">
Password Authentication:
<form onsubmit="matrixLogin.passwordLogin(); return false;">
<input id="user_id" size="32" type="text" placeholder="Matrix ID (e.g. bob)" autocapitalize="off" autocorrect="off" />
<br/>
<input id="password" size="32" type="password" placeholder="Password"/>
<br/>
<input type="submit" value="Log in"/>
</form>
</div>
<div id="no_login_types" type="button" class="login_flow" style="display: none;">
Log in currently unavailable.
</div>
</div>
</body>
</html>

2
setup/base/static/client/login/js/jquery-3.4.1.min.js vendored Normal file
View file

File diff suppressed because one or more lines are too long

291
setup/base/static/client/login/js/login.js Normal file
View file

@ -0,0 +1,291 @@
window.matrixLogin = {
endpoint: location.origin + "/_matrix/client/v3/login",
serverAcceptsPassword: false,
serverAcceptsSso: false,
};
// Titles get updated through the process to give users feedback.
const TITLE_PRE_AUTH = "Log in with one of the following methods";
const TITLE_POST_AUTH = "Logging in...";
// The cookie used to store the original query parameters when using SSO.
const COOKIE_KEY = "dendrite_login_fallback_qs";
/*
* Submit a login request.
*
* type: The login type as a string (e.g. "m.login.foo").
* data: An object of data specific to the login type.
* extra: (Optional) An object to search for extra information to send with the
* login request, e.g. device_id.
* callback: (Optional) Function to call on successful login.
*/
function submitLogin(type, data, extra, callback) {
console.log("Logging in with " + type);
setTitle(TITLE_POST_AUTH);
// Add the login type.
data.type = type;
// Add the device information, if it was provided.
if (extra.device_id) {
data.device_id = extra.device_id;
}
if (extra.initial_device_display_name) {
data.initial_device_display_name = extra.initial_device_display_name;
}
$.post(matrixLogin.endpoint, JSON.stringify(data), function(response) {
if (callback) {
callback();
}
matrixLogin.onLogin(response);
}).fail(errorFunc);
}
/*
* Display an error to the user and show the login form again.
*/
function errorFunc(err) {
// We want to show the error to the user rather than redirecting immediately to the
// SSO portal (if SSO is the only login option), so we inhibit the redirect.
showLogin(true);
if (err.responseJSON && err.responseJSON.error) {
setFeedbackString(err.responseJSON.error + " (" + err.responseJSON.errcode + ")");
}
else {
setFeedbackString("Request failed: " + err.status);
}
}
/*
* Display an error to the user.
*/
function setFeedbackString(text) {
$("#feedback").text(text);
}
/*
* (Maybe) Show the login forms.
*
* This actually does a few unrelated functions:
*
* * Configures the SSO redirect URL to come back to this page.
* * Configures and shows the SSO form, if the server supports SSO.
* * Otherwise, shows the password form.
*/
function showLogin(inhibitRedirect) {
setTitle(TITLE_PRE_AUTH);
// If inhibitRedirect is false, and SSO is the only supported login method,
// we can redirect straight to the SSO page.
if (matrixLogin.serverAcceptsSso) {
// Set the redirect to come back to this page, a login token will get
// added as a query parameter and handled after the redirect.
$("#sso_redirect_url").val(window.location.origin + window.location.pathname);
// Before submitting SSO, set the current query parameters into a cookie
// for retrieval later.
var qs = parseQsFromUrl();
setCookie(COOKIE_KEY, JSON.stringify(qs));
// If password is not supported and redirects are allowed, then submit
// the form (redirecting to the SSO provider).
if (!inhibitRedirect && !matrixLogin.serverAcceptsPassword) {
$("#sso_form").submit();
return;
}
// Otherwise, show the SSO form
$("#sso_flow").show();
}
if (matrixLogin.serverAcceptsPassword) {
$("#password_flow").show();
}
// If neither password or SSO are supported, show an error to the user.
if (!matrixLogin.serverAcceptsPassword && !matrixLogin.serverAcceptsSso) {
$("#no_login_types").show();
}
$("#loading").hide();
}
/*
* Hides the forms and shows a loading throbber.
*/
function showSpinner() {
$("#password_flow").hide();
$("#sso_flow").hide();
$("#no_login_types").hide();
$("#loading").show();
}
/*
* Helper to show the page's main title.
*/
function setTitle(title) {
$("#title").text(title);
}
/*
* Query the login endpoint for the homeserver's supported flows.
*
* This populates matrixLogin.serverAccepts* variables.
*/
function fetchLoginFlows(cb) {
$.get(matrixLogin.endpoint, function(response) {
for (var i = 0; i < response.flows.length; i++) {
var flow = response.flows[i];
if ("m.login.sso" === flow.type) {
matrixLogin.serverAcceptsSso = true;
console.log("Server accepts SSO");
}
if ("m.login.password" === flow.type) {
matrixLogin.serverAcceptsPassword = true;
console.log("Server accepts password");
}
}
cb();
}).fail(errorFunc);
}
/*
* Called on load to fetch login flows and attempt SSO login (if a token is available).
*/
matrixLogin.onLoad = function() {
fetchLoginFlows(function() {
// (Maybe) attempt logging in via SSO if a token is available.
if (!tryTokenLogin()) {
showLogin(false);
}
});
};
/*
* Submit simple user & password login.
*/
matrixLogin.passwordLogin = function() {
var user = $("#user_id").val();
var pwd = $("#password").val();
setFeedbackString("");
showSpinner();
submitLogin(
"m.login.password",
{user: user, password: pwd},
parseQsFromUrl());
};
/*
* The onLogin function gets called after a successful login.
*
* It is expected that implementations override this to be notified when the
* login is complete. The response to the login call is provided as the single
* parameter.
*/
matrixLogin.onLogin = function(response) {
// clobber this function
console.warn("onLogin - This function should be replaced to proceed.");
};
/*
* Process the query parameters from the current URL into an object.
*/
function parseQsFromUrl() {
var pos = window.location.href.indexOf("?");
if (pos == -1) {
return {};
}
var query = window.location.href.substr(pos + 1);
var result = {};
query.split("&").forEach(function(part) {
var item = part.split("=");
var key = item[0];
var val = item[1];
if (val) {
val = decodeURIComponent(val);
}
result[key] = val;
});
return result;
}
/*
* Process the cookies and return an object.
*/
function parseCookies() {
var allCookies = document.cookie;
var result = {};
allCookies.split(";").forEach(function(part) {
var item = part.split("=");
// Cookies might have arbitrary whitespace between them.
var key = item[0].trim();
// You can end up with a broken cookie that doesn't have an equals sign
// in it. Set to an empty value.
var val = (item[1] || "").trim();
// Values might be URI encoded.
if (val) {
val = decodeURIComponent(val);
}
result[key] = val;
});
return result;
}
/*
* Set a cookie that is valid for 1 hour.
*/
function setCookie(key, value) {
// The maximum age is set in seconds.
var maxAge = 60 * 60;
// Set the cookie, this defaults to the current domain and path.
document.cookie = key + "=" + encodeURIComponent(value) + ";max-age=" + maxAge + ";sameSite=lax";
}
/*
* Removes a cookie by key.
*/
function deleteCookie(key) {
// Delete a cookie by setting the expiration to 0. (Note that the value
// doesn't matter.)
document.cookie = key + "=deleted;expires=0";
}
/*
* Submits the login token if one is found in the query parameters. Returns a
* boolean of whether the login token was found or not.
*/
function tryTokenLogin() {
// Check if the login token is in the query parameters.
var qs = parseQsFromUrl();
var loginToken = qs.loginToken;
if (!loginToken) {
return false;
}
// Retrieve the original query parameters (from before the SSO redirect).
// They are stored as JSON in a cookie.
var cookies = parseCookies();
var originalQueryParams = JSON.parse(cookies[COOKIE_KEY] || "{}")
// If the login is successful, delete the cookie.
function callback() {
deleteCookie(COOKIE_KEY);
}
submitLogin(
"m.login.token",
{token: loginToken},
originalQueryParams,
callback);
return true;
}

BIN
setup/base/static/client/login/spinner.gif Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.8 KiB

79
setup/base/static/client/login/style.css Normal file
View file

@ -0,0 +1,79 @@
html {
height: 100%;
}
body {
height: 100%;
font-family: "Myriad Pro", "Myriad", Helvetica, Arial, sans-serif;
font-size: 12pt;
margin: 0px;
}
h1 {
font-size: 20pt;
}
a:link { color: #666; }
a:visited { color: #666; }
a:hover { color: #000; }
a:active { color: #000; }
input {
margin: 5px;
}
textbox, input[type="text"], input[type="password"] {
width: 90%;
}
form {
text-align: center;
margin: 10px 0 0 0;
}
ul.radiobuttons {
text-align: left;
list-style: none;
}
/*
* Add some padding to the viewport.
*/
#container {
padding: 10px;
}
/*
* Center all direct children of the main form.
*/
#container > * {
display: block;
margin-left: auto;
margin-right: auto;
text-align: center;
}
/*
* A wrapper around each login flow.
*/
.login_flow {
width: 300px;
text-align: left;
padding: 10px;
margin-bottom: 40px;
border-radius: 10px;
box-shadow: 0px 0px 20px 0px rgba(0,0,0,0.15);
background-color: #f8f8f8;
border: 1px #ccc solid;
}
/*
* Used to show error content.
*/
#feedback {
/* Red text. */
color: #ff0000;
/* A little space to not overlap the box-shadow. */
margin-bottom: 20px;
}

3
setup/config/config_appservice.go
View file

@ -40,6 +40,9 @@ type AppServiceAPI struct {
// on appservice endpoints. This is not recommended in production!
DisableTLSValidation bool `yaml:"disable_tls_validation"`
LegacyAuth bool `yaml:"legacy_auth"`
LegacyPaths bool `yaml:"legacy_paths"`
ConfigFiles []string `yaml:"config_files"`
}

7
setup/config/config_federationapi.go
View file

@ -18,6 +18,13 @@ type FederationAPI struct {
// The default value is 16 if not specified, which is circa 18 hours.
FederationMaxRetries uint32 `yaml:"send_max_retries"`
// P2P Feature: Whether relaying to specific nodes should be enabled.
// Defaults to false.
// Note: Enabling relays introduces a huge startup delay, if you are not using
// relays and have many servers to re-hydrate on start. Only enable this
// if you are using relays!
EnableRelays bool `yaml:"enable_relays"`
// P2P Feature: How many consecutive failures that we should tolerate when
// sending federation requests to a specific server until we should assume they
// are offline. If we assume they are offline then we will attempt to send

6
syncapi/routing/context.go
View file

@ -110,6 +110,7 @@ func Context(
}
stateFilter := synctypes.StateFilter{
Limit: filter.Limit,
NotSenders: filter.NotSenders,
NotTypes: filter.NotTypes,
Senders: filter.Senders,
@ -157,6 +158,11 @@ func Context(
}
}
// Limit is split up for before/after events
if filter.Limit > 1 {
filter.Limit = filter.Limit / 2
}
eventsBefore, err := snapshot.SelectContextBeforeEvent(ctx, id, roomID, filter)
if err != nil && err != sql.ErrNoRows {
logrus.WithError(err).Error("unable to fetch before events")

62
syncapi/routing/memberships.go
View file

@ -15,7 +15,6 @@
package routing
import (
"encoding/json"
"math"
"net/http"
@ -33,31 +32,13 @@ type getMembershipResponse struct {
Chunk []synctypes.ClientEvent `json:"chunk"`
}
// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members
type getJoinedMembersResponse struct {
Joined map[string]joinedMember `json:"joined"`
}
type joinedMember struct {
DisplayName string `json:"display_name"`
AvatarURL string `json:"avatar_url"`
}
// The database stores 'displayname' without an underscore.
// Deserialize into this and then change to the actual API response
type databaseJoinedMember struct {
DisplayName string `json:"displayname"`
AvatarURL string `json:"avatar_url"`
}
// GetMemberships implements
//
// GET /rooms/{roomId}/members
// GET /rooms/{roomId}/joined_members
func GetMemberships(
req *http.Request, device *userapi.Device, roomID string,
syncDB storage.Database, rsAPI api.SyncRoomserverAPI,
joinedOnly bool, membership, notMembership *string, at string,
membership, notMembership *string, at string,
) util.JSONResponse {
userID, err := spec.NewUserID(device.UserID, true)
if err != nil {
@ -87,13 +68,6 @@ func GetMemberships(
}
}
if joinedOnly && !queryRes.IsInRoom {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: spec.Forbidden("You aren't a member of the room and weren't previously a member of the room."),
}
}
db, err := syncDB.NewDatabaseSnapshot(req.Context())
if err != nil {
return util.JSONResponse{
@ -139,40 +113,6 @@ func GetMemberships(
result := qryRes.Events
if joinedOnly {
var res getJoinedMembersResponse
res.Joined = make(map[string]joinedMember)
for _, ev := range result {
var content databaseJoinedMember
if err := json.Unmarshal(ev.Content(), &content); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content")
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: spec.InternalServerError{},
}
}
userID, err := rsAPI.QueryUserIDForSender(req.Context(), ev.RoomID(), ev.SenderID())
if err != nil || userID == nil {
util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryUserIDForSender failed")
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: spec.InternalServerError{},
}
}
if err != nil {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: spec.Forbidden("You don't have permission to kick this user, unknown senderID"),
}
}
res.Joined[userID.String()] = joinedMember(content)
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: res,
}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: getMembershipResponse{synctypes.ToClientEvents(gomatrixserverlib.ToPDUs(result), synctypes.FormatAll, func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {

24
syncapi/routing/messages.go
View file

@ -135,13 +135,6 @@ func OnIncomingMessagesRequest(
var fromStream *types.StreamingToken
fromQuery := req.URL.Query().Get("from")
toQuery := req.URL.Query().Get("to")
emptyFromSupplied := fromQuery == ""
if emptyFromSupplied {
// NOTSPEC: We will pretend they used the latest sync token if no ?from= was provided.
// We do this to allow clients to get messages without having to call `/sync` e.g Cerulean
currPos := srp.Notifier.CurrentPosition()
fromQuery = currPos.String()
}
// Direction to return events from.
dir := req.URL.Query().Get("dir")
@ -155,6 +148,23 @@ func OnIncomingMessagesRequest(
// to have one of the two accepted values (so dir == "f" <=> !backwardOrdering).
backwardOrdering := (dir == "b")
emptyFromSupplied := fromQuery == ""
if emptyFromSupplied {
// If "from" isn't provided, it defaults to either the earliest stream
// position (if we're going forward) or to the latest one (if we're
// going backward).
var from types.TopologyToken
if backwardOrdering {
from = types.TopologyToken{Depth: math.MaxInt64, PDUPosition: math.MaxInt64}
} else {
// go 1 earlier than the first event so we correctly fetch the earliest event
// this is because Database.GetEventsInTopologicalRange is exclusive of the lower-bound.
from = types.TopologyToken{}
}
fromQuery = from.String()
}
from, err := types.NewTopologyTokenFromString(fromQuery)
if err != nil {
var streamToken types.StreamingToken

14
syncapi/routing/routing.go
View file

@ -197,19 +197,7 @@ func Setup(
}
at := req.URL.Query().Get("at")
return GetMemberships(req, device, vars["roomID"], syncDB, rsAPI, false, membership, notMembership, at)
return GetMemberships(req, device, vars["roomID"], syncDB, rsAPI, membership, notMembership, at)
}, httputil.WithAllowGuests()),
).Methods(http.MethodGet, http.MethodOptions)
v3mux.Handle("/rooms/{roomID}/joined_members",
httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
at := req.URL.Query().Get("at")
membership := spec.Join
return GetMemberships(req, device, vars["roomID"], syncDB, rsAPI, true, &membership, nil, at)
}),
).Methods(http.MethodGet, http.MethodOptions)
}

94
syncapi/streams/stream_pdu.go
View file

@ -203,6 +203,12 @@ func (p *PDUStreamProvider) IncrementalSync(
req.Log.WithError(err).Error("unable to update event filter with ignored users")
}
dbEvents, err := p.getRecentEvents(ctx, stateDeltas, r, eventFilter, snapshot)
if err != nil {
req.Log.WithError(err).Error("unable to get recent events")
return r.From
}
newPos = from
for _, delta := range stateDeltas {
newRange := r
@ -218,7 +224,7 @@ func (p *PDUStreamProvider) IncrementalSync(
}
}
var pos types.StreamPosition
if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req); err != nil {
if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req, dbEvents); err != nil {
req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed")
if err == context.DeadlineExceeded || err == context.Canceled || err == sql.ErrTxDone {
return newPos
@ -240,6 +246,66 @@ func (p *PDUStreamProvider) IncrementalSync(
return newPos
}
func (p *PDUStreamProvider) getRecentEvents(ctx context.Context, stateDeltas []types.StateDelta, r types.Range, eventFilter synctypes.RoomEventFilter, snapshot storage.DatabaseTransaction) (map[string]types.RecentEvents, error) {
var roomIDs []string
var newlyJoinedRoomIDs []string
for _, delta := range stateDeltas {
if delta.NewlyJoined {
newlyJoinedRoomIDs = append(newlyJoinedRoomIDs, delta.RoomID)
} else {
roomIDs = append(roomIDs, delta.RoomID)
}
}
dbEvents := make(map[string]types.RecentEvents)
if len(roomIDs) > 0 {
events, err := snapshot.RecentEvents(
ctx, roomIDs, r,
&eventFilter, true, true,
)
if err != nil {
if err != sql.ErrNoRows {
return nil, err
}
}
for k, v := range events {
dbEvents[k] = v
}
}
if len(newlyJoinedRoomIDs) > 0 {
// For rooms that were joined in this sync, try to fetch
// as much timeline events as allowed by the filter.
filter := eventFilter
// If we're going backwards, grep at least X events, this is mostly to satisfy Sytest
if eventFilter.Limit < recentEventBackwardsLimit {
filter.Limit = recentEventBackwardsLimit // TODO: Figure out a better way
diff := r.From - r.To
if diff > 0 && diff < recentEventBackwardsLimit {
filter.Limit = int(diff)
}
}
events, err := snapshot.RecentEvents(
ctx, newlyJoinedRoomIDs, types.Range{
From: r.To,
To: 0,
Backwards: true,
},
&filter, true, true,
)
if err != nil {
if err != sql.ErrNoRows {
return nil, err
}
}
for k, v := range events {
dbEvents[k] = v
}
}
return dbEvents, nil
}
// Limit the recent events to X when going backwards
const recentEventBackwardsLimit = 100
@ -253,29 +319,9 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
eventFilter *synctypes.RoomEventFilter,
stateFilter *synctypes.StateFilter,
req *types.SyncRequest,
dbEvents map[string]types.RecentEvents,
) (types.StreamPosition, error) {
var err error
originalLimit := eventFilter.Limit
// If we're going backwards, grep at least X events, this is mostly to satisfy Sytest
if r.Backwards && originalLimit < recentEventBackwardsLimit {
eventFilter.Limit = recentEventBackwardsLimit // TODO: Figure out a better way
diff := r.From - r.To
if diff > 0 && diff < recentEventBackwardsLimit {
eventFilter.Limit = int(diff)
}
}
dbEvents, err := snapshot.RecentEvents(
ctx, []string{delta.RoomID}, r,
eventFilter, true, true,
)
if err != nil {
if err == sql.ErrNoRows {
return r.To, nil
}
return r.From, fmt.Errorf("p.DB.RecentEvents: %w", err)
}
recentStreamEvents := dbEvents[delta.RoomID].Events
limited := dbEvents[delta.RoomID].Limited
@ -337,9 +383,9 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse(
logrus.WithError(err).Error("unable to apply history visibility filter")
}
if r.Backwards && len(events) > originalLimit {
if r.Backwards && len(events) > eventFilter.Limit {
// We're going backwards and the events are ordered chronologically, so take the last `limit` events
events = events[len(events)-originalLimit:]
events = events[len(events)-eventFilter.Limit:]
limited = true
}

35
syncapi/syncapi_test.go
View file

@ -753,24 +753,6 @@ func TestGetMembership(t *testing.T) {
},
wantOK: false,
},
{
name: "/joined_members - Bob never joined",
request: func(t *testing.T, room *test.Room) *http.Request {
return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
"access_token": bobDev.AccessToken,
}))
},
wantOK: false,
},
{
name: "/joined_members - Alice joined",
request: func(t *testing.T, room *test.Room) *http.Request {
return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
"access_token": aliceDev.AccessToken,
}))
},
wantOK: true,
},
{
name: "Alice leaves before Bob joins, should not be able to see Bob",
request: func(t *testing.T, room *test.Room) *http.Request {
@ -809,21 +791,6 @@ func TestGetMembership(t *testing.T) {
wantOK: true,
wantMemberCount: 2,
},
{
name: "/joined_members - Alice leaves, shouldn't be able to see members ",
request: func(t *testing.T, room *test.Room) *http.Request {
return test.NewRequest(t, "GET", fmt.Sprintf("/_matrix/client/v3/rooms/%s/joined_members", room.ID), test.WithQueryParams(map[string]string{
"access_token": aliceDev.AccessToken,
}))
},
additionalEvents: func(t *testing.T, room *test.Room) {
room.CreateAndInsert(t, alice, spec.MRoomMember, map[string]interface{}{
"membership": "leave",
}, test.WithStateKey(alice.ID))
},
useSleep: true,
wantOK: false,
},
{
name: "'at' specified, returns memberships before Bob joins",
request: func(t *testing.T, room *test.Room) *http.Request {
@ -1169,7 +1136,7 @@ func testContext(t *testing.T, dbType test.DBType) {
},
{
name: "events are not limited",
wantBeforeLength: 7,
wantBeforeLength: 5,
},
{
name: "all events are limited",

8
syncapi/types/types.go
View file

@ -286,8 +286,8 @@ func NewTopologyTokenFromString(tok string) (token TopologyToken, err error) {
if i > len(positions) {
break
}
var pos int
pos, err = strconv.Atoi(p)
var pos int64
pos, err = strconv.ParseInt(p, 10, 64)
if err != nil {
return
}
@ -318,8 +318,8 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) {
if i >= len(positions) {
break
}
var pos int
pos, err = strconv.Atoi(p)
var pos int64
pos, err = strconv.ParseInt(p, 10, 64)
if err != nil {
err = ErrMalformedSyncToken
return

17
syncapi/types/types_test.go
View file

@ -3,6 +3,7 @@ package types
import (
"context"
"encoding/json"
"math"
"reflect"
"testing"
@ -33,12 +34,28 @@ func TestSyncTokens(t *testing.T) {
"s3_1_0_0_0_0_2_0_5": StreamingToken{3, 1, 0, 0, 0, 0, 2, 0, 5}.String(),
"s3_1_2_3_5_0_0_0_6": StreamingToken{3, 1, 2, 3, 5, 0, 0, 0, 6}.String(),
"t3_1": TopologyToken{3, 1}.String(),
"t9223372036854775807_9223372036854775807": TopologyToken{Depth: math.MaxInt64, PDUPosition: math.MaxInt64}.String(),
"s9223372036854775807_1_2_3_5_0_0_0_6": StreamingToken{math.MaxInt64, 1, 2, 3, 5, 0, 0, 0, 6}.String(),
}
for a, b := range shouldPass {
if a != b {
t.Errorf("expected %q, got %q", a, b)
}
// parse as topology token
if a[0] == 't' {
if _, err := NewTopologyTokenFromString(a); err != nil {
t.Errorf("expected %q to pass, but got %q", a, err)
}
}
// parse as sync token
if a[0] == 's' {
if _, err := NewStreamTokenFromString(a); err != nil {
t.Errorf("expected %q to pass, but got %q", a, err)
}
}
}
shouldFail := []string{

8
userapi/api/api.go
View file

@ -379,6 +379,10 @@ type PerformDeviceCreationRequest struct {
// update for this account. Generally the only reason to do this is if the account
// is an appservice account.
NoDeviceListUpdate bool
// FromRegistration determines if this request comes from registering a new account
// and is in most cases false.
FromRegistration bool
}
// PerformDeviceCreationResponse is the response for PerformDeviceCreation
@ -803,6 +807,10 @@ type PerformUploadKeysRequest struct {
// itself doesn't change but it's easier to pretend upload new keys and reuse the same code paths.
// Without this flag, requests to modify device display names would delete device keys.
OnlyDisplayNameUpdates bool
// FromRegistration is set if this key upload comes right after creating an account
// and determines if we need to inform downstream components.
FromRegistration bool
}
// PerformUploadKeysResponse is the response to PerformUploadKeys

12
userapi/internal/key_api.go
View file

@ -711,9 +711,15 @@ func (a *UserInternalAPI) uploadLocalDeviceKeys(ctx context.Context, req *api.Pe
}
return
}
err = emitDeviceKeyChanges(a.KeyChangeProducer, existingKeys, keysToStore, req.OnlyDisplayNameUpdates)
if err != nil {
util.GetLogger(ctx).Errorf("Failed to emitDeviceKeyChanges: %s", err)
// If the request does _not_ come right after registering an account
// inform downstream components. However, we're fine with just creating the
// database entries above in other cases.
if !req.FromRegistration {
err = emitDeviceKeyChanges(a.KeyChangeProducer, existingKeys, keysToStore, req.OnlyDisplayNameUpdates)
if err != nil {
util.GetLogger(ctx).Errorf("Failed to emitDeviceKeyChanges: %s", err)
}
}
}

11
userapi/internal/user_api.go
View file

@ -316,7 +316,7 @@ func (a *UserInternalAPI) PerformDeviceCreation(ctx context.Context, req *api.Pe
return nil
}
// create empty device keys and upload them to trigger device list changes
return a.deviceListUpdate(dev.UserID, []string{dev.ID})
return a.deviceListUpdate(dev.UserID, []string{dev.ID}, req.FromRegistration)
}
func (a *UserInternalAPI) PerformDeviceDeletion(ctx context.Context, req *api.PerformDeviceDeletionRequest, res *api.PerformDeviceDeletionResponse) error {
@ -356,10 +356,10 @@ func (a *UserInternalAPI) PerformDeviceDeletion(ctx context.Context, req *api.Pe
return fmt.Errorf("a.KeyAPI.PerformDeleteKeys: %w", err)
}
// create empty device keys and upload them to delete what was once there and trigger device list changes
return a.deviceListUpdate(req.UserID, deletedDeviceIDs)
return a.deviceListUpdate(req.UserID, deletedDeviceIDs, false)
}
func (a *UserInternalAPI) deviceListUpdate(userID string, deviceIDs []string) error {
func (a *UserInternalAPI) deviceListUpdate(userID string, deviceIDs []string, fromRegistration bool) error {
deviceKeys := make([]api.DeviceKeys, len(deviceIDs))
for i, did := range deviceIDs {
deviceKeys[i] = api.DeviceKeys{
@ -371,8 +371,9 @@ func (a *UserInternalAPI) deviceListUpdate(userID string, deviceIDs []string) er
var uploadRes api.PerformUploadKeysResponse
if err := a.PerformUploadKeys(context.Background(), &api.PerformUploadKeysRequest{
UserID: userID,
DeviceKeys: deviceKeys,
UserID: userID,
DeviceKeys: deviceKeys,
FromRegistration: fromRegistration,
}, &uploadRes); err != nil {
return err
}