mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2025-12-26 08:13:09 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into neilalexander/pinecone

Browse source
This commit is contained in:
Neil Alexander 2021-01-26 13:38:43 +00:00
parent 2c9031e576 e14e9a1606
commit 51ceb4f9ab
No known key found for this signature in database
GPG key ID: A02A2019A2BB0944
313 changed files with 9594 additions and 3722 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

71
.github/workflows/docker-hub.yml vendored Normal file
View file

@ -0,0 +1,71 @@
# Based on https://github.com/docker/build-push-action
name: "Docker Hub"
on:
release:
types: [published]
env:
DOCKER_NAMESPACE: matrixdotorg
DOCKER_HUB_USER: dendritegithub
PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7
jobs:
Monolith:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Get release tag
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Build monolith image
id: docker_build_monolith
uses: docker/build-push-action@v2
with:
context: .
file: ./build/docker/Dockerfile.monolith
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:latest
${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ env.RELEASE_VERSION }}
Polylith:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Get release tag
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: ${{ env.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Build polylith image
id: docker_build_polylith
uses: docker/build-push-action@v2
with:
context: .
file: ./build/docker/Dockerfile.polylith
platforms: ${{ env.PLATFORMS }}
push: true
tags: |
${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:latest
${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }}

1
.golangci.yml
View file

@ -185,6 +185,7 @@ linters:
- gocyclo
- goimports # Does everything gofmt does
- gosimple
- govet
- ineffassign
- megacheck
- misspell # Check code comments, whereas misspell in CI checks *.md files

94
CHANGES.md
View file

@ -1,5 +1,99 @@
# Changelog
## Dendrite 0.3.7 (2021-01-26)
### Features
* Sync filtering support (for event types, senders and limits)
* In-process DNS caching support for deployments where a local DNS caching resolver is not available (disabled by default)
* Experimental support for MSC2444 (Peeking over Federation) has been merged
* Experimental federation support for MSC2946 (Spaces Summary) has been merged
### Fixes
* Dendrite will no longer load a given event more than once for state resolution, which may help to reduce memory usage and database I/O slightly in some cases
* Large well-known responses will no longer use significant amounts of memory
## Dendrite 0.3.6 (2021-01-18)
### Features
* Experimental support for MSC2946 (Spaces Summary) has been merged
* Send-to-device messages have been refactored and now take advantage of having their own stream position, making delivery more reliable
* Unstable features and MSCs are now listed in `/versions` (contributed by [sumitks866](https://github.com/sumitks866))
* Well-known and DNS SRV record results for federated servers are now cached properly, improving outbound federation performance and reducing traffic
### Fixes
* Updating forward extremities will no longer result in so many unnecessary state snapshots, reducing on-going disk usage in the roomserver database
* Pagination tokens for `/messages` have been fixed, which should improve the reliability of scrollback/pagination
* Dendrite now avoids returning `null`s in fields of the `/sync` response, and omitting some fields altogether when not needed, which should fix sync issues with Element Android
* Requests for user device lists now time out quicker, which prevents federated `/send` requests from also timing out in many cases
* Empty push rules are no longer sent over and over again in `/sync`
* An integer overflow in the device list updater which could result in panics on 32-bit platforms has been fixed (contributed by [Lesterpig](https://github.com/Lesterpig))
* Event IDs are now logged properly in federation sender and sync API consumer errors
## Dendrite 0.3.5 (2021-01-11)
### Features
* All `/sync` streams are now logically separate after a refactoring exercise
### Fixes
* Event references are now deeply checked properly when calculating forward extremities, reducing the amount of forward extremities in most cases, which improves RAM utilisation and reduces the work done by state resolution
* Sync no longer sends incorrect `next_batch` tokens with old stream positions, reducing flashbacks of old messages in clients
* The federation `/send` endpoint no longer uses the request context, which could result in some events failing to be persisted if the sending server gave up the HTTP connection
* Appservices can now auth as users in their namespaces properly
## Dendrite 0.3.4 (2020-12-18)
### Features
* The stream tokens for `/sync` have been refactored, giving PDUs, typing notifications, read receipts, invites and send-to-device messages their own respective stream positions, greatly improving the correctness of sync
* A new roominfo cache has been added, which results in less database hits in the roomserver
* Prometheus metrics have been added for sync requests, destination queues and client API event send perceived latency
### Fixes
* Event IDs are no longer recalculated so often in `/sync`, which reduces CPU usage
* Sync requests are now woken up correctly for our own device list updates
* The device list stream position is no longer lost, so unnecessary device updates no longer appear in every other sync
* A crash on concurrent map read/writes has been fixed in the stream token code
* The roomserver input API no longer starts more worker goroutines than needed
* The roomserver no longer uses the request context for queued tasks which could lead to send requests failing to be processed
* A new index has been added to the sync API current state table, which improves lookup performance significantly
* The client API `/joined_rooms` endpoint no longer incorrectly returns `null` if there are 0 rooms joined
* The roomserver will now query appservices when looking up a local room alias that isn't known
* The check on registration for appservice-exclusive namespaces has been fixed
## Dendrite 0.3.3 (2020-12-09)
### Features
* Federation sender should now use considerably less CPU cycles and RAM when sending events into large rooms
* The roomserver now uses considerably less CPU cycles by not calculating event IDs so often
* Experimental support for [MSC2836](https://github.com/matrix-org/matrix-doc/pull/2836) (threading) has been merged
* Dendrite will no longer hold federation HTTP connections open unnecessarily, which should help to reduce ambient CPU/RAM usage and hold fewer long-term file descriptors
### Fixes
* A bug in the latest event updater has been fixed, which should prevent the roomserver from losing forward extremities in some rare cases
* A panic has been fixed when federation is disabled (contributed by [kraem](https://github.com/kraem))
* The response format of the `/joined_members` endpoint has been fixed (contributed by [alexkursell](https://github.com/alexkursell))
## Dendrite 0.3.2 (2020-12-02)
### Features
* Federation can now be disabled with the `global.disable_federation` configuration option
### Fixes
* The `"since"` parameter is now checked more thoroughly in the sync API, which led to a bug that could cause forgotten rooms to reappear (contributed by [kaniini](https://github.com/kaniini))
* The polylith now proxies signing key requests through the federation sender correctly
* The code for checking if remote servers are allowed to see events now no longer wastes CPU time retrieving irrelevant state events
## Dendrite 0.3.1 (2020-11-20)
### Features

3
README.md
View file

@ -8,7 +8,6 @@ It intends to provide an **efficient**, **reliable** and **scalable** alternativ
a [brand new Go test suite](https://github.com/matrix-org/complement).
- Scalable: can run on multiple machines and eventually scale to massive homeserver deployments.
As of October 2020, Dendrite has now entered **beta** which means:
- Dendrite is ready for early adopters. We recommend running in Monolith mode with a PostgreSQL database.
- Dendrite has periodic semver releases. We intend to release new versions as we land significant features.
@ -24,7 +23,7 @@ This does not mean:
Currently, we expect Dendrite to function well for small (10s/100s of users) homeserver deployments as well as P2P Matrix nodes in-browser or on mobile devices.
In the future, we will be able to scale up to gigantic servers (equivalent to matrix.org) via polylith mode.
Join us in:
If you have further questions, please take a look at [our FAQ](docs/FAQ.md) or join us in:
- **[#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org)** - General chat about the Dendrite project, for users and server admins alike
- **[#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org)** - The place for developers, where all Dendrite development discussion happens

4
appservice/api/query.go
View file

@ -20,9 +20,9 @@ package api
import (
"context"
"database/sql"
"errors"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/gomatrixserverlib"
)
@ -109,7 +109,7 @@ func RetrieveUserProfile(
// If no user exists, return
if !userResp.UserIDExists {
return nil, eventutil.ErrProfileNoExists
return nil, errors.New("no known profile for given user ID")
}
// Try to query the user from the local database again

8
appservice/appservice.go
View file

@ -28,10 +28,10 @@ import (
"github.com/matrix-org/dendrite/appservice/storage"
"github.com/matrix-org/dendrite/appservice/types"
"github.com/matrix-org/dendrite/appservice/workers"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/internal/setup/kafka"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/kafka"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/sirupsen/logrus"
)
@ -89,7 +89,7 @@ func NewInternalAPI(
// We can't add ASes at runtime so this is safe to do.
if len(workerStates) > 0 {
consumer := consumers.NewOutputRoomEventConsumer(
base.Cfg, consumer, appserviceDB,
base.ProcessContext, base.Cfg, consumer, appserviceDB,
rsAPI, workerStates,
)
if err := consumer.Start(); err != nil {

5
appservice/consumers/roomserver.go
View file

@ -21,8 +21,9 @@ import (
"github.com/matrix-org/dendrite/appservice/storage"
"github.com/matrix-org/dendrite/appservice/types"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/process"
"github.com/matrix-org/gomatrixserverlib"
"github.com/Shopify/sarama"
@ -41,6 +42,7 @@ type OutputRoomEventConsumer struct {
// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call
// Start() to begin consuming from room servers.
func NewOutputRoomEventConsumer(
process *process.ProcessContext,
cfg *config.Dendrite,
kafkaConsumer sarama.Consumer,
appserviceDB storage.Database,
@ -48,6 +50,7 @@ func NewOutputRoomEventConsumer(
workerStates []types.ApplicationServiceWorkerState,
) *OutputRoomEventConsumer {
consumer := internal.ContinualConsumer{
Process: process,
ComponentName: "appservice/roomserver",
Topic: cfg.Global.Kafka.TopicFor(config.TopicOutputRoomEvent),
Consumer: kafkaConsumer,

2
appservice/query/query.go
View file

@ -23,7 +23,7 @@ import (
"time"
"github.com/matrix-org/dendrite/appservice/api"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
opentracing "github.com/opentracing/opentracing-go"
log "github.com/sirupsen/logrus"
)

2
appservice/storage/postgres/storage.go
View file

@ -21,8 +21,8 @@ import (
// Import postgres database driver
_ "github.com/lib/pq"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
)

2
appservice/storage/sqlite3/storage.go
View file

@ -20,8 +20,8 @@ import (
"database/sql"
// Import SQLite database driver
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
_ "github.com/mattn/go-sqlite3"
)

2
appservice/storage/storage.go
View file

@ -21,7 +21,7 @@ import (
"github.com/matrix-org/dendrite/appservice/storage/postgres"
"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
)
// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme)

2
appservice/storage/storage_wasm.go
View file

@ -18,7 +18,7 @@ import (
"fmt"
"github.com/matrix-org/dendrite/appservice/storage/sqlite3"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
)
func NewDatabase(dbProperties *config.DatabaseOptions) (Database, error) {

2
appservice/types/types.go
View file

@ -15,7 +15,7 @@ package types
import (
"sync"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
)
const (

10
appservice/workers/transaction_scheduler.go
View file

@ -26,7 +26,7 @@ import (
"github.com/matrix-org/dendrite/appservice/storage"
"github.com/matrix-org/dendrite/appservice/types"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
log "github.com/sirupsen/logrus"
)
@ -185,14 +185,14 @@ func createTransaction(
}
}
var ev []*gomatrixserverlib.Event
for _, e := range events {
ev = append(ev, e.Event)
var ev []*gomatrixserverlib.HeaderedEvent
for i := range events {
ev = append(ev, &events[i])
}
// Create a transaction and store the events inside
transaction := gomatrixserverlib.ApplicationServiceTransaction{
Events: ev,
Events: gomatrixserverlib.HeaderedToClientEvents(ev, gomatrixserverlib.FormatAll),
}
transactionJSON, err = json.Marshal(transaction)

6
build.sh
View file

@ -17,6 +17,8 @@ else
export FLAGS=""
fi
go install -trimpath -ldflags "$FLAGS" -v $PWD/`dirname $0`/cmd/...
mkdir -p bin
GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o main.wasm ./cmd/dendritejs
CGO_ENABLED=1 go build -trimpath -ldflags "$FLAGS" -v -o "bin/" ./cmd/...
CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -trimpath -ldflags "$FLAGS" -o bin/main.wasm ./cmd/dendritejs

10
build/docker/Dockerfile
View file

@ -1,10 +0,0 @@
FROM docker.io/golang:1.15-alpine AS builder
RUN apk --update --no-cache add bash build-base
WORKDIR /build
COPY . /build
RUN mkdir -p bin
RUN sh ./build.sh

19
build/docker/Dockerfile.monolith
View file

@ -1,11 +1,20 @@
FROM matrixdotorg/dendrite:latest AS base
FROM docker.io/golang:1.15-alpine AS base
RUN apk --update --no-cache add bash build-base
WORKDIR /build
COPY . /build
RUN mkdir -p bin
RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server
RUN go build -trimpath -o bin/ ./cmd/goose
RUN go build -trimpath -o bin/ ./cmd/create-account
RUN go build -trimpath -o bin/ ./cmd/generate-keys
FROM alpine:latest
COPY --from=base /build/bin/dendrite-monolith-server /usr/bin
COPY --from=base /build/bin/goose /usr/bin
COPY --from=base /build/bin/create-account /usr/bin
COPY --from=base /build/bin/generate-keys /usr/bin
COPY --from=base /build/bin/* /usr/bin
VOLUME /etc/dendrite
WORKDIR /etc/dendrite

19
build/docker/Dockerfile.polylith
View file

@ -1,11 +1,20 @@
FROM matrixdotorg/dendrite:latest AS base
FROM docker.io/golang:1.15-alpine AS base
RUN apk --update --no-cache add bash build-base
WORKDIR /build
COPY . /build
RUN mkdir -p bin
RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi
RUN go build -trimpath -o bin/ ./cmd/goose
RUN go build -trimpath -o bin/ ./cmd/create-account
RUN go build -trimpath -o bin/ ./cmd/generate-keys
FROM alpine:latest
COPY --from=base /build/bin/dendrite-polylith-multi /usr/bin
COPY --from=base /build/bin/goose /usr/bin
COPY --from=base /build/bin/create-account /usr/bin
COPY --from=base /build/bin/generate-keys /usr/bin
COPY --from=base /build/bin/* /usr/bin
VOLUME /etc/dendrite
WORKDIR /etc/dendrite

4
build/docker/README.md
View file

@ -4,8 +4,8 @@ These are Docker images for Dendrite!
They can be found on Docker Hub:
- [matrixdotorg/dendrite-monolith](https://hub.docker.com/repository/docker/matrixdotorg/dendrite-monolith) for monolith deployments
- [matrixdotorg/dendrite-polylith](https://hub.docker.com/repository/docker/matrixdotorg/dendrite-polylith) for polylith deployments
- [matrixdotorg/dendrite-monolith](https://hub.docker.com/r/matrixdotorg/dendrite-monolith) for monolith deployments
- [matrixdotorg/dendrite-polylith](https://hub.docker.com/r/matrixdotorg/dendrite-polylith) for polylith deployments
## Dockerfiles

31
build/docker/config/dendrite-config.yaml
View file

@ -77,7 +77,7 @@ global:
# Naffka database options. Not required when using Kafka.
naffka_database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_naffka?sslmode=disable
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
@ -91,6 +91,17 @@ global:
username: metrics
password: metrics
# DNS cache options. The DNS cache may reduce the load on DNS servers
# if there is no local caching resolver available for use.
dns_cache:
# Whether or not the DNS cache is enabled.
enabled: false
# Maximum number of entries to hold in the DNS cache, and
# for how long those items should be considered valid in seconds.
cache_size: 256
cache_lifetime: 300
# Configuration for the Appservice API.
app_service_api:
internal_api:
@ -98,7 +109,7 @@ app_service_api:
connect: http://appservice_api:7777
database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_appservice?sslmode=disable
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
@ -173,7 +184,7 @@ federation_sender:
connect: http://federation_sender:7775
database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_federationsender?sslmode=disable
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
@ -199,7 +210,7 @@ key_server:
connect: http://key_server:7779
database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_keyserver?sslmode=disable
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
@ -212,7 +223,7 @@ media_api:
listen: http://0.0.0.0:8074
database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_mediaapi?sslmode=disable
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
@ -248,7 +259,7 @@ room_server:
connect: http://room_server:7770
database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_roomserver?sslmode=disable
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
@ -259,7 +270,7 @@ signing_key_server:
connect: http://signing_key_server:7780
database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_signingkeyserver?sslmode=disable
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
@ -288,7 +299,7 @@ sync_api:
listen: http://0.0.0.0:8073
database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_syncapi?sslmode=disable
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
@ -299,12 +310,12 @@ user_api:
connect: http://user_api:7781
account_database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_account?sslmode=disable
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
device_database:
connection_string: postgresql://dendrite:itsasecret@postgres/dendrite_device?sslmode=disable
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1

2
build/docker/docker-compose.deps.yml
View file

@ -3,7 +3,7 @@ services:
# PostgreSQL is needed for both polylith and monolith modes.
postgres:
hostname: postgres
image: postgres:9.6
image: postgres:11
restart: always
volumes:
- ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh

2
build/docker/images-build.sh
View file

@ -6,7 +6,5 @@ TAG=${1:-latest}
echo "Building tag '${TAG}'"
docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} .
docker build -t matrixdotorg/dendrite-monolith:${TAG} -f build/docker/Dockerfile.monolith .
docker build -t matrixdotorg/dendrite-polylith:${TAG} -f build/docker/Dockerfile.polylith .

6
build/gobind-yggdrasil/monolith.go
View file

@ -17,11 +17,11 @@ import (
"github.com/matrix-org/dendrite/eduserver/cache"
"github.com/matrix-org/dendrite/federationsender"
"github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
@ -130,6 +130,7 @@ func (m *DendriteMonolith) Start() {
)
asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
rsAPI.SetAppserviceAPI(asAPI)
ygg.SetSessionFunc(func(address string) {
req := &api.PerformServersAliveRequest{
@ -165,6 +166,7 @@ func (m *DendriteMonolith) Start() {
),
}
monolith.AddAllPublicRoutes(
base.ProcessContext,
base.PublicClientAPIMux,
base.PublicFederationAPIMux,
base.PublicKeyAPIMux,

4
build/scripts/find-lint.sh
View file

@ -8,7 +8,7 @@
# - `DENDRITE_LINT_CONCURRENCY` - number of concurrent linters to run,
# golangci-lint defaults this to NumCPU
# - `GOGC` - how often to perform garbage collection during golangci-lint runs.
# Essentially a ratio of memory/speed. See https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint
# Essentially a ratio of memory/speed. See https://golangci-lint.run/usage/performance/#memory-usage
# for more info.
@ -24,8 +24,6 @@ fi
echo "Installing golangci-lint..."
# Make a backup of go.{mod,sum} first
# TODO: Once go 1.13 is out, use go get's -mod=readonly option
# https://github.com/golang/go/issues/30667
cp go.mod go.mod.bak && cp go.sum go.sum.bak
go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.19.1

2
clientapi/auth/password.go
View file

@ -20,7 +20,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/userutil"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/util"
)

2
clientapi/auth/user_interactive.go
View file

@ -20,7 +20,7 @@ import (
"net/http"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/util"
"github.com/sirupsen/logrus"

2
clientapi/auth/user_interactive_test.go
View file

@ -6,7 +6,7 @@ import (
"fmt"
"testing"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"

7
clientapi/clientapi.go
View file

@ -22,11 +22,11 @@ import (
"github.com/matrix-org/dendrite/clientapi/routing"
eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup/kafka"
"github.com/matrix-org/dendrite/internal/transactions"
keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/kafka"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/gomatrixserverlib"
@ -46,6 +46,7 @@ func AddPublicRoutes(
userAPI userapi.UserInternalAPI,
keyAPI keyserverAPI.KeyInternalAPI,
extRoomsProvider api.ExtraPublicRoomsProvider,
mscCfg *config.MSCs,
) {
_, producer := kafka.SetupConsumerProducer(&cfg.Matrix.Kafka)
@ -57,6 +58,6 @@ func AddPublicRoutes(
routing.Setup(
router, cfg, eduInputAPI, rsAPI, asAPI,
accountsDB, userAPI, federation,
syncProducer, transactionsCache, fsAPI, keyAPI, extRoomsProvider,
syncProducer, transactionsCache, fsAPI, keyAPI, extRoomsProvider, mscCfg,
)
}

2
clientapi/routing/auth_fallback.go
View file

@ -20,7 +20,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/util"
)

2
clientapi/routing/createroom.go
View file

@ -28,8 +28,8 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"

2
clientapi/routing/directory.go
View file

@ -21,8 +21,8 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/api"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"

2
clientapi/routing/directory_public.go
View file

@ -26,8 +26,8 @@ import (
"github.com/matrix-org/dendrite/clientapi/api"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)

2
clientapi/routing/getevent.go
View file

@ -18,8 +18,8 @@ import (
"net/http"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"

2
clientapi/routing/login.go
View file

@ -22,7 +22,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/userutil"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/gomatrixserverlib"

2
clientapi/routing/membership.go
View file

@ -25,10 +25,10 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/threepid"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/roomserver/api"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/gomatrixserverlib"

16
clientapi/routing/memberships.go
View file

@ -19,8 +19,8 @@ import (
"net/http"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
@ -44,6 +44,13 @@ type joinedMember struct {
AvatarURL string `json:"avatar_url"`
}
// The database stores 'displayname' without an underscore.
// Deserialize into this and then change to the actual API response
type databaseJoinedMember struct {
DisplayName string `json:"displayname"`
AvatarURL string `json:"avatar_url"`
}
// GetMemberships implements GET /rooms/{roomId}/members
func GetMemberships(
req *http.Request, device *userapi.Device, roomID string, joinedOnly bool,
@ -72,12 +79,12 @@ func GetMemberships(
var res getJoinedMembersResponse
res.Joined = make(map[string]joinedMember)
for _, ev := range queryRes.JoinEvents {
var content joinedMember
var content databaseJoinedMember
if err := json.Unmarshal(ev.Content, &content); err != nil {
util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content")
return jsonerror.InternalServerError()
}
res.Joined[ev.Sender] = content
res.Joined[ev.Sender] = joinedMember(content)
}
return util.JSONResponse{
Code: http.StatusOK,
@ -104,6 +111,9 @@ func GetJoinedRooms(
util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed")
return jsonerror.InternalServerError()
}
if res.RoomIDs == nil {
res.RoomIDs = []string{}
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: getJoinedRoomsResponse{res.RoomIDs},

2
clientapi/routing/password.go
View file

@ -7,7 +7,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/api"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage/accounts"

25
clientapi/routing/peekroom.go
View file

@ -77,3 +77,28 @@ func PeekRoomByIDOrAlias(
}{peekRes.RoomID},
}
}
func UnpeekRoomByID(
req *http.Request,
device *api.Device,
rsAPI roomserverAPI.RoomserverInternalAPI,
accountDB accounts.Database,
roomID string,
) util.JSONResponse {
unpeekReq := roomserverAPI.PerformUnpeekRequest{
RoomID: roomID,
UserID: device.UserID,
DeviceID: device.ID,
}
unpeekRes := roomserverAPI.PerformUnpeekResponse{}
rsAPI.PerformUnpeek(req.Context(), &unpeekReq, &unpeekRes)
if unpeekRes.Error != nil {
return unpeekRes.Error.JSONResponse()
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct{}{},
}
}

2
clientapi/routing/profile.go
View file

@ -23,9 +23,9 @@ import (
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/gomatrixserverlib"

2
clientapi/routing/rate_limiting.go
View file

@ -6,7 +6,7 @@ import (
"time"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/util"
)

2
clientapi/routing/redaction.go
View file

@ -21,10 +21,10 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/roomserver/api"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"

82
clientapi/routing/register.go
View file

@ -32,8 +32,8 @@ import (
"sync"
"time"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/clientapi/auth"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
@ -328,7 +328,22 @@ func UserIDIsWithinApplicationServiceNamespace(
userID string,
appservice *config.ApplicationService,
) bool {
var local, domain, err = gomatrixserverlib.SplitID('@', userID)
if err != nil {
// Not a valid userID
return false
}
if domain != cfg.Matrix.ServerName {
return false
}
if appservice != nil {
if appservice.SenderLocalpart == local {
return true
}
// Loop through given application service's namespaces and see if any match
for _, namespace := range appservice.NamespaceMap["users"] {
// AS namespaces are checked for validity in config
@ -341,6 +356,9 @@ func UserIDIsWithinApplicationServiceNamespace(
// Loop through all known application service's namespaces and see if any match
for _, knownAppService := range cfg.Derived.ApplicationServices {
if knownAppService.SenderLocalpart == local {
return true
}
for _, namespace := range knownAppService.NamespaceMap["users"] {
// AS namespaces are checked for validity in config
if namespace.RegexpObject.MatchString(userID) {
@ -488,17 +506,6 @@ func Register(
return *resErr
}
// Make sure normal user isn't registering under an exclusive application
// service namespace. Skip this check if no app services are registered.
if r.Auth.Type != authtypes.LoginTypeApplicationService &&
len(cfg.Derived.ApplicationServices) != 0 &&
UsernameMatchesExclusiveNamespaces(cfg, r.Username) {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.ASExclusive("This username is reserved by an application service."),
}
}
logger := util.GetLogger(req.Context())
logger.WithFields(log.Fields{
"username": r.Username,
@ -581,11 +588,33 @@ func handleRegistrationFlow(
// TODO: Handle mapping registrationRequest parameters into session parameters
// TODO: email / msisdn auth types.
accessToken, accessTokenErr := auth.ExtractAccessToken(req)
// Appservices are special and are not affected by disabled
// registration or user exclusivity.
if r.Auth.Type == authtypes.LoginTypeApplicationService ||
(r.Auth.Type == "" && accessTokenErr == nil) {
return handleApplicationServiceRegistration(
accessToken, accessTokenErr, req, r, cfg, userAPI,
)
}
if cfg.RegistrationDisabled && r.Auth.Type != authtypes.LoginTypeSharedSecret {
return util.MessageResponse(http.StatusForbidden, "Registration has been disabled")
}
// Make sure normal user isn't registering under an exclusive application
// service namespace. Skip this check if no app services are registered.
// If an access token is provided, ignore this check this is an appservice
// request and we will validate in validateApplicationService
if len(cfg.Derived.ApplicationServices) != 0 &&
UsernameMatchesExclusiveNamespaces(cfg, r.Username) {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.ASExclusive("This username is reserved by an application service."),
}
}
switch r.Auth.Type {
case authtypes.LoginTypeRecaptcha:
// Check given captcha response
@ -611,36 +640,15 @@ func handleRegistrationFlow(
// Add SharedSecret to the list of completed registration stages
AddCompletedSessionStage(sessionID, authtypes.LoginTypeSharedSecret)
case "":
// Extract the access token from the request, if there's one to extract
// (which we can know by checking whether the error is nil or not).
accessToken, err := auth.ExtractAccessToken(req)
// A missing auth type can mean either the registration is performed by
// an AS or the request is made as the first step of a registration
// using the User-Interactive Authentication API. This can be determined
// by whether the request contains an access token.
if err == nil {
return handleApplicationServiceRegistration(
accessToken, err, req, r, cfg, userAPI,
)
}
case authtypes.LoginTypeApplicationService:
// Extract the access token from the request.
accessToken, err := auth.ExtractAccessToken(req)
// Let the AS registration handler handle the process from here. We
// don't need a condition on that call since the registration is clearly
// stated as being AS-related.
return handleApplicationServiceRegistration(
accessToken, err, req, r, cfg, userAPI,
)
case authtypes.LoginTypeDummy:
// there is nothing to do
// Add Dummy to the list of completed registration stages
AddCompletedSessionStage(sessionID, authtypes.LoginTypeDummy)
case "":
// An empty auth type means that we want to fetch the available
// flows. It can also mean that we want to register as an appservice
// but that is handed above.
default:
return util.JSONResponse{
Code: http.StatusNotImplemented,

2
clientapi/routing/register_test.go
View file

@ -19,7 +19,7 @@ import (
"testing"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
)
var (

54
clientapi/routing/routing.go
View file

@ -28,11 +28,11 @@ import (
"github.com/matrix-org/dendrite/clientapi/producers"
eduServerAPI "github.com/matrix-org/dendrite/eduserver/api"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/transactions"
keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/gomatrixserverlib"
@ -58,17 +58,24 @@ func Setup(
federationSender federationSenderAPI.FederationSenderInternalAPI,
keyAPI keyserverAPI.KeyInternalAPI,
extRoomsProvider api.ExtraPublicRoomsProvider,
mscCfg *config.MSCs,
) {
rateLimits := newRateLimits(&cfg.RateLimiting)
userInteractiveAuth := auth.NewUserInteractive(accountDB.GetAccountByPassword, cfg)
unstableFeatures := make(map[string]bool)
for _, msc := range cfg.MSCs.MSCs {
unstableFeatures["org.matrix."+msc] = true
}
publicAPIMux.Handle("/versions",
httputil.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse {
return util.JSONResponse{
Code: http.StatusOK,
JSON: struct {
Versions []string `json:"versions"`
}{[]string{
Versions []string `json:"versions"`
UnstableFeatures map[string]bool `json:"unstable_features"`
}{Versions: []string{
"r0.0.1",
"r0.1.0",
"r0.2.0",
@ -76,7 +83,7 @@ func Setup(
"r0.4.0",
"r0.5.0",
"r0.6.1",
}},
}, UnstableFeatures: unstableFeatures},
}
}),
).Methods(http.MethodGet, http.MethodOptions)
@ -104,17 +111,23 @@ func Setup(
)
}),
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/peek/{roomIDOrAlias}",
httputil.MakeAuthAPI(gomatrixserverlib.Peek, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return PeekRoomByIDOrAlias(
req, device, rsAPI, accountDB, vars["roomIDOrAlias"],
)
}),
).Methods(http.MethodPost, http.MethodOptions)
if mscCfg.Enabled("msc2753") {
r0mux.Handle("/peek/{roomIDOrAlias}",
httputil.MakeAuthAPI(gomatrixserverlib.Peek, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
if r := rateLimits.rateLimit(req); r != nil {
return *r
}
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return PeekRoomByIDOrAlias(
req, device, rsAPI, accountDB, vars["roomIDOrAlias"],
)
}),
).Methods(http.MethodPost, http.MethodOptions)
}
r0mux.Handle("/joined_rooms",
httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return GetJoinedRooms(req, device, rsAPI)
@ -148,6 +161,17 @@ func Setup(
)
}),
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/unpeek",
httputil.MakeAuthAPI("unpeek", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
return UnpeekRoomByID(
req, device, rsAPI, accountDB, vars["roomID"],
)
}),
).Methods(http.MethodPost, http.MethodOptions)
r0mux.Handle("/rooms/{roomID}/ban",
httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))

32
clientapi/routing/sendevent.go
View file

@ -17,16 +17,18 @@ package routing
import (
"net/http"
"sync"
"time"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/internal/transactions"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
)
@ -40,6 +42,25 @@ var (
userRoomSendMutexes sync.Map // (roomID+userID) -> mutex. mutexes to ensure correct ordering of sendEvents
)
func init() {
prometheus.MustRegister(sendEventDuration)
}
var sendEventDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: "dendrite",
Subsystem: "clientapi",
Name: "sendevent_duration_millis",
Help: "How long it takes to build and submit a new event from the client API to the roomserver",
Buckets: []float64{ // milliseconds
5, 10, 25, 50, 75, 100, 250, 500,
1000, 2000, 3000, 4000, 5000, 6000,
7000, 8000, 9000, 10000, 15000, 20000,
},
},
[]string{"action"},
)
// SendEvent implements:
// /rooms/{roomID}/send/{eventType}
// /rooms/{roomID}/send/{eventType}/{txnID}
@ -75,10 +96,12 @@ func SendEvent(
mutex.(*sync.Mutex).Lock()
defer mutex.(*sync.Mutex).Unlock()
startedGeneratingEvent := time.Now()
e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, rsAPI)
if resErr != nil {
return *resErr
}
timeToGenerateEvent := time.Since(startedGeneratingEvent)
var txnAndSessionID *api.TransactionID
if txnID != nil {
@ -90,6 +113,7 @@ func SendEvent(
// pass the new event to the roomserver and receive the correct event ID
// event ID in case of duplicate transaction is discarded
startedSubmittingEvent := time.Now()
if err := api.SendEvents(
req.Context(), rsAPI,
api.KindNew,
@ -102,6 +126,7 @@ func SendEvent(
util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed")
return jsonerror.InternalServerError()
}
timeToSubmitEvent := time.Since(startedSubmittingEvent)
util.GetLogger(req.Context()).WithFields(logrus.Fields{
"event_id": e.EventID(),
"room_id": roomID,
@ -117,6 +142,11 @@ func SendEvent(
txnCache.AddTransaction(device.AccessToken, *txnID, &res)
}
// Take a note of how long it took to generate the event vs submit
// it to the roomserver.
sendEventDuration.With(prometheus.Labels{"action": "build"}).Observe(float64(timeToGenerateEvent.Milliseconds()))
sendEventDuration.With(prometheus.Labels{"action": "submit"}).Observe(float64(timeToSubmitEvent.Milliseconds()))
return res
}

2
clientapi/routing/threepid.go
View file

@ -21,7 +21,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/threepid"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage/accounts"

2
clientapi/routing/voip.go
View file

@ -23,7 +23,7 @@ import (
"time"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/util"

2
clientapi/threepid/invites.go
View file

@ -25,9 +25,9 @@ import (
"time"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/gomatrixserverlib"

2
clientapi/threepid/threepid.go
View file

@ -24,7 +24,7 @@ import (
"strconv"
"strings"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
)
// EmailAssociationRequest represents the request defined at https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-register-email-requesttoken

41
cmd/create-account/main.go
View file

@ -20,24 +20,27 @@ import (
"fmt"
"os"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/gomatrixserverlib"
"github.com/sirupsen/logrus"
)
const usage = `Usage: %s
Generate a new Matrix account for testing purposes.
Creates a new user account on the homeserver.
Example:
./create-account --config dendrite.yaml --username alice --password foobarbaz
Arguments:
`
var (
database = flag.String("database", "", "The location of the account database.")
username = flag.String("username", "", "The user ID localpart to register e.g 'alice' in '@alice:localhost'.")
password = flag.String("password", "", "Optional. The password to register with. If not specified, this account will be password-less.")
serverNameStr = flag.String("servername", "localhost", "The Matrix server domain which will form the domain part of the user ID.")
username = flag.String("username", "", "The username of the account to register (specify the localpart only, e.g. 'alice' for '@alice:domain.com')")
password = flag.String("password", "", "The password to associate with the account (optional, account will be password-less if not specified)")
)
func main() {
@ -45,36 +48,24 @@ func main() {
fmt.Fprintf(os.Stderr, usage, os.Args[0])
flag.PrintDefaults()
}
flag.Parse()
cfg := setup.ParseFlags(true)
if *username == "" {
flag.Usage()
fmt.Println("Missing --username")
os.Exit(1)
}
if *database == "" {
flag.Usage()
fmt.Println("Missing --database")
os.Exit(1)
}
serverName := gomatrixserverlib.ServerName(*serverNameStr)
accountDB, err := accounts.NewDatabase(&config.DatabaseOptions{
ConnectionString: config.DataSource(*database),
}, serverName)
ConnectionString: cfg.UserAPI.AccountDatabase.ConnectionString,
}, cfg.Global.ServerName)
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
logrus.Fatalln("Failed to connect to the database:", err.Error())
}
_, err = accountDB.CreateAccount(context.Background(), *username, *password, "")
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
logrus.Fatalln("Failed to create the account:", err.Error())
}
fmt.Println("Created account")
logrus.Infoln("Created account", *username)
}

23
cmd/dendrite-demo-libp2p/main.go
View file

@ -31,11 +31,12 @@ import (
"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed"
"github.com/matrix-org/dendrite/eduserver"
"github.com/matrix-org/dendrite/federationsender"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/mscs"
"github.com/matrix-org/dendrite/signingkeyserver"
"github.com/matrix-org/dendrite/userapi"
"github.com/matrix-org/gomatrixserverlib"
@ -75,9 +76,10 @@ func createFederationClient(
"matrix",
p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")),
)
return gomatrixserverlib.NewFederationClientWithTransport(
return gomatrixserverlib.NewFederationClient(
base.Base.Cfg.Global.ServerName, base.Base.Cfg.Global.KeyID,
base.Base.Cfg.Global.PrivateKey, true, tr,
base.Base.Cfg.Global.PrivateKey,
gomatrixserverlib.WithTransport(tr),
)
}
@ -89,7 +91,9 @@ func createClient(
"matrix",
p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")),
)
return gomatrixserverlib.NewClientWithTransport(tr)
return gomatrixserverlib.NewClient(
gomatrixserverlib.WithTransport(tr),
)
}
func main() {
@ -130,6 +134,8 @@ func main() {
cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-e2ekey.db", *instanceName))
cfg.MSCs.MSCs = []string{"msc2836"}
cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
if err = cfg.Derive(); err != nil {
panic(err)
}
@ -158,6 +164,7 @@ func main() {
&base.Base, cache.New(), userAPI,
)
asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI)
rsAPI.SetAppserviceAPI(asAPI)
fsAPI := federationsender.NewInternalAPI(
&base.Base, federation, rsAPI, keyRing,
)
@ -185,11 +192,15 @@ func main() {
ExtPublicRoomsProvider: provider,
}
monolith.AddAllPublicRoutes(
base.Base.ProcessContext,
base.Base.PublicClientAPIMux,
base.Base.PublicFederationAPIMux,
base.Base.PublicKeyAPIMux,
base.Base.PublicMediaAPIMux,
)
if err := mscs.Enable(&base.Base, &monolith); err != nil {
logrus.WithError(err).Fatalf("Failed to enable MSCs")
}
httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.Base.InternalAPIMux)
@ -224,5 +235,5 @@ func main() {
}
// We want to block forever to let the HTTP and HTTPS handler serve the APIs
select {}
base.Base.WaitForShutdown()
}

4
cmd/dendrite-demo-libp2p/p2pdendrite.go
View file

@ -22,7 +22,7 @@ import (
pstore "github.com/libp2p/go-libp2p-core/peerstore"
record "github.com/libp2p/go-libp2p-record"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/setup"
"github.com/libp2p/go-libp2p"
circuit "github.com/libp2p/go-libp2p-circuit"
@ -34,7 +34,7 @@ import (
pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
)
// P2PDendrite is a Peer-to-Peer variant of BaseDendrite.

15
cmd/dendrite-demo-yggdrasil/main.go
View file

@ -34,11 +34,12 @@ import (
"github.com/matrix-org/dendrite/federationsender"
"github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/mscs"
"github.com/matrix-org/dendrite/userapi"
"github.com/matrix-org/gomatrixserverlib"
@ -83,6 +84,8 @@ func main() {
cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName))
cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName))
cfg.MSCs.MSCs = []string{"msc2836"}
cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
if err = cfg.Derive(); err != nil {
panic(err)
}
@ -110,6 +113,7 @@ func main() {
)
asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI)
rsAPI.SetAppserviceAPI(asAPI)
fsAPI := federationsender.NewInternalAPI(
base, federation, rsAPI, keyRing,
)
@ -146,11 +150,15 @@ func main() {
),
}
monolith.AddAllPublicRoutes(
base.ProcessContext,
base.PublicClientAPIMux,
base.PublicFederationAPIMux,
base.PublicKeyAPIMux,
base.PublicMediaAPIMux,
)
if err := mscs.Enable(base, &monolith); err != nil {
logrus.WithError(err).Fatalf("Failed to enable MSCs")
}
httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
httpRouter.PathPrefix(httputil.InternalPathPrefix).Handler(base.InternalAPIMux)
@ -193,5 +201,6 @@ func main() {
}
}()
select {}
// We want to block forever to let the HTTP and HTTPS handler serve the APIs
base.WaitForShutdown()
}

11
cmd/dendrite-demo-yggdrasil/yggconn/client.go
View file

@ -4,7 +4,7 @@ import (
"net/http"
"time"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/gomatrixserverlib"
)
@ -33,7 +33,9 @@ func (n *Node) CreateClient(
},
},
)
return gomatrixserverlib.NewClientWithTransport(tr)
return gomatrixserverlib.NewClient(
gomatrixserverlib.WithTransport(tr),
)
}
func (n *Node) CreateFederationClient(
@ -53,8 +55,9 @@ func (n *Node) CreateFederationClient(
},
},
)
return gomatrixserverlib.NewFederationClientWithTransport(
return gomatrixserverlib.NewFederationClient(
base.Cfg.Global.ServerName, base.Cfg.Global.KeyID,
base.Cfg.Global.PrivateKey, true, tr,
base.Cfg.Global.PrivateKey,
gomatrixserverlib.WithTransport(tr),
)
}

10
cmd/dendrite-monolith-server/main.go
View file

@ -22,12 +22,12 @@ import (
"github.com/matrix-org/dendrite/eduserver"
"github.com/matrix-org/dendrite/eduserver/cache"
"github.com/matrix-org/dendrite/federationsender"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/mscs"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/mscs"
"github.com/matrix-org/dendrite/signingkeyserver"
"github.com/matrix-org/dendrite/userapi"
"github.com/sirupsen/logrus"
@ -126,6 +126,7 @@ func main() {
appservice.AddInternalRoutes(base.InternalAPIMux, asAPI)
asAPI = base.AppserviceHTTPClient()
}
rsAPI.SetAppserviceAPI(asAPI)
monolith := setup.Monolith{
Config: base.Cfg,
@ -143,6 +144,7 @@ func main() {
KeyAPI: keyAPI,
}
monolith.AddAllPublicRoutes(
base.ProcessContext,
base.PublicClientAPIMux,
base.PublicFederationAPIMux,
base.PublicKeyAPIMux,
@ -175,5 +177,5 @@ func main() {
}
// We want to block forever to let the HTTP and HTTPS handler serve the APIs
select {}
base.WaitForShutdown()
}

7
cmd/dendrite-polylith-multi/main.go
View file

@ -20,8 +20,8 @@ import (
"strings"
"github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
"github.com/sirupsen/logrus"
)
@ -74,5 +74,6 @@ func main() {
base := setup.NewBaseDendrite(cfg, component, false) // TODO
defer base.Close() // nolint: errcheck
start(base, cfg)
go start(base, cfg)
base.WaitForShutdown()
}

4
cmd/dendrite-polylith-multi/personalities/appservice.go
View file

@ -16,8 +16,8 @@ package personalities
import (
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
)
func Appservice(base *setup.BaseDendrite, cfg *config.Dendrite) {

5
cmd/dendrite-polylith-multi/personalities/clientapi.go
View file

@ -16,9 +16,9 @@ package personalities
import (
"github.com/matrix-org/dendrite/clientapi"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/internal/transactions"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
)
func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
@ -35,6 +35,7 @@ func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
clientapi.AddPublicRoutes(
base.PublicClientAPIMux, &base.Cfg.ClientAPI, accountDB, federation,
rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI, keyAPI, nil,
&cfg.MSCs,
)
base.SetupAndServeHTTP(

4
cmd/dendrite-polylith-multi/personalities/eduserver.go
View file

@ -17,8 +17,8 @@ package personalities
import (
"github.com/matrix-org/dendrite/eduserver"
"github.com/matrix-org/dendrite/eduserver/cache"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
)
func EDUServer(base *setup.BaseDendrite, cfg *config.Dendrite) {

5
cmd/dendrite-polylith-multi/personalities/federationapi.go
View file

@ -16,8 +16,8 @@ package personalities
import (
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
)
func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
@ -33,6 +33,7 @@ func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
base.PublicFederationAPIMux, base.PublicKeyAPIMux,
&base.Cfg.FederationAPI, userAPI, federation, keyRing,
rsAPI, fsAPI, base.EDUServerClient(), keyAPI,
&base.Cfg.MSCs,
)
base.SetupAndServeHTTP(

4
cmd/dendrite-polylith-multi/personalities/federationsender.go
View file

@ -16,8 +16,8 @@ package personalities
import (
"github.com/matrix-org/dendrite/federationsender"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
)
func FederationSender(base *setup.BaseDendrite, cfg *config.Dendrite) {

4
cmd/dendrite-polylith-multi/personalities/keyserver.go
View file

@ -15,9 +15,9 @@
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
)
func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {

4
cmd/dendrite-polylith-multi/personalities/mediaapi.go
View file

@ -15,9 +15,9 @@
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/mediaapi"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
)
func MediaAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {

6
cmd/dendrite-polylith-multi/personalities/roomserver.go
View file

@ -15,18 +15,20 @@
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
)
func RoomServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
serverKeyAPI := base.SigningKeyServerHTTPClient()
keyRing := serverKeyAPI.KeyRing()
asAPI := base.AppserviceHTTPClient()
fsAPI := base.FederationSenderHTTPClient()
rsAPI := roomserver.NewInternalAPI(base, keyRing)
rsAPI.SetFederationSenderAPI(fsAPI)
rsAPI.SetAppserviceAPI(asAPI)
roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI)
base.SetupAndServeHTTP(

4
cmd/dendrite-polylith-multi/personalities/signingkeyserver.go
View file

@ -15,8 +15,8 @@
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/signingkeyserver"
)

5
cmd/dendrite-polylith-multi/personalities/syncapi.go
View file

@ -15,8 +15,8 @@
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/syncapi"
)
@ -27,6 +27,7 @@ func SyncAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
rsAPI := base.RoomserverHTTPClient()
syncapi.AddPublicRoutes(
base.ProcessContext,
base.PublicClientAPIMux, userAPI, rsAPI,
base.KeyServerHTTPClient(),
federation, &cfg.SyncAPI,

4
cmd/dendrite-polylith-multi/personalities/userapi.go
View file

@ -15,8 +15,8 @@
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi"
)

14
cmd/dendritejs/main.go
View file

@ -26,11 +26,11 @@ import (
"github.com/matrix-org/dendrite/eduserver"
"github.com/matrix-org/dendrite/eduserver/cache"
"github.com/matrix-org/dendrite/federationsender"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/keyserver"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/userapi"
go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p"
@ -139,16 +139,18 @@ func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLoc
tr := go_http_js_libp2p.NewP2pTransport(node)
fed := gomatrixserverlib.NewFederationClient(
cfg.Global.ServerName, cfg.Global.KeyID, cfg.Global.PrivateKey, true,
cfg.Global.ServerName, cfg.Global.KeyID, cfg.Global.PrivateKey,
gomatrixserverlib.WithTransport(tr),
)
fed.Client = *gomatrixserverlib.NewClientWithTransport(tr)
return fed
}
func createClient(node *go_http_js_libp2p.P2pLocalNode) *gomatrixserverlib.Client {
tr := go_http_js_libp2p.NewP2pTransport(node)
return gomatrixserverlib.NewClientWithTransport(tr)
return gomatrixserverlib.NewClient(
gomatrixserverlib.WithTransport(tr),
)
}
func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http_js_libp2p.P2pLocalNode) {
@ -207,6 +209,7 @@ func main() {
asQuery := appservice.NewInternalAPI(
base, userAPI, rsAPI,
)
rsAPI.SetAppserviceAPI(asQuery)
fedSenderAPI := federationsender.NewInternalAPI(base, federation, rsAPI, &keyRing)
rsAPI.SetFederationSenderAPI(fedSenderAPI)
p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI, federation)
@ -228,6 +231,7 @@ func main() {
ExtPublicRoomsProvider: p2pPublicRoomProvider,
}
monolith.AddAllPublicRoutes(
base.ProcessContext,
base.PublicClientAPIMux,
base.PublicFederationAPIMux,
base.PublicKeyAPIMux,

1
cmd/furl/main.go
View file

@ -54,7 +54,6 @@ func main() {
gomatrixserverlib.ServerName(*requestFrom),
gomatrixserverlib.KeyID(keyBlock.Headers["Key-ID"]),
privateKey,
false,
)
u, err := url.Parse(flag.Arg(0))

6
cmd/generate-config/main.go
View file

@ -4,7 +4,7 @@ import (
"flag"
"fmt"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/setup/config"
"gopkg.in/yaml.v2"
)
@ -63,6 +63,10 @@ func main() {
if *defaultsForCI {
cfg.ClientAPI.RateLimiting.Enabled = false
cfg.FederationSender.DisableTLSValidation = true
cfg.MSCs.MSCs = []string{"msc2836", "msc2946", "msc2444", "msc2753"}
cfg.Logging[0].Level = "trace"
// don't hit matrix.org when running tests!!!
cfg.SigningKeyServer.KeyPerspectives = config.KeyPerspectives{}
}
j, err := yaml.Marshal(cfg)

2
cmd/resolve-state/main.go
View file

@ -8,10 +8,10 @@ import (
"strconv"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/roomserver/state"
"github.com/matrix-org/dendrite/roomserver/storage"
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/gomatrixserverlib"
)

2
cmd/roomserver-integration-tests/main.go
View file

@ -29,10 +29,10 @@ import (
"net/http"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/test"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/roomserver/inthttp"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
)

2
cmd/syncserver-integration-tests/main.go
View file

@ -24,9 +24,9 @@ import (
"path/filepath"
"time"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/test"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
)

48
dendrite-config.yaml
View file

@ -60,6 +60,10 @@ global:
- matrix.org
- vector.im
# Disables federation. Dendrite will not be able to make any outbound HTTP requests
# to other servers and the federation API will not be exposed.
disable_federation: false
# Configuration for Kafka/Naffka.
kafka:
# List of Kafka broker addresses to connect to. This is not needed if using
@ -85,7 +89,7 @@ global:
# Naffka database options. Not required when using Kafka.
naffka_database:
connection_string: file:naffka.db
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
@ -99,6 +103,17 @@ global:
username: metrics
password: metrics
# DNS cache options. The DNS cache may reduce the load on DNS servers
# if there is no local caching resolver available for use.
dns_cache:
# Whether or not the DNS cache is enabled.
enabled: false
# Maximum number of entries to hold in the DNS cache, and
# for how long those items should be considered valid in seconds.
cache_size: 256
cache_lifetime: 300
# Configuration for the Appservice API.
app_service_api:
internal_api:
@ -106,7 +121,7 @@ app_service_api:
connect: http://localhost:7777
database:
connection_string: file:appservice.db
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
@ -181,7 +196,7 @@ federation_sender:
connect: http://localhost:7775
database:
connection_string: file:federationsender.db
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
@ -207,7 +222,7 @@ key_server:
connect: http://localhost:7779
database:
connection_string: file:keyserver.db
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
@ -220,7 +235,7 @@ media_api:
listen: http://[::]:8074
database:
connection_string: file:mediaapi.db
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
@ -249,6 +264,19 @@ media_api:
height: 480
method: scale
# Configuration for experimental MSC's
mscs:
# A list of enabled MSC's
# Currently valid values are:
# - msc2836 (Threading, see https://github.com/matrix-org/matrix-doc/pull/2836)
# - msc2946 (Spaces Summary, see https://github.com/matrix-org/matrix-doc/pull/2946)
mscs: []
database:
connection_string: file:mscs.db
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
# Configuration for the Room Server.
room_server:
internal_api:
@ -256,7 +284,7 @@ room_server:
connect: http://localhost:7770
database:
connection_string: file:roomserver.db
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
@ -267,7 +295,7 @@ signing_key_server:
connect: http://localhost:7780
database:
connection_string: file:signingkeyserver.db
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
@ -296,7 +324,7 @@ sync_api:
listen: http://[::]:8073
database:
connection_string: file:syncapi.db
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
@ -312,12 +340,12 @@ user_api:
connect: http://localhost:7781
account_database:
connection_string: file:userapi_accounts.db
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1
device_database:
connection_string: file:userapi_devices.db
max_open_conns: 100
max_open_conns: 10
max_idle_conns: 2
conn_max_lifetime: -1

6
docs/CODE_STYLE.md
View file

@ -2,13 +2,13 @@
In addition to standard Go code style (`gofmt`, `goimports`), we use `golangci-lint`
to run a number of linters, the exact list can be found under linters in [.golangci.yml](.golangci.yml).
[Installation](https://github.com/golangci/golangci-lint#install) and [Editor
Integration](https://github.com/golangci/golangci-lint#editor-integration) for
[Installation](https://github.com/golangci/golangci-lint#install-golangci-lint) and [Editor
Integration](https://golangci-lint.run/usage/integrations/#editor-integration) for
it can be found in the readme of golangci-lint.
For rare cases where a linter is giving a spurious warning, it can be disabled
for that line or statement using a [comment
directive](https://github.com/golangci/golangci-lint#nolint), e.g. `var
directive](https://golangci-lint.run/usage/false-positives/#nolint), e.g. `var
bad_name int //nolint:golint,unused`. This should be used sparingly and only
when its clear that the lint warning is spurious.

2
docs/CONTRIBUTING.md
View file

@ -37,7 +37,7 @@ If a job fails, click the "details" button and you should be taken to the job's
logs.
![Click the details button on the failing build
step](docs/images/details-button-location.jpg)
step](https://raw.githubusercontent.com/matrix-org/dendrite/master/docs/images/details-button-location.jpg)
Scroll down to the failing step and you should see some log output. Scan the
logs until you find what it's complaining about, fix it, submit a new commit,

64
docs/FAQ.md Normal file
View file

@ -0,0 +1,64 @@
# Frequently Asked Questions
### Is Dendrite stable?
Mostly, although there are still bugs and missing features. If you are a confident power user and you are happy to spend some time debugging things when they go wrong, then please try out Dendrite. If you are a community, organisation or business that demands stability and uptime, then Dendrite is not for you yet - please install Synapse instead.
### Is Dendrite feature-complete?
No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the readme at the root of the repository for more information.
### Is there a migration path from Synapse to Dendrite?
No, not at present. There will be in the future when Dendrite reaches version 1.0.
### Should I run a monolith or a polylith deployment?
Monolith deployments are always preferred where possible, and at this time, are far better tested than polylith deployments are. The only reason to consider a polylith deployment is if you wish to run different Dendrite components on separate physical machines.
### I've installed Dendrite but federation isn't working
Check the [Federation Tester](https://federationtester.matrix.org). You need at least:
* A valid DNS name
* A valid TLS certificate for that DNS name
* Either DNS SRV records or well-known files
### Does Dendrite work with my favourite client?
It should do, although we are aware of some minor issues:
* **Element Android**: registration does not work, but logging in with an existing account does
* **Hydrogen**: occasionally sync can fail due to gaps in the `since` parameter, but clearing the cache fixes this
### Does Dendrite support push notifications?
No, not yet. This is a planned feature.
### Does Dendrite support application services/bridges?
Possibly - Dendrite does have some application service support but it is not well tested. Please let us know by raising a GitHub issue if you try it and run into problems.
### Is it possible to prevent communication with the outside world?
Yes, you can do this by disabling federation - set `disable_federation` to `true` in the `global` section of the Dendrite configuration file.
### Should I use PostgreSQL or SQLite for my databases?
Please use PostgreSQL wherever possible, especially if you are planning to run a homeserver that caters to more than a couple of users.
### Dendrite is using a lot of CPU
Generally speaking, you should expect to see some CPU spikes, particularly if you are joining or participating in large rooms. However, constant/sustained high CPU usage is not expected - if you are experiencing that, please join `#dendrite-dev:matrix.org` and let us know, or file a GitHub issue.
### Dendrite is using a lot of RAM
A lot of users report that Dendrite is using a lot of RAM, sometimes even gigabytes of it. This is usually due to Go's allocator behaviour, which tries to hold onto allocated memory until the operating system wants to reclaim it for something else. This can make the memory usage look significantly inflated in tools like `top`/`htop` when actually most of that memory is not really in use at all.
If you want to prevent this behaviour so that the Go runtime releases memory normally, start Dendrite using the `GODEBUG=madvdontneed=1` environment variable. It is also expected that the allocator behaviour will be changed again in Go 1.16 so that it does not hold onto memory unnecessarily in this way.
If you are running with `GODEBUG=madvdontneed=1` and still see hugely inflated memory usage then that's quite possibly a bug - please join `#dendrite-dev:matrix.org` and let us know, or file a GitHub issue.
### Dendrite is running out of PostgreSQL database connections
You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode!

49
docs/INSTALL.md
View file

@ -80,12 +80,6 @@ brew services start kafka
## Configuration
### SQLite database setup
Dendrite can use the built-in SQLite database engine for small setups.
The SQLite databases do not need to be pre-built - Dendrite will
create them automatically at startup.
### PostgreSQL database setup
Assuming that PostgreSQL 9.6 (or later) is installed:
@ -96,7 +90,23 @@ Assuming that PostgreSQL 9.6 (or later) is installed:
sudo -u postgres createuser -P dendrite
```
* Create the component databases:
At this point you have a choice on whether to run all of the Dendrite
components from a single database, or for each component to have its
own database. For most deployments, running from a single database will
be sufficient, although you may wish to separate them if you plan to
split out the databases across multiple machines in the future.
On macOS, omit `sudo -u postgres` from the below commands.
* If you want to run all Dendrite components from a single database:
```bash
sudo -u postgres createdb -O dendrite dendrite
```
... in which case your connection string will look like `postgres://user:pass@database/dendrite`.
* If you want to run each Dendrite component with its own database:
```bash
for i in mediaapi syncapi roomserver signingkeyserver federationsender appservice keyserver userapi_account userapi_device naffka; do
@ -104,14 +114,22 @@ Assuming that PostgreSQL 9.6 (or later) is installed:
done
```
(On macOS, omit `sudo -u postgres` from the above commands.)
... in which case your connection string will look like `postgres://user:pass@database/dendrite_componentname`.
### SQLite database setup
**WARNING:** SQLite is suitable for small experimental deployments only and should not be used in production - use PostgreSQL instead for any user-facing federating installation!
Dendrite can use the built-in SQLite database engine for small setups.
The SQLite databases do not need to be pre-built - Dendrite will
create them automatically at startup.
### Server key generation
Each Dendrite installation requires:
- A unique Matrix signing private key
- A valid and trusted TLS certificate and private key
* A unique Matrix signing private key
* A valid and trusted TLS certificate and private key
To generate a Matrix signing private key:
@ -119,7 +137,7 @@ To generate a Matrix signing private key:
./bin/generate-keys --private-key matrix_key.pem
```
**Warning:** Make sure take a safe backup of this key! You will likely need it if you want to reinstall Dendrite, or
**WARNING:** Make sure take a safe backup of this key! You will likely need it if you want to reinstall Dendrite, or
any other Matrix homeserver, on the same domain name in the future. If you lose this key, you may have trouble joining
federated rooms.
@ -129,8 +147,8 @@ For testing, you can generate a self-signed certificate and key, although this w
./bin/generate-keys --tls-cert server.crt --tls-key server.key
```
If you have server keys from an older Synapse instance,
[convert them](serverkeyformat.md#converting-synapse-keys) to Dendrite's PEM
If you have server keys from an older Synapse instance,
[convert them](serverkeyformat.md#converting-synapse-keys) to Dendrite's PEM
format and configure them as `old_private_keys` in your config.
### Configuration file
@ -140,7 +158,9 @@ Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Th
* The `server_name` entry to reflect the hostname of your Dendrite server
* The `database` lines with an updated connection string based on your
desired setup, e.g. replacing `database` with the name of the database:
* For Postgres: `postgres://dendrite:password@localhost/database`, e.g. `postgres://dendrite:password@localhost/dendrite_userapi_account.db`
* For Postgres: `postgres://dendrite:password@localhost/database`, e.g.
* `postgres://dendrite:password@localhost/dendrite_userapi_account` to connect to PostgreSQL with SSL/TLS
* `postgres://dendrite:password@localhost/dendrite_userapi_account?sslmode=disable` to connect to PostgreSQL without SSL/TLS
* For SQLite on disk: `file:component.db` or `file:///path/to/component.db`, e.g. `file:userapi_account.db`
* Postgres and SQLite can be mixed and matched on different components as desired.
* The `use_naffka` option if using Naffka in a monolith deployment
@ -295,4 +315,3 @@ amongst other things.
```bash
./bin/dendrite-polylith-multi --config=dendrite.yaml userapi
```

89
docs/PROFILING.md Normal file
View file

@ -0,0 +1,89 @@
# Profiling Dendrite
If you are running into problems with Dendrite using excessive resources (e.g. CPU or RAM) then you can use the profiler to work out what is happening.
Dendrite contains an embedded profiler called `pprof`, which is a part of the standard Go toolchain.
## Enable the profiler
To enable the profiler, start Dendrite with the `PPROFLISTEN` environment variable. This variable specifies which address and port to listen on, e.g.
```
PPROFLISTEN=localhost:65432 ./bin/dendrite-monolith-server ...
```
If pprof has been enabled successfully, a log line at startup will show that pprof is listening:
```
WARN[2020-12-03T13:32:33.669405000Z] [/Users/neilalexander/Desktop/dendrite/internal/log.go:87] SetupPprof
Starting pprof on localhost:65432
```
All examples from this point forward assume `PPROFLISTEN=localhost:65432` but you may need to adjust as necessary for your setup.
## Profiling CPU usage
To examine where CPU time is going, you can call the `profile` endpoint:
```
http://localhost:65432/debug/pprof/profile?seconds=30
```
The profile will run for the specified number of `seconds` and then will produce a result.
### Examine a profile using the Go toolchain
If you have Go installed and want to explore the profile, you can invoke `go tool pprof` to start the profile directly. The `-http=` parameter will instruct `go tool pprof` to start a web server providing a view of the captured profile:
```
go tool pprof -http=localhost:23456 http://localhost:65432/debug/pprof/profile?seconds=30
```
You can then visit `http://localhost:23456` in your web browser to see a visual representation of the profile. Particularly usefully, in the "View" menu, you can select "Flame Graph" to see a proportional interactive graph of CPU usage.
### Download a profile to send to someone else
If you don't have the Go tools installed but just want to capture the profile to send to someone else, you can instead use `curl` to download the profiler results:
```
curl -O http://localhost:65432/debug/pprof/profile?seconds=30
```
This will block for the specified number of seconds, capturing information about what Dendrite is doing, and then produces a `profile` file, which you can send onward.
## Profiling memory usage
To examine where memory usage is going, you can call the `heap` endpoint:
```
http://localhost:65432/debug/pprof/heap
```
The profile will return almost instantly.
### Examine a profile using the Go toolchain
If you have Go installed and want to explore the profile, you can invoke `go tool pprof` to start the profile directly. The `-http=` parameter will instruct `go tool pprof` to start a web server providing a view of the captured profile:
```
go tool pprof -http=localhost:23456 http://localhost:65432/debug/pprof/heap
```
You can then visit `http://localhost:23456` in your web browser to see a visual representation of the profile. The "Sample" menu lets you select between four different memory profiles:
* `inuse_space`: Shows how much actual heap memory is allocated per function (this is generally the most useful profile when diagnosing high memory usage)
* `inuse_objects`: Shows how many heap objects are allocated per function
* `alloc_space`: Shows how much memory has been allocated per function (although that memory may have since been deallocated)
* `alloc_objects`: Shows how many allocations have been made per function (although that memory may have since been deallocated)
Also in the "View" menu, you can select "Flame Graph" to see a proportional interactive graph of the memory usage.
### Download a profile to send to someone else
If you don't have the Go tools installed but just want to capture the profile to send to someone else, you can instead use `curl` to download the profiler results:
```
curl -O http://localhost:65432/debug/pprof/heap
```
This will almost instantly produce a `heap` file, which you can send onward.

17
docs/hiawatha/monolith-sample.conf Normal file
View file

@ -0,0 +1,17 @@
# Depending on which port is used for federation (.well-known/matrix/server or SRV record),
# ensure there's a binding for that port in the configuration. Replace "FEDPORT" with port
# number, (e.g. "8448"), and "IPV4" with your server's ipv4 address (separate binding for
# each ip address, e.g. if you use both ipv4 and ipv6 addresses).
Binding {
Port = FEDPORT
Interface = IPV4
TLScertFile = /path/to/fullchainandprivkey.pem
}
VirtualHost {
...
ReverseProxy = /_matrix http://localhost:8008 600
...
}

22
docs/hiawatha/polylith-sample.conf
View file

@ -1,3 +1,15 @@
# Depending on which port is used for federation (.well-known/matrix/server or SRV record),
# ensure there's a binding for that port in the configuration. Replace "FEDPORT" with port
# number, (e.g. "8448"), and "IPV4" with your server's ipv4 address (separate binding for
# each ip address, e.g. if you use both ipv4 and ipv6 addresses).
Binding {
Port = FEDPORT
Interface = IPV4
TLScertFile = /path/to/fullchainandprivkey.pem
}
VirtualHost {
...
# route requests to:
@ -7,10 +19,10 @@ VirtualHost {
# /_matrix/client/.*/keys/changes
# /_matrix/client/.*/rooms/{roomId}/messages
# to sync_api
ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages) http://localhost:8073
ReverseProxy = /_matrix/client http://localhost:8071
ReverseProxy = /_matrix/federation http://localhost:8072
ReverseProxy = /_matrix/key http://localhost:8072
ReverseProxy = /_matrix/media http://localhost:8074
ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages) http://localhost:8073 600
ReverseProxy = /_matrix/client http://localhost:8071 600
ReverseProxy = /_matrix/federation http://localhost:8072 600
ReverseProxy = /_matrix/key http://localhost:8072 600
ReverseProxy = /_matrix/media http://localhost:8074 600
...
}

21
docs/peeking.md
View file

@ -1,19 +1,26 @@
## Peeking
Peeking is implemented as per [MSC2753](https://github.com/matrix-org/matrix-doc/pull/2753).
Local peeking is implemented as per [MSC2753](https://github.com/matrix-org/matrix-doc/pull/2753).
Implementationwise, this means:
* Users call `/peek` and `/unpeek` on the clientapi from a given device.
* The clientapi delegates these via HTTP to the roomserver, which coordinates peeking in general for a given room
* The roomserver writes an NewPeek event into the kafka log headed to the syncserver
* The syncserver tracks the existence of the local peek in its DB, and then starts waking up the peeking devices for the room in question, putting it in the `peek` section of the /sync response.
* The syncserver tracks the existence of the local peek in the syncapi_peeks table in its DB, and then starts waking up the peeking devices for the room in question, putting it in the `peek` section of the /sync response.
Questions (given this is [my](https://github.com/ara4n) first time hacking on Dendrite):
* The whole clientapi -> roomserver -> syncapi flow to initiate a peek seems very indirect. Is there a reason not to just let syncapi itself host the implementation of `/peek`?
Peeking over federation is implemented as per [MSC2444](https://github.com/matrix-org/matrix-doc/pull/2444).
In future, peeking over federation will be added as per [MSC2444](https://github.com/matrix-org/matrix-doc/pull/2444).
* The `roomserver` will kick the `federationsender` much as it does for a federated `/join` in order to trigger a federated `/peek`
* The `federationsender` tracks the existence of the remote peek in question
For requests to peek our rooms ("inbound peeks"):
* Remote servers call `/peek` on federationapi
* The federationapi queries the federationsender to check if this is renewing an inbound peek or not.
* If not, it hits the PerformInboundPeek on the roomserver to ask it for the current state of the room.
* The roomserver atomically (in theory) adds a NewInboundPeek to its kafka stream to tell the federationserver to start peeking.
* The federationsender receives the event, tracks the inbound peek in the federationsender_inbound_peeks table, and starts sending events to the peeking server.
* The federationsender evicts stale inbound peeks which haven't been renewed.
For peeking into other server's rooms ("outbound peeks"):
* The `roomserver` will kick the `federationsender` much as it does for a federated `/join` in order to trigger a federated outbound `/peek`
* The `federationsender` tracks the existence of the outbound peek in in its federationsender_outbound_peeks table.
* The `federationsender` regularly renews the remote peek as long as there are still peeking devices syncing for it.
* TBD: how do we tell if there are no devices currently syncing for a given peeked room? The syncserver needs to tell the roomserver
somehow who then needs to warn the federationsender.

1
docs/systemd/monolith-example.service
View file

@ -5,6 +5,7 @@ After=network.target
After=postgresql.service
[Service]
Environment=GODEBUG=madvdontneed=1
RestartSec=2s
Type=simple
User=dendrite

1
docs/sytest.md
View file

@ -85,6 +85,7 @@ Set up the database:
```sh
sudo -u postgres psql -c "CREATE USER dendrite PASSWORD 'itsasecret'"
sudo -u postgres psql -c "ALTER USER dendrite CREATEDB"
for i in dendrite0 dendrite1 sytest_template; do sudo -u postgres psql -c "CREATE DATABASE $i OWNER dendrite;"; done
mkdir -p "server-0"
cat > "server-0/database.yaml" << EOF

13
eduserver/cache/cache.go vendored
View file

@ -113,19 +113,6 @@ func (t *EDUCache) AddTypingUser(
return t.GetLatestSyncPosition()
}
// AddSendToDeviceMessage increases the sync position for
// send-to-device updates.
// Returns the sync position before update, as the caller
// will use this to record the current stream position
// at the time that the send-to-device message was sent.
func (t *EDUCache) AddSendToDeviceMessage() int64 {
t.Lock()
defer t.Unlock()
latestSyncPosition := t.latestSyncPosition
t.latestSyncPosition++
return latestSyncPosition
}
// addUser with mutex lock & replace the previous timer.
// Returns the latest typing sync position after update.
func (t *EDUCache) addUser(

6
eduserver/eduserver.go
View file

@ -22,9 +22,9 @@ import (
"github.com/matrix-org/dendrite/eduserver/cache"
"github.com/matrix-org/dendrite/eduserver/input"
"github.com/matrix-org/dendrite/eduserver/inthttp"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/internal/setup/kafka"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/kafka"
userapi "github.com/matrix-org/dendrite/userapi/api"
)

5
federationapi/federationapi.go
View file

@ -18,9 +18,9 @@ import (
"github.com/gorilla/mux"
eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/dendrite/federationapi/routing"
@ -38,10 +38,11 @@ func AddPublicRoutes(
federationSenderAPI federationSenderAPI.FederationSenderInternalAPI,
eduAPI eduserverAPI.EDUServerInputAPI,
keyAPI keyserverAPI.KeyInternalAPI,
mscCfg *config.MSCs,
) {
routing.Setup(
fedRouter, keyRouter, cfg, rsAPI,
eduAPI, federationSenderAPI, keyRing,
federation, userAPI, keyAPI,
federation, userAPI, keyAPI, mscCfg,
)
}

11
federationapi/federationapi_test.go
View file

@ -7,9 +7,9 @@ import (
"testing"
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/internal/test"
"github.com/matrix-org/dendrite/setup"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/gomatrixserverlib"
)
@ -31,12 +31,15 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) {
fsAPI := base.FederationSenderHTTPClient()
// TODO: This is pretty fragile, as if anything calls anything on these nils this test will break.
// Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing.
federationapi.AddPublicRoutes(base.PublicFederationAPIMux, base.PublicKeyAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, nil)
federationapi.AddPublicRoutes(base.PublicFederationAPIMux, base.PublicKeyAPIMux, &cfg.FederationAPI, nil, nil, keyRing, nil, fsAPI, nil, nil, &cfg.MSCs)
baseURL, cancel := test.ListenAndServe(t, base.PublicFederationAPIMux, true)
defer cancel()
serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://"))
fedCli := gomatrixserverlib.NewFederationClient(serverName, cfg.Global.KeyID, cfg.Global.PrivateKey, true)
fedCli := gomatrixserverlib.NewFederationClient(
serverName, cfg.Global.KeyID, cfg.Global.PrivateKey,
gomatrixserverlib.WithSkipVerify(true),
)
testCases := []struct {
roomVer gomatrixserverlib.RoomVersion

2
federationapi/routing/backfill.go
View file

@ -22,8 +22,8 @@ import (
"time"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)

2
federationapi/routing/invite.go
View file

@ -21,9 +21,9 @@ import (
"net/http"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/roomserver/api"
roomserverVersion "github.com/matrix-org/dendrite/roomserver/version"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)

2
federationapi/routing/join.go
View file

@ -21,9 +21,9 @@ import (
"time"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)

2
federationapi/routing/keys.go
View file

@ -22,8 +22,8 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/keyserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/sirupsen/logrus"

2
federationapi/routing/leave.go
View file

@ -17,9 +17,9 @@ import (
"time"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)

102
federationapi/routing/peek.go Normal file
View file

@ -0,0 +1,102 @@
// Copyright 2020 New Vector Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package routing
import (
"net/http"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
)
// Peek implements the SS /peek API, handling inbound peeks
func Peek(
httpReq *http.Request,
request *gomatrixserverlib.FederationRequest,
cfg *config.FederationAPI,
rsAPI api.RoomserverInternalAPI,
roomID, peekID string,
remoteVersions []gomatrixserverlib.RoomVersion,
) util.JSONResponse {
// TODO: check if we're just refreshing an existing peek by querying the federationsender
verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID}
verRes := api.QueryRoomVersionForRoomResponse{}
if err := rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil {
return util.JSONResponse{
Code: http.StatusInternalServerError,
JSON: jsonerror.InternalServerError(),
}
}
// Check that the room that the peeking server is trying to peek is actually
// one of the room versions that they listed in their supported ?ver= in
// the peek URL.
remoteSupportsVersion := false
for _, v := range remoteVersions {
if v == verRes.RoomVersion {
remoteSupportsVersion = true
break
}
}
// If it isn't, stop trying to peek the room.
if !remoteSupportsVersion {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.IncompatibleRoomVersion(verRes.RoomVersion),
}
}
// TODO: Check history visibility
// tell the peeking server to renew every hour
renewalInterval := int64(60 * 60 * 1000 * 1000)
var response api.PerformInboundPeekResponse
err := rsAPI.PerformInboundPeek(
httpReq.Context(),
&api.PerformInboundPeekRequest{
RoomID: roomID,
PeekID: peekID,
ServerName: request.Origin(),
RenewalInterval: renewalInterval,
},
&response,
)
if err != nil {
resErr := util.ErrorResponse(err)
return resErr
}
if !response.RoomExists {
return util.JSONResponse{Code: http.StatusNotFound, JSON: nil}
}
respPeek := gomatrixserverlib.RespPeek{
StateEvents: gomatrixserverlib.UnwrapEventHeaders(response.StateEvents),
AuthEvents: gomatrixserverlib.UnwrapEventHeaders(response.AuthChainEvents),
RoomVersion: response.RoomVersion,
LatestEvent: response.LatestEvent.Unwrap(),
RenewalInterval: renewalInterval,
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: respPeek,
}
}

2
federationapi/routing/profile.go
View file

@ -19,8 +19,8 @@ import (
"net/http"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"

2
federationapi/routing/query.go
View file

@ -20,8 +20,8 @@ import (
"github.com/matrix-org/dendrite/clientapi/jsonerror"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrix"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"

43
federationapi/routing/routing.go
View file

@ -21,10 +21,10 @@ import (
"github.com/matrix-org/dendrite/clientapi/jsonerror"
eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/httputil"
keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
@ -48,6 +48,7 @@ func Setup(
federation *gomatrixserverlib.FederationClient,
userAPI userapi.UserInternalAPI,
keyAPI keyserverAPI.KeyInternalAPI,
mscCfg *config.MSCs,
) {
v2keysmux := keyMux.PathPrefix("/v2").Subrouter()
v1fedmux := fedMux.PathPrefix("/v1").Subrouter()
@ -229,7 +230,39 @@ func Setup(
},
)).Methods(http.MethodGet)
v1fedmux.Handle("/make_join/{roomID}/{eventID}", httputil.MakeFedAPI(
if mscCfg.Enabled("msc2444") {
v1fedmux.Handle("/peek/{roomID}/{peekID}", httputil.MakeFedAPI(
"federation_peek", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("Forbidden by server ACLs"),
}
}
roomID := vars["roomID"]
peekID := vars["peekID"]
queryVars := httpReq.URL.Query()
remoteVersions := []gomatrixserverlib.RoomVersion{}
if vers, ok := queryVars["ver"]; ok {
// The remote side supplied a ?ver= so use that to build up the list
// of supported room versions
for _, v := range vers {
remoteVersions = append(remoteVersions, gomatrixserverlib.RoomVersion(v))
}
} else {
// The remote side didn't supply a ?ver= so just assume that they only
// support room version 1
remoteVersions = append(remoteVersions, gomatrixserverlib.RoomVersionV1)
}
return Peek(
httpReq, request, cfg, rsAPI, roomID, peekID, remoteVersions,
)
},
)).Methods(http.MethodPut, http.MethodDelete)
}
v1fedmux.Handle("/make_join/{roomID}/{userID}", httputil.MakeFedAPI(
"federation_make_join", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) {
@ -239,11 +272,11 @@ func Setup(
}
}
roomID := vars["roomID"]
eventID := vars["eventID"]
userID := vars["userID"]
queryVars := httpReq.URL.Query()
remoteVersions := []gomatrixserverlib.RoomVersion{}
if vers, ok := queryVars["ver"]; ok {
// The remote side supplied a ?=ver so use that to build up the list
// The remote side supplied a ?ver= so use that to build up the list
// of supported room versions
for _, v := range vers {
remoteVersions = append(remoteVersions, gomatrixserverlib.RoomVersion(v))
@ -255,7 +288,7 @@ func Setup(
remoteVersions = append(remoteVersions, gomatrixserverlib.RoomVersionV1)
}
return MakeJoin(
httpReq, request, cfg, rsAPI, roomID, eventID, remoteVersions,
httpReq, request, cfg, rsAPI, roomID, userID, remoteVersions,
)
},
)).Methods(http.MethodGet)

141
federationapi/routing/send.go
View file

@ -25,9 +25,9 @@ import (
"github.com/matrix-org/dendrite/clientapi/jsonerror"
eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
"github.com/matrix-org/dendrite/internal/config"
keyapi "github.com/matrix-org/dendrite/keyserver/api"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/sirupsen/logrus"
@ -84,7 +84,7 @@ func Send(
util.GetLogger(httpReq.Context()).Infof("Received transaction %q from %q containing %d PDUs, %d EDUs", txnID, request.Origin(), len(t.PDUs), len(t.EDUs))
resp, jsonErr := t.processTransaction(httpReq.Context())
resp, jsonErr := t.processTransaction(context.Background())
if jsonErr != nil {
util.GetLogger(httpReq.Context()).WithField("jsonErr", jsonErr).Error("t.processTransaction failed")
return *jsonErr
@ -1005,79 +1005,82 @@ func (t *txnReq) lookupMissingStateViaStateIDs(ctx context.Context, roomID, even
return t.lookupMissingStateViaState(ctx, roomID, eventID, roomVersion)
}
util.GetLogger(ctx).WithFields(logrus.Fields{
"missing": missingCount,
"event_id": eventID,
"room_id": roomID,
"total_state": len(stateIDs.StateEventIDs),
"total_auth_events": len(stateIDs.AuthEventIDs),
"concurrent_requests": concurrentRequests,
}).Info("Fetching missing state at event")
if missingCount > 0 {
util.GetLogger(ctx).WithFields(logrus.Fields{
"missing": missingCount,
"event_id": eventID,
"room_id": roomID,
"total_state": len(stateIDs.StateEventIDs),
"total_auth_events": len(stateIDs.AuthEventIDs),
"concurrent_requests": concurrentRequests,
}).Info("Fetching missing state at event")
// Get a list of servers to fetch from.
servers := t.getServers(ctx, roomID)
if len(servers) > 5 {
servers = servers[:5]
}
// Create a queue containing all of the missing event IDs that we want
// to retrieve.
pending := make(chan string, missingCount)
for missingEventID := range missing {
pending <- missingEventID
}
close(pending)
// Define how many workers we should start to do this.
if missingCount < concurrentRequests {
concurrentRequests = missingCount
}
// Create the wait group.
var fetchgroup sync.WaitGroup
fetchgroup.Add(concurrentRequests)
// This is the only place where we'll write to t.haveEvents from
// multiple goroutines, and everywhere else is blocked on this
// synchronous function anyway.
var haveEventsMutex sync.Mutex
// Define what we'll do in order to fetch the missing event ID.
fetch := func(missingEventID string) {
var h *gomatrixserverlib.HeaderedEvent
h, err = t.lookupEvent(ctx, roomVersion, missingEventID, false, servers)
switch err.(type) {
case verifySigError:
return
case nil:
break
default:
util.GetLogger(ctx).WithFields(logrus.Fields{
"event_id": missingEventID,
"room_id": roomID,
}).Info("Failed to fetch missing event")
return
// Get a list of servers to fetch from.
servers := t.getServers(ctx, roomID)
if len(servers) > 5 {
servers = servers[:5]
}
haveEventsMutex.Lock()
t.haveEvents[h.EventID()] = h
haveEventsMutex.Unlock()
}
// Create the worker.
worker := func(ch <-chan string) {
defer fetchgroup.Done()
for missingEventID := range ch {
fetch(missingEventID)
// Create a queue containing all of the missing event IDs that we want
// to retrieve.
pending := make(chan string, missingCount)
for missingEventID := range missing {
pending <- missingEventID
}
close(pending)
// Define how many workers we should start to do this.
if missingCount < concurrentRequests {
concurrentRequests = missingCount
}
// Create the wait group.
var fetchgroup sync.WaitGroup
fetchgroup.Add(concurrentRequests)
// This is the only place where we'll write to t.haveEvents from
// multiple goroutines, and everywhere else is blocked on this
// synchronous function anyway.
var haveEventsMutex sync.Mutex
// Define what we'll do in order to fetch the missing event ID.
fetch := func(missingEventID string) {
var h *gomatrixserverlib.HeaderedEvent
h, err = t.lookupEvent(ctx, roomVersion, missingEventID, false, servers)
switch err.(type) {
case verifySigError:
return
case nil:
break
default:
util.GetLogger(ctx).WithFields(logrus.Fields{
"event_id": missingEventID,
"room_id": roomID,
}).Info("Failed to fetch missing event")
return
}
haveEventsMutex.Lock()
t.haveEvents[h.EventID()] = h
haveEventsMutex.Unlock()
}
// Create the worker.
worker := func(ch <-chan string) {
defer fetchgroup.Done()
for missingEventID := range ch {
fetch(missingEventID)
}
}
// Start the workers.
for i := 0; i < concurrentRequests; i++ {
go worker(pending)
}
// Wait for the workers to finish.
fetchgroup.Wait()
}
// Start the workers.
for i := 0; i < concurrentRequests; i++ {
go worker(pending)
}
// Wait for the workers to finish.
fetchgroup.Wait()
resp, err := t.createRespStateFromStateIDs(stateIDs)
return resp, err
}

45
federationapi/routing/send_test.go
View file

@ -9,7 +9,6 @@ import (
"time"
eduAPI "github.com/matrix-org/dendrite/eduserver/api"
fsAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal/test"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib"
@ -85,6 +84,7 @@ func (o *testEDUProducer) InputReceiptEvent(
}
type testRoomserverAPI struct {
api.RoomserverInternalAPITrace
inputRoomEvents []api.InputRoomEvent
queryMissingAuthPrevEvents func(*api.QueryMissingAuthPrevEventsRequest) api.QueryMissingAuthPrevEventsResponse
queryStateAfterEvents func(*api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse
@ -92,12 +92,6 @@ type testRoomserverAPI struct {
queryLatestEventsAndState func(*api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse
}
func (t *testRoomserverAPI) PerformForget(ctx context.Context, req *api.PerformForgetRequest, resp *api.PerformForgetResponse) error {
return nil
}
func (t *testRoomserverAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) {}
func (t *testRoomserverAPI) InputRoomEvents(
ctx context.Context,
request *api.InputRoomEventsRequest,
@ -109,43 +103,6 @@ func (t *testRoomserverAPI) InputRoomEvents(
}
}
func (t *testRoomserverAPI) PerformInvite(
ctx context.Context,
req *api.PerformInviteRequest,
res *api.PerformInviteResponse,
) error {
return nil
}
func (t *testRoomserverAPI) PerformJoin(
ctx context.Context,
req *api.PerformJoinRequest,
res *api.PerformJoinResponse,
) {
}
func (t *testRoomserverAPI) PerformPeek(
ctx context.Context,
req *api.PerformPeekRequest,
res *api.PerformPeekResponse,
) {
}
func (t *testRoomserverAPI) PerformPublish(
ctx context.Context,
req *api.PerformPublishRequest,
res *api.PerformPublishResponse,
) {
}
func (t *testRoomserverAPI) PerformLeave(
ctx context.Context,
req *api.PerformLeaveRequest,
res *api.PerformLeaveResponse,
) error {
return nil
}
// Query the latest events and state for a room from the room server.
func (t *testRoomserverAPI) QueryLatestEventsAndState(
ctx context.Context,

2
federationapi/routing/threepid.go
View file

@ -23,8 +23,8 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"

Some files were not shown because too many files have changed in this diff Show more