mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2025-12-23 23:03:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into neilalexander/h2c

Browse source
This commit is contained in:
Neil Alexander 2020-10-20 16:24:18 +01:00 committed by GitHub
parent 2b8b8dc8ea 39c7a8915c
commit 8e923a8407
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
32 changed files with 276 additions and 320 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
build/docker/Dockerfile
View file

@ -1,4 +1,4 @@
FROM docker.io/golang:1.13.7-alpine3.11 AS builder
FROM docker.io/golang:1.15-alpine AS builder
RUN apk --update --no-cache add bash build-base

13
build/docker/Dockerfile.component
View file

@ -1,13 +0,0 @@
FROM matrixdotorg/dendrite:latest AS base
FROM alpine:latest
ARG component=monolith
ENV entrypoint=${component}
COPY --from=base /build/bin/${component} /usr/bin
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT /usr/bin/${entrypoint} $@

13
build/docker/Dockerfile.monolith Normal file
View file

@ -0,0 +1,13 @@
FROM matrixdotorg/dendrite:latest AS base
FROM alpine:latest
COPY --from=base /build/bin/dendrite-monolith-server /usr/bin
COPY --from=base /build/bin/goose /usr/bin
COPY --from=base /build/bin/create-account /usr/bin
COPY --from=base /build/bin/generate-keys /usr/bin
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]

13
build/docker/Dockerfile.polylith Normal file
View file

@ -0,0 +1,13 @@
FROM matrixdotorg/dendrite:latest AS base
FROM alpine:latest
COPY --from=base /build/bin/dendrite-polylith-multi /usr/bin
COPY --from=base /build/bin/goose /usr/bin
COPY --from=base /build/bin/create-account /usr/bin
COPY --from=base /build/bin/generate-keys /usr/bin
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]

2
build/docker/docker-compose.deps.yml
View file

@ -2,7 +2,7 @@ version: "3.4"
services:
postgres:
hostname: postgres
image: postgres:9.5
image: postgres:9.6
restart: always
volumes:
- ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh

3
build/docker/docker-compose.monolith.yml
View file

@ -2,9 +2,8 @@ version: "3.4"
services:
monolith:
hostname: monolith
image: matrixdotorg/dendrite:monolith
image: matrixdotorg/dendrite-monolith:latest
command: [
"--config=dendrite.yaml",
"--tls-cert=server.crt",
"--tls-key=server.key"
]

106
build/docker/docker-compose.polylith.yml
View file

@ -1,43 +1,18 @@
version: "3.4"
services:
client_api_proxy:
hostname: client_api_proxy
image: matrixdotorg/dendrite:clientproxy
command: [
"--bind-address=:8008",
"--client-api-server-url=http://client_api:8071",
"--sync-api-server-url=http://sync_api:8073",
"--media-api-server-url=http://media_api:8074"
]
volumes:
- ./config:/etc/dendrite
networks:
- internal
depends_on:
- sync_api
- client_api
- media_api
ports:
- "8008:8008"
client_api:
hostname: client_api
image: matrixdotorg/dendrite:clientapi
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: clientapi
volumes:
- ./config:/etc/dendrite
- room_server
networks:
- internal
media_api:
hostname: media_api
image: matrixdotorg/dendrite:mediaapi
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: mediaapi
volumes:
- ./config:/etc/dendrite
networks:
@ -45,10 +20,8 @@ services:
sync_api:
hostname: sync_api
image: matrixdotorg/dendrite:syncapi
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: syncapi
volumes:
- ./config:/etc/dendrite
networks:
@ -56,10 +29,8 @@ services:
room_server:
hostname: room_server
image: matrixdotorg/dendrite:roomserver
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: roomserver
volumes:
- ./config:/etc/dendrite
networks:
@ -67,40 +38,17 @@ services:
edu_server:
hostname: edu_server
image: matrixdotorg/dendrite:eduserver
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: eduserver
volumes:
- ./config:/etc/dendrite
networks:
- internal
federation_api_proxy:
hostname: federation_api_proxy
image: matrixdotorg/dendrite:federationproxy
command: [
"--bind-address=:8448",
"--federation-api-url=http://federation_api:8072",
"--media-api-server-url=http://media_api:8074"
]
volumes:
- ./config:/etc/dendrite
depends_on:
- federation_api
- federation_sender
- media_api
networks:
- internal
ports:
- "8448:8448"
federation_api:
hostname: federation_api
image: matrixdotorg/dendrite:federationapi
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: federationapi
volumes:
- ./config:/etc/dendrite
networks:
@ -108,10 +56,8 @@ services:
federation_sender:
hostname: federation_sender
image: matrixdotorg/dendrite:federationsender
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: federationsender
volumes:
- ./config:/etc/dendrite
networks:
@ -119,10 +65,8 @@ services:
key_server:
hostname: key_server
image: matrixdotorg/dendrite:keyserver
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: keyserver
volumes:
- ./config:/etc/dendrite
networks:
@ -130,10 +74,8 @@ services:
signing_key_server:
hostname: signing_key_server
image: matrixdotorg/dendrite:signingkeyserver
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: signingkeyserver
volumes:
- ./config:/etc/dendrite
networks:
@ -141,10 +83,8 @@ services:
user_api:
hostname: user_api
image: matrixdotorg/dendrite:userapi
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: userapi
volumes:
- ./config:/etc/dendrite
networks:
@ -152,10 +92,8 @@ services:
appservice_api:
hostname: appservice_api
image: matrixdotorg/dendrite:appservice
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: appservice
volumes:
- ./config:/etc/dendrite
networks:

21
build/docker/images-build.sh
View file

@ -2,20 +2,11 @@
cd $(git rev-parse --show-toplevel)
docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:latest .
TAG=${1:-latest}
docker build -t matrixdotorg/dendrite:monolith --build-arg component=dendrite-monolith-server -f build/docker/Dockerfile.component .
echo "Building tag '${TAG}'"
docker build -t matrixdotorg/dendrite:appservice --build-arg component=dendrite-appservice-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:clientapi --build-arg component=dendrite-client-api-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:clientproxy --build-arg component=client-api-proxy -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:eduserver --build-arg component=dendrite-edu-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:federationapi --build-arg component=dendrite-federation-api-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:federationsender --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:federationproxy --build-arg component=federation-api-proxy -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:keyserver --build-arg component=dendrite-key-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:signingkeyserver --build-arg component=dendrite-signing-key-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:userapi --build-arg component=dendrite-user-api-server -f build/docker/Dockerfile.component .
docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} .
docker build -t matrixdotorg/dendrite-monolith:${TAG} -f build/docker/Dockerfile.monolith .
docker build -t matrixdotorg/dendrite-polylith:${TAG} -f build/docker/Dockerfile.polylith .

19
build/docker/images-pull.sh
View file

@ -1,17 +1,8 @@
#!/bin/bash
docker pull matrixdotorg/dendrite:monolith
TAG=${1:-latest}
docker pull matrixdotorg/dendrite:appservice
docker pull matrixdotorg/dendrite:clientapi
docker pull matrixdotorg/dendrite:clientproxy
docker pull matrixdotorg/dendrite:eduserver
docker pull matrixdotorg/dendrite:federationapi
docker pull matrixdotorg/dendrite:federationsender
docker pull matrixdotorg/dendrite:federationproxy
docker pull matrixdotorg/dendrite:keyserver
docker pull matrixdotorg/dendrite:mediaapi
docker pull matrixdotorg/dendrite:roomserver
docker pull matrixdotorg/dendrite:syncapi
docker pull matrixdotorg/dendrite:signingkeyserver
docker pull matrixdotorg/dendrite:userapi
echo "Pulling tag '${TAG}'"
docker pull matrixdotorg/dendrite-monolith:${TAG}
docker pull matrixdotorg/dendrite-polylith:${TAG}

19
build/docker/images-push.sh
View file

@ -1,17 +1,8 @@
#!/bin/bash
docker push matrixdotorg/dendrite:monolith
TAG=${1:-latest}
docker push matrixdotorg/dendrite:appservice
docker push matrixdotorg/dendrite:clientapi
docker push matrixdotorg/dendrite:clientproxy
docker push matrixdotorg/dendrite:eduserver
docker push matrixdotorg/dendrite:federationapi
docker push matrixdotorg/dendrite:federationsender
docker push matrixdotorg/dendrite:federationproxy
docker push matrixdotorg/dendrite:keyserver
docker push matrixdotorg/dendrite:mediaapi
docker push matrixdotorg/dendrite:roomserver
docker push matrixdotorg/dendrite:syncapi
docker push matrixdotorg/dendrite:signingkeyserver
docker push matrixdotorg/dendrite:userapi
echo "Pushing tag '${TAG}'"
docker push matrixdotorg/dendrite-monolith:${TAG}
docker push matrixdotorg/dendrite-polylith:${TAG}

2
build/docker/postgres/create_db.sh
View file

@ -1,5 +1,5 @@
#!/bin/sh
for db in account device mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice e2ekey naffka; do
for db in account device mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice naffka; do
createdb -U dendrite -O dendrite dendrite_$db
done

2
clientapi/routing/state.go
View file

@ -141,7 +141,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
util.GetLogger(ctx).WithError(err).Error("Failed to QueryMembershipForUser")
return jsonerror.InternalServerError()
}
for _, ev := range stateRes.StateEvents {
for _, ev := range stateAfterRes.StateEvents {
stateEvents = append(
stateEvents,
gomatrixserverlib.HeaderedToClientEvent(ev, gomatrixserverlib.FormatAll),

28
cmd/create-account/main.go
View file

@ -22,7 +22,6 @@ import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/dendrite/userapi/storage/devices"
"github.com/matrix-org/gomatrixserverlib"
)
@ -39,7 +38,6 @@ var (
username = flag.String("username", "", "The user ID localpart to register e.g 'alice' in '@alice:localhost'.")
password = flag.String("password", "", "Optional. The password to register with. If not specified, this account will be password-less.")
serverNameStr = flag.String("servername", "localhost", "The Matrix server domain which will form the domain part of the user ID.")
accessToken = flag.String("token", "", "Optional. The desired access_token to have. If not specified, a random access_token will be made.")
)
func main() {
@ -78,29 +76,5 @@ func main() {
os.Exit(1)
}
deviceDB, err := devices.NewDatabase(&config.DatabaseOptions{
ConnectionString: config.DataSource(*database),
}, serverName)
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
if *accessToken == "" {
t := "token_" + *username
accessToken = &t
}
device, err := deviceDB.CreateDevice(
context.Background(), *username, nil, *accessToken, nil, "127.0.0.1", "",
)
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
fmt.Println("Created account:")
fmt.Printf("user_id = %s\n", device.UserID)
fmt.Printf("device_id = %s\n", device.ID)
fmt.Printf("access_token = %s\n", device.AccessToken)
fmt.Println("Created account")
}

78
cmd/dendrite-polylith-multi/main.go Normal file
View file

@ -0,0 +1,78 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"flag"
"os"
"strings"
"github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/sirupsen/logrus"
)
type entrypoint func(base *setup.BaseDendrite, cfg *config.Dendrite)
func main() {
cfg := setup.ParseFlags(true)
component := ""
if flag.NFlag() > 0 {
component = flag.Arg(0) // ./dendrite-polylith-multi --config=... clientapi
} else if len(os.Args) > 1 {
component = os.Args[1] // ./dendrite-polylith-multi clientapi
}
components := map[string]entrypoint{
"appservice": personalities.Appservice,
"clientapi": personalities.ClientAPI,
"eduserver": personalities.EDUServer,
"federationapi": personalities.FederationAPI,
"federationsender": personalities.FederationSender,
"keyserver": personalities.KeyServer,
"mediaapi": personalities.MediaAPI,
"roomserver": personalities.RoomServer,
"signingkeyserver": personalities.SigningKeyServer,
"syncapi": personalities.SyncAPI,
"userapi": personalities.UserAPI,
}
start, ok := components[component]
if !ok {
if component == "" {
logrus.Errorf("No component specified")
logrus.Info("The first argument on the command line must be the name of the component to run")
} else {
logrus.Errorf("Unknown component %q specified", component)
}
var list []string
for c := range components {
list = append(list, c)
}
logrus.Infof("Valid components: %s", strings.Join(list, ", "))
os.Exit(1)
}
logrus.Infof("Starting %q component", component)
base := setup.NewBaseDendrite(cfg, component, false) // TODO
defer base.Close() // nolint: errcheck
start(base, cfg)
}

11
cmd/dendrite-appservice-server/main.go → cmd/dendrite-polylith-multi/personalities/appservice.go
View file

@ -1,4 +1,4 @@
// Copyright 2018 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,18 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "AppServiceAPI", true)
defer base.Close() // nolint: errcheck
func Appservice(base *setup.BaseDendrite, cfg *config.Dendrite) {
userAPI := base.UserAPIClient()
rsAPI := base.RoomserverHTTPClient()

12
cmd/dendrite-client-api-server/main.go → cmd/dendrite-polylith-multi/personalities/clientapi.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,20 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/clientapi"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/internal/transactions"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "ClientAPI", true)
defer base.Close() // nolint: errcheck
func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
accountDB := base.CreateAccountsDB()
federation := base.CreateFederationClient()

18
cmd/dendrite-edu-server/main.go → cmd/dendrite-polylith-multi/personalities/eduserver.go
View file

@ -1,3 +1,5 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
@ -10,26 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
_ "net/http/pprof"
"github.com/matrix-org/dendrite/eduserver"
"github.com/matrix-org/dendrite/eduserver/cache"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/sirupsen/logrus"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "EDUServerAPI", true)
defer func() {
if err := base.Close(); err != nil {
logrus.WithError(err).Warn("BaseDendrite close failed")
}
}()
func EDUServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient())
eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI)

11
cmd/dendrite-federation-api-server/main.go → cmd/dendrite-polylith-multi/personalities/federationapi.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,18 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "FederationAPI", true)
defer base.Close() // nolint: errcheck
func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
userAPI := base.UserAPIClient()
federation := base.CreateFederationClient()
serverKeyAPI := base.SigningKeyServerHTTPClient()

11
cmd/dendrite-federation-sender-server/main.go → cmd/dendrite-polylith-multi/personalities/federationsender.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,18 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/federationsender"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "FederationSender", true)
defer base.Close() // nolint: errcheck
func FederationSender(base *setup.BaseDendrite, cfg *config.Dendrite) {
federation := base.CreateFederationClient()
serverKeyAPI := base.SigningKeyServerHTTPClient()

9
cmd/dendrite-key-server/main.go → cmd/dendrite-polylith-multi/personalities/keyserver.go
View file

@ -12,18 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/keyserver"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "KeyServer", true)
defer base.Close() // nolint: errcheck
func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient())
intAPI.SetUserAPI(base.UserAPIClient())

11
cmd/dendrite-media-api-server/main.go → cmd/dendrite-polylith-multi/personalities/mediaapi.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,18 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/mediaapi"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "MediaAPI", true)
defer base.Close() // nolint: errcheck
func MediaAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
userAPI := base.UserAPIClient()
client := base.CreateClient()

11
cmd/dendrite-room-server/main.go → cmd/dendrite-polylith-multi/personalities/roomserver.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,18 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/roomserver"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "RoomServerAPI", true)
defer base.Close() // nolint: errcheck
func RoomServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
serverKeyAPI := base.SigningKeyServerHTTPClient()
keyRing := serverKeyAPI.KeyRing()

9
cmd/dendrite-signing-key-server/main.go → cmd/dendrite-polylith-multi/personalities/signingkeyserver.go
View file

@ -12,18 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/signingkeyserver"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "SigningKeyServer", true)
defer base.Close() // nolint: errcheck
func SigningKeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
federation := base.CreateFederationClient()
intAPI := signingkeyserver.NewInternalAPI(&base.Cfg.SigningKeyServer, federation, base.Caches)

12
cmd/dendrite-sync-api-server/main.go → cmd/dendrite-polylith-multi/personalities/syncapi.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,19 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/syncapi"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "SyncAPI", true)
defer base.Close() // nolint: errcheck
func SyncAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
userAPI := base.UserAPIClient()
federation := base.CreateFederationClient()

11
cmd/dendrite-user-api-server/main.go → cmd/dendrite-polylith-multi/personalities/userapi.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,18 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/userapi"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "UserAPI", true)
defer base.Close() // nolint: errcheck
func UserAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
accountDB := base.CreateAccountsDB()
userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient())

67
docs/INSTALL.md
View file

@ -24,7 +24,7 @@ use in production environments just yet!
Dendrite requires:
* Go 1.13 or higher
* Postgres 9.5 or higher (if using Postgres databases, not needed for SQLite)
* Postgres 9.6 or higher (if using Postgres databases, not needed for SQLite)
If you want to run a polylith deployment, you also need:
@ -93,12 +93,12 @@ brew services start kafka
### SQLite database setup
Dendrite can use the built-in SQLite database engine for small setups.
The SQLite databases do not need to be preconfigured - Dendrite will
The SQLite databases do not need to be pre-built - Dendrite will
create them automatically at startup.
### Postgres database setup
Assuming that Postgres 9.5 (or later) is installed:
Assuming that Postgres 9.6 (or later) is installed:
* Create role, choosing a new password when prompted:
@ -109,7 +109,7 @@ Assuming that Postgres 9.5 (or later) is installed:
* Create the component databases:
```bash
for i in account device mediaapi syncapi roomserver signingkeyserver federationsender appservice e2ekey naffka; do
for i in mediaapi syncapi roomserver signingkeyserver federationsender appservice keyserver userapi_account userapi_device naffka; do
sudo -u postgres createdb -O dendrite dendrite_$i
done
```
@ -135,8 +135,8 @@ Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Th
* The `server_name` entry to reflect the hostname of your Dendrite server
* The `database` lines with an updated connection string based on your
desired setup, e.g. replacing `component` with the name of the component:
* For Postgres: `postgres://dendrite:password@localhost/component`
desired setup, e.g. replacing `database` with the name of the database:
* For Postgres: `postgres://dendrite:password@localhost/database`
* For SQLite on disk: `file:component.db` or `file:///path/to/component.db`
* Postgres and SQLite can be mixed and matched.
* The `use_naffka` option if using Naffka in a monolith deployment
@ -147,6 +147,10 @@ then configuring `key_perspectives` (like `matrix.org` in the sample) can
help to improve reliability considerably by allowing your homeserver to fetch
public keys for dead homeservers from somewhere else.
**WARNING:** Dendrite supports running all components from the same database in
Postgres mode, but this is **NOT** a supported configuration with SQLite. When
using SQLite, all components **MUST** use their own database file.
## Starting a monolith server
It is possible to use Naffka as an in-process replacement to Kafka when using
@ -167,30 +171,17 @@ as shown below, it will also listen for HTTPS connections on port 8448.
The following contains scripts which will run all the required processes in order to point a Matrix client at Dendrite.
### Client proxy
### nginx (or other reverse proxy)
This is what Matrix clients will talk to. If you use the script below, point
your client at `http://localhost:8008`.
This is what your clients and federated hosts will talk to. It must forward
requests onto the correct API server based on URL:
```bash
./bin/client-api-proxy \
--bind-address ":8008" \
--client-api-server-url "http://localhost:7771" \
--sync-api-server-url "http://localhost:7773" \
--media-api-server-url "http://localhost:7774" \
```
* `/_matrix/client` to the client API server
* `/_matrix/federation` to the federation API server
* `/_matrix/key` to the federation API server
* `/_matrix/media` to the media API server
### Federation proxy
This is what Matrix servers will talk to. This is only required if you want
to support federation.
```bash
./bin/federation-api-proxy \
--bind-address ":8448" \
--federation-api-url "http://localhost:7772" \
--media-api-server-url "http://localhost:7774" \
```
See `docs/nginx/polylith-sample.conf` for a sample configuration.
### Client API server
@ -198,7 +189,7 @@ This is what implements CS API endpoints. Clients talk to this via the proxy in
order to send messages, create and join rooms, etc.
```bash
./bin/dendrite-client-api-server --config=dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml clientapi
```
### Sync server
@ -207,7 +198,7 @@ This is what implements `/sync` requests. Clients talk to this via the proxy
in order to receive messages.
```bash
./bin/dendrite-sync-api-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml syncapi
```
### Media server
@ -216,7 +207,7 @@ This implements `/media` requests. Clients talk to this via the proxy in
order to upload and retrieve media.
```bash
./bin/dendrite-media-api-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml mediaapi
```
### Federation API server
@ -226,7 +217,7 @@ order to send transactions. This is only required if you want to support
federation.
```bash
./bin/dendrite-federation-api-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml federationapi
```
### Internal components
@ -239,7 +230,7 @@ contacted by other components. This includes the following components.
This is what implements the room DAG. Clients do not talk to this.
```bash
./bin/dendrite-room-server --config=dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml roomserver
```
#### Federation sender
@ -248,7 +239,7 @@ This sends events from our users to other servers. This is only required if
you want to support federation.
```bash
./bin/dendrite-federation-sender-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml federationsender
```
#### Appservice server
@ -259,7 +250,7 @@ running locally. This is only required if you want to support running
application services on your homeserver.
```bash
./bin/dendrite-appservice-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml appservice
```
#### Key server
@ -267,7 +258,7 @@ application services on your homeserver.
This manages end-to-end encryption keys for users.
```bash
./bin/dendrite-key-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml keyserver
```
#### Signing key server
@ -275,7 +266,7 @@ This manages end-to-end encryption keys for users.
This manages signing keys for servers.
```bash
./bin/dendrite-signing-key-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml signingkeyserver
```
#### EDU server
@ -283,7 +274,7 @@ This manages signing keys for servers.
This manages processing EDUs such as typing, send-to-device events and presence. Clients do not talk to
```bash
./bin/dendrite-edu-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml eduserver
```
#### User server
@ -292,6 +283,6 @@ This manages user accounts, device access tokens and user account data,
amongst other things.
```bash
./bin/dendrite-user-api-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml userapi
```

24
roomserver/internal/input/input_events.go
View file

@ -17,6 +17,7 @@
package input
import (
"bytes"
"context"
"fmt"
@ -26,6 +27,7 @@ import (
"github.com/matrix-org/dendrite/roomserver/state"
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/sirupsen/logrus"
)
@ -44,6 +46,28 @@ func (r *Inputer) processRoomEvent(
headered := input.Event
event := headered.Unwrap()
// if we have already got this event then do not process it again, if the input kind is an outlier.
// Outliers contain no extra information which may warrant a re-processing.
if input.Kind == api.KindOutlier {
evs, err2 := r.DB.EventsFromIDs(ctx, []string{event.EventID()})
if err2 == nil && len(evs) == 1 {
// check hash matches if we're on early room versions where the event ID was a random string
idFormat, err2 := headered.RoomVersion.EventIDFormat()
if err2 == nil {
switch idFormat {
case gomatrixserverlib.EventIDFormatV1:
if bytes.Equal(event.EventReference().EventSHA256, evs[0].EventReference().EventSHA256) {
util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring")
return event.EventID(), nil
}
default:
util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring")
return event.EventID(), nil
}
}
}
}
// Check that the event passes authentication checks and work out
// the numeric IDs for the auth events.
isRejected := false

12
roomserver/internal/input/input_latest_events.go
View file

@ -233,18 +233,6 @@ func (u *latestEventsUpdater) latestState() error {
if err != nil {
return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err)
}
if len(u.removed) > len(u.added) {
// This really shouldn't happen.
// TODO: What is ultimately the best way to handle this situation?
logrus.Errorf(
"Invalid state delta on event %q wants to remove %d state but only add %d state (between state snapshots %d and %d)",
u.event.EventID(), len(u.removed), len(u.added), u.oldStateNID, u.newStateNID,
)
u.added = u.added[:0]
u.removed = u.removed[:0]
u.newStateNID = u.oldStateNID
return nil
}
// Also work out the state before the event removes and the event
// adds.

14
roomserver/storage/shared/latest_events_updater.go
View file

@ -70,16 +70,14 @@ func (u *LatestEventsUpdater) CurrentStateSnapshotNID() types.StateSnapshotNID {
return u.currentStateSnapshotNID
}
// StorePreviousEvents implements types.RoomRecentEventsUpdater
// StorePreviousEvents implements types.RoomRecentEventsUpdater - This must be called from a Writer
func (u *LatestEventsUpdater) StorePreviousEvents(eventNID types.EventNID, previousEventReferences []gomatrixserverlib.EventReference) error {
return u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error {
for _, ref := range previousEventReferences {
if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, txn, ref.EventID, ref.EventSHA256, eventNID); err != nil {
return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err)
}
for _, ref := range previousEventReferences {
if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, u.txn, ref.EventID, ref.EventSHA256, eventNID); err != nil {
return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err)
}
return nil
})
}
return nil
}
// IsReferenced implements types.RoomRecentEventsUpdater

25
roomserver/storage/shared/storage.go
View file

@ -492,15 +492,32 @@ func (d *Database) StoreEvent(
if roomInfo == nil && len(prevEvents) > 0 {
return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("expected room %q to exist", event.RoomID())
}
// Create an updater - NB: on sqlite this WILL create a txn as we are directly calling the shared DB form of
// GetLatestEventsForUpdate - not via the SQLiteDatabase form which has `nil` txns. This
// function only does SELECTs though so the created txn (at this point) is just a read txn like
// any other so this is fine. If we ever update GetLatestEventsForUpdate or NewLatestEventsUpdater
// to do writes however then this will need to go inside `Writer.Do`.
updater, err = d.GetLatestEventsForUpdate(ctx, *roomInfo)
if err != nil {
return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("NewLatestEventsUpdater: %w", err)
}
if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil {
return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("updater.StorePreviousEvents: %w", err)
// Ensure that we atomically store prev events AND commit them. If we don't wrap StorePreviousEvents
// and EndTransaction in a writer then it's possible for a new write txn to be made between the two
// function calls which will then fail with 'database is locked'. This new write txn would HAVE to be
// something like SetRoomAlias/RemoveRoomAlias as normal input events are already done sequentially due to
// SupportsConcurrentRoomInputs() == false on sqlite, though this does not apply to setting room aliases
// as they don't go via InputRoomEvents
err = d.Writer.Do(d.DB, updater.txn, func(txn *sql.Tx) error {
if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil {
return fmt.Errorf("updater.StorePreviousEvents: %w", err)
}
succeeded := true
err = sqlutil.EndTransaction(updater, &succeeded)
return err
})
if err != nil {
return 0, types.StateAtEvent{}, nil, "", err
}
succeeded := true
err = sqlutil.EndTransaction(updater, &succeeded)
}
return roomNID, types.StateAtEvent{

5
sytest-blacklist
View file

@ -57,4 +57,7 @@ The only membership state included in a gapped incremental sync is for senders i
# Blacklisted out of flakiness after #1479
Invited user can reject local invite after originator leaves
Invited user can reject invite for empty room
If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes
If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes
# Blacklisted due to flakiness
A prev_batch token from incremental sync can be used in the v1 messages API

2
sytest-whitelist
View file

@ -481,7 +481,7 @@ m.room.history_visibility == "joined" allows/forbids appropriately for Guest use
m.room.history_visibility == "joined" allows/forbids appropriately for Real users
POST rejects invalid utf-8 in JSON
Users cannot kick users who have already left a room
A prev_batch token from incremental sync can be used in the v1 messages API
Event with an invalid signature in the send_join response should not cause room join to fail
Inbound federation rejects typing notifications from wrong remote
Should not be able to take over the room by pretending there is no PL event
Can get rooms/{roomId}/state for a departed room (SPEC-216)