mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2025-12-23 23:03:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into neilalexander/excludesync

Browse source
This commit is contained in:
Neil Alexander 2020-10-20 16:24:09 +01:00 committed by GitHub
parent b40dc6d778 39c7a8915c
commit 97f2a78670
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 270 additions and 317 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
build/docker/Dockerfile
View file

@ -1,4 +1,4 @@
FROM docker.io/golang:1.13.7-alpine3.11 AS builder
FROM docker.io/golang:1.15-alpine AS builder
RUN apk --update --no-cache add bash build-base

13
build/docker/Dockerfile.component
View file

@ -1,13 +0,0 @@
FROM matrixdotorg/dendrite:latest AS base
FROM alpine:latest
ARG component=monolith
ENV entrypoint=${component}
COPY --from=base /build/bin/${component} /usr/bin
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT /usr/bin/${entrypoint} $@

13
build/docker/Dockerfile.monolith Normal file
View file

@ -0,0 +1,13 @@
FROM matrixdotorg/dendrite:latest AS base
FROM alpine:latest
COPY --from=base /build/bin/dendrite-monolith-server /usr/bin
COPY --from=base /build/bin/goose /usr/bin
COPY --from=base /build/bin/create-account /usr/bin
COPY --from=base /build/bin/generate-keys /usr/bin
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]

13
build/docker/Dockerfile.polylith Normal file
View file

@ -0,0 +1,13 @@
FROM matrixdotorg/dendrite:latest AS base
FROM alpine:latest
COPY --from=base /build/bin/dendrite-polylith-multi /usr/bin
COPY --from=base /build/bin/goose /usr/bin
COPY --from=base /build/bin/create-account /usr/bin
COPY --from=base /build/bin/generate-keys /usr/bin
VOLUME /etc/dendrite
WORKDIR /etc/dendrite
ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]

2
build/docker/docker-compose.deps.yml
View file

@ -2,7 +2,7 @@ version: "3.4"
services:
postgres:
hostname: postgres
image: postgres:9.5
image: postgres:9.6
restart: always
volumes:
- ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh

3
build/docker/docker-compose.monolith.yml
View file

@ -2,9 +2,8 @@ version: "3.4"
services:
monolith:
hostname: monolith
image: matrixdotorg/dendrite:monolith
image: matrixdotorg/dendrite-monolith:latest
command: [
"--config=dendrite.yaml",
"--tls-cert=server.crt",
"--tls-key=server.key"
]

106
build/docker/docker-compose.polylith.yml
View file

@ -1,43 +1,18 @@
version: "3.4"
services:
client_api_proxy:
hostname: client_api_proxy
image: matrixdotorg/dendrite:clientproxy
command: [
"--bind-address=:8008",
"--client-api-server-url=http://client_api:8071",
"--sync-api-server-url=http://sync_api:8073",
"--media-api-server-url=http://media_api:8074"
]
volumes:
- ./config:/etc/dendrite
networks:
- internal
depends_on:
- sync_api
- client_api
- media_api
ports:
- "8008:8008"
client_api:
hostname: client_api
image: matrixdotorg/dendrite:clientapi
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: clientapi
volumes:
- ./config:/etc/dendrite
- room_server
networks:
- internal
media_api:
hostname: media_api
image: matrixdotorg/dendrite:mediaapi
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: mediaapi
volumes:
- ./config:/etc/dendrite
networks:
@ -45,10 +20,8 @@ services:
sync_api:
hostname: sync_api
image: matrixdotorg/dendrite:syncapi
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: syncapi
volumes:
- ./config:/etc/dendrite
networks:
@ -56,10 +29,8 @@ services:
room_server:
hostname: room_server
image: matrixdotorg/dendrite:roomserver
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: roomserver
volumes:
- ./config:/etc/dendrite
networks:
@ -67,40 +38,17 @@ services:
edu_server:
hostname: edu_server
image: matrixdotorg/dendrite:eduserver
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: eduserver
volumes:
- ./config:/etc/dendrite
networks:
- internal
federation_api_proxy:
hostname: federation_api_proxy
image: matrixdotorg/dendrite:federationproxy
command: [
"--bind-address=:8448",
"--federation-api-url=http://federation_api:8072",
"--media-api-server-url=http://media_api:8074"
]
volumes:
- ./config:/etc/dendrite
depends_on:
- federation_api
- federation_sender
- media_api
networks:
- internal
ports:
- "8448:8448"
federation_api:
hostname: federation_api
image: matrixdotorg/dendrite:federationapi
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: federationapi
volumes:
- ./config:/etc/dendrite
networks:
@ -108,10 +56,8 @@ services:
federation_sender:
hostname: federation_sender
image: matrixdotorg/dendrite:federationsender
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: federationsender
volumes:
- ./config:/etc/dendrite
networks:
@ -119,10 +65,8 @@ services:
key_server:
hostname: key_server
image: matrixdotorg/dendrite:keyserver
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: keyserver
volumes:
- ./config:/etc/dendrite
networks:
@ -130,10 +74,8 @@ services:
signing_key_server:
hostname: signing_key_server
image: matrixdotorg/dendrite:signingkeyserver
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: signingkeyserver
volumes:
- ./config:/etc/dendrite
networks:
@ -141,10 +83,8 @@ services:
user_api:
hostname: user_api
image: matrixdotorg/dendrite:userapi
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: userapi
volumes:
- ./config:/etc/dendrite
networks:
@ -152,10 +92,8 @@ services:
appservice_api:
hostname: appservice_api
image: matrixdotorg/dendrite:appservice
command: [
"--config=dendrite.yaml"
]
image: matrixdotorg/dendrite-polylith:latest
command: appservice
volumes:
- ./config:/etc/dendrite
networks:

21
build/docker/images-build.sh
View file

@ -2,20 +2,11 @@
cd $(git rev-parse --show-toplevel)
docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:latest .
TAG=${1:-latest}
docker build -t matrixdotorg/dendrite:monolith --build-arg component=dendrite-monolith-server -f build/docker/Dockerfile.component .
echo "Building tag '${TAG}'"
docker build -t matrixdotorg/dendrite:appservice --build-arg component=dendrite-appservice-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:clientapi --build-arg component=dendrite-client-api-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:clientproxy --build-arg component=client-api-proxy -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:eduserver --build-arg component=dendrite-edu-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:federationapi --build-arg component=dendrite-federation-api-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:federationsender --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:federationproxy --build-arg component=federation-api-proxy -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:keyserver --build-arg component=dendrite-key-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:signingkeyserver --build-arg component=dendrite-signing-key-server -f build/docker/Dockerfile.component .
docker build -t matrixdotorg/dendrite:userapi --build-arg component=dendrite-user-api-server -f build/docker/Dockerfile.component .
docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} .
docker build -t matrixdotorg/dendrite-monolith:${TAG} -f build/docker/Dockerfile.monolith .
docker build -t matrixdotorg/dendrite-polylith:${TAG} -f build/docker/Dockerfile.polylith .

19
build/docker/images-pull.sh
View file

@ -1,17 +1,8 @@
#!/bin/bash
docker pull matrixdotorg/dendrite:monolith
TAG=${1:-latest}
docker pull matrixdotorg/dendrite:appservice
docker pull matrixdotorg/dendrite:clientapi
docker pull matrixdotorg/dendrite:clientproxy
docker pull matrixdotorg/dendrite:eduserver
docker pull matrixdotorg/dendrite:federationapi
docker pull matrixdotorg/dendrite:federationsender
docker pull matrixdotorg/dendrite:federationproxy
docker pull matrixdotorg/dendrite:keyserver
docker pull matrixdotorg/dendrite:mediaapi
docker pull matrixdotorg/dendrite:roomserver
docker pull matrixdotorg/dendrite:syncapi
docker pull matrixdotorg/dendrite:signingkeyserver
docker pull matrixdotorg/dendrite:userapi
echo "Pulling tag '${TAG}'"
docker pull matrixdotorg/dendrite-monolith:${TAG}
docker pull matrixdotorg/dendrite-polylith:${TAG}

19
build/docker/images-push.sh
View file

@ -1,17 +1,8 @@
#!/bin/bash
docker push matrixdotorg/dendrite:monolith
TAG=${1:-latest}
docker push matrixdotorg/dendrite:appservice
docker push matrixdotorg/dendrite:clientapi
docker push matrixdotorg/dendrite:clientproxy
docker push matrixdotorg/dendrite:eduserver
docker push matrixdotorg/dendrite:federationapi
docker push matrixdotorg/dendrite:federationsender
docker push matrixdotorg/dendrite:federationproxy
docker push matrixdotorg/dendrite:keyserver
docker push matrixdotorg/dendrite:mediaapi
docker push matrixdotorg/dendrite:roomserver
docker push matrixdotorg/dendrite:syncapi
docker push matrixdotorg/dendrite:signingkeyserver
docker push matrixdotorg/dendrite:userapi
echo "Pushing tag '${TAG}'"
docker push matrixdotorg/dendrite-monolith:${TAG}
docker push matrixdotorg/dendrite-polylith:${TAG}

2
build/docker/postgres/create_db.sh
View file

@ -1,5 +1,5 @@
#!/bin/sh
for db in account device mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice e2ekey naffka; do
for db in account device mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice naffka; do
createdb -U dendrite -O dendrite dendrite_$db
done

28
cmd/create-account/main.go
View file

@ -22,7 +22,6 @@ import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/userapi/storage/accounts"
"github.com/matrix-org/dendrite/userapi/storage/devices"
"github.com/matrix-org/gomatrixserverlib"
)
@ -39,7 +38,6 @@ var (
username = flag.String("username", "", "The user ID localpart to register e.g 'alice' in '@alice:localhost'.")
password = flag.String("password", "", "Optional. The password to register with. If not specified, this account will be password-less.")
serverNameStr = flag.String("servername", "localhost", "The Matrix server domain which will form the domain part of the user ID.")
accessToken = flag.String("token", "", "Optional. The desired access_token to have. If not specified, a random access_token will be made.")
)
func main() {
@ -78,29 +76,5 @@ func main() {
os.Exit(1)
}
deviceDB, err := devices.NewDatabase(&config.DatabaseOptions{
ConnectionString: config.DataSource(*database),
}, serverName)
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
if *accessToken == "" {
t := "token_" + *username
accessToken = &t
}
device, err := deviceDB.CreateDevice(
context.Background(), *username, nil, *accessToken, nil, "127.0.0.1", "",
)
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
fmt.Println("Created account:")
fmt.Printf("user_id = %s\n", device.UserID)
fmt.Printf("device_id = %s\n", device.ID)
fmt.Printf("access_token = %s\n", device.AccessToken)
fmt.Println("Created account")
}

78
cmd/dendrite-polylith-multi/main.go Normal file
View file

@ -0,0 +1,78 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"flag"
"os"
"strings"
"github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/sirupsen/logrus"
)
type entrypoint func(base *setup.BaseDendrite, cfg *config.Dendrite)
func main() {
cfg := setup.ParseFlags(true)
component := ""
if flag.NFlag() > 0 {
component = flag.Arg(0) // ./dendrite-polylith-multi --config=... clientapi
} else if len(os.Args) > 1 {
component = os.Args[1] // ./dendrite-polylith-multi clientapi
}
components := map[string]entrypoint{
"appservice": personalities.Appservice,
"clientapi": personalities.ClientAPI,
"eduserver": personalities.EDUServer,
"federationapi": personalities.FederationAPI,
"federationsender": personalities.FederationSender,
"keyserver": personalities.KeyServer,
"mediaapi": personalities.MediaAPI,
"roomserver": personalities.RoomServer,
"signingkeyserver": personalities.SigningKeyServer,
"syncapi": personalities.SyncAPI,
"userapi": personalities.UserAPI,
}
start, ok := components[component]
if !ok {
if component == "" {
logrus.Errorf("No component specified")
logrus.Info("The first argument on the command line must be the name of the component to run")
} else {
logrus.Errorf("Unknown component %q specified", component)
}
var list []string
for c := range components {
list = append(list, c)
}
logrus.Infof("Valid components: %s", strings.Join(list, ", "))
os.Exit(1)
}
logrus.Infof("Starting %q component", component)
base := setup.NewBaseDendrite(cfg, component, false) // TODO
defer base.Close() // nolint: errcheck
start(base, cfg)
}

11
cmd/dendrite-appservice-server/main.go → cmd/dendrite-polylith-multi/personalities/appservice.go
View file

@ -1,4 +1,4 @@
// Copyright 2018 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,18 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/appservice"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "AppServiceAPI", true)
defer base.Close() // nolint: errcheck
func Appservice(base *setup.BaseDendrite, cfg *config.Dendrite) {
userAPI := base.UserAPIClient()
rsAPI := base.RoomserverHTTPClient()

12
cmd/dendrite-client-api-server/main.go → cmd/dendrite-polylith-multi/personalities/clientapi.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,20 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/clientapi"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/internal/transactions"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "ClientAPI", true)
defer base.Close() // nolint: errcheck
func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
accountDB := base.CreateAccountsDB()
federation := base.CreateFederationClient()

18
cmd/dendrite-edu-server/main.go → cmd/dendrite-polylith-multi/personalities/eduserver.go
View file

@ -1,3 +1,5 @@
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
@ -10,26 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
_ "net/http/pprof"
"github.com/matrix-org/dendrite/eduserver"
"github.com/matrix-org/dendrite/eduserver/cache"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/sirupsen/logrus"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "EDUServerAPI", true)
defer func() {
if err := base.Close(); err != nil {
logrus.WithError(err).Warn("BaseDendrite close failed")
}
}()
func EDUServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient())
eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI)

11
cmd/dendrite-federation-api-server/main.go → cmd/dendrite-polylith-multi/personalities/federationapi.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,18 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "FederationAPI", true)
defer base.Close() // nolint: errcheck
func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
userAPI := base.UserAPIClient()
federation := base.CreateFederationClient()
serverKeyAPI := base.SigningKeyServerHTTPClient()

11
cmd/dendrite-federation-sender-server/main.go → cmd/dendrite-polylith-multi/personalities/federationsender.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,18 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/federationsender"
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "FederationSender", true)
defer base.Close() // nolint: errcheck
func FederationSender(base *setup.BaseDendrite, cfg *config.Dendrite) {
federation := base.CreateFederationClient()
serverKeyAPI := base.SigningKeyServerHTTPClient()

9
cmd/dendrite-key-server/main.go → cmd/dendrite-polylith-multi/personalities/keyserver.go
View file

@ -12,18 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/keyserver"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "KeyServer", true)
defer base.Close() // nolint: errcheck
func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient())
intAPI.SetUserAPI(base.UserAPIClient())

11
cmd/dendrite-media-api-server/main.go → cmd/dendrite-polylith-multi/personalities/mediaapi.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,18 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/mediaapi"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "MediaAPI", true)
defer base.Close() // nolint: errcheck
func MediaAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
userAPI := base.UserAPIClient()
client := base.CreateClient()

11
cmd/dendrite-room-server/main.go → cmd/dendrite-polylith-multi/personalities/roomserver.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,18 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/roomserver"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "RoomServerAPI", true)
defer base.Close() // nolint: errcheck
func RoomServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
serverKeyAPI := base.SigningKeyServerHTTPClient()
keyRing := serverKeyAPI.KeyRing()

9
cmd/dendrite-signing-key-server/main.go → cmd/dendrite-polylith-multi/personalities/signingkeyserver.go
View file

@ -12,18 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/signingkeyserver"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "SigningKeyServer", true)
defer base.Close() // nolint: errcheck
func SigningKeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
federation := base.CreateFederationClient()
intAPI := signingkeyserver.NewInternalAPI(&base.Cfg.SigningKeyServer, federation, base.Caches)

12
cmd/dendrite-sync-api-server/main.go → cmd/dendrite-polylith-multi/personalities/syncapi.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,19 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/syncapi"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "SyncAPI", true)
defer base.Close() // nolint: errcheck
func SyncAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
userAPI := base.UserAPIClient()
federation := base.CreateFederationClient()

11
cmd/dendrite-user-api-server/main.go → cmd/dendrite-polylith-multi/personalities/userapi.go
View file

@ -1,4 +1,4 @@
// Copyright 2017 Vector Creations Ltd
// Copyright 2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,18 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
package personalities
import (
"github.com/matrix-org/dendrite/internal/config"
"github.com/matrix-org/dendrite/internal/setup"
"github.com/matrix-org/dendrite/userapi"
)
func main() {
cfg := setup.ParseFlags(false)
base := setup.NewBaseDendrite(cfg, "UserAPI", true)
defer base.Close() // nolint: errcheck
func UserAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
accountDB := base.CreateAccountsDB()
userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient())

67
docs/INSTALL.md
View file

@ -24,7 +24,7 @@ use in production environments just yet!
Dendrite requires:
* Go 1.13 or higher
* Postgres 9.5 or higher (if using Postgres databases, not needed for SQLite)
* Postgres 9.6 or higher (if using Postgres databases, not needed for SQLite)
If you want to run a polylith deployment, you also need:
@ -93,12 +93,12 @@ brew services start kafka
### SQLite database setup
Dendrite can use the built-in SQLite database engine for small setups.
The SQLite databases do not need to be preconfigured - Dendrite will
The SQLite databases do not need to be pre-built - Dendrite will
create them automatically at startup.
### Postgres database setup
Assuming that Postgres 9.5 (or later) is installed:
Assuming that Postgres 9.6 (or later) is installed:
* Create role, choosing a new password when prompted:
@ -109,7 +109,7 @@ Assuming that Postgres 9.5 (or later) is installed:
* Create the component databases:
```bash
for i in account device mediaapi syncapi roomserver signingkeyserver federationsender appservice e2ekey naffka; do
for i in mediaapi syncapi roomserver signingkeyserver federationsender appservice keyserver userapi_account userapi_device naffka; do
sudo -u postgres createdb -O dendrite dendrite_$i
done
```
@ -135,8 +135,8 @@ Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Th
* The `server_name` entry to reflect the hostname of your Dendrite server
* The `database` lines with an updated connection string based on your
desired setup, e.g. replacing `component` with the name of the component:
* For Postgres: `postgres://dendrite:password@localhost/component`
desired setup, e.g. replacing `database` with the name of the database:
* For Postgres: `postgres://dendrite:password@localhost/database`
* For SQLite on disk: `file:component.db` or `file:///path/to/component.db`
* Postgres and SQLite can be mixed and matched.
* The `use_naffka` option if using Naffka in a monolith deployment
@ -147,6 +147,10 @@ then configuring `key_perspectives` (like `matrix.org` in the sample) can
help to improve reliability considerably by allowing your homeserver to fetch
public keys for dead homeservers from somewhere else.
**WARNING:** Dendrite supports running all components from the same database in
Postgres mode, but this is **NOT** a supported configuration with SQLite. When
using SQLite, all components **MUST** use their own database file.
## Starting a monolith server
It is possible to use Naffka as an in-process replacement to Kafka when using
@ -167,30 +171,17 @@ as shown below, it will also listen for HTTPS connections on port 8448.
The following contains scripts which will run all the required processes in order to point a Matrix client at Dendrite.
### Client proxy
### nginx (or other reverse proxy)
This is what Matrix clients will talk to. If you use the script below, point
your client at `http://localhost:8008`.
This is what your clients and federated hosts will talk to. It must forward
requests onto the correct API server based on URL:
```bash
./bin/client-api-proxy \
--bind-address ":8008" \
--client-api-server-url "http://localhost:7771" \
--sync-api-server-url "http://localhost:7773" \
--media-api-server-url "http://localhost:7774" \
```
* `/_matrix/client` to the client API server
* `/_matrix/federation` to the federation API server
* `/_matrix/key` to the federation API server
* `/_matrix/media` to the media API server
### Federation proxy
This is what Matrix servers will talk to. This is only required if you want
to support federation.
```bash
./bin/federation-api-proxy \
--bind-address ":8448" \
--federation-api-url "http://localhost:7772" \
--media-api-server-url "http://localhost:7774" \
```
See `docs/nginx/polylith-sample.conf` for a sample configuration.
### Client API server
@ -198,7 +189,7 @@ This is what implements CS API endpoints. Clients talk to this via the proxy in
order to send messages, create and join rooms, etc.
```bash
./bin/dendrite-client-api-server --config=dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml clientapi
```
### Sync server
@ -207,7 +198,7 @@ This is what implements `/sync` requests. Clients talk to this via the proxy
in order to receive messages.
```bash
./bin/dendrite-sync-api-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml syncapi
```
### Media server
@ -216,7 +207,7 @@ This implements `/media` requests. Clients talk to this via the proxy in
order to upload and retrieve media.
```bash
./bin/dendrite-media-api-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml mediaapi
```
### Federation API server
@ -226,7 +217,7 @@ order to send transactions. This is only required if you want to support
federation.
```bash
./bin/dendrite-federation-api-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml federationapi
```
### Internal components
@ -239,7 +230,7 @@ contacted by other components. This includes the following components.
This is what implements the room DAG. Clients do not talk to this.
```bash
./bin/dendrite-room-server --config=dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml roomserver
```
#### Federation sender
@ -248,7 +239,7 @@ This sends events from our users to other servers. This is only required if
you want to support federation.
```bash
./bin/dendrite-federation-sender-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml federationsender
```
#### Appservice server
@ -259,7 +250,7 @@ running locally. This is only required if you want to support running
application services on your homeserver.
```bash
./bin/dendrite-appservice-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml appservice
```
#### Key server
@ -267,7 +258,7 @@ application services on your homeserver.
This manages end-to-end encryption keys for users.
```bash
./bin/dendrite-key-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml keyserver
```
#### Signing key server
@ -275,7 +266,7 @@ This manages end-to-end encryption keys for users.
This manages signing keys for servers.
```bash
./bin/dendrite-signing-key-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml signingkeyserver
```
#### EDU server
@ -283,7 +274,7 @@ This manages signing keys for servers.
This manages processing EDUs such as typing, send-to-device events and presence. Clients do not talk to
```bash
./bin/dendrite-edu-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml eduserver
```
#### User server
@ -292,6 +283,6 @@ This manages user accounts, device access tokens and user account data,
amongst other things.
```bash
./bin/dendrite-user-api-server --config dendrite.yaml
./bin/dendrite-polylith-multi --config=dendrite.yaml userapi
```

24
roomserver/internal/input/input_events.go
View file

@ -17,6 +17,7 @@
package input
import (
"bytes"
"context"
"fmt"
@ -26,6 +27,7 @@ import (
"github.com/matrix-org/dendrite/roomserver/state"
"github.com/matrix-org/dendrite/roomserver/types"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/sirupsen/logrus"
)
@ -44,6 +46,28 @@ func (r *Inputer) processRoomEvent(
headered := input.Event
event := headered.Unwrap()
// if we have already got this event then do not process it again, if the input kind is an outlier.
// Outliers contain no extra information which may warrant a re-processing.
if input.Kind == api.KindOutlier {
evs, err2 := r.DB.EventsFromIDs(ctx, []string{event.EventID()})
if err2 == nil && len(evs) == 1 {
// check hash matches if we're on early room versions where the event ID was a random string
idFormat, err2 := headered.RoomVersion.EventIDFormat()
if err2 == nil {
switch idFormat {
case gomatrixserverlib.EventIDFormatV1:
if bytes.Equal(event.EventReference().EventSHA256, evs[0].EventReference().EventSHA256) {
util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring")
return event.EventID(), nil
}
default:
util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring")
return event.EventID(), nil
}
}
}
}
// Check that the event passes authentication checks and work out
// the numeric IDs for the auth events.
isRejected := false

12
roomserver/internal/input/input_latest_events.go
View file

@ -233,18 +233,6 @@ func (u *latestEventsUpdater) latestState() error {
if err != nil {
return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err)
}
if len(u.removed) > len(u.added) {
// This really shouldn't happen.
// TODO: What is ultimately the best way to handle this situation?
logrus.Errorf(
"Invalid state delta on event %q wants to remove %d state but only add %d state (between state snapshots %d and %d)",
u.event.EventID(), len(u.removed), len(u.added), u.oldStateNID, u.newStateNID,
)
u.added = u.added[:0]
u.removed = u.removed[:0]
u.newStateNID = u.oldStateNID
return nil
}
// Also work out the state before the event removes and the event
// adds.

14
roomserver/storage/shared/latest_events_updater.go
View file

@ -70,16 +70,14 @@ func (u *LatestEventsUpdater) CurrentStateSnapshotNID() types.StateSnapshotNID {
return u.currentStateSnapshotNID
}
// StorePreviousEvents implements types.RoomRecentEventsUpdater
// StorePreviousEvents implements types.RoomRecentEventsUpdater - This must be called from a Writer
func (u *LatestEventsUpdater) StorePreviousEvents(eventNID types.EventNID, previousEventReferences []gomatrixserverlib.EventReference) error {
return u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error {
for _, ref := range previousEventReferences {
if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, txn, ref.EventID, ref.EventSHA256, eventNID); err != nil {
return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err)
}
for _, ref := range previousEventReferences {
if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, u.txn, ref.EventID, ref.EventSHA256, eventNID); err != nil {
return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err)
}
return nil
})
}
return nil
}
// IsReferenced implements types.RoomRecentEventsUpdater

25
roomserver/storage/shared/storage.go
View file

@ -492,15 +492,32 @@ func (d *Database) StoreEvent(
if roomInfo == nil && len(prevEvents) > 0 {
return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("expected room %q to exist", event.RoomID())
}
// Create an updater - NB: on sqlite this WILL create a txn as we are directly calling the shared DB form of
// GetLatestEventsForUpdate - not via the SQLiteDatabase form which has `nil` txns. This
// function only does SELECTs though so the created txn (at this point) is just a read txn like
// any other so this is fine. If we ever update GetLatestEventsForUpdate or NewLatestEventsUpdater
// to do writes however then this will need to go inside `Writer.Do`.
updater, err = d.GetLatestEventsForUpdate(ctx, *roomInfo)
if err != nil {
return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("NewLatestEventsUpdater: %w", err)
}
if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil {
return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("updater.StorePreviousEvents: %w", err)
// Ensure that we atomically store prev events AND commit them. If we don't wrap StorePreviousEvents
// and EndTransaction in a writer then it's possible for a new write txn to be made between the two
// function calls which will then fail with 'database is locked'. This new write txn would HAVE to be
// something like SetRoomAlias/RemoveRoomAlias as normal input events are already done sequentially due to
// SupportsConcurrentRoomInputs() == false on sqlite, though this does not apply to setting room aliases
// as they don't go via InputRoomEvents
err = d.Writer.Do(d.DB, updater.txn, func(txn *sql.Tx) error {
if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil {
return fmt.Errorf("updater.StorePreviousEvents: %w", err)
}
succeeded := true
err = sqlutil.EndTransaction(updater, &succeeded)
return err
})
if err != nil {
return 0, types.StateAtEvent{}, nil, "", err
}
succeeded := true
err = sqlutil.EndTransaction(updater, &succeeded)
}
return roomNID, types.StateAtEvent{