Merge branch 'master' into neilalexander/h2c

build/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/golang:1.13.7-alpine3.11 AS builder
+FROM docker.io/golang:1.15-alpine AS builder
 
 RUN apk --update --no-cache add bash build-base
 

build/docker/Dockerfile.component

@@ -1,13 +0,0 @@
-FROM matrixdotorg/dendrite:latest AS base
-
-FROM alpine:latest
-
-ARG component=monolith
-ENV entrypoint=${component}
-
-COPY --from=base /build/bin/${component} /usr/bin
-
-VOLUME /etc/dendrite
-WORKDIR /etc/dendrite
-
-ENTRYPOINT /usr/bin/${entrypoint} $@

build/docker/Dockerfile.monolith

@@ -0,0 +1,13 @@
+FROM matrixdotorg/dendrite:latest AS base
+
+FROM alpine:latest
+
+COPY --from=base /build/bin/dendrite-monolith-server /usr/bin
+COPY --from=base /build/bin/goose /usr/bin
+COPY --from=base /build/bin/create-account /usr/bin
+COPY --from=base /build/bin/generate-keys /usr/bin
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT ["/usr/bin/dendrite-monolith-server"]

build/docker/Dockerfile.polylith

@@ -0,0 +1,13 @@
+FROM matrixdotorg/dendrite:latest AS base
+
+FROM alpine:latest
+
+COPY --from=base /build/bin/dendrite-polylith-multi /usr/bin
+COPY --from=base /build/bin/goose /usr/bin
+COPY --from=base /build/bin/create-account /usr/bin
+COPY --from=base /build/bin/generate-keys /usr/bin
+
+VOLUME /etc/dendrite
+WORKDIR /etc/dendrite
+
+ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"]

build/docker/docker-compose.deps.yml

@@ -2,7 +2,7 @@ version: "3.4"
 services:
   postgres:
     hostname: postgres
-    image: postgres:9.5
+    image: postgres:9.6
     restart: always
     volumes:
       - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh

build/docker/docker-compose.monolith.yml

@@ -2,9 +2,8 @@ version: "3.4"
 services:
   monolith:
     hostname: monolith
-    image: matrixdotorg/dendrite:monolith
+    image: matrixdotorg/dendrite-monolith:latest
     command: [
-      "--config=dendrite.yaml",
       "--tls-cert=server.crt",
       "--tls-key=server.key"
     ]

build/docker/docker-compose.polylith.yml

@@ -1,43 +1,18 @@
 version: "3.4"
 services:
-  client_api_proxy:
-    hostname: client_api_proxy
-    image: matrixdotorg/dendrite:clientproxy
-    command: [
-      "--bind-address=:8008",
-      "--client-api-server-url=http://client_api:8071",
-      "--sync-api-server-url=http://sync_api:8073",
-      "--media-api-server-url=http://media_api:8074"
-    ]
-    volumes:
-      - ./config:/etc/dendrite
-    networks:
-      - internal
-    depends_on:
-      - sync_api
-      - client_api
-      - media_api
-    ports:
-      - "8008:8008"
-
   client_api:
     hostname: client_api
-    image: matrixdotorg/dendrite:clientapi
+    image: matrixdotorg/dendrite-polylith:latest
-    command: [
+    command: clientapi
-      "--config=dendrite.yaml"
-    ]
     volumes:
       - ./config:/etc/dendrite
-      - room_server
     networks:
       - internal
 
   media_api:
     hostname: media_api
-    image: matrixdotorg/dendrite:mediaapi
+    image: matrixdotorg/dendrite-polylith:latest
-    command: [
+    command: mediaapi
-      "--config=dendrite.yaml"
-    ]
     volumes:
       - ./config:/etc/dendrite
     networks:
@@ -45,10 +20,8 @@ services:
 
   sync_api:
     hostname: sync_api
-    image: matrixdotorg/dendrite:syncapi
+    image: matrixdotorg/dendrite-polylith:latest
-    command: [
+    command: syncapi
-      "--config=dendrite.yaml"
-    ]
     volumes:
       - ./config:/etc/dendrite
     networks:
@@ -56,10 +29,8 @@ services:
 
   room_server:
     hostname: room_server
-    image: matrixdotorg/dendrite:roomserver
+    image: matrixdotorg/dendrite-polylith:latest
-    command: [
+    command: roomserver
-      "--config=dendrite.yaml"
-    ]
     volumes:
       - ./config:/etc/dendrite
     networks:
@@ -67,40 +38,17 @@ services:
 
   edu_server:
     hostname: edu_server
-    image: matrixdotorg/dendrite:eduserver
+    image: matrixdotorg/dendrite-polylith:latest
-    command: [
+    command: eduserver
-      "--config=dendrite.yaml"
-    ]
     volumes:
       - ./config:/etc/dendrite
     networks:
       - internal
 
-  federation_api_proxy:
-    hostname: federation_api_proxy
-    image: matrixdotorg/dendrite:federationproxy
-    command: [
-      "--bind-address=:8448",
-      "--federation-api-url=http://federation_api:8072",
-      "--media-api-server-url=http://media_api:8074"
-    ]
-    volumes:
-      - ./config:/etc/dendrite
-    depends_on:
-      - federation_api
-      - federation_sender
-      - media_api
-    networks:
-      - internal
-    ports:
-      - "8448:8448"
-
   federation_api:
     hostname: federation_api
-    image: matrixdotorg/dendrite:federationapi
+    image: matrixdotorg/dendrite-polylith:latest
-    command: [
+    command: federationapi
-      "--config=dendrite.yaml"
-    ]
     volumes:
       - ./config:/etc/dendrite
     networks:
@@ -108,10 +56,8 @@ services:
 
   federation_sender:
     hostname: federation_sender
-    image: matrixdotorg/dendrite:federationsender
+    image: matrixdotorg/dendrite-polylith:latest
-    command: [
+    command: federationsender
-      "--config=dendrite.yaml"
-    ]
     volumes:
       - ./config:/etc/dendrite
     networks:
@@ -119,10 +65,8 @@ services:
       
   key_server:
     hostname: key_server
-    image: matrixdotorg/dendrite:keyserver
+    image: matrixdotorg/dendrite-polylith:latest
-    command: [
+    command: keyserver
-      "--config=dendrite.yaml"
-    ]
     volumes:
       - ./config:/etc/dendrite
     networks:
@@ -130,10 +74,8 @@ services:
 
   signing_key_server:
     hostname: signing_key_server
-    image: matrixdotorg/dendrite:signingkeyserver
+    image: matrixdotorg/dendrite-polylith:latest
-    command: [
+    command: signingkeyserver
-      "--config=dendrite.yaml"
-    ]
     volumes:
       - ./config:/etc/dendrite
     networks:
@@ -141,10 +83,8 @@ services:
 
   user_api:
     hostname: user_api
-    image: matrixdotorg/dendrite:userapi
+    image: matrixdotorg/dendrite-polylith:latest
-    command: [
+    command: userapi
-        "--config=dendrite.yaml"
-    ]
     volumes:
         - ./config:/etc/dendrite
     networks:
@@ -152,10 +92,8 @@ services:
 
   appservice_api:
     hostname: appservice_api
-    image: matrixdotorg/dendrite:appservice
+    image: matrixdotorg/dendrite-polylith:latest
-    command: [
+    command: appservice
-      "--config=dendrite.yaml"
-    ]
     volumes:
       - ./config:/etc/dendrite
     networks:

build/docker/images-build.sh

@@ -2,20 +2,11 @@
 
 cd $(git rev-parse --show-toplevel)
 
-docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:latest .
+TAG=${1:-latest}
 
-docker build -t matrixdotorg/dendrite:monolith          --build-arg component=dendrite-monolith-server          -f build/docker/Dockerfile.component .
+echo "Building tag '${TAG}'"
 
-docker build -t matrixdotorg/dendrite:appservice        --build-arg component=dendrite-appservice-server        -f build/docker/Dockerfile.component .
+docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} .
-docker build -t matrixdotorg/dendrite:clientapi         --build-arg component=dendrite-client-api-server        -f build/docker/Dockerfile.component .
+
-docker build -t matrixdotorg/dendrite:clientproxy       --build-arg component=client-api-proxy                  -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-monolith:${TAG}  -f build/docker/Dockerfile.monolith .
-docker build -t matrixdotorg/dendrite:eduserver         --build-arg component=dendrite-edu-server               -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-polylith:${TAG}  -f build/docker/Dockerfile.polylith .
-docker build -t matrixdotorg/dendrite:federationapi     --build-arg component=dendrite-federation-api-server    -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:federationsender  --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:federationproxy   --build-arg component=federation-api-proxy              -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:keyserver         --build-arg component=dendrite-key-server               -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:mediaapi          --build-arg component=dendrite-media-api-server         -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:roomserver        --build-arg component=dendrite-room-server              -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:syncapi           --build-arg component=dendrite-sync-api-server          -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:signingkeyserver      --build-arg component=dendrite-signing-key-server    -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:userapi           --build-arg component=dendrite-user-api-server          -f build/docker/Dockerfile.component .

build/docker/images-pull.sh

@@ -1,17 +1,8 @@
 #!/bin/bash
 
-docker pull matrixdotorg/dendrite:monolith
+TAG=${1:-latest}
 
-docker pull matrixdotorg/dendrite:appservice
+echo "Pulling tag '${TAG}'"
-docker pull matrixdotorg/dendrite:clientapi
+
-docker pull matrixdotorg/dendrite:clientproxy
+docker pull matrixdotorg/dendrite-monolith:${TAG}
-docker pull matrixdotorg/dendrite:eduserver
+docker pull matrixdotorg/dendrite-polylith:${TAG}
-docker pull matrixdotorg/dendrite:federationapi
-docker pull matrixdotorg/dendrite:federationsender
-docker pull matrixdotorg/dendrite:federationproxy
-docker pull matrixdotorg/dendrite:keyserver
-docker pull matrixdotorg/dendrite:mediaapi
-docker pull matrixdotorg/dendrite:roomserver
-docker pull matrixdotorg/dendrite:syncapi
-docker pull matrixdotorg/dendrite:signingkeyserver
-docker pull matrixdotorg/dendrite:userapi

build/docker/images-push.sh

@@ -1,17 +1,8 @@
 #!/bin/bash
 
-docker push matrixdotorg/dendrite:monolith
+TAG=${1:-latest}
 
-docker push matrixdotorg/dendrite:appservice
+echo "Pushing tag '${TAG}'"
-docker push matrixdotorg/dendrite:clientapi
+
-docker push matrixdotorg/dendrite:clientproxy
+docker push matrixdotorg/dendrite-monolith:${TAG}
-docker push matrixdotorg/dendrite:eduserver
+docker push matrixdotorg/dendrite-polylith:${TAG}
-docker push matrixdotorg/dendrite:federationapi
-docker push matrixdotorg/dendrite:federationsender
-docker push matrixdotorg/dendrite:federationproxy
-docker push matrixdotorg/dendrite:keyserver
-docker push matrixdotorg/dendrite:mediaapi
-docker push matrixdotorg/dendrite:roomserver
-docker push matrixdotorg/dendrite:syncapi
-docker push matrixdotorg/dendrite:signingkeyserver
-docker push matrixdotorg/dendrite:userapi

build/docker/postgres/create_db.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
 
-for db in account device mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice e2ekey naffka; do
+for db in account device mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice naffka; do
     createdb -U dendrite -O dendrite dendrite_$db
 done

clientapi/routing/state.go

@@ -141,7 +141,7 @@ func OnIncomingStateRequest(ctx context.Context, device *userapi.Device, rsAPI a
 			util.GetLogger(ctx).WithError(err).Error("Failed to QueryMembershipForUser")
 			return jsonerror.InternalServerError()
 		}
-		for _, ev := range stateRes.StateEvents {
+		for _, ev := range stateAfterRes.StateEvents {
 			stateEvents = append(
 				stateEvents,
 				gomatrixserverlib.HeaderedToClientEvent(ev, gomatrixserverlib.FormatAll),

cmd/create-account/main.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/userapi/storage/accounts"
-	"github.com/matrix-org/dendrite/userapi/storage/devices"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -39,7 +38,6 @@ var (
 	username      = flag.String("username", "", "The user ID localpart to register e.g 'alice' in '@alice:localhost'.")
 	password      = flag.String("password", "", "Optional. The password to register with. If not specified, this account will be password-less.")
 	serverNameStr = flag.String("servername", "localhost", "The Matrix server domain which will form the domain part of the user ID.")
-	accessToken   = flag.String("token", "", "Optional. The desired access_token to have. If not specified, a random access_token will be made.")
 )
 
 func main() {
@@ -78,29 +76,5 @@ func main() {
 		os.Exit(1)
 	}
 
-	deviceDB, err := devices.NewDatabase(&config.DatabaseOptions{
+	fmt.Println("Created account")
-		ConnectionString: config.DataSource(*database),
-	}, serverName)
-	if err != nil {
-		fmt.Println(err.Error())
-		os.Exit(1)
-	}
-
-	if *accessToken == "" {
-		t := "token_" + *username
-		accessToken = &t
-	}
-
-	device, err := deviceDB.CreateDevice(
-		context.Background(), *username, nil, *accessToken, nil, "127.0.0.1", "",
-	)
-	if err != nil {
-		fmt.Println(err.Error())
-		os.Exit(1)
-	}
-
-	fmt.Println("Created account:")
-	fmt.Printf("user_id      = %s\n", device.UserID)
-	fmt.Printf("device_id    = %s\n", device.ID)
-	fmt.Printf("access_token = %s\n", device.AccessToken)
 }

cmd/dendrite-polylith-multi/main.go

@@ -0,0 +1,78 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"os"
+	"strings"
+
+	"github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities"
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/matrix-org/dendrite/internal/setup"
+	"github.com/sirupsen/logrus"
+)
+
+type entrypoint func(base *setup.BaseDendrite, cfg *config.Dendrite)
+
+func main() {
+	cfg := setup.ParseFlags(true)
+
+	component := ""
+	if flag.NFlag() > 0 {
+		component = flag.Arg(0) // ./dendrite-polylith-multi --config=... clientapi
+	} else if len(os.Args) > 1 {
+		component = os.Args[1] // ./dendrite-polylith-multi clientapi
+	}
+
+	components := map[string]entrypoint{
+		"appservice":       personalities.Appservice,
+		"clientapi":        personalities.ClientAPI,
+		"eduserver":        personalities.EDUServer,
+		"federationapi":    personalities.FederationAPI,
+		"federationsender": personalities.FederationSender,
+		"keyserver":        personalities.KeyServer,
+		"mediaapi":         personalities.MediaAPI,
+		"roomserver":       personalities.RoomServer,
+		"signingkeyserver": personalities.SigningKeyServer,
+		"syncapi":          personalities.SyncAPI,
+		"userapi":          personalities.UserAPI,
+	}
+
+	start, ok := components[component]
+	if !ok {
+		if component == "" {
+			logrus.Errorf("No component specified")
+			logrus.Info("The first argument on the command line must be the name of the component to run")
+		} else {
+			logrus.Errorf("Unknown component %q specified", component)
+		}
+
+		var list []string
+		for c := range components {
+			list = append(list, c)
+		}
+		logrus.Infof("Valid components: %s", strings.Join(list, ", "))
+
+		os.Exit(1)
+	}
+
+	logrus.Infof("Starting %q component", component)
+
+	base := setup.NewBaseDendrite(cfg, component, false) // TODO
+	defer base.Close()                                   // nolint: errcheck
+
+	start(base, cfg)
+}

cmd/dendrite-polylith-multi/personalities/appservice.go

@@ -1,4 +1,4 @@
-// Copyright 2018 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
 	"github.com/matrix-org/dendrite/appservice"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 )
 
-func main() {
+func Appservice(base *setup.BaseDendrite, cfg *config.Dendrite) {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "AppServiceAPI", true)
-
-	defer base.Close() // nolint: errcheck
 	userAPI := base.UserAPIClient()
 	rsAPI := base.RoomserverHTTPClient()
 

cmd/dendrite-polylith-multi/personalities/clientapi.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,20 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
 	"github.com/matrix-org/dendrite/clientapi"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/internal/transactions"
 )
 
-func main() {
+func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
-	cfg := setup.ParseFlags(false)
-
-	base := setup.NewBaseDendrite(cfg, "ClientAPI", true)
-	defer base.Close() // nolint: errcheck
-
 	accountDB := base.CreateAccountsDB()
 	federation := base.CreateFederationClient()
 

cmd/dendrite-polylith-multi/personalities/eduserver.go

@@ -1,3 +1,5 @@
+// Copyright 2020 The Matrix.org Foundation C.I.C.
+//
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -10,26 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
-	_ "net/http/pprof"
-
 	"github.com/matrix-org/dendrite/eduserver"
 	"github.com/matrix-org/dendrite/eduserver/cache"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
-	"github.com/sirupsen/logrus"
 )
 
-func main() {
+func EDUServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "EDUServerAPI", true)
-	defer func() {
-		if err := base.Close(); err != nil {
-			logrus.WithError(err).Warn("BaseDendrite close failed")
-		}
-	}()
-
 	intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient())
 	eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI)
 

cmd/dendrite-polylith-multi/personalities/federationapi.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
 	"github.com/matrix-org/dendrite/federationapi"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 )
 
-func main() {
+func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "FederationAPI", true)
-	defer base.Close() // nolint: errcheck
-
 	userAPI := base.UserAPIClient()
 	federation := base.CreateFederationClient()
 	serverKeyAPI := base.SigningKeyServerHTTPClient()

cmd/dendrite-polylith-multi/personalities/federationsender.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
 	"github.com/matrix-org/dendrite/federationsender"
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 )
 
-func main() {
+func FederationSender(base *setup.BaseDendrite, cfg *config.Dendrite) {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "FederationSender", true)
-	defer base.Close() // nolint: errcheck
-
 	federation := base.CreateFederationClient()
 
 	serverKeyAPI := base.SigningKeyServerHTTPClient()

cmd/dendrite-polylith-multi/personalities/keyserver.go

@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/keyserver"
 )
 
-func main() {
+func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "KeyServer", true)
-	defer base.Close() // nolint: errcheck
-
 	intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient())
 	intAPI.SetUserAPI(base.UserAPIClient())
 

cmd/dendrite-polylith-multi/personalities/mediaapi.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/mediaapi"
 )
 
-func main() {
+func MediaAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "MediaAPI", true)
-	defer base.Close() // nolint: errcheck
-
 	userAPI := base.UserAPIClient()
 	client := base.CreateClient()
 

cmd/dendrite-polylith-multi/personalities/roomserver.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/roomserver"
 )
 
-func main() {
+func RoomServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "RoomServerAPI", true)
-	defer base.Close() // nolint: errcheck
-
 	serverKeyAPI := base.SigningKeyServerHTTPClient()
 	keyRing := serverKeyAPI.KeyRing()
 

cmd/dendrite-polylith-multi/personalities/signingkeyserver.go

@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/signingkeyserver"
 )
 
-func main() {
+func SigningKeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "SigningKeyServer", true)
-	defer base.Close() // nolint: errcheck
-
 	federation := base.CreateFederationClient()
 
 	intAPI := signingkeyserver.NewInternalAPI(&base.Cfg.SigningKeyServer, federation, base.Caches)

cmd/dendrite-polylith-multi/personalities/syncapi.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,19 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/syncapi"
 )
 
-func main() {
+func SyncAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
-	cfg := setup.ParseFlags(false)
-
-	base := setup.NewBaseDendrite(cfg, "SyncAPI", true)
-	defer base.Close() // nolint: errcheck
-
 	userAPI := base.UserAPIClient()
 	federation := base.CreateFederationClient()
 

cmd/dendrite-polylith-multi/personalities/userapi.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Vector Creations Ltd
+// Copyright 2020 The Matrix.org Foundation C.I.C.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package personalities
 
 import (
+	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/setup"
 	"github.com/matrix-org/dendrite/userapi"
 )
 
-func main() {
+func UserAPI(base *setup.BaseDendrite, cfg *config.Dendrite) {
-	cfg := setup.ParseFlags(false)
-	base := setup.NewBaseDendrite(cfg, "UserAPI", true)
-	defer base.Close() // nolint: errcheck
-
 	accountDB := base.CreateAccountsDB()
 
 	userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient())

docs/INSTALL.md

@@ -24,7 +24,7 @@ use in production environments just yet!
 Dendrite requires:
 
 * Go 1.13 or higher
-* Postgres 9.5 or higher (if using Postgres databases, not needed for SQLite)
+* Postgres 9.6 or higher (if using Postgres databases, not needed for SQLite)
 
 If you want to run a polylith deployment, you also need:
 
@@ -93,12 +93,12 @@ brew services start kafka
 ### SQLite database setup
 
 Dendrite can use the built-in SQLite database engine for small setups.
-The SQLite databases do not need to be preconfigured - Dendrite will
+The SQLite databases do not need to be pre-built - Dendrite will
 create them automatically at startup.
 
 ### Postgres database setup
 
-Assuming that Postgres 9.5 (or later) is installed:
+Assuming that Postgres 9.6 (or later) is installed:
 
 * Create role, choosing a new password when prompted:
 
@@ -109,7 +109,7 @@ Assuming that Postgres 9.5 (or later) is installed:
 * Create the component databases:
 
   ```bash
-  for i in account device mediaapi syncapi roomserver signingkeyserver federationsender appservice e2ekey naffka; do
+  for i in mediaapi syncapi roomserver signingkeyserver federationsender appservice keyserver userapi_account userapi_device naffka; do
       sudo -u postgres createdb -O dendrite dendrite_$i
   done
   ```
@@ -135,8 +135,8 @@ Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Th
 
 * The `server_name` entry to reflect the hostname of your Dendrite server
 * The `database` lines with an updated connection string based on your
-  desired setup, e.g. replacing `component` with the name of the component:
+  desired setup, e.g. replacing `database` with the name of the database:
-  * For Postgres: `postgres://dendrite:password@localhost/component`
+  * For Postgres: `postgres://dendrite:password@localhost/database`
   * For SQLite on disk: `file:component.db` or `file:///path/to/component.db`
   * Postgres and SQLite can be mixed and matched.
 * The `use_naffka` option if using Naffka in a monolith deployment
@@ -147,6 +147,10 @@ then configuring `key_perspectives` (like `matrix.org` in the sample) can
 help to improve reliability considerably by allowing your homeserver to fetch
 public keys for dead homeservers from somewhere else.
 
+**WARNING:** Dendrite supports running all components from the same database in
+Postgres mode, but this is **NOT** a supported configuration with SQLite. When
+using SQLite, all components **MUST** use their own database file.
+
 ## Starting a monolith server
 
 It is possible to use Naffka as an in-process replacement to Kafka when using
@@ -167,30 +171,17 @@ as shown below, it will also listen for HTTPS connections on port 8448.
 
 The following contains scripts which will run all the required processes in order to point a Matrix client at Dendrite.
 
-### Client proxy
+### nginx (or other reverse proxy)
 
-This is what Matrix clients will talk to. If you use the script below, point
+This is what your clients and federated hosts will talk to. It must forward
-your client at `http://localhost:8008`.
+requests onto the correct API server based on URL:
 
-```bash
+* `/_matrix/client` to the client API server
-./bin/client-api-proxy \
+* `/_matrix/federation` to the federation API server
---bind-address ":8008" \
+* `/_matrix/key` to the federation API server
---client-api-server-url "http://localhost:7771" \
+* `/_matrix/media` to the media API server
---sync-api-server-url "http://localhost:7773" \
---media-api-server-url "http://localhost:7774" \
-```
 
-### Federation proxy
+See `docs/nginx/polylith-sample.conf` for a sample configuration.
-
-This is what Matrix servers will talk to. This is only required if you want
-to support federation.
-
-```bash
-./bin/federation-api-proxy \
---bind-address ":8448" \
---federation-api-url "http://localhost:7772" \
---media-api-server-url "http://localhost:7774" \
-```
 
 ### Client API server
 
@@ -198,7 +189,7 @@ This is what implements CS API endpoints. Clients talk to this via the proxy in
 order to send messages, create and join rooms, etc.
 
 ```bash
-./bin/dendrite-client-api-server --config=dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml clientapi
 ```
 
 ### Sync server
@@ -207,7 +198,7 @@ This is what implements `/sync` requests. Clients talk to this via the proxy
 in order to receive messages.
 
 ```bash
-./bin/dendrite-sync-api-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml syncapi
 ```
 
 ### Media server
@@ -216,7 +207,7 @@ This implements `/media` requests. Clients talk to this via the proxy in
 order to upload and retrieve media.
 
 ```bash
-./bin/dendrite-media-api-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml mediaapi
 ```
 
 ### Federation API server
@@ -226,7 +217,7 @@ order to send transactions.  This is only required if you want to support
 federation.
 
 ```bash
-./bin/dendrite-federation-api-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml federationapi
 ```
 
 ### Internal components
@@ -239,7 +230,7 @@ contacted by other components. This includes the following components.
 This is what implements the room DAG. Clients do not talk to this.
 
 ```bash
-./bin/dendrite-room-server --config=dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml roomserver
 ```
 
 #### Federation sender
@@ -248,7 +239,7 @@ This sends events from our users to other servers.  This is only required if
 you want to support federation.
 
 ```bash
-./bin/dendrite-federation-sender-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml federationsender
 ```
 
 #### Appservice server
@@ -259,7 +250,7 @@ running locally.  This is only required if you want to support running
 application services on your homeserver.
 
 ```bash
-./bin/dendrite-appservice-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml appservice
 ```
 
 #### Key server
@@ -267,7 +258,7 @@ application services on your homeserver.
 This manages end-to-end encryption keys for users.
 
 ```bash
-./bin/dendrite-key-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml keyserver
 ```
 
 #### Signing key server
@@ -275,7 +266,7 @@ This manages end-to-end encryption keys for users.
 This manages signing keys for servers.
 
 ```bash
-./bin/dendrite-signing-key-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml signingkeyserver
 ```
 
 #### EDU server
@@ -283,7 +274,7 @@ This manages signing keys for servers.
 This manages processing EDUs such as typing, send-to-device events and presence. Clients do not talk to
 
 ```bash
-./bin/dendrite-edu-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml eduserver
 ```
 
 #### User server
@@ -292,6 +283,6 @@ This manages user accounts, device access tokens and user account data,
 amongst other things.
 
 ```bash
-./bin/dendrite-user-api-server --config dendrite.yaml
+./bin/dendrite-polylith-multi --config=dendrite.yaml userapi
 ```
 

roomserver/internal/input/input_events.go

@@ -17,6 +17,7 @@
 package input
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 
@@ -26,6 +27,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 )
 
@@ -44,6 +46,28 @@ func (r *Inputer) processRoomEvent(
 	headered := input.Event
 	event := headered.Unwrap()
 
+	// if we have already got this event then do not process it again, if the input kind is an outlier.
+	// Outliers contain no extra information which may warrant a re-processing.
+	if input.Kind == api.KindOutlier {
+		evs, err2 := r.DB.EventsFromIDs(ctx, []string{event.EventID()})
+		if err2 == nil && len(evs) == 1 {
+			// check hash matches if we're on early room versions where the event ID was a random string
+			idFormat, err2 := headered.RoomVersion.EventIDFormat()
+			if err2 == nil {
+				switch idFormat {
+				case gomatrixserverlib.EventIDFormatV1:
+					if bytes.Equal(event.EventReference().EventSHA256, evs[0].EventReference().EventSHA256) {
+						util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring")
+						return event.EventID(), nil
+					}
+				default:
+					util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring")
+					return event.EventID(), nil
+				}
+			}
+		}
+	}
+
 	// Check that the event passes authentication checks and work out
 	// the numeric IDs for the auth events.
 	isRejected := false

roomserver/internal/input/input_latest_events.go

@@ -233,18 +233,6 @@ func (u *latestEventsUpdater) latestState() error {
 	if err != nil {
 		return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err)
 	}
-	if len(u.removed) > len(u.added) {
-		// This really shouldn't happen.
-		// TODO: What is ultimately the best way to handle this situation?
-		logrus.Errorf(
-			"Invalid state delta on event %q wants to remove %d state but only add %d state (between state snapshots %d and %d)",
-			u.event.EventID(), len(u.removed), len(u.added), u.oldStateNID, u.newStateNID,
-		)
-		u.added = u.added[:0]
-		u.removed = u.removed[:0]
-		u.newStateNID = u.oldStateNID
-		return nil
-	}
 
 	// Also work out the state before the event removes and the event
 	// adds.

roomserver/storage/shared/latest_events_updater.go

@@ -70,16 +70,14 @@ func (u *LatestEventsUpdater) CurrentStateSnapshotNID() types.StateSnapshotNID {
 	return u.currentStateSnapshotNID
 }
 
-// StorePreviousEvents implements types.RoomRecentEventsUpdater
+// StorePreviousEvents implements types.RoomRecentEventsUpdater - This must be called from a Writer
 func (u *LatestEventsUpdater) StorePreviousEvents(eventNID types.EventNID, previousEventReferences []gomatrixserverlib.EventReference) error {
-	return u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error {
+	for _, ref := range previousEventReferences {
-		for _, ref := range previousEventReferences {
+		if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, u.txn, ref.EventID, ref.EventSHA256, eventNID); err != nil {
-			if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, txn, ref.EventID, ref.EventSHA256, eventNID); err != nil {
+			return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err)
-				return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err)
-			}
 		}
-		return nil
+	}
-	})
+	return nil
 }
 
 // IsReferenced implements types.RoomRecentEventsUpdater

roomserver/storage/shared/storage.go

@@ -492,15 +492,32 @@ func (d *Database) StoreEvent(
 		if roomInfo == nil && len(prevEvents) > 0 {
 			return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("expected room %q to exist", event.RoomID())
 		}
+		// Create an updater - NB: on sqlite this WILL create a txn as we are directly calling the shared DB form of
+		// GetLatestEventsForUpdate - not via the SQLiteDatabase form which has `nil` txns. This
+		// function only does SELECTs though so the created txn (at this point) is just a read txn like
+		// any other so this is fine. If we ever update GetLatestEventsForUpdate or NewLatestEventsUpdater
+		// to do writes however then this will need to go inside `Writer.Do`.
 		updater, err = d.GetLatestEventsForUpdate(ctx, *roomInfo)
 		if err != nil {
 			return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("NewLatestEventsUpdater: %w", err)
 		}
-		if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil {
+		// Ensure that we atomically store prev events AND commit them. If we don't wrap StorePreviousEvents
-			return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("updater.StorePreviousEvents: %w", err)
+		// and EndTransaction in a writer then it's possible for a new write txn to be made between the two
+		// function calls which will then fail with 'database is locked'. This new write txn would HAVE to be
+		// something like SetRoomAlias/RemoveRoomAlias as normal input events are already done sequentially due to
+		// SupportsConcurrentRoomInputs() == false on sqlite, though this does not apply to setting room aliases
+		// as they don't go via InputRoomEvents
+		err = d.Writer.Do(d.DB, updater.txn, func(txn *sql.Tx) error {
+			if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil {
+				return fmt.Errorf("updater.StorePreviousEvents: %w", err)
+			}
+			succeeded := true
+			err = sqlutil.EndTransaction(updater, &succeeded)
+			return err
+		})
+		if err != nil {
+			return 0, types.StateAtEvent{}, nil, "", err
 		}
-		succeeded := true
-		err = sqlutil.EndTransaction(updater, &succeeded)
 	}
 
 	return roomNID, types.StateAtEvent{

sytest-blacklist

@@ -58,3 +58,6 @@ The only membership state included in a gapped incremental sync is for senders i
 Invited user can reject local invite after originator leaves
 Invited user can reject invite for empty room
 If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes
+
+# Blacklisted due to flakiness
+A prev_batch token from incremental sync can be used in the v1 messages API

sytest-whitelist

@@ -481,7 +481,7 @@ m.room.history_visibility == "joined" allows/forbids appropriately for Guest use
 m.room.history_visibility == "joined" allows/forbids appropriately for Real users
 POST rejects invalid utf-8 in JSON
 Users cannot kick users who have already left a room
-A prev_batch token from incremental sync can be used in the v1 messages API
 Event with an invalid signature in the send_join response should not cause room join to fail
 Inbound federation rejects typing notifications from wrong remote
 Should not be able to take over the room by pretending there is no PL event
+Can get rooms/{roomId}/state for a departed room (SPEC-216)