From a71360d0992b4bc120e9a98af7755e72e2fc3eb5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 20 Oct 2020 11:22:46 +0100 Subject: [PATCH 1/3] Update INSTALL.md (#1549) --- docs/INSTALL.md | 45 ++++++++++++++++++--------------------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index be12d7b86..d246e9221 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -93,7 +93,7 @@ brew services start kafka ### SQLite database setup Dendrite can use the built-in SQLite database engine for small setups. -The SQLite databases do not need to be preconfigured - Dendrite will +The SQLite databases do not need to be pre-built - Dendrite will create them automatically at startup. ### Postgres database setup @@ -109,7 +109,7 @@ Assuming that Postgres 9.5 (or later) is installed: * Create the component databases: ```bash - for i in account device mediaapi syncapi roomserver signingkeyserver federationsender appservice e2ekey naffka; do + for i in mediaapi syncapi roomserver signingkeyserver federationsender appservice keyserver userapi_account userapi_device naffka; do sudo -u postgres createdb -O dendrite dendrite_$i done ``` @@ -135,8 +135,8 @@ Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Th * The `server_name` entry to reflect the hostname of your Dendrite server * The `database` lines with an updated connection string based on your - desired setup, e.g. replacing `component` with the name of the component: - * For Postgres: `postgres://dendrite:password@localhost/component` + desired setup, e.g. replacing `database` with the name of the database: + * For Postgres: `postgres://dendrite:password@localhost/database` * For SQLite on disk: `file:component.db` or `file:///path/to/component.db` * Postgres and SQLite can be mixed and matched. * The `use_naffka` option if using Naffka in a monolith deployment @@ -147,6 +147,10 @@ then configuring `key_perspectives` (like `matrix.org` in the sample) can help to improve reliability considerably by allowing your homeserver to fetch public keys for dead homeservers from somewhere else. +**WARNING:** Dendrite supports running all components from the same database in +Postgres mode, but this is **NOT** a supported configuration with SQLite. When +using SQLite, all components **MUST** use their own database file. + ## Starting a monolith server It is possible to use Naffka as an in-process replacement to Kafka when using @@ -167,30 +171,17 @@ as shown below, it will also listen for HTTPS connections on port 8448. The following contains scripts which will run all the required processes in order to point a Matrix client at Dendrite. -### Client proxy +### nginx (or other reverse proxy) -This is what Matrix clients will talk to. If you use the script below, point -your client at `http://localhost:8008`. +This is what your clients and federated hosts will talk to. It must forward +requests onto the correct API server based on URL: -```bash -./bin/client-api-proxy \ ---bind-address ":8008" \ ---client-api-server-url "http://localhost:7771" \ ---sync-api-server-url "http://localhost:7773" \ ---media-api-server-url "http://localhost:7774" \ -``` +* `/_matrix/client` to the client API server +* `/_matrix/federation` to the federation API server +* `/_matrix/key` to the federation API server +* `/_matrix/media` to the media API server -### Federation proxy - -This is what Matrix servers will talk to. This is only required if you want -to support federation. - -```bash -./bin/federation-api-proxy \ ---bind-address ":8448" \ ---federation-api-url "http://localhost:7772" \ ---media-api-server-url "http://localhost:7774" \ -``` +See `docs/nginx/polylith-sample.conf` for a sample configuration. ### Client API server @@ -198,7 +189,7 @@ This is what implements CS API endpoints. Clients talk to this via the proxy in order to send messages, create and join rooms, etc. ```bash -./bin/dendrite-client-api-server --config=dendrite.yaml +./bin/dendrite-client-api-server --config dendrite.yaml ``` ### Sync server @@ -239,7 +230,7 @@ contacted by other components. This includes the following components. This is what implements the room DAG. Clients do not talk to this. ```bash -./bin/dendrite-room-server --config=dendrite.yaml +./bin/dendrite-room-server --config dendrite.yaml ``` #### Federation sender From 92982a402ff553fe6d9179859a96ec33e2f09929 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 20 Oct 2020 11:34:22 +0100 Subject: [PATCH 2/3] Update Docker (#1542) * Separate Docker images, rather than tags * Allow specifying tag to build/push/pull * Include goose in Docker builds --- build/docker/Dockerfile.component | 1 + build/docker/docker-compose.monolith.yml | 2 +- build/docker/docker-compose.polylith.yml | 61 +++++------------------- build/docker/images-build.sh | 32 +++++++------ build/docker/images-pull.sh | 30 ++++++------ build/docker/images-push.sh | 30 ++++++------ 6 files changed, 62 insertions(+), 94 deletions(-) diff --git a/build/docker/Dockerfile.component b/build/docker/Dockerfile.component index 13634391a..1acf510fb 100644 --- a/build/docker/Dockerfile.component +++ b/build/docker/Dockerfile.component @@ -6,6 +6,7 @@ ARG component=monolith ENV entrypoint=${component} COPY --from=base /build/bin/${component} /usr/bin +COPY --from=base /build/bin/goose /usr/bin VOLUME /etc/dendrite WORKDIR /etc/dendrite diff --git a/build/docker/docker-compose.monolith.yml b/build/docker/docker-compose.monolith.yml index 336a43984..7d63e1c65 100644 --- a/build/docker/docker-compose.monolith.yml +++ b/build/docker/docker-compose.monolith.yml @@ -2,7 +2,7 @@ version: "3.4" services: monolith: hostname: monolith - image: matrixdotorg/dendrite:monolith + image: matrixdotorg/dendrite-monolith:latest command: [ "--config=dendrite.yaml", "--tls-cert=server.crt", diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index 8a4c50e06..e8da9c24a 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -1,28 +1,8 @@ version: "3.4" services: - client_api_proxy: - hostname: client_api_proxy - image: matrixdotorg/dendrite:clientproxy - command: [ - "--bind-address=:8008", - "--client-api-server-url=http://client_api:8071", - "--sync-api-server-url=http://sync_api:8073", - "--media-api-server-url=http://media_api:8074" - ] - volumes: - - ./config:/etc/dendrite - networks: - - internal - depends_on: - - sync_api - - client_api - - media_api - ports: - - "8008:8008" - client_api: hostname: client_api - image: matrixdotorg/dendrite:clientapi + image: matrixdotorg/dendrite-clientapi:latest command: [ "--config=dendrite.yaml" ] @@ -34,7 +14,7 @@ services: media_api: hostname: media_api - image: matrixdotorg/dendrite:mediaapi + image: matrixdotorg/dendrite-mediaapi:latest command: [ "--config=dendrite.yaml" ] @@ -45,7 +25,7 @@ services: sync_api: hostname: sync_api - image: matrixdotorg/dendrite:syncapi + image: matrixdotorg/dendrite-syncapi:latest command: [ "--config=dendrite.yaml" ] @@ -56,7 +36,7 @@ services: room_server: hostname: room_server - image: matrixdotorg/dendrite:roomserver + image: matrixdotorg/dendrite-roomserver:latest command: [ "--config=dendrite.yaml" ] @@ -67,7 +47,7 @@ services: edu_server: hostname: edu_server - image: matrixdotorg/dendrite:eduserver + image: matrixdotorg/dendrite-eduserver:latest command: [ "--config=dendrite.yaml" ] @@ -76,28 +56,9 @@ services: networks: - internal - federation_api_proxy: - hostname: federation_api_proxy - image: matrixdotorg/dendrite:federationproxy - command: [ - "--bind-address=:8448", - "--federation-api-url=http://federation_api:8072", - "--media-api-server-url=http://media_api:8074" - ] - volumes: - - ./config:/etc/dendrite - depends_on: - - federation_api - - federation_sender - - media_api - networks: - - internal - ports: - - "8448:8448" - federation_api: hostname: federation_api - image: matrixdotorg/dendrite:federationapi + image: matrixdotorg/dendrite-federationapi:latest command: [ "--config=dendrite.yaml" ] @@ -108,7 +69,7 @@ services: federation_sender: hostname: federation_sender - image: matrixdotorg/dendrite:federationsender + image: matrixdotorg/dendrite-federationsender:latest command: [ "--config=dendrite.yaml" ] @@ -119,7 +80,7 @@ services: key_server: hostname: key_server - image: matrixdotorg/dendrite:keyserver + image: matrixdotorg/dendrite-keyserver:latest command: [ "--config=dendrite.yaml" ] @@ -130,7 +91,7 @@ services: signing_key_server: hostname: signing_key_server - image: matrixdotorg/dendrite:signingkeyserver + image: matrixdotorg/dendrite-signingkeyserver:latest command: [ "--config=dendrite.yaml" ] @@ -141,7 +102,7 @@ services: user_api: hostname: user_api - image: matrixdotorg/dendrite:userapi + image: matrixdotorg/dendrite-userapi:latest command: [ "--config=dendrite.yaml" ] @@ -152,7 +113,7 @@ services: appservice_api: hostname: appservice_api - image: matrixdotorg/dendrite:appservice + image: matrixdotorg/dendrite-appservice:latest command: [ "--config=dendrite.yaml" ] diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index d72bac214..daad63be0 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -2,20 +2,22 @@ cd $(git rev-parse --show-toplevel) -docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:latest . +TAG=${1:-latest} -docker build -t matrixdotorg/dendrite:monolith --build-arg component=dendrite-monolith-server -f build/docker/Dockerfile.component . +echo "Building tag '${TAG}'" -docker build -t matrixdotorg/dendrite:appservice --build-arg component=dendrite-appservice-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:clientapi --build-arg component=dendrite-client-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:clientproxy --build-arg component=client-api-proxy -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:eduserver --build-arg component=dendrite-edu-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationapi --build-arg component=dendrite-federation-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationsender --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationproxy --build-arg component=federation-api-proxy -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:keyserver --build-arg component=dendrite-key-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:signingkeyserver --build-arg component=dendrite-signing-key-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:userapi --build-arg component=dendrite-user-api-server -f build/docker/Dockerfile.component . +docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} . + +docker build -t matrixdotorg/dendrite-monolith:${TAG} --build-arg component=dendrite-monolith-server -f build/docker/Dockerfile.component . + +docker build -t matrixdotorg/dendrite-appservice:${TAG} --build-arg component=dendrite-appservice-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-clientapi:${TAG} --build-arg component=dendrite-client-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-eduserver:${TAG} --build-arg component=dendrite-edu-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-federationapi:${TAG} --build-arg component=dendrite-federation-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-federationsender:${TAG} --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-keyserver:${TAG} --build-arg component=dendrite-key-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-mediaapi:${TAG} --build-arg component=dendrite-media-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-roomserver:${TAG} --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-syncapi:${TAG} --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-signingkeyserver:${TAG} --build-arg component=dendrite-signing-key-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-userapi:${TAG} --build-arg component=dendrite-user-api-server -f build/docker/Dockerfile.component . diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh index be9185464..e3284a2a6 100755 --- a/build/docker/images-pull.sh +++ b/build/docker/images-pull.sh @@ -1,17 +1,19 @@ #!/bin/bash -docker pull matrixdotorg/dendrite:monolith +TAG=${1:-latest} -docker pull matrixdotorg/dendrite:appservice -docker pull matrixdotorg/dendrite:clientapi -docker pull matrixdotorg/dendrite:clientproxy -docker pull matrixdotorg/dendrite:eduserver -docker pull matrixdotorg/dendrite:federationapi -docker pull matrixdotorg/dendrite:federationsender -docker pull matrixdotorg/dendrite:federationproxy -docker pull matrixdotorg/dendrite:keyserver -docker pull matrixdotorg/dendrite:mediaapi -docker pull matrixdotorg/dendrite:roomserver -docker pull matrixdotorg/dendrite:syncapi -docker pull matrixdotorg/dendrite:signingkeyserver -docker pull matrixdotorg/dendrite:userapi +echo "Pulling tag '${TAG}'" + +docker pull matrixdotorg/dendrite-monolith:${TAG} + +docker pull matrixdotorg/dendrite-appservice:${TAG} +docker pull matrixdotorg/dendrite-clientapi:${TAG} +docker pull matrixdotorg/dendrite-eduserver:${TAG} +docker pull matrixdotorg/dendrite-federationapi:${TAG} +docker pull matrixdotorg/dendrite-federationsender:${TAG} +docker pull matrixdotorg/dendrite-keyserver:${TAG} +docker pull matrixdotorg/dendrite-mediaapi:${TAG} +docker pull matrixdotorg/dendrite-roomserver:${TAG} +docker pull matrixdotorg/dendrite-syncapi:${TAG} +docker pull matrixdotorg/dendrite-signingkeyserver:${TAG} +docker pull matrixdotorg/dendrite-userapi:${TAG} diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index 64920171b..e4eb773ac 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -1,17 +1,19 @@ #!/bin/bash -docker push matrixdotorg/dendrite:monolith +TAG=${1:-latest} -docker push matrixdotorg/dendrite:appservice -docker push matrixdotorg/dendrite:clientapi -docker push matrixdotorg/dendrite:clientproxy -docker push matrixdotorg/dendrite:eduserver -docker push matrixdotorg/dendrite:federationapi -docker push matrixdotorg/dendrite:federationsender -docker push matrixdotorg/dendrite:federationproxy -docker push matrixdotorg/dendrite:keyserver -docker push matrixdotorg/dendrite:mediaapi -docker push matrixdotorg/dendrite:roomserver -docker push matrixdotorg/dendrite:syncapi -docker push matrixdotorg/dendrite:signingkeyserver -docker push matrixdotorg/dendrite:userapi +echo "Pushing tag '${TAG}'" + +docker push matrixdotorg/dendrite-monolith:${TAG} + +docker push matrixdotorg/dendrite-appservice:${TAG} +docker push matrixdotorg/dendrite-clientapi:${TAG} +docker push matrixdotorg/dendrite-eduserver:${TAG} +docker push matrixdotorg/dendrite-federationapi:${TAG} +docker push matrixdotorg/dendrite-federationsender:${TAG} +docker push matrixdotorg/dendrite-keyserver:${TAG} +docker push matrixdotorg/dendrite-mediaapi:${TAG} +docker push matrixdotorg/dendrite-roomserver:${TAG} +docker push matrixdotorg/dendrite-syncapi:${TAG} +docker push matrixdotorg/dendrite-signingkeyserver:${TAG} +docker push matrixdotorg/dendrite-userapi:${TAG} From eb86e2b336e8feaf54eb7fe688c896aa11bd5329 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 20 Oct 2020 11:42:54 +0100 Subject: [PATCH 3/3] Fix sqlite locking bugs present on sytest (#1543) * Fix sqite locking bugs present on sytest Comments do the explaining. * Fix deadlock in sqlite mode Caused by starting a writer whilst within a writer * Only complain about invalid state deltas for non-overwrite events * Do not re-process outlier unnecessarily --- roomserver/internal/input/input_events.go | 24 ++++++++++++++++++ .../internal/input/input_latest_events.go | 2 +- .../storage/shared/latest_events_updater.go | 14 +++++------ roomserver/storage/shared/storage.go | 25 ++++++++++++++++--- 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 67031609f..6a5d9d264 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -17,6 +17,7 @@ package input import ( + "bytes" "context" "fmt" @@ -26,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) @@ -44,6 +46,28 @@ func (r *Inputer) processRoomEvent( headered := input.Event event := headered.Unwrap() + // if we have already got this event then do not process it again, if the input kind is an outlier. + // Outliers contain no extra information which may warrant a re-processing. + if input.Kind == api.KindOutlier { + evs, err := r.DB.EventsFromIDs(ctx, []string{event.EventID()}) + if err == nil && len(evs) == 1 { + // check hash matches if we're on early room versions where the event ID was a random string + idFormat, err := headered.RoomVersion.EventIDFormat() + if err == nil { + switch idFormat { + case gomatrixserverlib.EventIDFormatV1: + if bytes.Equal(event.EventReference().EventSHA256, evs[0].EventReference().EventSHA256) { + util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring") + return event.EventID(), nil + } + default: + util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring") + return event.EventID(), nil + } + } + } + } + // Check that the event passes authentication checks and work out // the numeric IDs for the auth events. isRejected := false diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index 5adcd0877..f76b0a0b4 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -233,7 +233,7 @@ func (u *latestEventsUpdater) latestState() error { if err != nil { return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err) } - if len(u.removed) > len(u.added) { + if !u.stateAtEvent.Overwrite && len(u.removed) > len(u.added) { // This really shouldn't happen. // TODO: What is ultimately the best way to handle this situation? logrus.Errorf( diff --git a/roomserver/storage/shared/latest_events_updater.go b/roomserver/storage/shared/latest_events_updater.go index b316f639d..8825dc464 100644 --- a/roomserver/storage/shared/latest_events_updater.go +++ b/roomserver/storage/shared/latest_events_updater.go @@ -70,16 +70,14 @@ func (u *LatestEventsUpdater) CurrentStateSnapshotNID() types.StateSnapshotNID { return u.currentStateSnapshotNID } -// StorePreviousEvents implements types.RoomRecentEventsUpdater +// StorePreviousEvents implements types.RoomRecentEventsUpdater - This must be called from a Writer func (u *LatestEventsUpdater) StorePreviousEvents(eventNID types.EventNID, previousEventReferences []gomatrixserverlib.EventReference) error { - return u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error { - for _, ref := range previousEventReferences { - if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { - return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err) - } + for _, ref := range previousEventReferences { + if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, u.txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { + return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err) } - return nil - }) + } + return nil } // IsReferenced implements types.RoomRecentEventsUpdater diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index f2be8b3cf..51dcb8887 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -492,15 +492,32 @@ func (d *Database) StoreEvent( if roomInfo == nil && len(prevEvents) > 0 { return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("expected room %q to exist", event.RoomID()) } + // Create an updater - NB: on sqlite this WILL create a txn as we are directly calling the shared DB form of + // GetLatestEventsForUpdate - not via the SQLiteDatabase form which has `nil` txns. This + // function only does SELECTs though so the created txn (at this point) is just a read txn like + // any other so this is fine. If we ever update GetLatestEventsForUpdate or NewLatestEventsUpdater + // to do writes however then this will need to go inside `Writer.Do`. updater, err = d.GetLatestEventsForUpdate(ctx, *roomInfo) if err != nil { return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("NewLatestEventsUpdater: %w", err) } - if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil { - return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("updater.StorePreviousEvents: %w", err) + // Ensure that we atomically store prev events AND commit them. If we don't wrap StorePreviousEvents + // and EndTransaction in a writer then it's possible for a new write txn to be made between the two + // function calls which will then fail with 'database is locked'. This new write txn would HAVE to be + // something like SetRoomAlias/RemoveRoomAlias as normal input events are already done sequentially due to + // SupportsConcurrentRoomInputs() == false on sqlite, though this does not apply to setting room aliases + // as they don't go via InputRoomEvents + err = d.Writer.Do(d.DB, updater.txn, func(txn *sql.Tx) error { + if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil { + return fmt.Errorf("updater.StorePreviousEvents: %w", err) + } + succeeded := true + err = sqlutil.EndTransaction(updater, &succeeded) + return err + }) + if err != nil { + return 0, types.StateAtEvent{}, nil, "", err } - succeeded := true - err = sqlutil.EndTransaction(updater, &succeeded) } return roomNID, types.StateAtEvent{