From a71360d0992b4bc120e9a98af7755e72e2fc3eb5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 20 Oct 2020 11:22:46 +0100 Subject: [PATCH 01/10] Update INSTALL.md (#1549) --- docs/INSTALL.md | 45 ++++++++++++++++++--------------------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index be12d7b86..d246e9221 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -93,7 +93,7 @@ brew services start kafka ### SQLite database setup Dendrite can use the built-in SQLite database engine for small setups. -The SQLite databases do not need to be preconfigured - Dendrite will +The SQLite databases do not need to be pre-built - Dendrite will create them automatically at startup. ### Postgres database setup @@ -109,7 +109,7 @@ Assuming that Postgres 9.5 (or later) is installed: * Create the component databases: ```bash - for i in account device mediaapi syncapi roomserver signingkeyserver federationsender appservice e2ekey naffka; do + for i in mediaapi syncapi roomserver signingkeyserver federationsender appservice keyserver userapi_account userapi_device naffka; do sudo -u postgres createdb -O dendrite dendrite_$i done ``` @@ -135,8 +135,8 @@ Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Th * The `server_name` entry to reflect the hostname of your Dendrite server * The `database` lines with an updated connection string based on your - desired setup, e.g. replacing `component` with the name of the component: - * For Postgres: `postgres://dendrite:password@localhost/component` + desired setup, e.g. replacing `database` with the name of the database: + * For Postgres: `postgres://dendrite:password@localhost/database` * For SQLite on disk: `file:component.db` or `file:///path/to/component.db` * Postgres and SQLite can be mixed and matched. * The `use_naffka` option if using Naffka in a monolith deployment @@ -147,6 +147,10 @@ then configuring `key_perspectives` (like `matrix.org` in the sample) can help to improve reliability considerably by allowing your homeserver to fetch public keys for dead homeservers from somewhere else. +**WARNING:** Dendrite supports running all components from the same database in +Postgres mode, but this is **NOT** a supported configuration with SQLite. When +using SQLite, all components **MUST** use their own database file. + ## Starting a monolith server It is possible to use Naffka as an in-process replacement to Kafka when using @@ -167,30 +171,17 @@ as shown below, it will also listen for HTTPS connections on port 8448. The following contains scripts which will run all the required processes in order to point a Matrix client at Dendrite. -### Client proxy +### nginx (or other reverse proxy) -This is what Matrix clients will talk to. If you use the script below, point -your client at `http://localhost:8008`. +This is what your clients and federated hosts will talk to. It must forward +requests onto the correct API server based on URL: -```bash -./bin/client-api-proxy \ ---bind-address ":8008" \ ---client-api-server-url "http://localhost:7771" \ ---sync-api-server-url "http://localhost:7773" \ ---media-api-server-url "http://localhost:7774" \ -``` +* `/_matrix/client` to the client API server +* `/_matrix/federation` to the federation API server +* `/_matrix/key` to the federation API server +* `/_matrix/media` to the media API server -### Federation proxy - -This is what Matrix servers will talk to. This is only required if you want -to support federation. - -```bash -./bin/federation-api-proxy \ ---bind-address ":8448" \ ---federation-api-url "http://localhost:7772" \ ---media-api-server-url "http://localhost:7774" \ -``` +See `docs/nginx/polylith-sample.conf` for a sample configuration. ### Client API server @@ -198,7 +189,7 @@ This is what implements CS API endpoints. Clients talk to this via the proxy in order to send messages, create and join rooms, etc. ```bash -./bin/dendrite-client-api-server --config=dendrite.yaml +./bin/dendrite-client-api-server --config dendrite.yaml ``` ### Sync server @@ -239,7 +230,7 @@ contacted by other components. This includes the following components. This is what implements the room DAG. Clients do not talk to this. ```bash -./bin/dendrite-room-server --config=dendrite.yaml +./bin/dendrite-room-server --config dendrite.yaml ``` #### Federation sender From 92982a402ff553fe6d9179859a96ec33e2f09929 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 20 Oct 2020 11:34:22 +0100 Subject: [PATCH 02/10] Update Docker (#1542) * Separate Docker images, rather than tags * Allow specifying tag to build/push/pull * Include goose in Docker builds --- build/docker/Dockerfile.component | 1 + build/docker/docker-compose.monolith.yml | 2 +- build/docker/docker-compose.polylith.yml | 61 +++++------------------- build/docker/images-build.sh | 32 +++++++------ build/docker/images-pull.sh | 30 ++++++------ build/docker/images-push.sh | 30 ++++++------ 6 files changed, 62 insertions(+), 94 deletions(-) diff --git a/build/docker/Dockerfile.component b/build/docker/Dockerfile.component index 13634391a..1acf510fb 100644 --- a/build/docker/Dockerfile.component +++ b/build/docker/Dockerfile.component @@ -6,6 +6,7 @@ ARG component=monolith ENV entrypoint=${component} COPY --from=base /build/bin/${component} /usr/bin +COPY --from=base /build/bin/goose /usr/bin VOLUME /etc/dendrite WORKDIR /etc/dendrite diff --git a/build/docker/docker-compose.monolith.yml b/build/docker/docker-compose.monolith.yml index 336a43984..7d63e1c65 100644 --- a/build/docker/docker-compose.monolith.yml +++ b/build/docker/docker-compose.monolith.yml @@ -2,7 +2,7 @@ version: "3.4" services: monolith: hostname: monolith - image: matrixdotorg/dendrite:monolith + image: matrixdotorg/dendrite-monolith:latest command: [ "--config=dendrite.yaml", "--tls-cert=server.crt", diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index 8a4c50e06..e8da9c24a 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -1,28 +1,8 @@ version: "3.4" services: - client_api_proxy: - hostname: client_api_proxy - image: matrixdotorg/dendrite:clientproxy - command: [ - "--bind-address=:8008", - "--client-api-server-url=http://client_api:8071", - "--sync-api-server-url=http://sync_api:8073", - "--media-api-server-url=http://media_api:8074" - ] - volumes: - - ./config:/etc/dendrite - networks: - - internal - depends_on: - - sync_api - - client_api - - media_api - ports: - - "8008:8008" - client_api: hostname: client_api - image: matrixdotorg/dendrite:clientapi + image: matrixdotorg/dendrite-clientapi:latest command: [ "--config=dendrite.yaml" ] @@ -34,7 +14,7 @@ services: media_api: hostname: media_api - image: matrixdotorg/dendrite:mediaapi + image: matrixdotorg/dendrite-mediaapi:latest command: [ "--config=dendrite.yaml" ] @@ -45,7 +25,7 @@ services: sync_api: hostname: sync_api - image: matrixdotorg/dendrite:syncapi + image: matrixdotorg/dendrite-syncapi:latest command: [ "--config=dendrite.yaml" ] @@ -56,7 +36,7 @@ services: room_server: hostname: room_server - image: matrixdotorg/dendrite:roomserver + image: matrixdotorg/dendrite-roomserver:latest command: [ "--config=dendrite.yaml" ] @@ -67,7 +47,7 @@ services: edu_server: hostname: edu_server - image: matrixdotorg/dendrite:eduserver + image: matrixdotorg/dendrite-eduserver:latest command: [ "--config=dendrite.yaml" ] @@ -76,28 +56,9 @@ services: networks: - internal - federation_api_proxy: - hostname: federation_api_proxy - image: matrixdotorg/dendrite:federationproxy - command: [ - "--bind-address=:8448", - "--federation-api-url=http://federation_api:8072", - "--media-api-server-url=http://media_api:8074" - ] - volumes: - - ./config:/etc/dendrite - depends_on: - - federation_api - - federation_sender - - media_api - networks: - - internal - ports: - - "8448:8448" - federation_api: hostname: federation_api - image: matrixdotorg/dendrite:federationapi + image: matrixdotorg/dendrite-federationapi:latest command: [ "--config=dendrite.yaml" ] @@ -108,7 +69,7 @@ services: federation_sender: hostname: federation_sender - image: matrixdotorg/dendrite:federationsender + image: matrixdotorg/dendrite-federationsender:latest command: [ "--config=dendrite.yaml" ] @@ -119,7 +80,7 @@ services: key_server: hostname: key_server - image: matrixdotorg/dendrite:keyserver + image: matrixdotorg/dendrite-keyserver:latest command: [ "--config=dendrite.yaml" ] @@ -130,7 +91,7 @@ services: signing_key_server: hostname: signing_key_server - image: matrixdotorg/dendrite:signingkeyserver + image: matrixdotorg/dendrite-signingkeyserver:latest command: [ "--config=dendrite.yaml" ] @@ -141,7 +102,7 @@ services: user_api: hostname: user_api - image: matrixdotorg/dendrite:userapi + image: matrixdotorg/dendrite-userapi:latest command: [ "--config=dendrite.yaml" ] @@ -152,7 +113,7 @@ services: appservice_api: hostname: appservice_api - image: matrixdotorg/dendrite:appservice + image: matrixdotorg/dendrite-appservice:latest command: [ "--config=dendrite.yaml" ] diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index d72bac214..daad63be0 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -2,20 +2,22 @@ cd $(git rev-parse --show-toplevel) -docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:latest . +TAG=${1:-latest} -docker build -t matrixdotorg/dendrite:monolith --build-arg component=dendrite-monolith-server -f build/docker/Dockerfile.component . +echo "Building tag '${TAG}'" -docker build -t matrixdotorg/dendrite:appservice --build-arg component=dendrite-appservice-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:clientapi --build-arg component=dendrite-client-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:clientproxy --build-arg component=client-api-proxy -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:eduserver --build-arg component=dendrite-edu-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationapi --build-arg component=dendrite-federation-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationsender --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationproxy --build-arg component=federation-api-proxy -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:keyserver --build-arg component=dendrite-key-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:signingkeyserver --build-arg component=dendrite-signing-key-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite:userapi --build-arg component=dendrite-user-api-server -f build/docker/Dockerfile.component . +docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} . + +docker build -t matrixdotorg/dendrite-monolith:${TAG} --build-arg component=dendrite-monolith-server -f build/docker/Dockerfile.component . + +docker build -t matrixdotorg/dendrite-appservice:${TAG} --build-arg component=dendrite-appservice-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-clientapi:${TAG} --build-arg component=dendrite-client-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-eduserver:${TAG} --build-arg component=dendrite-edu-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-federationapi:${TAG} --build-arg component=dendrite-federation-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-federationsender:${TAG} --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-keyserver:${TAG} --build-arg component=dendrite-key-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-mediaapi:${TAG} --build-arg component=dendrite-media-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-roomserver:${TAG} --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-syncapi:${TAG} --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-signingkeyserver:${TAG} --build-arg component=dendrite-signing-key-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-userapi:${TAG} --build-arg component=dendrite-user-api-server -f build/docker/Dockerfile.component . diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh index be9185464..e3284a2a6 100755 --- a/build/docker/images-pull.sh +++ b/build/docker/images-pull.sh @@ -1,17 +1,19 @@ #!/bin/bash -docker pull matrixdotorg/dendrite:monolith +TAG=${1:-latest} -docker pull matrixdotorg/dendrite:appservice -docker pull matrixdotorg/dendrite:clientapi -docker pull matrixdotorg/dendrite:clientproxy -docker pull matrixdotorg/dendrite:eduserver -docker pull matrixdotorg/dendrite:federationapi -docker pull matrixdotorg/dendrite:federationsender -docker pull matrixdotorg/dendrite:federationproxy -docker pull matrixdotorg/dendrite:keyserver -docker pull matrixdotorg/dendrite:mediaapi -docker pull matrixdotorg/dendrite:roomserver -docker pull matrixdotorg/dendrite:syncapi -docker pull matrixdotorg/dendrite:signingkeyserver -docker pull matrixdotorg/dendrite:userapi +echo "Pulling tag '${TAG}'" + +docker pull matrixdotorg/dendrite-monolith:${TAG} + +docker pull matrixdotorg/dendrite-appservice:${TAG} +docker pull matrixdotorg/dendrite-clientapi:${TAG} +docker pull matrixdotorg/dendrite-eduserver:${TAG} +docker pull matrixdotorg/dendrite-federationapi:${TAG} +docker pull matrixdotorg/dendrite-federationsender:${TAG} +docker pull matrixdotorg/dendrite-keyserver:${TAG} +docker pull matrixdotorg/dendrite-mediaapi:${TAG} +docker pull matrixdotorg/dendrite-roomserver:${TAG} +docker pull matrixdotorg/dendrite-syncapi:${TAG} +docker pull matrixdotorg/dendrite-signingkeyserver:${TAG} +docker pull matrixdotorg/dendrite-userapi:${TAG} diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index 64920171b..e4eb773ac 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -1,17 +1,19 @@ #!/bin/bash -docker push matrixdotorg/dendrite:monolith +TAG=${1:-latest} -docker push matrixdotorg/dendrite:appservice -docker push matrixdotorg/dendrite:clientapi -docker push matrixdotorg/dendrite:clientproxy -docker push matrixdotorg/dendrite:eduserver -docker push matrixdotorg/dendrite:federationapi -docker push matrixdotorg/dendrite:federationsender -docker push matrixdotorg/dendrite:federationproxy -docker push matrixdotorg/dendrite:keyserver -docker push matrixdotorg/dendrite:mediaapi -docker push matrixdotorg/dendrite:roomserver -docker push matrixdotorg/dendrite:syncapi -docker push matrixdotorg/dendrite:signingkeyserver -docker push matrixdotorg/dendrite:userapi +echo "Pushing tag '${TAG}'" + +docker push matrixdotorg/dendrite-monolith:${TAG} + +docker push matrixdotorg/dendrite-appservice:${TAG} +docker push matrixdotorg/dendrite-clientapi:${TAG} +docker push matrixdotorg/dendrite-eduserver:${TAG} +docker push matrixdotorg/dendrite-federationapi:${TAG} +docker push matrixdotorg/dendrite-federationsender:${TAG} +docker push matrixdotorg/dendrite-keyserver:${TAG} +docker push matrixdotorg/dendrite-mediaapi:${TAG} +docker push matrixdotorg/dendrite-roomserver:${TAG} +docker push matrixdotorg/dendrite-syncapi:${TAG} +docker push matrixdotorg/dendrite-signingkeyserver:${TAG} +docker push matrixdotorg/dendrite-userapi:${TAG} From eb86e2b336e8feaf54eb7fe688c896aa11bd5329 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 20 Oct 2020 11:42:54 +0100 Subject: [PATCH 03/10] Fix sqlite locking bugs present on sytest (#1543) * Fix sqite locking bugs present on sytest Comments do the explaining. * Fix deadlock in sqlite mode Caused by starting a writer whilst within a writer * Only complain about invalid state deltas for non-overwrite events * Do not re-process outlier unnecessarily --- roomserver/internal/input/input_events.go | 24 ++++++++++++++++++ .../internal/input/input_latest_events.go | 2 +- .../storage/shared/latest_events_updater.go | 14 +++++------ roomserver/storage/shared/storage.go | 25 ++++++++++++++++--- 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 67031609f..6a5d9d264 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -17,6 +17,7 @@ package input import ( + "bytes" "context" "fmt" @@ -26,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) @@ -44,6 +46,28 @@ func (r *Inputer) processRoomEvent( headered := input.Event event := headered.Unwrap() + // if we have already got this event then do not process it again, if the input kind is an outlier. + // Outliers contain no extra information which may warrant a re-processing. + if input.Kind == api.KindOutlier { + evs, err := r.DB.EventsFromIDs(ctx, []string{event.EventID()}) + if err == nil && len(evs) == 1 { + // check hash matches if we're on early room versions where the event ID was a random string + idFormat, err := headered.RoomVersion.EventIDFormat() + if err == nil { + switch idFormat { + case gomatrixserverlib.EventIDFormatV1: + if bytes.Equal(event.EventReference().EventSHA256, evs[0].EventReference().EventSHA256) { + util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring") + return event.EventID(), nil + } + default: + util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring") + return event.EventID(), nil + } + } + } + } + // Check that the event passes authentication checks and work out // the numeric IDs for the auth events. isRejected := false diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index 5adcd0877..f76b0a0b4 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -233,7 +233,7 @@ func (u *latestEventsUpdater) latestState() error { if err != nil { return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err) } - if len(u.removed) > len(u.added) { + if !u.stateAtEvent.Overwrite && len(u.removed) > len(u.added) { // This really shouldn't happen. // TODO: What is ultimately the best way to handle this situation? logrus.Errorf( diff --git a/roomserver/storage/shared/latest_events_updater.go b/roomserver/storage/shared/latest_events_updater.go index b316f639d..8825dc464 100644 --- a/roomserver/storage/shared/latest_events_updater.go +++ b/roomserver/storage/shared/latest_events_updater.go @@ -70,16 +70,14 @@ func (u *LatestEventsUpdater) CurrentStateSnapshotNID() types.StateSnapshotNID { return u.currentStateSnapshotNID } -// StorePreviousEvents implements types.RoomRecentEventsUpdater +// StorePreviousEvents implements types.RoomRecentEventsUpdater - This must be called from a Writer func (u *LatestEventsUpdater) StorePreviousEvents(eventNID types.EventNID, previousEventReferences []gomatrixserverlib.EventReference) error { - return u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error { - for _, ref := range previousEventReferences { - if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { - return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err) - } + for _, ref := range previousEventReferences { + if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, u.txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { + return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err) } - return nil - }) + } + return nil } // IsReferenced implements types.RoomRecentEventsUpdater diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index f2be8b3cf..51dcb8887 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -492,15 +492,32 @@ func (d *Database) StoreEvent( if roomInfo == nil && len(prevEvents) > 0 { return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("expected room %q to exist", event.RoomID()) } + // Create an updater - NB: on sqlite this WILL create a txn as we are directly calling the shared DB form of + // GetLatestEventsForUpdate - not via the SQLiteDatabase form which has `nil` txns. This + // function only does SELECTs though so the created txn (at this point) is just a read txn like + // any other so this is fine. If we ever update GetLatestEventsForUpdate or NewLatestEventsUpdater + // to do writes however then this will need to go inside `Writer.Do`. updater, err = d.GetLatestEventsForUpdate(ctx, *roomInfo) if err != nil { return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("NewLatestEventsUpdater: %w", err) } - if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil { - return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("updater.StorePreviousEvents: %w", err) + // Ensure that we atomically store prev events AND commit them. If we don't wrap StorePreviousEvents + // and EndTransaction in a writer then it's possible for a new write txn to be made between the two + // function calls which will then fail with 'database is locked'. This new write txn would HAVE to be + // something like SetRoomAlias/RemoveRoomAlias as normal input events are already done sequentially due to + // SupportsConcurrentRoomInputs() == false on sqlite, though this does not apply to setting room aliases + // as they don't go via InputRoomEvents + err = d.Writer.Do(d.DB, updater.txn, func(txn *sql.Tx) error { + if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil { + return fmt.Errorf("updater.StorePreviousEvents: %w", err) + } + succeeded := true + err = sqlutil.EndTransaction(updater, &succeeded) + return err + }) + if err != nil { + return 0, types.StateAtEvent{}, nil, "", err } - succeeded := true - err = sqlutil.EndTransaction(updater, &succeeded) } return roomNID, types.StateAtEvent{ From 53a745f3332c4bb35dddd80ff8f83e7eb11456ca Mon Sep 17 00:00:00 2001 From: Pika <15848969+ThatNerdyPikachu@users.noreply.github.com> Date: Tue, 20 Oct 2020 06:47:37 -0400 Subject: [PATCH 04/10] fix create-account (#1546) --- cmd/create-account/main.go | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index a9bd92794..f6de2d0d4 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -22,7 +22,6 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/userapi/storage/accounts" - "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/matrix-org/gomatrixserverlib" ) @@ -39,7 +38,6 @@ var ( username = flag.String("username", "", "The user ID localpart to register e.g 'alice' in '@alice:localhost'.") password = flag.String("password", "", "Optional. The password to register with. If not specified, this account will be password-less.") serverNameStr = flag.String("servername", "localhost", "The Matrix server domain which will form the domain part of the user ID.") - accessToken = flag.String("token", "", "Optional. The desired access_token to have. If not specified, a random access_token will be made.") ) func main() { @@ -78,29 +76,5 @@ func main() { os.Exit(1) } - deviceDB, err := devices.NewDatabase(&config.DatabaseOptions{ - ConnectionString: config.DataSource(*database), - }, serverName) - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - - if *accessToken == "" { - t := "token_" + *username - accessToken = &t - } - - device, err := deviceDB.CreateDevice( - context.Background(), *username, nil, *accessToken, nil, "127.0.0.1", "", - ) - if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - - fmt.Println("Created account:") - fmt.Printf("user_id = %s\n", device.UserID) - fmt.Printf("device_id = %s\n", device.ID) - fmt.Printf("access_token = %s\n", device.AccessToken) + fmt.Println("Created account") } From 837c295c26aa42faf1087368d05ceca2e42d95a0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 20 Oct 2020 12:29:53 +0100 Subject: [PATCH 05/10] Linting --- roomserver/internal/input/input_events.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 6a5d9d264..c055289c9 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -49,11 +49,11 @@ func (r *Inputer) processRoomEvent( // if we have already got this event then do not process it again, if the input kind is an outlier. // Outliers contain no extra information which may warrant a re-processing. if input.Kind == api.KindOutlier { - evs, err := r.DB.EventsFromIDs(ctx, []string{event.EventID()}) - if err == nil && len(evs) == 1 { + evs, err2 := r.DB.EventsFromIDs(ctx, []string{event.EventID()}) + if err2 == nil && len(evs) == 1 { // check hash matches if we're on early room versions where the event ID was a random string - idFormat, err := headered.RoomVersion.EventIDFormat() - if err == nil { + idFormat, err2 := headered.RoomVersion.EventIDFormat() + if err2 == nil { switch idFormat { case gomatrixserverlib.EventIDFormatV1: if bytes.Equal(event.EventReference().EventSHA256, evs[0].EventReference().EventSHA256) { From 6c3c621de09a62c7ec8d89b2d8e6a9d73833fd0b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 20 Oct 2020 12:36:16 +0100 Subject: [PATCH 06/10] Remove invalid state delta check (#1550) --- roomserver/internal/input/input_latest_events.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/roomserver/internal/input/input_latest_events.go b/roomserver/internal/input/input_latest_events.go index f76b0a0b4..5631959b7 100644 --- a/roomserver/internal/input/input_latest_events.go +++ b/roomserver/internal/input/input_latest_events.go @@ -233,18 +233,6 @@ func (u *latestEventsUpdater) latestState() error { if err != nil { return fmt.Errorf("roomState.DifferenceBetweenStateSnapshots: %w", err) } - if !u.stateAtEvent.Overwrite && len(u.removed) > len(u.added) { - // This really shouldn't happen. - // TODO: What is ultimately the best way to handle this situation? - logrus.Errorf( - "Invalid state delta on event %q wants to remove %d state but only add %d state (between state snapshots %d and %d)", - u.event.EventID(), len(u.removed), len(u.added), u.oldStateNID, u.newStateNID, - ) - u.added = u.added[:0] - u.removed = u.removed[:0] - u.newStateNID = u.oldStateNID - return nil - } // Also work out the state before the event removes and the event // adds. From 39c7a8915ccba3ac4d281ae7938f2e912ad966b4 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 20 Oct 2020 16:11:24 +0100 Subject: [PATCH 07/10] Multi-personality polylith binary (#1552) * Initial work oon multipersonality binary * Remove old binaries * Monolith and polylith binaries * Better logging * dendrite-poly-multi * Fix path * Copyright notices etc * Tweaks * Update Docker, INSTALL.md * Take first argument if flags package doesn't find any args * Postgres 9.6 or later, fix some more Docker stuff * Don't create unnecessary e2ekey DB * Run go mod tidy --- build/docker/Dockerfile | 2 +- build/docker/Dockerfile.component | 14 ---- build/docker/Dockerfile.monolith | 13 ++++ build/docker/Dockerfile.polylith | 13 ++++ build/docker/docker-compose.deps.yml | 2 +- build/docker/docker-compose.monolith.yml | 1 - build/docker/docker-compose.polylith.yml | 67 ++++++---------- build/docker/images-build.sh | 15 +--- build/docker/images-pull.sh | 13 +--- build/docker/images-push.sh | 13 +--- build/docker/postgres/create_db.sh | 2 +- cmd/dendrite-polylith-multi/main.go | 78 +++++++++++++++++++ .../personalities/appservice.go} | 11 +-- .../personalities/clientapi.go} | 12 +-- .../personalities/eduserver.go} | 18 ++--- .../personalities/federationapi.go} | 11 +-- .../personalities/federationsender.go} | 11 +-- .../personalities/keyserver.go} | 9 +-- .../personalities/mediaapi.go} | 11 +-- .../personalities/roomserver.go} | 11 +-- .../personalities/signingkeyserver.go} | 9 +-- .../personalities/syncapi.go} | 12 +-- .../personalities/userapi.go} | 11 +-- docs/INSTALL.md | 26 +++---- 24 files changed, 189 insertions(+), 196 deletions(-) delete mode 100644 build/docker/Dockerfile.component create mode 100644 build/docker/Dockerfile.monolith create mode 100644 build/docker/Dockerfile.polylith create mode 100644 cmd/dendrite-polylith-multi/main.go rename cmd/{dendrite-appservice-server/main.go => dendrite-polylith-multi/personalities/appservice.go} (83%) rename cmd/{dendrite-client-api-server/main.go => dendrite-polylith-multi/personalities/clientapi.go} (87%) rename cmd/{dendrite-edu-server/main.go => dendrite-polylith-multi/personalities/eduserver.go} (77%) rename cmd/{dendrite-federation-api-server/main.go => dendrite-polylith-multi/personalities/federationapi.go} (86%) rename cmd/{dendrite-federation-sender-server/main.go => dendrite-polylith-multi/personalities/federationsender.go} (85%) rename cmd/{dendrite-key-server/main.go => dendrite-polylith-multi/personalities/keyserver.go} (87%) rename cmd/{dendrite-media-api-server/main.go => dendrite-polylith-multi/personalities/mediaapi.go} (82%) rename cmd/{dendrite-room-server/main.go => dendrite-polylith-multi/personalities/roomserver.go} (84%) rename cmd/{dendrite-signing-key-server/main.go => dendrite-polylith-multi/personalities/signingkeyserver.go} (86%) rename cmd/{dendrite-sync-api-server/main.go => dendrite-polylith-multi/personalities/syncapi.go} (84%) rename cmd/{dendrite-user-api-server/main.go => dendrite-polylith-multi/personalities/userapi.go} (84%) diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index d8e07681f..5cab0530f 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.13.7-alpine3.11 AS builder +FROM docker.io/golang:1.15-alpine AS builder RUN apk --update --no-cache add bash build-base diff --git a/build/docker/Dockerfile.component b/build/docker/Dockerfile.component deleted file mode 100644 index 1acf510fb..000000000 --- a/build/docker/Dockerfile.component +++ /dev/null @@ -1,14 +0,0 @@ -FROM matrixdotorg/dendrite:latest AS base - -FROM alpine:latest - -ARG component=monolith -ENV entrypoint=${component} - -COPY --from=base /build/bin/${component} /usr/bin -COPY --from=base /build/bin/goose /usr/bin - -VOLUME /etc/dendrite -WORKDIR /etc/dendrite - -ENTRYPOINT /usr/bin/${entrypoint} $@ \ No newline at end of file diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith new file mode 100644 index 000000000..3e9d0cba4 --- /dev/null +++ b/build/docker/Dockerfile.monolith @@ -0,0 +1,13 @@ +FROM matrixdotorg/dendrite:latest AS base + +FROM alpine:latest + +COPY --from=base /build/bin/dendrite-monolith-server /usr/bin +COPY --from=base /build/bin/goose /usr/bin +COPY --from=base /build/bin/create-account /usr/bin +COPY --from=base /build/bin/generate-keys /usr/bin + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] \ No newline at end of file diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith new file mode 100644 index 000000000..dd4cbd38f --- /dev/null +++ b/build/docker/Dockerfile.polylith @@ -0,0 +1,13 @@ +FROM matrixdotorg/dendrite:latest AS base + +FROM alpine:latest + +COPY --from=base /build/bin/dendrite-polylith-multi /usr/bin +COPY --from=base /build/bin/goose /usr/bin +COPY --from=base /build/bin/create-account /usr/bin +COPY --from=base /build/bin/generate-keys /usr/bin + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] \ No newline at end of file diff --git a/build/docker/docker-compose.deps.yml b/build/docker/docker-compose.deps.yml index 74e478a8d..1a27ffac0 100644 --- a/build/docker/docker-compose.deps.yml +++ b/build/docker/docker-compose.deps.yml @@ -2,7 +2,7 @@ version: "3.4" services: postgres: hostname: postgres - image: postgres:9.5 + image: postgres:9.6 restart: always volumes: - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh diff --git a/build/docker/docker-compose.monolith.yml b/build/docker/docker-compose.monolith.yml index 7d63e1c65..8fb798343 100644 --- a/build/docker/docker-compose.monolith.yml +++ b/build/docker/docker-compose.monolith.yml @@ -4,7 +4,6 @@ services: hostname: monolith image: matrixdotorg/dendrite-monolith:latest command: [ - "--config=dendrite.yaml", "--tls-cert=server.crt", "--tls-key=server.key" ] diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index e8da9c24a..f377e36fc 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -2,22 +2,17 @@ version: "3.4" services: client_api: hostname: client_api - image: matrixdotorg/dendrite-clientapi:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: clientapi volumes: - ./config:/etc/dendrite - - room_server networks: - internal media_api: hostname: media_api - image: matrixdotorg/dendrite-mediaapi:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: mediaapi volumes: - ./config:/etc/dendrite networks: @@ -25,10 +20,8 @@ services: sync_api: hostname: sync_api - image: matrixdotorg/dendrite-syncapi:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: syncapi volumes: - ./config:/etc/dendrite networks: @@ -36,10 +29,8 @@ services: room_server: hostname: room_server - image: matrixdotorg/dendrite-roomserver:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: roomserver volumes: - ./config:/etc/dendrite networks: @@ -47,10 +38,8 @@ services: edu_server: hostname: edu_server - image: matrixdotorg/dendrite-eduserver:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: eduserver volumes: - ./config:/etc/dendrite networks: @@ -58,10 +47,8 @@ services: federation_api: hostname: federation_api - image: matrixdotorg/dendrite-federationapi:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: federationapi volumes: - ./config:/etc/dendrite networks: @@ -69,10 +56,8 @@ services: federation_sender: hostname: federation_sender - image: matrixdotorg/dendrite-federationsender:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: federationsender volumes: - ./config:/etc/dendrite networks: @@ -80,10 +65,8 @@ services: key_server: hostname: key_server - image: matrixdotorg/dendrite-keyserver:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: keyserver volumes: - ./config:/etc/dendrite networks: @@ -91,10 +74,8 @@ services: signing_key_server: hostname: signing_key_server - image: matrixdotorg/dendrite-signingkeyserver:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: signingkeyserver volumes: - ./config:/etc/dendrite networks: @@ -102,10 +83,8 @@ services: user_api: hostname: user_api - image: matrixdotorg/dendrite-userapi:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: userapi volumes: - ./config:/etc/dendrite networks: @@ -113,10 +92,8 @@ services: appservice_api: hostname: appservice_api - image: matrixdotorg/dendrite-appservice:latest - command: [ - "--config=dendrite.yaml" - ] + image: matrixdotorg/dendrite-polylith:latest + command: appservice volumes: - ./config:/etc/dendrite networks: diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index daad63be0..f80f6bed2 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -8,16 +8,5 @@ echo "Building tag '${TAG}'" docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} . -docker build -t matrixdotorg/dendrite-monolith:${TAG} --build-arg component=dendrite-monolith-server -f build/docker/Dockerfile.component . - -docker build -t matrixdotorg/dendrite-appservice:${TAG} --build-arg component=dendrite-appservice-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-clientapi:${TAG} --build-arg component=dendrite-client-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-eduserver:${TAG} --build-arg component=dendrite-edu-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-federationapi:${TAG} --build-arg component=dendrite-federation-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-federationsender:${TAG} --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-keyserver:${TAG} --build-arg component=dendrite-key-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-mediaapi:${TAG} --build-arg component=dendrite-media-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-roomserver:${TAG} --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-syncapi:${TAG} --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-signingkeyserver:${TAG} --build-arg component=dendrite-signing-key-server -f build/docker/Dockerfile.component . -docker build -t matrixdotorg/dendrite-userapi:${TAG} --build-arg component=dendrite-user-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite-monolith:${TAG} -f build/docker/Dockerfile.monolith . +docker build -t matrixdotorg/dendrite-polylith:${TAG} -f build/docker/Dockerfile.polylith . \ No newline at end of file diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh index e3284a2a6..496e80067 100755 --- a/build/docker/images-pull.sh +++ b/build/docker/images-pull.sh @@ -5,15 +5,4 @@ TAG=${1:-latest} echo "Pulling tag '${TAG}'" docker pull matrixdotorg/dendrite-monolith:${TAG} - -docker pull matrixdotorg/dendrite-appservice:${TAG} -docker pull matrixdotorg/dendrite-clientapi:${TAG} -docker pull matrixdotorg/dendrite-eduserver:${TAG} -docker pull matrixdotorg/dendrite-federationapi:${TAG} -docker pull matrixdotorg/dendrite-federationsender:${TAG} -docker pull matrixdotorg/dendrite-keyserver:${TAG} -docker pull matrixdotorg/dendrite-mediaapi:${TAG} -docker pull matrixdotorg/dendrite-roomserver:${TAG} -docker pull matrixdotorg/dendrite-syncapi:${TAG} -docker pull matrixdotorg/dendrite-signingkeyserver:${TAG} -docker pull matrixdotorg/dendrite-userapi:${TAG} +docker pull matrixdotorg/dendrite-polylith:${TAG} \ No newline at end of file diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index e4eb773ac..fd9b999ea 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -5,15 +5,4 @@ TAG=${1:-latest} echo "Pushing tag '${TAG}'" docker push matrixdotorg/dendrite-monolith:${TAG} - -docker push matrixdotorg/dendrite-appservice:${TAG} -docker push matrixdotorg/dendrite-clientapi:${TAG} -docker push matrixdotorg/dendrite-eduserver:${TAG} -docker push matrixdotorg/dendrite-federationapi:${TAG} -docker push matrixdotorg/dendrite-federationsender:${TAG} -docker push matrixdotorg/dendrite-keyserver:${TAG} -docker push matrixdotorg/dendrite-mediaapi:${TAG} -docker push matrixdotorg/dendrite-roomserver:${TAG} -docker push matrixdotorg/dendrite-syncapi:${TAG} -docker push matrixdotorg/dendrite-signingkeyserver:${TAG} -docker push matrixdotorg/dendrite-userapi:${TAG} +docker push matrixdotorg/dendrite-polylith:${TAG} \ No newline at end of file diff --git a/build/docker/postgres/create_db.sh b/build/docker/postgres/create_db.sh index 97514467b..7495a3978 100755 --- a/build/docker/postgres/create_db.sh +++ b/build/docker/postgres/create_db.sh @@ -1,5 +1,5 @@ #!/bin/sh -for db in account device mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice e2ekey naffka; do +for db in account device mediaapi syncapi roomserver signingkeyserver keyserver federationsender appservice naffka; do createdb -U dendrite -O dendrite dendrite_$db done diff --git a/cmd/dendrite-polylith-multi/main.go b/cmd/dendrite-polylith-multi/main.go new file mode 100644 index 000000000..0d6406c01 --- /dev/null +++ b/cmd/dendrite-polylith-multi/main.go @@ -0,0 +1,78 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "os" + "strings" + + "github.com/matrix-org/dendrite/cmd/dendrite-polylith-multi/personalities" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/setup" + "github.com/sirupsen/logrus" +) + +type entrypoint func(base *setup.BaseDendrite, cfg *config.Dendrite) + +func main() { + cfg := setup.ParseFlags(true) + + component := "" + if flag.NFlag() > 0 { + component = flag.Arg(0) // ./dendrite-polylith-multi --config=... clientapi + } else if len(os.Args) > 1 { + component = os.Args[1] // ./dendrite-polylith-multi clientapi + } + + components := map[string]entrypoint{ + "appservice": personalities.Appservice, + "clientapi": personalities.ClientAPI, + "eduserver": personalities.EDUServer, + "federationapi": personalities.FederationAPI, + "federationsender": personalities.FederationSender, + "keyserver": personalities.KeyServer, + "mediaapi": personalities.MediaAPI, + "roomserver": personalities.RoomServer, + "signingkeyserver": personalities.SigningKeyServer, + "syncapi": personalities.SyncAPI, + "userapi": personalities.UserAPI, + } + + start, ok := components[component] + if !ok { + if component == "" { + logrus.Errorf("No component specified") + logrus.Info("The first argument on the command line must be the name of the component to run") + } else { + logrus.Errorf("Unknown component %q specified", component) + } + + var list []string + for c := range components { + list = append(list, c) + } + logrus.Infof("Valid components: %s", strings.Join(list, ", ")) + + os.Exit(1) + } + + logrus.Infof("Starting %q component", component) + + base := setup.NewBaseDendrite(cfg, component, false) // TODO + defer base.Close() // nolint: errcheck + + start(base, cfg) +} diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-polylith-multi/personalities/appservice.go similarity index 83% rename from cmd/dendrite-appservice-server/main.go rename to cmd/dendrite-polylith-multi/personalities/appservice.go index 6adbdb17c..7fa87b115 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/appservice.go @@ -1,4 +1,4 @@ -// Copyright 2018 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "AppServiceAPI", true) - - defer base.Close() // nolint: errcheck +func Appservice(base *setup.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() rsAPI := base.RoomserverHTTPClient() diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/clientapi.go similarity index 87% rename from cmd/dendrite-client-api-server/main.go rename to cmd/dendrite-polylith-multi/personalities/clientapi.go index 0061de74f..09fc63ab3 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/clientapi.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,20 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( "github.com/matrix-org/dendrite/clientapi" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/internal/transactions" ) -func main() { - cfg := setup.ParseFlags(false) - - base := setup.NewBaseDendrite(cfg, "ClientAPI", true) - defer base.Close() // nolint: errcheck - +func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { accountDB := base.CreateAccountsDB() federation := base.CreateFederationClient() diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-polylith-multi/personalities/eduserver.go similarity index 77% rename from cmd/dendrite-edu-server/main.go rename to cmd/dendrite-polylith-multi/personalities/eduserver.go index 3a34b9a68..a5d2926f1 100644 --- a/cmd/dendrite-edu-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/eduserver.go @@ -1,3 +1,5 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -10,26 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( - _ "net/http/pprof" - "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" - "github.com/sirupsen/logrus" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "EDUServerAPI", true) - defer func() { - if err := base.Close(); err != nil { - logrus.WithError(err).Warn("BaseDendrite close failed") - } - }() - +func EDUServer(base *setup.BaseDendrite, cfg *config.Dendrite) { intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient()) eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI) diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/federationapi.go similarity index 86% rename from cmd/dendrite-federation-api-server/main.go rename to cmd/dendrite-polylith-multi/personalities/federationapi.go index 3ebb16f4b..a1bbeafad 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/federationapi.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "FederationAPI", true) - defer base.Close() // nolint: errcheck - +func FederationAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() federation := base.CreateFederationClient() serverKeyAPI := base.SigningKeyServerHTTPClient() diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-polylith-multi/personalities/federationsender.go similarity index 85% rename from cmd/dendrite-federation-sender-server/main.go rename to cmd/dendrite-polylith-multi/personalities/federationsender.go index 99b416c45..052523789 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/federationsender.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "FederationSender", true) - defer base.Close() // nolint: errcheck - +func FederationSender(base *setup.BaseDendrite, cfg *config.Dendrite) { federation := base.CreateFederationClient() serverKeyAPI := base.SigningKeyServerHTTPClient() diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-polylith-multi/personalities/keyserver.go similarity index 87% rename from cmd/dendrite-key-server/main.go rename to cmd/dendrite-polylith-multi/personalities/keyserver.go index ff5b22236..8c159ad06 100644 --- a/cmd/dendrite-key-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/keyserver.go @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/keyserver" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "KeyServer", true) - defer base.Close() // nolint: errcheck - +func KeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) { intAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, base.CreateFederationClient()) intAPI.SetUserAPI(base.UserAPIClient()) diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/mediaapi.go similarity index 82% rename from cmd/dendrite-media-api-server/main.go rename to cmd/dendrite-polylith-multi/personalities/mediaapi.go index 2c2fe3b36..64e5bc312 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/mediaapi.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/mediaapi" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "MediaAPI", true) - defer base.Close() // nolint: errcheck - +func MediaAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() client := base.CreateClient() diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-polylith-multi/personalities/roomserver.go similarity index 84% rename from cmd/dendrite-room-server/main.go rename to cmd/dendrite-polylith-multi/personalities/roomserver.go index d3f145745..91027506d 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/roomserver.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/roomserver" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "RoomServerAPI", true) - defer base.Close() // nolint: errcheck - +func RoomServer(base *setup.BaseDendrite, cfg *config.Dendrite) { serverKeyAPI := base.SigningKeyServerHTTPClient() keyRing := serverKeyAPI.KeyRing() diff --git a/cmd/dendrite-signing-key-server/main.go b/cmd/dendrite-polylith-multi/personalities/signingkeyserver.go similarity index 86% rename from cmd/dendrite-signing-key-server/main.go rename to cmd/dendrite-polylith-multi/personalities/signingkeyserver.go index a4d48d361..a7bfff10b 100644 --- a/cmd/dendrite-signing-key-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/signingkeyserver.go @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/signingkeyserver" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "SigningKeyServer", true) - defer base.Close() // nolint: errcheck - +func SigningKeyServer(base *setup.BaseDendrite, cfg *config.Dendrite) { federation := base.CreateFederationClient() intAPI := signingkeyserver.NewInternalAPI(&base.Cfg.SigningKeyServer, federation, base.Caches) diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/syncapi.go similarity index 84% rename from cmd/dendrite-sync-api-server/main.go rename to cmd/dendrite-polylith-multi/personalities/syncapi.go index 351dbc5f4..2d5c0b525 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/syncapi.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,19 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/syncapi" ) -func main() { - cfg := setup.ParseFlags(false) - - base := setup.NewBaseDendrite(cfg, "SyncAPI", true) - defer base.Close() // nolint: errcheck - +func SyncAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { userAPI := base.UserAPIClient() federation := base.CreateFederationClient() diff --git a/cmd/dendrite-user-api-server/main.go b/cmd/dendrite-polylith-multi/personalities/userapi.go similarity index 84% rename from cmd/dendrite-user-api-server/main.go rename to cmd/dendrite-polylith-multi/personalities/userapi.go index fb65fefbc..fe5e4fbd0 100644 --- a/cmd/dendrite-user-api-server/main.go +++ b/cmd/dendrite-polylith-multi/personalities/userapi.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package personalities import ( + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/userapi" ) -func main() { - cfg := setup.ParseFlags(false) - base := setup.NewBaseDendrite(cfg, "UserAPI", true) - defer base.Close() // nolint: errcheck - +func UserAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { accountDB := base.CreateAccountsDB() userAPI := userapi.NewInternalAPI(accountDB, &cfg.UserAPI, cfg.Derived.ApplicationServices, base.KeyServerHTTPClient()) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index d246e9221..f804193cd 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -24,7 +24,7 @@ use in production environments just yet! Dendrite requires: * Go 1.13 or higher -* Postgres 9.5 or higher (if using Postgres databases, not needed for SQLite) +* Postgres 9.6 or higher (if using Postgres databases, not needed for SQLite) If you want to run a polylith deployment, you also need: @@ -98,7 +98,7 @@ create them automatically at startup. ### Postgres database setup -Assuming that Postgres 9.5 (or later) is installed: +Assuming that Postgres 9.6 (or later) is installed: * Create role, choosing a new password when prompted: @@ -189,7 +189,7 @@ This is what implements CS API endpoints. Clients talk to this via the proxy in order to send messages, create and join rooms, etc. ```bash -./bin/dendrite-client-api-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml clientapi ``` ### Sync server @@ -198,7 +198,7 @@ This is what implements `/sync` requests. Clients talk to this via the proxy in order to receive messages. ```bash -./bin/dendrite-sync-api-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml syncapi ``` ### Media server @@ -207,7 +207,7 @@ This implements `/media` requests. Clients talk to this via the proxy in order to upload and retrieve media. ```bash -./bin/dendrite-media-api-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml mediaapi ``` ### Federation API server @@ -217,7 +217,7 @@ order to send transactions. This is only required if you want to support federation. ```bash -./bin/dendrite-federation-api-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml federationapi ``` ### Internal components @@ -230,7 +230,7 @@ contacted by other components. This includes the following components. This is what implements the room DAG. Clients do not talk to this. ```bash -./bin/dendrite-room-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml roomserver ``` #### Federation sender @@ -239,7 +239,7 @@ This sends events from our users to other servers. This is only required if you want to support federation. ```bash -./bin/dendrite-federation-sender-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml federationsender ``` #### Appservice server @@ -250,7 +250,7 @@ running locally. This is only required if you want to support running application services on your homeserver. ```bash -./bin/dendrite-appservice-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml appservice ``` #### Key server @@ -258,7 +258,7 @@ application services on your homeserver. This manages end-to-end encryption keys for users. ```bash -./bin/dendrite-key-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml keyserver ``` #### Signing key server @@ -266,7 +266,7 @@ This manages end-to-end encryption keys for users. This manages signing keys for servers. ```bash -./bin/dendrite-signing-key-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml signingkeyserver ``` #### EDU server @@ -274,7 +274,7 @@ This manages signing keys for servers. This manages processing EDUs such as typing, send-to-device events and presence. Clients do not talk to ```bash -./bin/dendrite-edu-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml eduserver ``` #### User server @@ -283,6 +283,6 @@ This manages user accounts, device access tokens and user account data, amongst other things. ```bash -./bin/dendrite-user-api-server --config dendrite.yaml +./bin/dendrite-polylith-multi --config=dendrite.yaml userapi ``` From 6a16d46fba9edf82203b2b3606d9a16ed6ef4ae0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 20 Oct 2020 16:46:53 +0100 Subject: [PATCH 08/10] Version 0.2.0 (#1551) * v0.2.0-rc2 * Update CHANGES.md (also with some markdown lint suggestions) * Update version number * Update CHANGES.md * Update CHANGES.md * Update CHANGES.md * Add known issue * Update CHANGES.md --- CHANGES.md | 194 +++++++++++++++++++++++++++++--------------- internal/version.go | 2 +- 2 files changed, 128 insertions(+), 68 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 17fb75bed..d05b871ac 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,89 +1,149 @@ -# Dendrite 0.1.0 (2020-10-08) +# Changelog + +## Dendrite 0.2.0 (2020-10-20) + +### Important + +* This release makes breaking changes for polylith deployments, since they now use the multi-personality binary rather than separate binary files + * Users of polylith deployments should revise their setups to use the new binary - see the Features section below +* This release also makes breaking changes for Docker deployments, as are now publishing images to Docker Hub in separate repositories for monolith and polylith + * New repositories are as follows: [matrixdotorg/dendrite-monolith](https://hub.docker.com/repository/docker/matrixdotorg/dendrite-monolith) and [matrixdotorg/dendrite-polylith](https://hub.docker.com/repository/docker/matrixdotorg/dendrite-polylith) + * The new `latest` tag will be updated with the latest release, and new versioned tags, e.g. `v0.2.0`, will preserve specific release versions + * [Sample Compose configs](https://github.com/matrix-org/dendrite/tree/master/build/docker) have been updated - if you are running a Docker deployment, please review the changes + * Images for the client API proxy and federation API proxy are no longer provided as they are unsupported - please use [nginx](docs/nginx/) (or another reverse proxy) instead + +### Features + +* Dendrite polylith deployments now use a special multi-personality binary, rather than separate binaries + * This is cleaner, builds faster and simplifies deployment + * The first command line argument states the component to run, e.g. `./dendrite-polylith-multi roomserver` +* Database migrations are now run at startup +* Invalid UTF-8 in requests is now rejected (contributed by [Pestdoktor](https://github.com/Pestdoktor)) +* Fully read markers are now implemented in the client API (contributed by [Lesterpig](https://github.com/Lesterpig)) +* Missing auth events are now retrieved from other servers in the room, rather than just the event origin +* `m.room.create` events are now validated properly when processing a `/send_join` response +* The roomserver now implements `KindOld` for handling historic events without them becoming forward extremity candidates, i.e. for backfilled or missing events + +### Fixes + +* State resolution v2 performance has been improved dramatically when dealing with large state sets +* The roomserver no longer processes outlier events if they are already known +* A SQLite locking issue in the previous events updater has been fixed +* The client API `/state` endpoint now correctly returns state after the leave event, if the user has left the room +* The client API `/createRoom` endpoint now sends cumulative state to the roomserver for the initial room events +* The federation API `/send` endpoint now correctly requests the entire room state from the roomserver when needed +* Some internal HTTP API paths have been fixed in the user API (contributed by [S7evinK](https://github.com/S7evinK)) +* A race condition in the rate limiting code resulting in concurrent map writes has been fixed +* Each component now correctly starts a consumer/producer connection in monolith mode (when using Kafka) +* State resolution is no longer run for single trusted state snapshots that have been verified before +* A crash when rolling back the transaction in the latest events updater has been fixed +* Typing events are now ignored when the sender domain does not match the origin server +* Duplicate redaction entries no longer result in database errors +* Recursion has been removed from the code path for retrieving missing events +* `QueryMissingAuthPrevEvents` now returns events that have no associated state as if they are missing +* Signing key fetchers no longer ignore keys for the local domain, if retrieving a key that is not known in the local config +* Federation timeouts have been adjusted so we don't give up on remote requests so quickly +* `create-account` no longer relies on the device database (contributed by [ThatNerdyPikachu](https://github.com/ThatNerdyPikachu)) + +### Known issues + +* Old events can incorrectly appear in `/sync` as if they are new when retrieving missing events from federated servers, causing them to appear at the bottom of the timeline in clients + +## Dendrite 0.1.0 (2020-10-08) First versioned release of Dendrite. ## Client-Server API Features ### Account registration and management -- Registration: By password only. -- Login: By password only. No fallback. -- Logout: Yes. -- Change password: Yes. -- Link email/msisdn to account: No. -- Deactivate account: Yes. -- Check if username is available: Yes. -- Account data: Yes. -- OpenID: No. + +* Registration: By password only. +* Login: By password only. No fallback. +* Logout: Yes. +* Change password: Yes. +* Link email/msisdn to account: No. +* Deactivate account: Yes. +* Check if username is available: Yes. +* Account data: Yes. +* OpenID: No. ### Rooms -- Room creation: Yes, including presets. -- Joining rooms: Yes, including by alias or `?server_name=`. -- Event sending: Yes, including transaction IDs. -- Aliases: Yes. -- Published room directory: Yes. -- Kicking users: Yes. -- Banning users: Yes. -- Inviting users: Yes, but not third-party invites. -- Forgetting rooms: No. -- Room versions: All (v1 - v6) -- Tagging: Yes. + +* Room creation: Yes, including presets. +* Joining rooms: Yes, including by alias or `?server_name=`. +* Event sending: Yes, including transaction IDs. +* Aliases: Yes. +* Published room directory: Yes. +* Kicking users: Yes. +* Banning users: Yes. +* Inviting users: Yes, but not third-party invites. +* Forgetting rooms: No. +* Room versions: All (v1 * v6) +* Tagging: Yes. ### User management -- User directory: Basic support. -- Ignoring users: No. -- Groups/Communities: No. + +* User directory: Basic support. +* Ignoring users: No. +* Groups/Communities: No. ### Device management -- Creating devices: Yes. -- Deleting devices: Yes. -- Send-to-device messaging: Yes. + +* Creating devices: Yes. +* Deleting devices: Yes. +* Send-to-device messaging: Yes. ### Sync -- Filters: Timeline limit only. Rest unimplemented. -- Deprecated `/events` and `/initialSync`: No. + +* Filters: Timeline limit only. Rest unimplemented. +* Deprecated `/events` and `/initialSync`: No. ### Room events -- Typing: Yes. -- Receipts: No. -- Read Markers: No. -- Presence: No. -- Content repository (attachments): Yes. -- History visibility: No, defaults to `joined`. -- Push notifications: No. -- Event context: No. -- Reporting content: No. + +* Typing: Yes. +* Receipts: No. +* Read Markers: No. +* Presence: No. +* Content repository (attachments): Yes. +* History visibility: No, defaults to `joined`. +* Push notifications: No. +* Event context: No. +* Reporting content: No. ### End-to-End Encryption -- Uploading device keys: Yes. -- Downloading device keys: Yes. -- Claiming one-time keys: Yes. -- Querying key changes: Yes. -- Cross-Signing: No. + +* Uploading device keys: Yes. +* Downloading device keys: Yes. +* Claiming one-time keys: Yes. +* Querying key changes: Yes. +* Cross-Signing: No. ### Misc -- Server-side search: No. -- Guest access: Partial. -- Room previews: No, partial support for Peeking via MSC2753. -- Third-Party networks: No. -- Server notices: No. -- Policy lists: No. + +* Server-side search: No. +* Guest access: Partial. +* Room previews: No, partial support for Peeking via MSC2753. +* Third-Party networks: No. +* Server notices: No. +* Policy lists: No. ## Federation Features -- Querying keys (incl. notary): Yes. -- Server ACLs: Yes. -- Sending transactions: Yes. -- Joining rooms: Yes. -- Inviting to rooms: Yes, but not third-party invites. -- Leaving rooms: Yes. -- Content repository: Yes. -- Backfilling / get_missing_events: Yes. -- Retrieving state of the room (`/state` and `/state_ids`): Yes. -- Public rooms: Yes. -- Querying profile data: Yes. -- Device management: Yes. -- Send-to-Device messaging: Yes. -- Querying/Claiming E2E Keys: Yes. -- Typing: Yes. -- Presence: No. -- Receipts: No. -- OpenID: No. \ No newline at end of file + +* Querying keys (incl. notary): Yes. +* Server ACLs: Yes. +* Sending transactions: Yes. +* Joining rooms: Yes. +* Inviting to rooms: Yes, but not third-party invites. +* Leaving rooms: Yes. +* Content repository: Yes. +* Backfilling / get_missing_events: Yes. +* Retrieving state of the room (`/state` and `/state_ids`): Yes. +* Public rooms: Yes. +* Querying profile data: Yes. +* Device management: Yes. +* Send-to-Device messaging: Yes. +* Querying/Claiming E2E Keys: Yes. +* Typing: Yes. +* Presence: No. +* Receipts: No. +* OpenID: No. \ No newline at end of file diff --git a/internal/version.go b/internal/version.go index a9e245d44..040ffa32a 100644 --- a/internal/version.go +++ b/internal/version.go @@ -16,7 +16,7 @@ var build string const ( VersionMajor = 0 - VersionMinor = 1 + VersionMinor = 2 VersionPatch = 0 VersionTag = "" // example: "rc1" ) From 7ca89ef5116bd918b50c275f6a52d171ea1ccd96 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 20 Oct 2020 17:10:37 +0100 Subject: [PATCH 09/10] Update gomatrixserverlib --- go.mod | 2 +- go.sum | 4 ++-- sytest-whitelist | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d3060fa0f..afbe15f14 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8 + github.com/matrix-org/gomatrixserverlib v0.0.0-20201020160945-66a6d4926351 github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.2 diff --git a/go.sum b/go.sum index 377a2e093..eaec548ba 100644 --- a/go.sum +++ b/go.sum @@ -569,8 +569,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8 h1:GF1PxbvImWDoz1DQZNMoaYtIqQXtyLAtmQOzwwmw1OI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20201015151920-aa4f62b827b8/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201020160945-66a6d4926351 h1:Yg+fvSEvDLjMFkkE4W0e1yBTtCMVmH3tztw/RhTLPcA= +github.com/matrix-org/gomatrixserverlib v0.0.0-20201020160945-66a6d4926351/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91 h1:HJ6U3S3ljJqNffYMcIeAncp5qT/i+ZMiJ2JC2F0aXP4= github.com/matrix-org/naffka v0.0.0-20200901083833-bcdd62999a91/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/sytest-whitelist b/sytest-whitelist index cecf24f75..1a12b591b 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -485,3 +485,4 @@ Event with an invalid signature in the send_join response should not cause room Inbound federation rejects typing notifications from wrong remote Should not be able to take over the room by pretending there is no PL event Can get rooms/{roomId}/state for a departed room (SPEC-216) +Users cannot set notifications powerlevel higher than their own From 24e38c413567355a88c0b92f5a325d7b77325ad1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 20 Oct 2020 17:13:12 +0100 Subject: [PATCH 10/10] Internal HTTP APIs over H2C (#1541) * H2C on internal HTTP because SCIENCE * Update comments --- go.mod | 1 + internal/setup/base.go | 30 ++++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index afbe15f14..d12efd4e0 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20201006093556-760d9a7fd5ee go.uber.org/atomic v1.6.0 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a + golang.org/x/net v0.0.0-20200528225125-3c3fba18258b gopkg.in/h2non/bimg.v1 v1.1.4 gopkg.in/yaml.v2 v2.3.0 ) diff --git a/internal/setup/base.go b/internal/setup/base.go index 8bc4ae17a..4e1cee479 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -15,8 +15,10 @@ package setup import ( + "crypto/tls" "fmt" "io" + "net" "net/http" "net/url" "time" @@ -25,6 +27,8 @@ import ( "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/gomatrixserverlib" "github.com/prometheus/client_golang/prometheus/promhttp" + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/userapi/storage/accounts" @@ -107,7 +111,22 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo logrus.WithError(err).Warnf("Failed to create cache") } - apiClient := http.Client{Timeout: time.Minute * 10} + apiClient := http.Client{ + Timeout: time.Minute * 10, + Transport: &http2.Transport{ + AllowHTTP: true, + DialTLS: func(network, addr string, _ *tls.Config) (net.Conn, error) { + // Ordinarily HTTP/2 would expect TLS, but the remote listener is + // H2C-enabled (HTTP/2 without encryption). Overriding the DialTLS + // function with a plain Dial allows us to trick the HTTP client + // into establishing a HTTP/2 connection without TLS. + // TODO: Eventually we will want to look at authenticating and + // encrypting these internal HTTP APIs, at which point we will have + // to reconsider H2C and change all this anyway. + return net.Dial(network, addr) + }, + }, + } client := http.Client{Timeout: HTTPClientTimeout} if cfg.FederationSender.Proxy.Enabled { client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{ @@ -269,10 +288,17 @@ func (b *BaseDendrite) SetupAndServeHTTP( internalServ := externalServ if internalAddr != NoListener && externalAddr != internalAddr { + // H2C allows us to accept HTTP/2 connections without TLS + // encryption. Since we don't currently require any form of + // authentication or encryption on these internal HTTP APIs, + // H2C gives us all of the advantages of HTTP/2 (such as + // stream multiplexing and avoiding head-of-line blocking) + // without enabling TLS. + internalH2S := &http2.Server{} internalRouter = mux.NewRouter().SkipClean(true).UseEncodedPath() internalServ = &http.Server{ Addr: string(internalAddr), - Handler: internalRouter, + Handler: h2c.NewHandler(internalRouter, internalH2S), } }