Merge branch 'master' into neilalexander/rsfixes

2025-12-23 23:03:10 -06:00 · 2020-10-20 11:43:37 +01:00 · 2020-10-20 11:43:37 +01:00 · 1a7b82329b
parent b91b6bd602 eb86e2b336
commit 1a7b82329b
10 changed files with 131 additions and 133 deletions
--- a/build/docker/Dockerfile.component
+++ b/build/docker/Dockerfile.component
@ -6,6 +6,7 @@ ARG component=monolith
 ENV entrypoint=${component}

 COPY --from=base /build/bin/${component} /usr/bin
+COPY --from=base /build/bin/goose /usr/bin

 VOLUME /etc/dendrite
 WORKDIR /etc/dendrite
--- a/build/docker/docker-compose.monolith.yml
+++ b/build/docker/docker-compose.monolith.yml
@ -2,7 +2,7 @@ version: "3.4"
 services:
  monolith:
    hostname: monolith
-    image: matrixdotorg/dendrite:monolith
+    image: matrixdotorg/dendrite-monolith:latest
    command: [
      "--config=dendrite.yaml",
      "--tls-cert=server.crt",
--- a/build/docker/docker-compose.polylith.yml
+++ b/build/docker/docker-compose.polylith.yml
@ -1,28 +1,8 @@
 version: "3.4"
 services:
-  client_api_proxy:
-    hostname: client_api_proxy
-    image: matrixdotorg/dendrite:clientproxy
-    command: [
-      "--bind-address=:8008",
-      "--client-api-server-url=http://client_api:8071",
-      "--sync-api-server-url=http://sync_api:8073",
-      "--media-api-server-url=http://media_api:8074"
-    ]
-    volumes:
-      - ./config:/etc/dendrite
-    networks:
-      - internal
-    depends_on:
-      - sync_api
-      - client_api
-      - media_api
-    ports:
-      - "8008:8008"
-
  client_api:
    hostname: client_api
-    image: matrixdotorg/dendrite:clientapi
+    image: matrixdotorg/dendrite-clientapi:latest
    command: [
      "--config=dendrite.yaml"
    ]
@ -34,7 +14,7 @@ services:

  media_api:
    hostname: media_api
-    image: matrixdotorg/dendrite:mediaapi
+    image: matrixdotorg/dendrite-mediaapi:latest
    command: [
      "--config=dendrite.yaml"
    ]
@ -45,7 +25,7 @@ services:

  sync_api:
    hostname: sync_api
-    image: matrixdotorg/dendrite:syncapi
+    image: matrixdotorg/dendrite-syncapi:latest
    command: [
      "--config=dendrite.yaml"
    ]
@ -56,7 +36,7 @@ services:

  room_server:
    hostname: room_server
-    image: matrixdotorg/dendrite:roomserver
+    image: matrixdotorg/dendrite-roomserver:latest
    command: [
      "--config=dendrite.yaml"
    ]
@ -67,7 +47,7 @@ services:

  edu_server:
    hostname: edu_server
-    image: matrixdotorg/dendrite:eduserver
+    image: matrixdotorg/dendrite-eduserver:latest
    command: [
      "--config=dendrite.yaml"
    ]
@ -76,28 +56,9 @@ services:
    networks:
      - internal

-  federation_api_proxy:
-    hostname: federation_api_proxy
-    image: matrixdotorg/dendrite:federationproxy
-    command: [
-      "--bind-address=:8448",
-      "--federation-api-url=http://federation_api:8072",
-      "--media-api-server-url=http://media_api:8074"
-    ]
-    volumes:
-      - ./config:/etc/dendrite
-    depends_on:
-      - federation_api
-      - federation_sender
-      - media_api
-    networks:
-      - internal
-    ports:
-      - "8448:8448"
-
  federation_api:
    hostname: federation_api
-    image: matrixdotorg/dendrite:federationapi
+    image: matrixdotorg/dendrite-federationapi:latest
    command: [
      "--config=dendrite.yaml"
    ]
@ -108,7 +69,7 @@ services:

  federation_sender:
    hostname: federation_sender
-    image: matrixdotorg/dendrite:federationsender
+    image: matrixdotorg/dendrite-federationsender:latest
    command: [
      "--config=dendrite.yaml"
    ]
@ -119,7 +80,7 @@ services:
      
  key_server:
    hostname: key_server
-    image: matrixdotorg/dendrite:keyserver
+    image: matrixdotorg/dendrite-keyserver:latest
    command: [
      "--config=dendrite.yaml"
    ]
@ -130,7 +91,7 @@ services:

  signing_key_server:
    hostname: signing_key_server
-    image: matrixdotorg/dendrite:signingkeyserver
+    image: matrixdotorg/dendrite-signingkeyserver:latest
    command: [
      "--config=dendrite.yaml"
    ]
@ -141,7 +102,7 @@ services:

  user_api:
    hostname: user_api
-    image: matrixdotorg/dendrite:userapi
+    image: matrixdotorg/dendrite-userapi:latest
    command: [
        "--config=dendrite.yaml"
    ]
@ -152,7 +113,7 @@ services:

  appservice_api:
    hostname: appservice_api
-    image: matrixdotorg/dendrite:appservice
+    image: matrixdotorg/dendrite-appservice:latest
    command: [
      "--config=dendrite.yaml"
    ]
--- a/build/docker/images-build.sh
+++ b/build/docker/images-build.sh
@ -2,20 +2,22 @@

 cd $(git rev-parse --show-toplevel)

-docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:latest .
+TAG=${1:-latest}

-docker build -t matrixdotorg/dendrite:monolith          --build-arg component=dendrite-monolith-server          -f build/docker/Dockerfile.component .
+echo "Building tag '${TAG}'"

-docker build -t matrixdotorg/dendrite:appservice        --build-arg component=dendrite-appservice-server        -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:clientapi         --build-arg component=dendrite-client-api-server        -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:clientproxy       --build-arg component=client-api-proxy                  -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:eduserver         --build-arg component=dendrite-edu-server               -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:federationapi     --build-arg component=dendrite-federation-api-server    -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:federationsender  --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:federationproxy   --build-arg component=federation-api-proxy              -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:keyserver         --build-arg component=dendrite-key-server               -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:mediaapi          --build-arg component=dendrite-media-api-server         -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:roomserver        --build-arg component=dendrite-room-server              -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:syncapi           --build-arg component=dendrite-sync-api-server          -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:signingkeyserver      --build-arg component=dendrite-signing-key-server    -f build/docker/Dockerfile.component .
-docker build -t matrixdotorg/dendrite:userapi           --build-arg component=dendrite-user-api-server          -f build/docker/Dockerfile.component .
+docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:${TAG} .
+
+docker build -t matrixdotorg/dendrite-monolith:${TAG}          --build-arg component=dendrite-monolith-server          -f build/docker/Dockerfile.component .
+
+docker build -t matrixdotorg/dendrite-appservice:${TAG}        --build-arg component=dendrite-appservice-server        -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-clientapi:${TAG}         --build-arg component=dendrite-client-api-server        -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-eduserver:${TAG}         --build-arg component=dendrite-edu-server               -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-federationapi:${TAG}     --build-arg component=dendrite-federation-api-server    -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-federationsender:${TAG}  --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-keyserver:${TAG}         --build-arg component=dendrite-key-server               -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-mediaapi:${TAG}          --build-arg component=dendrite-media-api-server         -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-roomserver:${TAG}        --build-arg component=dendrite-room-server              -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-syncapi:${TAG}           --build-arg component=dendrite-sync-api-server          -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-signingkeyserver:${TAG}  --build-arg component=dendrite-signing-key-server       -f build/docker/Dockerfile.component .
+docker build -t matrixdotorg/dendrite-userapi:${TAG}           --build-arg component=dendrite-user-api-server          -f build/docker/Dockerfile.component .
--- a/build/docker/images-pull.sh
+++ b/build/docker/images-pull.sh
@ -1,17 +1,19 @@
 #!/bin/bash

-docker pull matrixdotorg/dendrite:monolith
+TAG=${1:-latest}

-docker pull matrixdotorg/dendrite:appservice
-docker pull matrixdotorg/dendrite:clientapi
-docker pull matrixdotorg/dendrite:clientproxy
-docker pull matrixdotorg/dendrite:eduserver
-docker pull matrixdotorg/dendrite:federationapi
-docker pull matrixdotorg/dendrite:federationsender
-docker pull matrixdotorg/dendrite:federationproxy
-docker pull matrixdotorg/dendrite:keyserver
-docker pull matrixdotorg/dendrite:mediaapi
-docker pull matrixdotorg/dendrite:roomserver
-docker pull matrixdotorg/dendrite:syncapi
-docker pull matrixdotorg/dendrite:signingkeyserver
-docker pull matrixdotorg/dendrite:userapi
+echo "Pulling tag '${TAG}'"
+
+docker pull matrixdotorg/dendrite-monolith:${TAG}
+
+docker pull matrixdotorg/dendrite-appservice:${TAG}
+docker pull matrixdotorg/dendrite-clientapi:${TAG}
+docker pull matrixdotorg/dendrite-eduserver:${TAG}
+docker pull matrixdotorg/dendrite-federationapi:${TAG}
+docker pull matrixdotorg/dendrite-federationsender:${TAG}
+docker pull matrixdotorg/dendrite-keyserver:${TAG}
+docker pull matrixdotorg/dendrite-mediaapi:${TAG}
+docker pull matrixdotorg/dendrite-roomserver:${TAG}
+docker pull matrixdotorg/dendrite-syncapi:${TAG}
+docker pull matrixdotorg/dendrite-signingkeyserver:${TAG}
+docker pull matrixdotorg/dendrite-userapi:${TAG}
--- a/build/docker/images-push.sh
+++ b/build/docker/images-push.sh
@ -1,17 +1,19 @@
 #!/bin/bash

-docker push matrixdotorg/dendrite:monolith
+TAG=${1:-latest}

-docker push matrixdotorg/dendrite:appservice
-docker push matrixdotorg/dendrite:clientapi
-docker push matrixdotorg/dendrite:clientproxy
-docker push matrixdotorg/dendrite:eduserver
-docker push matrixdotorg/dendrite:federationapi
-docker push matrixdotorg/dendrite:federationsender
-docker push matrixdotorg/dendrite:federationproxy
-docker push matrixdotorg/dendrite:keyserver
-docker push matrixdotorg/dendrite:mediaapi
-docker push matrixdotorg/dendrite:roomserver
-docker push matrixdotorg/dendrite:syncapi
-docker push matrixdotorg/dendrite:signingkeyserver
-docker push matrixdotorg/dendrite:userapi
+echo "Pushing tag '${TAG}'"
+
+docker push matrixdotorg/dendrite-monolith:${TAG}
+
+docker push matrixdotorg/dendrite-appservice:${TAG}
+docker push matrixdotorg/dendrite-clientapi:${TAG}
+docker push matrixdotorg/dendrite-eduserver:${TAG}
+docker push matrixdotorg/dendrite-federationapi:${TAG}
+docker push matrixdotorg/dendrite-federationsender:${TAG}
+docker push matrixdotorg/dendrite-keyserver:${TAG}
+docker push matrixdotorg/dendrite-mediaapi:${TAG}
+docker push matrixdotorg/dendrite-roomserver:${TAG}
+docker push matrixdotorg/dendrite-syncapi:${TAG}
+docker push matrixdotorg/dendrite-signingkeyserver:${TAG}
+docker push matrixdotorg/dendrite-userapi:${TAG}
--- a/docs/INSTALL.md
+++ b/docs/INSTALL.md
@ -93,7 +93,7 @@ brew services start kafka
 ### SQLite database setup

 Dendrite can use the built-in SQLite database engine for small setups.
-The SQLite databases do not need to be preconfigured - Dendrite will
+The SQLite databases do not need to be pre-built - Dendrite will
 create them automatically at startup.

 ### Postgres database setup
@ -109,7 +109,7 @@ Assuming that Postgres 9.5 (or later) is installed:
 * Create the component databases:

  ```bash
-  for i in account device mediaapi syncapi roomserver signingkeyserver federationsender appservice e2ekey naffka; do
+  for i in mediaapi syncapi roomserver signingkeyserver federationsender appservice keyserver userapi_account userapi_device naffka; do
      sudo -u postgres createdb -O dendrite dendrite_$i
  done
  ```
@ -135,8 +135,8 @@ Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Th

 * The `server_name` entry to reflect the hostname of your Dendrite server
 * The `database` lines with an updated connection string based on your
-  desired setup, e.g. replacing `component` with the name of the component:
-  * For Postgres: `postgres://dendrite:password@localhost/component`
+  desired setup, e.g. replacing `database` with the name of the database:
+  * For Postgres: `postgres://dendrite:password@localhost/database`
  * For SQLite on disk: `file:component.db` or `file:///path/to/component.db`
  * Postgres and SQLite can be mixed and matched.
 * The `use_naffka` option if using Naffka in a monolith deployment
@ -147,6 +147,10 @@ then configuring `key_perspectives` (like `matrix.org` in the sample) can
 help to improve reliability considerably by allowing your homeserver to fetch
 public keys for dead homeservers from somewhere else.

+**WARNING:** Dendrite supports running all components from the same database in
+Postgres mode, but this is **NOT** a supported configuration with SQLite. When
+using SQLite, all components **MUST** use their own database file.
+
 ## Starting a monolith server

 It is possible to use Naffka as an in-process replacement to Kafka when using
@ -167,30 +171,17 @@ as shown below, it will also listen for HTTPS connections on port 8448.

 The following contains scripts which will run all the required processes in order to point a Matrix client at Dendrite.

-### Client proxy
+### nginx (or other reverse proxy)

-This is what Matrix clients will talk to. If you use the script below, point
-your client at `http://localhost:8008`.
+This is what your clients and federated hosts will talk to. It must forward
+requests onto the correct API server based on URL:

-```bash
-./bin/client-api-proxy \
--bind-address ":8008" \
--client-api-server-url "http://localhost:7771" \
--sync-api-server-url "http://localhost:7773" \
--media-api-server-url "http://localhost:7774" \
-```
+* `/_matrix/client` to the client API server
+* `/_matrix/federation` to the federation API server
+* `/_matrix/key` to the federation API server
+* `/_matrix/media` to the media API server

-### Federation proxy
-
-This is what Matrix servers will talk to. This is only required if you want
-to support federation.
-
-```bash
-./bin/federation-api-proxy \
--bind-address ":8448" \
--federation-api-url "http://localhost:7772" \
--media-api-server-url "http://localhost:7774" \
-```
+See `docs/nginx/polylith-sample.conf` for a sample configuration.

 ### Client API server

@ -198,7 +189,7 @@ This is what implements CS API endpoints. Clients talk to this via the proxy in
 order to send messages, create and join rooms, etc.

 ```bash
-./bin/dendrite-client-api-server --config=dendrite.yaml
+./bin/dendrite-client-api-server --config dendrite.yaml
 ```

 ### Sync server
@ -239,7 +230,7 @@ contacted by other components. This includes the following components.
 This is what implements the room DAG. Clients do not talk to this.

 ```bash
-./bin/dendrite-room-server --config=dendrite.yaml
+./bin/dendrite-room-server --config dendrite.yaml
 ```

 #### Federation sender
--- a/roomserver/internal/input/input_events.go
+++ b/roomserver/internal/input/input_events.go
@ -17,6 +17,7 @@
 package input

 import (
+	"bytes"
 	"context"
 	"fmt"

@ -26,6 +27,7 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/state"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
 	"github.com/sirupsen/logrus"
 )

@ -44,6 +46,28 @@ func (r *Inputer) processRoomEvent(
 	headered := input.Event
 	event := headered.Unwrap()

+	// if we have already got this event then do not process it again, if the input kind is an outlier.
+	// Outliers contain no extra information which may warrant a re-processing.
+	if input.Kind == api.KindOutlier {
+		evs, err := r.DB.EventsFromIDs(ctx, []string{event.EventID()})
+		if err == nil && len(evs) == 1 {
+			// check hash matches if we're on early room versions where the event ID was a random string
+			idFormat, err := headered.RoomVersion.EventIDFormat()
+			if err == nil {
+				switch idFormat {
+				case gomatrixserverlib.EventIDFormatV1:
+					if bytes.Equal(event.EventReference().EventSHA256, evs[0].EventReference().EventSHA256) {
+						util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring")
+						return event.EventID(), nil
+					}
+				default:
+					util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring")
+					return event.EventID(), nil
+				}
+			}
+		}
+	}
+
 	// Check that the event passes authentication checks and work out
 	// the numeric IDs for the auth events.
 	isRejected := false
--- a/roomserver/storage/shared/latest_events_updater.go
+++ b/roomserver/storage/shared/latest_events_updater.go
@ -70,16 +70,14 @@ func (u *LatestEventsUpdater) CurrentStateSnapshotNID() types.StateSnapshotNID {
 	return u.currentStateSnapshotNID
 }

-// StorePreviousEvents implements types.RoomRecentEventsUpdater
+// StorePreviousEvents implements types.RoomRecentEventsUpdater - This must be called from a Writer
 func (u *LatestEventsUpdater) StorePreviousEvents(eventNID types.EventNID, previousEventReferences []gomatrixserverlib.EventReference) error {
-	return u.d.Writer.Do(u.d.DB, u.txn, func(txn *sql.Tx) error {
-		for _, ref := range previousEventReferences {
-			if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, txn, ref.EventID, ref.EventSHA256, eventNID); err != nil {
-				return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err)
-			}
+	for _, ref := range previousEventReferences {
+		if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, u.txn, ref.EventID, ref.EventSHA256, eventNID); err != nil {
+			return fmt.Errorf("u.d.PrevEventsTable.InsertPreviousEvent: %w", err)
 		}
-		return nil
-	})
+	}
+	return nil
 }

 // IsReferenced implements types.RoomRecentEventsUpdater
--- a/roomserver/storage/shared/storage.go
+++ b/roomserver/storage/shared/storage.go
@ -492,15 +492,32 @@ func (d *Database) StoreEvent(
 		if roomInfo == nil && len(prevEvents) > 0 {
 			return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("expected room %q to exist", event.RoomID())
 		}
+		// Create an updater - NB: on sqlite this WILL create a txn as we are directly calling the shared DB form of
+		// GetLatestEventsForUpdate - not via the SQLiteDatabase form which has `nil` txns. This
+		// function only does SELECTs though so the created txn (at this point) is just a read txn like
+		// any other so this is fine. If we ever update GetLatestEventsForUpdate or NewLatestEventsUpdater
+		// to do writes however then this will need to go inside `Writer.Do`.
 		updater, err = d.GetLatestEventsForUpdate(ctx, *roomInfo)
 		if err != nil {
 			return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("NewLatestEventsUpdater: %w", err)
 		}
-		if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil {
-			return 0, types.StateAtEvent{}, nil, "", fmt.Errorf("updater.StorePreviousEvents: %w", err)
+		// Ensure that we atomically store prev events AND commit them. If we don't wrap StorePreviousEvents
+		// and EndTransaction in a writer then it's possible for a new write txn to be made between the two
+		// function calls which will then fail with 'database is locked'. This new write txn would HAVE to be
+		// something like SetRoomAlias/RemoveRoomAlias as normal input events are already done sequentially due to
+		// SupportsConcurrentRoomInputs() == false on sqlite, though this does not apply to setting room aliases
+		// as they don't go via InputRoomEvents
+		err = d.Writer.Do(d.DB, updater.txn, func(txn *sql.Tx) error {
+			if err = updater.StorePreviousEvents(eventNID, prevEvents); err != nil {
+				return fmt.Errorf("updater.StorePreviousEvents: %w", err)
+			}
+			succeeded := true
+			err = sqlutil.EndTransaction(updater, &succeeded)
+			return err
+		})
+		if err != nil {
+			return 0, types.StateAtEvent{}, nil, "", err
 		}
-		succeeded := true
-		err = sqlutil.EndTransaction(updater, &succeeded)
 	}

 	return roomNID, types.StateAtEvent{