Merge branch 'main' of github.com:matrix-org/dendrite into s7evink/consent-tracking

cmd/dendrite-demo-yggdrasil/main.go

@@ -21,8 +21,11 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"os"
 	"time"
 
+	"github.com/matrix-org/gomatrixserverlib"
+
 	"github.com/gorilla/mux"
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed"
@@ -42,8 +45,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/setup/mscs"
 	"github.com/matrix-org/dendrite/userapi"
-	"github.com/matrix-org/gomatrixserverlib"
-
 	"github.com/sirupsen/logrus"
 
 	_ "github.com/mattn/go-sqlite3"
@@ -63,33 +64,42 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	/*
+
-		ygg.SetMulticastEnabled(true)
+	// iterate through the cli args and check if the config flag was set
-		if instancePeer != nil && *instancePeer != "" {
+	configFlagSet := false
-			if err = ygg.SetStaticPeer(*instancePeer); err != nil {
+	for _, arg := range os.Args {
-				logrus.WithError(err).Error("Failed to set static peer")
+		if arg == "--config" || arg == "-config" {
-			}
+			configFlagSet = true
+			break
 		}
-	*/
+	}
 
 	cfg := &config.Dendrite{}
-	cfg.Defaults(true)
+
+	// use custom config if config flag is set
+	if configFlagSet {
+		cfg = setup.ParseFlags(true)
+	} else {
+		cfg.Defaults(true)
+		cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
+		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
+		cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
+		cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
+		cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
+		cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
+		cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
+		cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
+		cfg.MSCs.MSCs = []string{"msc2836"}
+		cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
+		if err = cfg.Derive(); err != nil {
+			panic(err)
+		}
+	}
+
+	// always override ServerName, PrivateKey and KeyID
 	cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName())
 	cfg.Global.PrivateKey = ygg.PrivateKey()
-	cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID)
+	cfg.Global.KeyID = signing.KeyID
-	cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName))
-	cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName))
-	cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName))
-	cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName))
-	cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName))
-	cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName))
-	cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName))
-	cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName))
-	cfg.MSCs.MSCs = []string{"msc2836"}
-	cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName))
-	if err = cfg.Derive(); err != nil {
-		panic(err)
-	}
 
 	base := base.NewBaseDendrite(cfg, "Monolith")
 	defer base.Close() // nolint: errcheck

roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go

@@ -256,23 +256,17 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 		return fmt.Errorf("assertion query failed: %s", err)
 	}
 	if count > 0 {
-		var debugEventID, debugRoomID string
+		var res sql.Result
-		var debugEventTypeNID, debugStateKeyNID, debugSnapNID, debugDepth int64
+		var c int64
-		err = tx.QueryRow(
+		res, err = tx.Exec(`UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid)
-			`SELECT event_id, event_type_nid, event_state_key_nid, roomserver_events.state_snapshot_nid, depth, room_id FROM roomserver_events
+		if err != nil && err != sql.ErrNoRows {
-				JOIN roomserver_rooms ON roomserver_rooms.room_nid = roomserver_events.room_nid WHERE roomserver_events.state_snapshot_nid < $1 AND roomserver_events.state_snapshot_nid != 0`, maxsnapshotid,
+			return fmt.Errorf("failed to reset invalid state snapshots: %w", err)
-		).Scan(&debugEventID, &debugEventTypeNID, &debugStateKeyNID, &debugSnapNID, &debugDepth, &debugRoomID)
+		}
-		if err != nil {
+		if c, err = res.RowsAffected(); err != nil {
-			logrus.Errorf("cannot extract debug info: %v", err)
+			return fmt.Errorf("failed to get row count for invalid state snapshots updated: %w", err)
-		} else {
+		} else if c != count {
-			logrus.Errorf(
+			return fmt.Errorf("expected to reset %d event(s) but only updated %d event(s)", count, c)
-				"Affected row: event_id=%v room_id=%v type=%v state_key=%v snapshot=%v depth=%v",
-				debugEventID, debugRoomID, debugEventTypeNID, debugStateKeyNID, debugSnapNID, debugDepth,
-			)
-			logrus.Errorf("To fix this manually, run this query first then retry the migration: "+
-				"UPDATE roomserver_events SET state_snapshot_nid=0 WHERE event_id='%v'", debugEventID)
 		}
-		return fmt.Errorf("%d events exist in roomserver_events which have not been converted to a new state_snapshot_nid; this is a bug, please report", count)
 	}
 	if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid).Scan(&count); err != nil {
 		return fmt.Errorf("assertion query failed: %s", err)

roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go

@@ -179,7 +179,17 @@ func UpStateBlocksRefactor(tx *sql.Tx) error {
 		return fmt.Errorf("assertion query failed: %s", err)
 	}
 	if count > 0 {
-		return fmt.Errorf("%d events exist in roomserver_events which have not been converted to a new state_snapshot_nid; this is a bug, please report", count)
+		var res sql.Result
+		var c int64
+		res, err = tx.Exec(`UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, oldMaxSnapshotID)
+		if err != nil && err != sql.ErrNoRows {
+			return fmt.Errorf("failed to reset invalid state snapshots: %w", err)
+		}
+		if c, err = res.RowsAffected(); err != nil {
+			return fmt.Errorf("failed to get row count for invalid state snapshots updated: %w", err)
+		} else if c != count {
+			return fmt.Errorf("expected to reset %d event(s) but only updated %d event(s)", count, c)
+		}
 	}
 	if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, oldMaxSnapshotID).Scan(&count); err != nil {
 		return fmt.Errorf("assertion query failed: %s", err)