From a02dd7721d8555391597444d185d402f94b626ae Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 21 Feb 2022 15:25:54 +0000 Subject: [PATCH 1/2] Reset invalid state snapshots for events during state storage refactor migration (#2209) This should help with #2204. We can't do this for rooms, only events. --- .../2021041615092700_state_blocks_refactor.go | 26 +++++++------------ .../2021041615092700_state_blocks_refactor.go | 12 ++++++++- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go b/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go index 06740dc8b..06442a4c3 100644 --- a/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go +++ b/roomserver/storage/postgres/deltas/2021041615092700_state_blocks_refactor.go @@ -256,23 +256,17 @@ func UpStateBlocksRefactor(tx *sql.Tx) error { return fmt.Errorf("assertion query failed: %s", err) } if count > 0 { - var debugEventID, debugRoomID string - var debugEventTypeNID, debugStateKeyNID, debugSnapNID, debugDepth int64 - err = tx.QueryRow( - `SELECT event_id, event_type_nid, event_state_key_nid, roomserver_events.state_snapshot_nid, depth, room_id FROM roomserver_events - JOIN roomserver_rooms ON roomserver_rooms.room_nid = roomserver_events.room_nid WHERE roomserver_events.state_snapshot_nid < $1 AND roomserver_events.state_snapshot_nid != 0`, maxsnapshotid, - ).Scan(&debugEventID, &debugEventTypeNID, &debugStateKeyNID, &debugSnapNID, &debugDepth, &debugRoomID) - if err != nil { - logrus.Errorf("cannot extract debug info: %v", err) - } else { - logrus.Errorf( - "Affected row: event_id=%v room_id=%v type=%v state_key=%v snapshot=%v depth=%v", - debugEventID, debugRoomID, debugEventTypeNID, debugStateKeyNID, debugSnapNID, debugDepth, - ) - logrus.Errorf("To fix this manually, run this query first then retry the migration: "+ - "UPDATE roomserver_events SET state_snapshot_nid=0 WHERE event_id='%v'", debugEventID) + var res sql.Result + var c int64 + res, err = tx.Exec(`UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid) + if err != nil && err != sql.ErrNoRows { + return fmt.Errorf("failed to reset invalid state snapshots: %w", err) + } + if c, err = res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get row count for invalid state snapshots updated: %w", err) + } else if c != count { + return fmt.Errorf("expected to reset %d event(s) but only updated %d event(s)", count, c) } - return fmt.Errorf("%d events exist in roomserver_events which have not been converted to a new state_snapshot_nid; this is a bug, please report", count) } if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, maxsnapshotid).Scan(&count); err != nil { return fmt.Errorf("assertion query failed: %s", err) diff --git a/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go b/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go index 8d0331748..8f5ab8fc5 100644 --- a/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go +++ b/roomserver/storage/sqlite3/deltas/2021041615092700_state_blocks_refactor.go @@ -179,7 +179,17 @@ func UpStateBlocksRefactor(tx *sql.Tx) error { return fmt.Errorf("assertion query failed: %s", err) } if count > 0 { - return fmt.Errorf("%d events exist in roomserver_events which have not been converted to a new state_snapshot_nid; this is a bug, please report", count) + var res sql.Result + var c int64 + res, err = tx.Exec(`UPDATE roomserver_events SET state_snapshot_nid = 0 WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, oldMaxSnapshotID) + if err != nil && err != sql.ErrNoRows { + return fmt.Errorf("failed to reset invalid state snapshots: %w", err) + } + if c, err = res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get row count for invalid state snapshots updated: %w", err) + } else if c != count { + return fmt.Errorf("expected to reset %d event(s) but only updated %d event(s)", count, c) + } } if err = tx.QueryRow(`SELECT COUNT(*) FROM roomserver_rooms WHERE state_snapshot_nid < $1 AND state_snapshot_nid != 0`, oldMaxSnapshotID).Scan(&count); err != nil { return fmt.Errorf("assertion query failed: %s", err) From 280e9b19a195e3ce19f0fa5bc0e94bb09e397a23 Mon Sep 17 00:00:00 2001 From: Benjamin Nater Date: Mon, 21 Feb 2022 16:36:03 +0100 Subject: [PATCH 2/2] Don't ignore config flag for yggdrasil demo if set (#2070) * Don't ignore config flag for yggdrasil demo if set Signed-off-by: bn4t * run goimports Signed-off-by: bn4t * always override ServerName, PrivateKey and KeyID Co-authored-by: Neil Alexander --- cmd/dendrite-demo-yggdrasil/main.go | 58 +++++++++++++++++------------ 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 49e096bd1..d16f0e9e5 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -21,8 +21,11 @@ import ( "fmt" "net" "net/http" + "os" "time" + "github.com/matrix-org/gomatrixserverlib" + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed" @@ -42,8 +45,6 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/mscs" "github.com/matrix-org/dendrite/userapi" - "github.com/matrix-org/gomatrixserverlib" - "github.com/sirupsen/logrus" _ "github.com/mattn/go-sqlite3" @@ -63,33 +64,42 @@ func main() { if err != nil { panic(err) } - /* - ygg.SetMulticastEnabled(true) - if instancePeer != nil && *instancePeer != "" { - if err = ygg.SetStaticPeer(*instancePeer); err != nil { - logrus.WithError(err).Error("Failed to set static peer") - } + + // iterate through the cli args and check if the config flag was set + configFlagSet := false + for _, arg := range os.Args { + if arg == "--config" || arg == "-config" { + configFlagSet = true + break } - */ + } cfg := &config.Dendrite{} - cfg.Defaults(true) + + // use custom config if config flag is set + if configFlagSet { + cfg = setup.ParseFlags(true) + } else { + cfg.Defaults(true) + cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName)) + cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) + cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) + cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) + cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) + cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName)) + cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName)) + cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) + cfg.MSCs.MSCs = []string{"msc2836"} + cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName)) + if err = cfg.Derive(); err != nil { + panic(err) + } + } + + // always override ServerName, PrivateKey and KeyID cfg.Global.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName()) cfg.Global.PrivateKey = ygg.PrivateKey() - cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) - cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName)) - cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) - cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) - cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) - cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) - cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName)) - cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName)) - cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) - cfg.MSCs.MSCs = []string{"msc2836"} - cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName)) - if err = cfg.Derive(); err != nil { - panic(err) - } + cfg.Global.KeyID = signing.KeyID base := base.NewBaseDendrite(cfg, "Monolith") defer base.Close() // nolint: errcheck