diff --git a/internal/sqlutil/migrate.go b/internal/sqlutil/migrate.go
index bf67e8df5..b74c8bb9f 100644
--- a/internal/sqlutil/migrate.go
+++ b/internal/sqlutil/migrate.go
@@ -145,7 +145,11 @@ func (m *Migrator) ExecutedMigrations(ctx context.Context) (map[string]struct{},
 // InsertMigration inserts a migration given there name to the database.
 // This should only be used when manually inserting migrations.
 func InsertMigration(ctx context.Context, db *sql.DB, migrationName string) error {
-	_, err := db.ExecContext(ctx, insertVersionSQL,
+	_, err := db.ExecContext(ctx, createDBMigrationsSQL)
+	if err != nil {
+		return fmt.Errorf("unable to create db_migrations: %w", err)
+	}
+	_, err = db.ExecContext(ctx, insertVersionSQL,
 		migrationName,
 		time.Now().Format(time.RFC3339),
 		internal.VersionString(),
diff --git a/keyserver/storage/postgres/key_changes_table.go b/keyserver/storage/postgres/key_changes_table.go
index 38b8a98a0..17a8559c2 100644
--- a/keyserver/storage/postgres/key_changes_table.go
+++ b/keyserver/storage/postgres/key_changes_table.go
@@ -75,17 +75,8 @@ func executeMigration(ctx context.Context, db *sql.DB) error {
 	// This forces an error, which indicates the migration is already applied, since the
 	// column partition was removed from the table
 	migrationName := "keyserver: refactor key changes"
-	var migrationCount int
 
-	err := db.QueryRowContext(ctx, "SELECT count(*) FROM db_migrations WHERE version = $1", migrationName).Scan(&migrationCount)
-	if err != nil {
-		return err
-	}
-	if migrationCount > 0 {
-		return nil
-	}
-
-	err = db.QueryRowContext(ctx, "select column_name from information_schema.columns where table_name = 'keyserver_key_changes' AND column_name = 'partition'").Err()
+	err := db.QueryRowContext(ctx, "select column_name from information_schema.columns where table_name = 'keyserver_key_changes' AND column_name = 'partition'").Err()
 	if err != nil {
 		if errors.Is(err, sql.ErrNoRows) { // migration was already executed, as the column was removed
 			if err = sqlutil.InsertMigration(ctx, db, migrationName); err != nil {
diff --git a/keyserver/storage/sqlite3/key_changes_table.go b/keyserver/storage/sqlite3/key_changes_table.go
index 9e0cb9ef5..40012fccf 100644
--- a/keyserver/storage/sqlite3/key_changes_table.go
+++ b/keyserver/storage/sqlite3/key_changes_table.go
@@ -74,17 +74,8 @@ func executeMigration(ctx context.Context, db *sql.DB) error {
 	// This forces an error, which indicates the migration is already applied, since the
 	// column partition was removed from the table
 	migrationName := "keyserver: refactor key changes"
-	var migrationCount int
 
-	err := db.QueryRowContext(ctx, "SELECT count(*) FROM db_migrations WHERE version = $1", migrationName).Scan(&migrationCount)
-	if err != nil {
-		return err
-	}
-	if migrationCount > 0 {
-		return nil
-	}
-
-	err = db.QueryRowContext(ctx, `SELECT p.name FROM sqlite_master AS m JOIN pragma_table_info(m.name) AS p WHERE m.name = 'keyserver_key_changes' AND p.name = 'partition'`).Err()
+	err := db.QueryRowContext(ctx, `SELECT p.name FROM sqlite_master AS m JOIN pragma_table_info(m.name) AS p WHERE m.name = 'keyserver_key_changes' AND p.name = 'partition'`).Err()
 	if err != nil {
 		if errors.Is(err, sql.ErrNoRows) { // migration was already executed, as the column was removed
 			if err = sqlutil.InsertMigration(ctx, db, migrationName); err != nil {
diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go
index 141d027ab..547e729c1 100644
--- a/roomserver/storage/postgres/storage.go
+++ b/roomserver/storage/postgres/storage.go
@@ -73,17 +73,8 @@ func executeMigration(ctx context.Context, db *sql.DB) error {
 	// This forces an error, which indicates the migration is already applied, since the
 	// column event_nid was removed from the table
 	migrationName := "roomserver: state blocks refactor"
-	var migrationCount int
 
-	err := db.QueryRowContext(ctx, "SELECT count(*) FROM db_migrations WHERE version = $1", migrationName).Scan(&migrationCount)
-	if err != nil {
-		return err
-	}
-	if migrationCount > 0 {
-		return nil
-	}
-
-	err = db.QueryRowContext(ctx, "select column_name from information_schema.columns where table_name = 'roomserver_state_block' AND column_name = 'event_nid'").Err()
+	err := db.QueryRowContext(ctx, "select column_name from information_schema.columns where table_name = 'roomserver_state_block' AND column_name = 'event_nid'").Err()
 	if err != nil {
 		if errors.Is(err, sql.ErrNoRows) { // migration was already executed, as the column was removed
 			if err = sqlutil.InsertMigration(ctx, db, migrationName); err != nil {
diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go
index c1b3b41de..208936a4f 100644
--- a/roomserver/storage/sqlite3/storage.go
+++ b/roomserver/storage/sqlite3/storage.go
@@ -81,17 +81,8 @@ func executeMigration(ctx context.Context, db *sql.DB) error {
 	// This forces an error, which indicates the migration is already applied, since the
 	// column event_nid was removed from the table
 	migrationName := "roomserver: state blocks refactor"
-	var migrationCount int
 
-	err := db.QueryRowContext(ctx, "SELECT count(*) FROM db_migrations WHERE version = $1", migrationName).Scan(&migrationCount)
-	if err != nil {
-		return err
-	}
-	if migrationCount > 0 {
-		return nil
-	}
-
-	err = db.QueryRowContext(ctx, `SELECT p.name FROM sqlite_master AS m JOIN pragma_table_info(m.name) AS p WHERE m.name = 'roomserver_state_block' AND p.name = 'event_nid'`).Err()
+	err := db.QueryRowContext(ctx, `SELECT p.name FROM sqlite_master AS m JOIN pragma_table_info(m.name) AS p WHERE m.name = 'roomserver_state_block' AND p.name = 'event_nid'`).Err()
 	if err != nil {
 		if errors.Is(err, sql.ErrNoRows) { // migration was already executed, as the column was removed
 			if err = sqlutil.InsertMigration(ctx, db, migrationName); err != nil {