Support auto-upgrading accounts DB

internal/sqlutil/migrate.go

@@ -0,0 +1,126 @@
+package sqlutil
+
+import (
+	"database/sql"
+	"fmt"
+	"runtime"
+	"sort"
+
+	"github.com/matrix-org/dendrite/internal/config"
+	"github.com/pressly/goose"
+)
+
+type Migrations struct {
+	registeredGoMigrations map[int64]*goose.Migration
+}
+
+func NewMigrations() *Migrations {
+	return &Migrations{
+		registeredGoMigrations: make(map[int64]*goose.Migration),
+	}
+}
+
+// Copy-pasted from goose directly to store migrations into a map we control
+
+// AddMigration adds a migration.
+func (m *Migrations) AddMigration(up func(*sql.Tx) error, down func(*sql.Tx) error) {
+	_, filename, _, _ := runtime.Caller(1)
+	m.AddNamedMigration(filename, up, down)
+}
+
+// AddNamedMigration : Add a named migration.
+func (m *Migrations) AddNamedMigration(filename string, up func(*sql.Tx) error, down func(*sql.Tx) error) {
+	v, _ := goose.NumericComponent(filename)
+	migration := &goose.Migration{Version: v, Next: -1, Previous: -1, Registered: true, UpFn: up, DownFn: down, Source: filename}
+
+	if existing, ok := m.registeredGoMigrations[v]; ok {
+		panic(fmt.Sprintf("failed to add migration %q: version conflicts with %q", filename, existing.Source))
+	}
+
+	m.registeredGoMigrations[v] = migration
+}
+
+// RunDeltas up to the latest version.
+func (m *Migrations) RunDeltas(db *sql.DB, props *config.DatabaseOptions) error {
+	maxVer := goose.MaxVersion
+	minVer := int64(0)
+	migrations, err := m.collect(minVer, maxVer)
+	if err != nil {
+		return fmt.Errorf("RunDeltas: Failed to collect migrations: %w", err)
+	}
+	if props.ConnectionString.IsPostgres() {
+		goose.SetDialect("postgres")
+	} else if props.ConnectionString.IsSQLite() {
+		goose.SetDialect("sqlite3")
+	} else {
+		return fmt.Errorf("Unknown connection string: %s", props.ConnectionString)
+	}
+	for {
+		current, err := goose.EnsureDBVersion(db)
+		if err != nil {
+			return fmt.Errorf("RunDeltas: Failed to EnsureDBVersion: %w", err)
+		}
+
+		next, err := migrations.Next(current)
+		if err != nil {
+			if err == goose.ErrNoNextVersion {
+				return nil
+			}
+
+			return fmt.Errorf("RunDeltas: Failed to load next migration to %+v : %w", next, err)
+		}
+
+		if err = next.Up(db); err != nil {
+			return fmt.Errorf("RunDeltas: Failed run migration: %w", err)
+		}
+	}
+}
+
+func (m *Migrations) collect(current, target int64) (goose.Migrations, error) {
+	var migrations goose.Migrations
+
+	// Go migrations registered via goose.AddMigration().
+	for _, migration := range m.registeredGoMigrations {
+		v, err := goose.NumericComponent(migration.Source)
+		if err != nil {
+			return nil, err
+		}
+		if versionFilter(v, current, target) {
+			migrations = append(migrations, migration)
+		}
+	}
+
+	migrations = sortAndConnectMigrations(migrations)
+
+	return migrations, nil
+}
+
+func sortAndConnectMigrations(migrations goose.Migrations) goose.Migrations {
+	sort.Sort(migrations)
+
+	// now that we're sorted in the appropriate direction,
+	// populate next and previous for each migration
+	for i, m := range migrations {
+		prev := int64(-1)
+		if i > 0 {
+			prev = migrations[i-1].Version
+			migrations[i-1].Next = m.Version
+		}
+		migrations[i].Previous = prev
+	}
+
+	return migrations
+}
+
+func versionFilter(v, current, target int64) bool {
+
+	if target > current {
+		return v > current && v <= target
+	}
+
+	if target < current {
+		return v <= current && v > target
+	}
+
+	return false
+}

userapi/storage/accounts/postgres/deltas/20200929203058_is_active.go

@@ -0,0 +1,28 @@
+package deltas
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+func LoadIsActive(m *sqlutil.Migrations) {
+	m.AddMigration(UpIsActive, DownIsActive)
+}
+
+func UpIsActive(tx *sql.Tx) error {
+	_, err := tx.Exec("ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS is_deactivated BOOLEAN DEFAULT FALSE;")
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}
+
+func DownIsActive(tx *sql.Tx) error {
+	_, err := tx.Exec("ALTER TABLE account_accounts DROP COLUMN is_deactivated;")
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	return nil
+}

userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql

@@ -1,9 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS is_deactivated BOOLEAN DEFAULT FALSE;
--- +goose StatementEnd
-
--- +goose Down
--- +goose StatementBegin
-ALTER TABLE account_accounts DROP COLUMN is_deactivated;
--- +goose StatementEnd

userapi/storage/accounts/postgres/storage.go

@@ -25,6 +25,8 @@ import (
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts/postgres/deltas"
+	_ "github.com/matrix-org/dendrite/userapi/storage/accounts/postgres/deltas"
 	"github.com/matrix-org/gomatrixserverlib"
 	"golang.org/x/crypto/bcrypt"
 
@@ -70,7 +72,11 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver
 	if err = d.threepids.prepare(db); err != nil {
 		return nil, err
 	}
-	return d, nil
+
+	m := sqlutil.NewMigrations()
+	deltas.LoadIsActive(m)
+
+	return d, m.RunDeltas(db, dbProperties)
 }
 
 // GetAccountByPassword returns the account associated with the given localpart and password.

userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.go

@@ -1,5 +1,18 @@
--- +goose Up
+package deltas
--- +goose StatementBegin
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/matrix-org/dendrite/internal/sqlutil"
+)
+
+func LoadIsActive(m *sqlutil.Migrations) {
+	m.AddMigration(UpIsActive, DownIsActive)
+}
+
+func UpIsActive(tx *sql.Tx) error {
+	_, err := tx.Exec(`
 	ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
 CREATE TABLE account_accounts (
     localpart TEXT NOT NULL PRIMARY KEY,
@@ -15,11 +28,15 @@ INSERT
         localpart, created_ts, password_hash, appservice_id
     FROM account_accounts_tmp
 ;
-DROP TABLE account_accounts_tmp;
+DROP TABLE account_accounts_tmp;`)
--- +goose StatementEnd
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}
 
--- +goose Down
+func DownIsActive(tx *sql.Tx) error {
--- +goose StatementBegin
+	_, err := tx.Exec(`
 	ALTER TABLE account_accounts RENAME TO account_accounts_tmp;
 CREATE TABLE account_accounts (
     localpart TEXT NOT NULL PRIMARY KEY,
@@ -34,5 +51,9 @@ INSERT
         localpart, created_ts, password_hash, appservice_id
     FROM account_accounts_tmp
 ;
-DROP TABLE account_accounts_tmp;
+DROP TABLE account_accounts_tmp;`)
--- +goose StatementEnd
+	if err != nil {
+		return fmt.Errorf("failed to execute downgrade: %w", err)
+	}
+	return nil
+}

userapi/storage/accounts/sqlite3/storage.go

@@ -26,6 +26,7 @@ import (
 	"github.com/matrix-org/dendrite/internal/config"
 	"github.com/matrix-org/dendrite/internal/sqlutil"
 	"github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/dendrite/userapi/storage/accounts/sqlite3/deltas"
 	"github.com/matrix-org/gomatrixserverlib"
 	"golang.org/x/crypto/bcrypt"
 	// Import the sqlite3 database driver.
@@ -76,7 +77,10 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver
 	if err = d.threepids.prepare(db); err != nil {
 		return nil, err
 	}
-	return d, nil
+	m := sqlutil.NewMigrations()
+	deltas.LoadIsActive(m)
+
+	return d, m.RunDeltas(db, dbProperties)
 }
 
 // GetAccountByPassword returns the account associated with the given localpart and password.