From 4b797f28be8343680a2ef273c9a91712a674e133 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 15 Oct 2020 14:42:14 +0100 Subject: [PATCH] Support auto-upgrading accounts DB --- internal/sqlutil/migrate.go | 126 ++++++++++++++++++ .../deltas/20200929203058_is_active.go | 28 ++++ .../deltas/20200929203058_is_active.sql | 9 -- userapi/storage/accounts/postgres/storage.go | 8 +- ...active.sql => 20200929203058_is_active.go} | 41 ++++-- userapi/storage/accounts/sqlite3/storage.go | 6 +- 6 files changed, 197 insertions(+), 21 deletions(-) create mode 100644 internal/sqlutil/migrate.go create mode 100644 userapi/storage/accounts/postgres/deltas/20200929203058_is_active.go delete mode 100644 userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql rename userapi/storage/accounts/sqlite3/deltas/{20200929203058_is_active.sql => 20200929203058_is_active.go} (52%) diff --git a/internal/sqlutil/migrate.go b/internal/sqlutil/migrate.go new file mode 100644 index 000000000..da1a7cbe8 --- /dev/null +++ b/internal/sqlutil/migrate.go @@ -0,0 +1,126 @@ +package sqlutil + +import ( + "database/sql" + "fmt" + "runtime" + "sort" + + "github.com/matrix-org/dendrite/internal/config" + "github.com/pressly/goose" +) + +type Migrations struct { + registeredGoMigrations map[int64]*goose.Migration +} + +func NewMigrations() *Migrations { + return &Migrations{ + registeredGoMigrations: make(map[int64]*goose.Migration), + } +} + +// Copy-pasted from goose directly to store migrations into a map we control + +// AddMigration adds a migration. +func (m *Migrations) AddMigration(up func(*sql.Tx) error, down func(*sql.Tx) error) { + _, filename, _, _ := runtime.Caller(1) + m.AddNamedMigration(filename, up, down) +} + +// AddNamedMigration : Add a named migration. +func (m *Migrations) AddNamedMigration(filename string, up func(*sql.Tx) error, down func(*sql.Tx) error) { + v, _ := goose.NumericComponent(filename) + migration := &goose.Migration{Version: v, Next: -1, Previous: -1, Registered: true, UpFn: up, DownFn: down, Source: filename} + + if existing, ok := m.registeredGoMigrations[v]; ok { + panic(fmt.Sprintf("failed to add migration %q: version conflicts with %q", filename, existing.Source)) + } + + m.registeredGoMigrations[v] = migration +} + +// RunDeltas up to the latest version. +func (m *Migrations) RunDeltas(db *sql.DB, props *config.DatabaseOptions) error { + maxVer := goose.MaxVersion + minVer := int64(0) + migrations, err := m.collect(minVer, maxVer) + if err != nil { + return fmt.Errorf("RunDeltas: Failed to collect migrations: %w", err) + } + if props.ConnectionString.IsPostgres() { + goose.SetDialect("postgres") + } else if props.ConnectionString.IsSQLite() { + goose.SetDialect("sqlite3") + } else { + return fmt.Errorf("Unknown connection string: %s", props.ConnectionString) + } + for { + current, err := goose.EnsureDBVersion(db) + if err != nil { + return fmt.Errorf("RunDeltas: Failed to EnsureDBVersion: %w", err) + } + + next, err := migrations.Next(current) + if err != nil { + if err == goose.ErrNoNextVersion { + return nil + } + + return fmt.Errorf("RunDeltas: Failed to load next migration to %+v : %w", next, err) + } + + if err = next.Up(db); err != nil { + return fmt.Errorf("RunDeltas: Failed run migration: %w", err) + } + } +} + +func (m *Migrations) collect(current, target int64) (goose.Migrations, error) { + var migrations goose.Migrations + + // Go migrations registered via goose.AddMigration(). + for _, migration := range m.registeredGoMigrations { + v, err := goose.NumericComponent(migration.Source) + if err != nil { + return nil, err + } + if versionFilter(v, current, target) { + migrations = append(migrations, migration) + } + } + + migrations = sortAndConnectMigrations(migrations) + + return migrations, nil +} + +func sortAndConnectMigrations(migrations goose.Migrations) goose.Migrations { + sort.Sort(migrations) + + // now that we're sorted in the appropriate direction, + // populate next and previous for each migration + for i, m := range migrations { + prev := int64(-1) + if i > 0 { + prev = migrations[i-1].Version + migrations[i-1].Next = m.Version + } + migrations[i].Previous = prev + } + + return migrations +} + +func versionFilter(v, current, target int64) bool { + + if target > current { + return v > current && v <= target + } + + if target < current { + return v <= current && v > target + } + + return false +} diff --git a/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.go b/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.go new file mode 100644 index 000000000..1e3e78567 --- /dev/null +++ b/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.go @@ -0,0 +1,28 @@ +package deltas + +import ( + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal/sqlutil" +) + +func LoadIsActive(m *sqlutil.Migrations) { + m.AddMigration(UpIsActive, DownIsActive) +} + +func UpIsActive(tx *sql.Tx) error { + _, err := tx.Exec("ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS is_deactivated BOOLEAN DEFAULT FALSE;") + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} + +func DownIsActive(tx *sql.Tx) error { + _, err := tx.Exec("ALTER TABLE account_accounts DROP COLUMN is_deactivated;") + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + return nil +} diff --git a/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql b/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql deleted file mode 100644 index 32e6e1664..000000000 --- a/userapi/storage/accounts/postgres/deltas/20200929203058_is_active.sql +++ /dev/null @@ -1,9 +0,0 @@ --- +goose Up --- +goose StatementBegin -ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS is_deactivated BOOLEAN DEFAULT FALSE; --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin -ALTER TABLE account_accounts DROP COLUMN is_deactivated; --- +goose StatementEnd diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go index 2230f7e79..768bd208b 100644 --- a/userapi/storage/accounts/postgres/storage.go +++ b/userapi/storage/accounts/postgres/storage.go @@ -25,6 +25,8 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts/postgres/deltas" + _ "github.com/matrix-org/dendrite/userapi/storage/accounts/postgres/deltas" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" @@ -70,7 +72,11 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err = d.threepids.prepare(db); err != nil { return nil, err } - return d, nil + + m := sqlutil.NewMigrations() + deltas.LoadIsActive(m) + + return d, m.RunDeltas(db, dbProperties) } // GetAccountByPassword returns the account associated with the given localpart and password. diff --git a/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql b/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.go similarity index 52% rename from userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql rename to userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.go index 51e9bae3c..5c4c9c5e3 100644 --- a/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.sql +++ b/userapi/storage/accounts/sqlite3/deltas/20200929203058_is_active.go @@ -1,6 +1,19 @@ --- +goose Up --- +goose StatementBegin -ALTER TABLE account_accounts RENAME TO account_accounts_tmp; +package deltas + +import ( + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal/sqlutil" +) + +func LoadIsActive(m *sqlutil.Migrations) { + m.AddMigration(UpIsActive, DownIsActive) +} + +func UpIsActive(tx *sql.Tx) error { + _, err := tx.Exec(` + ALTER TABLE account_accounts RENAME TO account_accounts_tmp; CREATE TABLE account_accounts ( localpart TEXT NOT NULL PRIMARY KEY, created_ts BIGINT NOT NULL, @@ -15,12 +28,16 @@ INSERT localpart, created_ts, password_hash, appservice_id FROM account_accounts_tmp ; -DROP TABLE account_accounts_tmp; --- +goose StatementEnd +DROP TABLE account_accounts_tmp;`) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} --- +goose Down --- +goose StatementBegin -ALTER TABLE account_accounts RENAME TO account_accounts_tmp; +func DownIsActive(tx *sql.Tx) error { + _, err := tx.Exec(` + ALTER TABLE account_accounts RENAME TO account_accounts_tmp; CREATE TABLE account_accounts ( localpart TEXT NOT NULL PRIMARY KEY, created_ts BIGINT NOT NULL, @@ -34,5 +51,9 @@ INSERT localpart, created_ts, password_hash, appservice_id FROM account_accounts_tmp ; -DROP TABLE account_accounts_tmp; --- +goose StatementEnd +DROP TABLE account_accounts_tmp;`) + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + return nil +} diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go index 7a2830a93..fe0d2eacc 100644 --- a/userapi/storage/accounts/sqlite3/storage.go +++ b/userapi/storage/accounts/sqlite3/storage.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts/sqlite3/deltas" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" // Import the sqlite3 database driver. @@ -76,7 +77,10 @@ func NewDatabase(dbProperties *config.DatabaseOptions, serverName gomatrixserver if err = d.threepids.prepare(db); err != nil { return nil, err } - return d, nil + m := sqlutil.NewMigrations() + deltas.LoadIsActive(m) + + return d, m.RunDeltas(db, dbProperties) } // GetAccountByPassword returns the account associated with the given localpart and password.