diff --git a/CHANGES.md b/CHANGES.md index 79d2fe2e1..657ca1920 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,26 @@ # Changelog +## Dendrite 0.9.9 (2022-09-22) + +### Features + +* Dendrite will now try to keep HTTP connections open to remote federated servers for a few minutes after a request and attempt to reuse those connections where possible + * This should reduce the amount of time spent on TLS handshakes and often speed up requests to remote servers + * This new behaviour can be disabled with the `federation_api.disable_http_keepalives` option if needed +* A number of dependencies have been updated + +### Fixes + +* A bug where the roomserver did not correctly propagate rewritten room state to downstream components (like the federation API and sync API) has been fixed, which could cause issues when performing a federated join to a previously left room +* Event auth now correctly parses the `join_authorised_via_users_server` field in the membership event content +* Database migrations should no longer produce unique constraint errors at Dendrite startup +* The `origin` of device list updates should now be populated correctly +* Send-to-device messages will no longer be dropped if we fail to publish them to specific devices +* The roomserver query to find state after events will now always resolve state if there are multiple prev events +* The roomserver will now return no memberships if querying history visibility for an event which has no state snapshot +* The device list updater will now mark a device list as stale if a requesting device ID is not known +* Transactions sent to appservices should no longer have accidental duplicated transaction IDs (contributed by [tak-hntlabs](https://github.com/tak-hntlabs)) + ## Dendrite 0.9.8 (2022-09-12) ### Important diff --git a/federationapi/federationapi_keys_test.go b/federationapi/federationapi_keys_test.go index 4469da35b..85cc43aa5 100644 --- a/federationapi/federationapi_keys_test.go +++ b/federationapi/federationapi_keys_test.go @@ -12,12 +12,13 @@ import ( "testing" "time" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/gomatrixserverlib" ) type server struct { @@ -86,7 +87,12 @@ func TestMain(m *testing.M) { cfg.Global.JetStream.StoragePath = config.Path(d) cfg.Global.KeyID = serverKeyID cfg.Global.KeyValidityPeriod = s.validity - cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:") + f, err := os.CreateTemp(d, "federation_keys_test*.db") + if err != nil { + return -1 + } + defer f.Close() + cfg.FederationAPI.Database.ConnectionString = config.DataSource("file:" + f.Name()) s.config = &cfg.FederationAPI // Create a transport which redirects federation requests to diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index 15f7a6840..e923143a7 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -10,6 +10,10 @@ import ( "testing" "time" + "github.com/matrix-org/gomatrix" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/internal" @@ -20,9 +24,6 @@ import ( "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" - "github.com/matrix-org/gomatrix" - "github.com/matrix-org/gomatrixserverlib" - "github.com/nats-io/nats.go" ) type fedRoomserverAPI struct { @@ -271,7 +272,6 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) { cfg.Global.ServerName = gomatrixserverlib.ServerName("localhost") cfg.Global.PrivateKey = privKey cfg.Global.JetStream.InMemory = true - cfg.FederationAPI.Database.ConnectionString = config.DataSource("file::memory:") base := base.NewBaseDendrite(cfg, "Monolith") keyRing := &test.NopJSONVerifier{} // TODO: This is pretty fragile, as if anything calls anything on these nils this test will break. diff --git a/go.mod b/go.mod index 48f6a1a3f..68b5cef84 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220920133029-eb0633c739ef + github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3 github.com/matrix-org/pinecone v0.0.0-20220915154206-df85cb5026fc github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 diff --git a/go.sum b/go.sum index 73c20e4d1..216b4a5fc 100644 --- a/go.sum +++ b/go.sum @@ -384,8 +384,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220920133029-eb0633c739ef h1:qnDDZVkqKAB8if91AUX2VtQGCGnxLBYYt2tG972SYPk= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220920133029-eb0633c739ef/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3 h1:u3FKZmXxfhv3XhD8RziBlt96QTt8eHFhg1upCloBh2g= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= github.com/matrix-org/pinecone v0.0.0-20220915154206-df85cb5026fc h1:NAf6Xec/RoYD+/UT3m2q58yZAu0pYA9C4gc/RbS89ls= github.com/matrix-org/pinecone v0.0.0-20220915154206-df85cb5026fc/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= diff --git a/internal/sqlutil/migrate.go b/internal/sqlutil/migrate.go index b6a8b1f25..a66a75826 100644 --- a/internal/sqlutil/migrate.go +++ b/internal/sqlutil/migrate.go @@ -49,12 +49,13 @@ type Migration struct { Down func(ctx context.Context, txn *sql.Tx) error } -// Migrator +// Migrator contains fields required to run migrations. type Migrator struct { db *sql.DB migrations []Migration knownMigrations map[string]struct{} mutex *sync.Mutex + insertStmt *sql.Stmt } // NewMigrator creates a new DB migrator. @@ -82,35 +83,26 @@ func (m *Migrator) AddMigrations(migrations ...Migration) { // Up executes all migrations in order they were added. func (m *Migrator) Up(ctx context.Context) error { - var ( - err error - dendriteVersion = internal.VersionString() - ) // ensure there is a table for known migrations executedMigrations, err := m.ExecutedMigrations(ctx) if err != nil { return fmt.Errorf("unable to create/get migrations: %w", err) } - + // ensure we close the insert statement, as it's not needed anymore + defer m.close() return WithTransaction(m.db, func(txn *sql.Tx) error { for i := range m.migrations { - now := time.Now().UTC().Format(time.RFC3339) migration := m.migrations[i] // Skip migration if it was already executed if _, ok := executedMigrations[migration.Version]; ok { continue } logrus.Debugf("Executing database migration '%s'", migration.Version) - err = migration.Up(ctx, txn) - if err != nil { + + if err = migration.Up(ctx, txn); err != nil { return fmt.Errorf("unable to execute migration '%s': %w", migration.Version, err) } - _, err = txn.ExecContext(ctx, insertVersionSQL, - migration.Version, - now, - dendriteVersion, - ) - if err != nil { + if err = m.insertMigration(ctx, txn, migration.Version); err != nil { return fmt.Errorf("unable to insert executed migrations: %w", err) } } @@ -118,6 +110,23 @@ func (m *Migrator) Up(ctx context.Context) error { }) } +func (m *Migrator) insertMigration(ctx context.Context, txn *sql.Tx, migrationName string) error { + if m.insertStmt == nil { + stmt, err := m.db.Prepare(insertVersionSQL) + if err != nil { + return fmt.Errorf("unable to prepare insert statement: %w", err) + } + m.insertStmt = stmt + } + stmt := TxStmtContext(ctx, txn, m.insertStmt) + _, err := stmt.ExecContext(ctx, + migrationName, + time.Now().Format(time.RFC3339), + internal.VersionString(), + ) + return err +} + // ExecutedMigrations returns a map with already executed migrations in addition to creating the // migrations table, if it doesn't exist. func (m *Migrator) ExecutedMigrations(ctx context.Context) (map[string]struct{}, error) { @@ -146,19 +155,20 @@ func (m *Migrator) ExecutedMigrations(ctx context.Context) (map[string]struct{}, // inserts a migration given their name to the database. // This should only be used when manually inserting migrations. func InsertMigration(ctx context.Context, db *sql.DB, migrationName string) error { - _, err := db.ExecContext(ctx, createDBMigrationsSQL) + m := NewMigrator(db) + defer m.close() + existingMigrations, err := m.ExecutedMigrations(ctx) if err != nil { - return fmt.Errorf("unable to create db_migrations: %w", err) + return err } - _, err = db.ExecContext(ctx, insertVersionSQL, - migrationName, - time.Now().Format(time.RFC3339), - internal.VersionString(), - ) - // If the migration was already executed, we'll get a unique constraint error, - // return nil instead, to avoid unnecessary logging. - if IsUniqueConstraintViolationErr(err) { + if _, ok := existingMigrations[migrationName]; ok { return nil } - return err + return m.insertMigration(ctx, nil, migrationName) +} + +func (m *Migrator) close() { + if m.insertStmt != nil { + internal.CloseAndLogIfError(context.Background(), m.insertStmt, "unable to close insert statement") + } } diff --git a/internal/sqlutil/migrate_test.go b/internal/sqlutil/migrate_test.go index d8bcae196..5116237a1 100644 --- a/internal/sqlutil/migrate_test.go +++ b/internal/sqlutil/migrate_test.go @@ -7,9 +7,10 @@ import ( "reflect" "testing" + _ "github.com/mattn/go-sqlite3" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/test" - _ "github.com/mattn/go-sqlite3" ) var dummyMigrations = []sqlutil.Migration{ @@ -81,11 +82,12 @@ func Test_migrations_Up(t *testing.T) { } ctx := context.Background() - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - conStr, close := test.PrepareDBConnectionString(t, dbType) - defer close() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + conStr, close := test.PrepareDBConnectionString(t, dbType) + defer close() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { driverName := "sqlite3" if dbType == test.DBTypePostgres { driverName = "postgres" @@ -107,6 +109,30 @@ func Test_migrations_Up(t *testing.T) { t.Errorf("expected: %+v, got %v", tt.wantResult, result) } }) - }) - } + } + }) +} + +func Test_insertMigration(t *testing.T) { + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + conStr, close := test.PrepareDBConnectionString(t, dbType) + defer close() + driverName := "sqlite3" + if dbType == test.DBTypePostgres { + driverName = "postgres" + } + + db, err := sql.Open(driverName, conStr) + if err != nil { + t.Errorf("unable to open database: %v", err) + } + + if err := sqlutil.InsertMigration(context.Background(), db, "testing"); err != nil { + t.Fatalf("unable to insert migration: %s", err) + } + // Second insert should not return an error, as it was already executed. + if err := sqlutil.InsertMigration(context.Background(), db, "testing"); err != nil { + t.Fatalf("unable to insert migration: %s", err) + } + }) } diff --git a/internal/version.go b/internal/version.go index ef7b879c2..f9b101702 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 9 - VersionPatch = 8 + VersionPatch = 9 VersionTag = "" // example: "rc1" ) diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index c2c9616e8..b2ea105ff 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -308,6 +308,12 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. if syncReq.Since.IsEmpty() { // Complete sync syncReq.Response.NextBatch = types.StreamingToken{ + // Get the current DeviceListPosition first, as the currentPosition + // might advance while processing other streams, resulting in flakey + // tests. + DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync( + syncReq.Context, syncReq, + ), PDUPosition: rp.streams.PDUStreamProvider.CompleteSync( syncReq.Context, syncReq, ), @@ -329,9 +335,6 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. NotificationDataPosition: rp.streams.NotificationDataStreamProvider.CompleteSync( syncReq.Context, syncReq, ), - DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync( - syncReq.Context, syncReq, - ), PresencePosition: rp.streams.PresenceStreamProvider.CompleteSync( syncReq.Context, syncReq, ),