Merge branch 'main' into mailbox

cmd/generate-config/main.go

@@ -70,6 +70,7 @@ func main() {
 			cfg.AppServiceAPI.DisableTLSValidation = true
 			cfg.ClientAPI.RateLimiting.Enabled = false
 			cfg.FederationAPI.DisableTLSValidation = false
+			cfg.FederationAPI.DisableHTTPKeepalives = true
 			// don't hit matrix.org when running tests!!!
 			cfg.FederationAPI.KeyPerspectives = config.KeyPerspectives{}
 			cfg.MediaAPI.BasePath = config.Path(filepath.Join(*dirPath, "media"))

internal/log.go

@@ -24,6 +24,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync"
 
 	"github.com/matrix-org/util"
 
@@ -37,6 +38,7 @@ import (
 // this unfortunately results in us adding the same hook multiple times.
 // This map ensures we only ever add one level hook.
 var stdLevelLogAdded = make(map[logrus.Level]bool)
+var levelLogAddedMu = &sync.Mutex{}
 
 type utcFormatter struct {
 	logrus.Formatter

internal/log_unix.go

@@ -85,6 +85,8 @@ func checkSyslogHookParams(params map[string]interface{}) {
 }
 
 func setupStdLogHook(level logrus.Level) {
+	levelLogAddedMu.Lock()
+	defer levelLogAddedMu.Unlock()
 	if stdLevelLogAdded[level] {
 		return
 	}

setup/jetstream/helpers.go

@@ -77,6 +77,11 @@ func JetStreamConsumer(
 					// The consumer was deleted so stop.
 					return
 				} else {
+					// Unfortunately, there's no ErrServerShutdown or similar, so we need to compare the string
+					if err.Error() == "nats: Server Shutdown" {
+						logrus.WithContext(ctx).Warn("nats server shutting down")
+						return
+					}
 					// Something else went wrong, so we'll panic.
 					sentry.CaptureException(err)
 					logrus.WithContext(ctx).WithField("subject", subj).Fatal(err)

sytest-blacklist

@@ -7,6 +7,7 @@ AS-ghosted users can use rooms via AS
 Events in rooms with AS-hosted room aliases are sent to AS server
 Inviting an AS-hosted user asks the AS server
 Accesing an AS-hosted room alias asks the AS server
+If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes
 
 # This will fail in HTTP API mode, so blacklisted for now
 If a device list update goes missing, the server resyncs on the next one

sytest-whitelist

@@ -778,4 +778,9 @@ Can receive redactions from regular users over federation in room version 10
 New federated private chats get full presence information (SYN-115)
 /state returns M_NOT_FOUND for an outlier
 /state_ids returns M_NOT_FOUND for an outlier
-Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state
+Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state
+Invited user can reject invite for empty room
+Invited user can reject local invite after originator leaves
+Guest users can join guest_access rooms
+Forgotten room messages cannot be paginated
+Local device key changes get to remote servers with correct prev_id

test/db.go

@@ -19,12 +19,11 @@ import (
 	"database/sql"
 	"encoding/hex"
 	"fmt"
-	"math/rand"
 	"os"
 	"os/exec"
 	"os/user"
+	"path/filepath"
 	"testing"
-	"time"
 
 	"github.com/lib/pq"
 )
@@ -104,15 +103,21 @@ func currentUser() string {
 // unless close() is called.
 func PrepareDBConnectionString(t *testing.T, dbType DBType) (connStr string, close func()) {
 	if dbType == DBTypeSQLite {
-		rand.Seed(time.Now().UnixNano())
-		randBytes := make([]byte, 32)
-		rand.Read(randBytes)
-		dbName := fmt.Sprintf("dendrite_test_%s.db", hex.EncodeToString(randBytes[:16]))
-		return fmt.Sprintf("file:%s", dbName), func() {
-			err := os.Remove(dbName)
-			if err != nil {
-				t.Fatalf("failed to cleanup sqlite db '%s': %s", dbName, err)
-			}
+		// TODO (devon): remove?
+		// rand.Seed(time.Now().UnixNano())
+		// randBytes := make([]byte, 32)
+		// rand.Read(randBytes)
+		// dbName := fmt.Sprintf("dendrite_test_%s.db", hex.EncodeToString(randBytes[:16]))
+		// return fmt.Sprintf("file:%s", dbName), func() {
+		// 	err := os.Remove(dbName)
+		// 	if err != nil {
+		// 		t.Fatalf("failed to cleanup sqlite db '%s': %s", dbName, err)
+		// 	}
+
+		// this will be made in the t.TempDir, which is unique per test
+		dbname := filepath.Join(t.TempDir(), "dendrite_test.db")
+		return fmt.Sprintf("file:%s", dbname), func() {
+			t.Cleanup(func() {}) // removes the t.TempDir
 		}
 	}
 

test/testrig/base.go

@@ -15,18 +15,14 @@
 package testrig
 
 import (
-	"errors"
 	"fmt"
-	"io/fs"
-	"os"
-	"strings"
+	"path/filepath"
 	"testing"
 
-	"github.com/nats-io/nats.go"
-
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	"github.com/matrix-org/dendrite/test"
+	"github.com/nats-io/nats.go"
 )
 
 func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, func()) {
@@ -75,49 +71,45 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f
 		})
 		cfg.Global.ServerName = "test"
 
-		// Configure each components db connection to be unique so tests can run in parallel
-		connStr, _ := test.PrepareDBConnectionString(t, dbType)
-		cfg.FederationAPI.Database.ConnectionString = config.DataSource(connStr)
-		connStr, _ = test.PrepareDBConnectionString(t, dbType)
-		cfg.KeyServer.Database.ConnectionString = config.DataSource(connStr)
-		connStr, _ = test.PrepareDBConnectionString(t, dbType)
-		cfg.MSCs.Database.ConnectionString = config.DataSource(connStr)
-		connStr, _ = test.PrepareDBConnectionString(t, dbType)
-		cfg.MediaAPI.Database.ConnectionString = config.DataSource(connStr)
-		connStr, _ = test.PrepareDBConnectionString(t, dbType)
-		cfg.RoomServer.Database.ConnectionString = config.DataSource(connStr)
-		connStr, _ = test.PrepareDBConnectionString(t, dbType)
-		cfg.SyncAPI.Database.ConnectionString = config.DataSource(connStr)
-		connStr, _ = test.PrepareDBConnectionString(t, dbType)
-		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(connStr)
-		connStr, _ = test.PrepareDBConnectionString(t, dbType)
-		cfg.RelayAPI.Database.ConnectionString = config.DataSource(connStr)
+		// TODO (devon): remove?
+		// // Configure each components db connection to be unique so tests can run in parallel
+		// connStr, _ := test.PrepareDBConnectionString(t, dbType)
+		// cfg.FederationAPI.Database.ConnectionString = config.DataSource(connStr)
+		// connStr, _ = test.PrepareDBConnectionString(t, dbType)
+		// cfg.KeyServer.Database.ConnectionString = config.DataSource(connStr)
+		// connStr, _ = test.PrepareDBConnectionString(t, dbType)
+		// cfg.MSCs.Database.ConnectionString = config.DataSource(connStr)
+		// connStr, _ = test.PrepareDBConnectionString(t, dbType)
+		// cfg.MediaAPI.Database.ConnectionString = config.DataSource(connStr)
+		// connStr, _ = test.PrepareDBConnectionString(t, dbType)
+		// cfg.RoomServer.Database.ConnectionString = config.DataSource(connStr)
+		// connStr, _ = test.PrepareDBConnectionString(t, dbType)
+		// cfg.SyncAPI.Database.ConnectionString = config.DataSource(connStr)
+		// connStr, _ = test.PrepareDBConnectionString(t, dbType)
+		// cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(connStr)
+		// connStr, _ = test.PrepareDBConnectionString(t, dbType)
+		// cfg.RelayAPI.Database.ConnectionString = config.DataSource(connStr)
 
 		// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
 		// the file system event with InMemory=true :(
 		cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType)
+
+		// Use a temp dir provided by go for tests, this will be cleanup by a call to t.CleanUp()
+		tempDir := t.TempDir()
+		cfg.FederationAPI.Database.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "federationapi.db"))
+		cfg.KeyServer.Database.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "keyserver.db"))
+		cfg.MSCs.Database.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "mscs.db"))
+		cfg.MediaAPI.Database.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "mediaapi.db"))
+		cfg.RoomServer.Database.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "roomserver.db"))
+		cfg.SyncAPI.Database.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "syncapi.db"))
+		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "userapi.db"))
+		cfg.RelayAPI.Database.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "relayapi.db"))
+
 		base := base.NewBaseDendrite(&cfg, "Test", base.DisableMetrics)
 		return base, func() {
 			base.ShutdownDendrite()
 			base.WaitForShutdown()
-			// cleanup db files. This risks getting out of sync as we add more database strings :(
-			dbFiles := []config.DataSource{
-				cfg.FederationAPI.Database.ConnectionString,
-				cfg.KeyServer.Database.ConnectionString,
-				cfg.MSCs.Database.ConnectionString,
-				cfg.MediaAPI.Database.ConnectionString,
-				cfg.RoomServer.Database.ConnectionString,
-				cfg.SyncAPI.Database.ConnectionString,
-				cfg.UserAPI.AccountDatabase.ConnectionString,
-				cfg.RelayAPI.Database.ConnectionString,
-			}
-			for _, fileURI := range dbFiles {
-				path := strings.TrimPrefix(string(fileURI), "file:")
-				err := os.Remove(path)
-				if err != nil && !errors.Is(err, fs.ErrNotExist) {
-					t.Fatalf("failed to cleanup sqlite db '%s': %s", fileURI, err)
-				}
-			}
+			t.Cleanup(func() {}) // removes t.TempDir, where all database files are created
 		}
 	default:
 		t.Fatalf("unknown db type: %v", dbType)