Fix sqlite unit test parallelization issues

.gitignore

@@ -56,6 +56,7 @@ dendrite.yaml
 
 # Database files
 *.db
+*.db-journal
 
 # Log files
 *.log*

federationapi/routing/routing.go

@@ -67,9 +67,11 @@ func Setup(
 	servers federationAPI.ServersInRoomProvider,
 	producer *producers.SyncAPIProducer,
 ) {
-	prometheus.MustRegister(
+	if cfg.Matrix.Metrics.Enabled {
-		internal.PDUCountTotal, internal.EDUCountTotal,
+		prometheus.MustRegister(
-	)
+			internal.PDUCountTotal, internal.EDUCountTotal,
+		)
+	}
 
 	v2keysmux := keyMux.PathPrefix("/v2").Subrouter()
 	v1fedmux := fedMux.PathPrefix("/v1").Subrouter()

federationapi/routing/send_test.go

@@ -546,48 +546,53 @@ type sendContent struct {
 }
 
 func TestHandleSend(t *testing.T) {
-	base, close := testrig.CreateBaseDendrite(t, test.DBTypeSQLite)
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-	defer close()
+		base, close := testrig.CreateBaseDendrite(t, dbType)
+		defer close()
 
-	fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath()
+		fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath()
-	keyMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicKeyPathPrefix).Subrouter().UseEncodedPath()
+		keyMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicKeyPathPrefix).Subrouter().UseEncodedPath()
-	cfg := config.FederationAPI{
+		cfg := config.FederationAPI{
-		Matrix: &config.Global{
+			Matrix: &config.Global{
-			SigningIdentity: gomatrixserverlib.SigningIdentity{
+				SigningIdentity: gomatrixserverlib.SigningIdentity{
-				ServerName: "remote",
+					ServerName: "remote",
+				},
+				Metrics: config.Metrics{
+					Enabled: false,
+				},
 			},
-		},
+		}
-	}
+		fedapi := fedAPI.NewInternalAPI(base, nil, nil, nil, nil, true)
-	fedapi := fedAPI.NewInternalAPI(base, nil, nil, nil, nil, true)
+		serverKeyAPI := &signing.YggdrasilKeys{}
-	serverKeyAPI := &signing.YggdrasilKeys{}
+		keyRing := serverKeyAPI.KeyRing()
-	keyRing := serverKeyAPI.KeyRing()
+		r, ok := fedapi.(*fedInternal.FederationInternalAPI)
-	r, ok := fedapi.(*fedInternal.FederationInternalAPI)
+		if !ok {
-	if !ok {
+			panic("This is a programming error.")
-		panic("This is a programming error.")
+		}
-	}
+		routing.Setup(fedMux, keyMux, nil, &cfg, nil, r, keyRing, nil, nil, nil, &base.Cfg.MSCs, nil, nil)
-	routing.Setup(fedMux, keyMux, nil, &cfg, nil, r, keyRing, nil, nil, nil, &base.Cfg.MSCs, nil, nil)
 
-	handler := fedMux.Get(routing.SendRouteName).GetHandler().ServeHTTP
+		handler := fedMux.Get(routing.SendRouteName).GetHandler().ServeHTTP
-	_, sk, _ := ed25519.GenerateKey(nil)
+		_, sk, _ := ed25519.GenerateKey(nil)
-	keyID := signing.KeyID
+		keyID := signing.KeyID
-	pk := sk.Public().(ed25519.PublicKey)
+		pk := sk.Public().(ed25519.PublicKey)
-	serverName := gomatrixserverlib.ServerName(hex.EncodeToString(pk))
+		serverName := gomatrixserverlib.ServerName(hex.EncodeToString(pk))
-	req := gomatrixserverlib.NewFederationRequest("PUT", serverName, "remote", "/send/1234")
+		req := gomatrixserverlib.NewFederationRequest("PUT", serverName, "remote", "/send/1234")
-	content := sendContent{}
+		content := sendContent{}
-	err := req.SetContent(content)
+		err := req.SetContent(content)
-	if err != nil {
+		if err != nil {
-		t.Fatalf("Error: %s", err.Error())
+			t.Fatalf("Error: %s", err.Error())
-	}
+		}
-	req.Sign(serverName, gomatrixserverlib.KeyID(keyID), sk)
+		req.Sign(serverName, gomatrixserverlib.KeyID(keyID), sk)
-	httpReq, err := req.HTTPRequest()
+		httpReq, err := req.HTTPRequest()
-	if err != nil {
+		if err != nil {
-		t.Fatalf("Error: %s", err.Error())
+			t.Fatalf("Error: %s", err.Error())
-	}
+		}
-	vars := map[string]string{"txnID": "1234"}
+		vars := map[string]string{"txnID": "1234"}
-	w := httptest.NewRecorder()
+		w := httptest.NewRecorder()
-	httpReq = mux.SetURLVars(httpReq, vars)
+		httpReq = mux.SetURLVars(httpReq, vars)
-	handler(w, httpReq)
+		handler(w, httpReq)
 
-	res := w.Result()
+		res := w.Result()
-	assert.Equal(t, 200, res.StatusCode)
+		assert.Equal(t, 200, res.StatusCode)
+	})
 }

relayapi/relayapi_test.go

@@ -104,49 +104,51 @@ func createSendRelayTxnHTTPRequest(serverName gomatrixserverlib.ServerName, txnI
 }
 
 func TestCreateRelayPublicRoutes(t *testing.T) {
-	base, close := testrig.CreateBaseDendrite(t, test.DBTypeSQLite)
+	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
-	defer close()
+		base, close := testrig.CreateBaseDendrite(t, dbType)
+		defer close()
 
-	relayAPI := relayapi.NewRelayInternalAPI(base, nil, nil, nil, nil)
+		relayAPI := relayapi.NewRelayInternalAPI(base, nil, nil, nil, nil)
-	assert.NotNil(t, relayAPI)
+		assert.NotNil(t, relayAPI)
 
-	serverKeyAPI := &signing.YggdrasilKeys{}
+		serverKeyAPI := &signing.YggdrasilKeys{}
-	keyRing := serverKeyAPI.KeyRing()
+		keyRing := serverKeyAPI.KeyRing()
-	relayapi.AddPublicRoutes(base, keyRing, relayAPI)
+		relayapi.AddPublicRoutes(base, keyRing, relayAPI)
 
-	testCases := []struct {
+		testCases := []struct {
-		name            string
+			name            string
-		req             *http.Request
+			req             *http.Request
-		wantCode        int
+			wantCode        int
-		wantJoinedRooms []string
+			wantJoinedRooms []string
-	}{
+		}{
-		{
+			{
-			name:     "relay_txn invalid user id",
+				name:     "relay_txn invalid user id",
-			req:      createGetRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "user:local"),
+				req:      createGetRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "user:local"),
-			wantCode: 400,
+				wantCode: 400,
-		},
+			},
-		{
+			{
-			name:     "relay_txn valid user id",
+				name:     "relay_txn valid user id",
-			req:      createGetRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "@user:local"),
+				req:      createGetRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "@user:local"),
-			wantCode: 200,
+				wantCode: 200,
-		},
+			},
-		{
+			{
-			name:     "send_relay invalid user id",
+				name:     "send_relay invalid user id",
-			req:      createSendRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "123", "user:local"),
+				req:      createSendRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "123", "user:local"),
-			wantCode: 400,
+				wantCode: 400,
-		},
+			},
-		{
+			{
-			name:     "send_relay valid user id",
+				name:     "send_relay valid user id",
-			req:      createSendRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "123", "@user:local"),
+				req:      createSendRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "123", "@user:local"),
-			wantCode: 200,
+				wantCode: 200,
-		},
+			},
-	}
-
-	for _, tc := range testCases {
-		w := httptest.NewRecorder()
-		base.PublicFederationAPIMux.ServeHTTP(w, tc.req)
-		if w.Code != tc.wantCode {
-			t.Fatalf("%s: got HTTP %d want %d", tc.name, w.Code, tc.wantCode)
 		}
-	}
+
+		for _, tc := range testCases {
+			w := httptest.NewRecorder()
+			base.PublicFederationAPIMux.ServeHTTP(w, tc.req)
+			if w.Code != tc.wantCode {
+				t.Fatalf("%s: got HTTP %d want %d", tc.name, w.Code, tc.wantCode)
+			}
+		}
+	})
 }

test/db.go

@@ -19,10 +19,12 @@ import (
 	"database/sql"
 	"encoding/hex"
 	"fmt"
+	"math/rand"
 	"os"
 	"os/exec"
 	"os/user"
 	"testing"
+	"time"
 
 	"github.com/lib/pq"
 )
@@ -100,15 +102,16 @@ func currentUser() string {
 // Returns the connection string to use and a close function which must be called when the test finishes.
 // Calling this function twice will return the same database, which will have data from previous tests
 // unless close() is called.
-// TODO: namespace for concurrent package tests
 func PrepareDBConnectionString(t *testing.T, dbType DBType) (connStr string, close func()) {
 	if dbType == DBTypeSQLite {
-		// this will be made in the current working directory which namespaces concurrent package runs correctly
+		rand.Seed(time.Now().UnixNano())
-		dbname := "dendrite_test.db"
+		randBytes := make([]byte, 32)
-		return fmt.Sprintf("file:%s", dbname), func() {
+		rand.Read(randBytes)
-			err := os.Remove(dbname)
+		dbName := fmt.Sprintf("dendrite_test_%s.db", hex.EncodeToString(randBytes[:16]))
+		return fmt.Sprintf("file:%s", dbName), func() {
+			err := os.Remove(dbName)
 			if err != nil {
-				t.Fatalf("failed to cleanup sqlite db '%s': %s", dbname, err)
+				t.Fatalf("failed to cleanup sqlite db '%s': %s", dbName, err)
 			}
 		}
 	}
@@ -176,7 +179,7 @@ func WithAllDatabases(t *testing.T, testFn func(t *testing.T, db DBType)) {
 	for dbName, dbType := range dbs {
 		dbt := dbType
 		t.Run(dbName, func(tt *testing.T) {
-			//tt.Parallel()
+			tt.Parallel()
 			testFn(tt, dbt)
 		})
 	}

test/testrig/base.go

@@ -69,6 +69,22 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f
 			Monolithic: false, // because we need a database per component
 		})
 		cfg.Global.ServerName = "test"
+		connStr, _ := test.PrepareDBConnectionString(t, dbType)
+		cfg.FederationAPI.Database.ConnectionString = config.DataSource(connStr)
+		connStr, _ = test.PrepareDBConnectionString(t, dbType)
+		cfg.KeyServer.Database.ConnectionString = config.DataSource(connStr)
+		connStr, _ = test.PrepareDBConnectionString(t, dbType)
+		cfg.MSCs.Database.ConnectionString = config.DataSource(connStr)
+		connStr, _ = test.PrepareDBConnectionString(t, dbType)
+		cfg.MediaAPI.Database.ConnectionString = config.DataSource(connStr)
+		connStr, _ = test.PrepareDBConnectionString(t, dbType)
+		cfg.RoomServer.Database.ConnectionString = config.DataSource(connStr)
+		connStr, _ = test.PrepareDBConnectionString(t, dbType)
+		cfg.SyncAPI.Database.ConnectionString = config.DataSource(connStr)
+		connStr, _ = test.PrepareDBConnectionString(t, dbType)
+		cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(connStr)
+		connStr, _ = test.PrepareDBConnectionString(t, dbType)
+		cfg.RelayAPI.Database.ConnectionString = config.DataSource(connStr)
 		// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
 		// the file system event with InMemory=true :(
 		cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType)
@@ -82,6 +98,7 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f
 				cfg.RoomServer.Database.ConnectionString,
 				cfg.SyncAPI.Database.ConnectionString,
 				cfg.UserAPI.AccountDatabase.ConnectionString,
+				cfg.RelayAPI.Database.ConnectionString,
 			}
 			for _, fileURI := range dbFiles {
 				path := strings.TrimPrefix(string(fileURI), "file:")