mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2026-01-16 18:43:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

Fix sqlite unit test parallelization issues

Browse source
This commit is contained in:
Devon Hudson 2023-01-18 16:02:32 -07:00
parent 703d4371f5
commit 91126e6efa
No known key found for this signature in database
GPG key ID: CD06B18E77F6A628
6 changed files with 121 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -56,6 +56,7 @@ dendrite.yaml
# Database files
*.db
*.db-journal
# Log files
*.log*

8
federationapi/routing/routing.go
View file

@ -67,9 +67,11 @@ func Setup(
servers federationAPI.ServersInRoomProvider,
producer *producers.SyncAPIProducer,
) {
prometheus.MustRegister(
internal.PDUCountTotal, internal.EDUCountTotal,
)
if cfg.Matrix.Metrics.Enabled {
prometheus.MustRegister(
internal.PDUCountTotal, internal.EDUCountTotal,
)
}
v2keysmux := keyMux.PathPrefix("/v2").Subrouter()
v1fedmux := fedMux.PathPrefix("/v1").Subrouter()

85
federationapi/routing/send_test.go
View file

@ -546,48 +546,53 @@ type sendContent struct {
}
func TestHandleSend(t *testing.T) {
base, close := testrig.CreateBaseDendrite(t, test.DBTypeSQLite)
defer close()
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
base, close := testrig.CreateBaseDendrite(t, dbType)
defer close()
fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath()
keyMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicKeyPathPrefix).Subrouter().UseEncodedPath()
cfg := config.FederationAPI{
Matrix: &config.Global{
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: "remote",
fedMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicFederationPathPrefix).Subrouter().UseEncodedPath()
keyMux := mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicKeyPathPrefix).Subrouter().UseEncodedPath()
cfg := config.FederationAPI{
Matrix: &config.Global{
SigningIdentity: gomatrixserverlib.SigningIdentity{
ServerName: "remote",
},
Metrics: config.Metrics{
Enabled: false,
},
},
},
}
fedapi := fedAPI.NewInternalAPI(base, nil, nil, nil, nil, true)
serverKeyAPI := &signing.YggdrasilKeys{}
keyRing := serverKeyAPI.KeyRing()
r, ok := fedapi.(*fedInternal.FederationInternalAPI)
if !ok {
panic("This is a programming error.")
}
routing.Setup(fedMux, keyMux, nil, &cfg, nil, r, keyRing, nil, nil, nil, &base.Cfg.MSCs, nil, nil)
}
fedapi := fedAPI.NewInternalAPI(base, nil, nil, nil, nil, true)
serverKeyAPI := &signing.YggdrasilKeys{}
keyRing := serverKeyAPI.KeyRing()
r, ok := fedapi.(*fedInternal.FederationInternalAPI)
if !ok {
panic("This is a programming error.")
}
routing.Setup(fedMux, keyMux, nil, &cfg, nil, r, keyRing, nil, nil, nil, &base.Cfg.MSCs, nil, nil)
handler := fedMux.Get(routing.SendRouteName).GetHandler().ServeHTTP
_, sk, _ := ed25519.GenerateKey(nil)
keyID := signing.KeyID
pk := sk.Public().(ed25519.PublicKey)
serverName := gomatrixserverlib.ServerName(hex.EncodeToString(pk))
req := gomatrixserverlib.NewFederationRequest("PUT", serverName, "remote", "/send/1234")
content := sendContent{}
err := req.SetContent(content)
if err != nil {
t.Fatalf("Error: %s", err.Error())
}
req.Sign(serverName, gomatrixserverlib.KeyID(keyID), sk)
httpReq, err := req.HTTPRequest()
if err != nil {
t.Fatalf("Error: %s", err.Error())
}
vars := map[string]string{"txnID": "1234"}
w := httptest.NewRecorder()
httpReq = mux.SetURLVars(httpReq, vars)
handler(w, httpReq)
handler := fedMux.Get(routing.SendRouteName).GetHandler().ServeHTTP
_, sk, _ := ed25519.GenerateKey(nil)
keyID := signing.KeyID
pk := sk.Public().(ed25519.PublicKey)
serverName := gomatrixserverlib.ServerName(hex.EncodeToString(pk))
req := gomatrixserverlib.NewFederationRequest("PUT", serverName, "remote", "/send/1234")
content := sendContent{}
err := req.SetContent(content)
if err != nil {
t.Fatalf("Error: %s", err.Error())
}
req.Sign(serverName, gomatrixserverlib.KeyID(keyID), sk)
httpReq, err := req.HTTPRequest()
if err != nil {
t.Fatalf("Error: %s", err.Error())
}
vars := map[string]string{"txnID": "1234"}
w := httptest.NewRecorder()
httpReq = mux.SetURLVars(httpReq, vars)
handler(w, httpReq)
res := w.Result()
assert.Equal(t, 200, res.StatusCode)
res := w.Result()
assert.Equal(t, 200, res.StatusCode)
})
}

84
relayapi/relayapi_test.go
View file

@ -104,49 +104,51 @@ func createSendRelayTxnHTTPRequest(serverName gomatrixserverlib.ServerName, txnI
}
func TestCreateRelayPublicRoutes(t *testing.T) {
base, close := testrig.CreateBaseDendrite(t, test.DBTypeSQLite)
defer close()
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
base, close := testrig.CreateBaseDendrite(t, dbType)
defer close()
relayAPI := relayapi.NewRelayInternalAPI(base, nil, nil, nil, nil)
assert.NotNil(t, relayAPI)
relayAPI := relayapi.NewRelayInternalAPI(base, nil, nil, nil, nil)
assert.NotNil(t, relayAPI)
serverKeyAPI := &signing.YggdrasilKeys{}
keyRing := serverKeyAPI.KeyRing()
relayapi.AddPublicRoutes(base, keyRing, relayAPI)
serverKeyAPI := &signing.YggdrasilKeys{}
keyRing := serverKeyAPI.KeyRing()
relayapi.AddPublicRoutes(base, keyRing, relayAPI)
testCases := []struct {
name string
req *http.Request
wantCode int
wantJoinedRooms []string
}{
{
name: "relay_txn invalid user id",
req: createGetRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "user:local"),
wantCode: 400,
},
{
name: "relay_txn valid user id",
req: createGetRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "@user:local"),
wantCode: 200,
},
{
name: "send_relay invalid user id",
req: createSendRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "123", "user:local"),
wantCode: 400,
},
{
name: "send_relay valid user id",
req: createSendRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "123", "@user:local"),
wantCode: 200,
},
}
for _, tc := range testCases {
w := httptest.NewRecorder()
base.PublicFederationAPIMux.ServeHTTP(w, tc.req)
if w.Code != tc.wantCode {
t.Fatalf("%s: got HTTP %d want %d", tc.name, w.Code, tc.wantCode)
testCases := []struct {
name string
req *http.Request
wantCode int
wantJoinedRooms []string
}{
{
name: "relay_txn invalid user id",
req: createGetRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "user:local"),
wantCode: 400,
},
{
name: "relay_txn valid user id",
req: createGetRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "@user:local"),
wantCode: 200,
},
{
name: "send_relay invalid user id",
req: createSendRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "123", "user:local"),
wantCode: 400,
},
{
name: "send_relay valid user id",
req: createSendRelayTxnHTTPRequest(base.Cfg.Global.ServerName, "123", "@user:local"),
wantCode: 200,
},
}
}
for _, tc := range testCases {
w := httptest.NewRecorder()
base.PublicFederationAPIMux.ServeHTTP(w, tc.req)
if w.Code != tc.wantCode {
t.Fatalf("%s: got HTTP %d want %d", tc.name, w.Code, tc.wantCode)
}
}
})
}

17
test/db.go
View file

@ -19,10 +19,12 @@ import (
"database/sql"
"encoding/hex"
"fmt"
"math/rand"
"os"
"os/exec"
"os/user"
"testing"
"time"
"github.com/lib/pq"
)
@ -100,15 +102,16 @@ func currentUser() string {
// Returns the connection string to use and a close function which must be called when the test finishes.
// Calling this function twice will return the same database, which will have data from previous tests
// unless close() is called.
// TODO: namespace for concurrent package tests
func PrepareDBConnectionString(t *testing.T, dbType DBType) (connStr string, close func()) {
if dbType == DBTypeSQLite {
// this will be made in the current working directory which namespaces concurrent package runs correctly
dbname := "dendrite_test.db"
return fmt.Sprintf("file:%s", dbname), func() {
err := os.Remove(dbname)
rand.Seed(time.Now().UnixNano())
randBytes := make([]byte, 32)
rand.Read(randBytes)
dbName := fmt.Sprintf("dendrite_test_%s.db", hex.EncodeToString(randBytes[:16]))
return fmt.Sprintf("file:%s", dbName), func() {
err := os.Remove(dbName)
if err != nil {
t.Fatalf("failed to cleanup sqlite db '%s': %s", dbname, err)
t.Fatalf("failed to cleanup sqlite db '%s': %s", dbName, err)
}
}
}
@ -176,7 +179,7 @@ func WithAllDatabases(t *testing.T, testFn func(t *testing.T, db DBType)) {
for dbName, dbType := range dbs {
dbt := dbType
t.Run(dbName, func(tt *testing.T) {
//tt.Parallel()
tt.Parallel()
testFn(tt, dbt)
})
}

17
test/testrig/base.go
View file

@ -69,6 +69,22 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f
Monolithic: false, // because we need a database per component
})
cfg.Global.ServerName = "test"
connStr, _ := test.PrepareDBConnectionString(t, dbType)
cfg.FederationAPI.Database.ConnectionString = config.DataSource(connStr)
connStr, _ = test.PrepareDBConnectionString(t, dbType)
cfg.KeyServer.Database.ConnectionString = config.DataSource(connStr)
connStr, _ = test.PrepareDBConnectionString(t, dbType)
cfg.MSCs.Database.ConnectionString = config.DataSource(connStr)
connStr, _ = test.PrepareDBConnectionString(t, dbType)
cfg.MediaAPI.Database.ConnectionString = config.DataSource(connStr)
connStr, _ = test.PrepareDBConnectionString(t, dbType)
cfg.RoomServer.Database.ConnectionString = config.DataSource(connStr)
connStr, _ = test.PrepareDBConnectionString(t, dbType)
cfg.SyncAPI.Database.ConnectionString = config.DataSource(connStr)
connStr, _ = test.PrepareDBConnectionString(t, dbType)
cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(connStr)
connStr, _ = test.PrepareDBConnectionString(t, dbType)
cfg.RelayAPI.Database.ConnectionString = config.DataSource(connStr)
// use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use
// the file system event with InMemory=true :(
cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType)
@ -82,6 +98,7 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f
cfg.RoomServer.Database.ConnectionString,
cfg.SyncAPI.Database.ConnectionString,
cfg.UserAPI.AccountDatabase.ConnectionString,
cfg.RelayAPI.Database.ConnectionString,
}
for _, fileURI := range dbFiles {
path := strings.TrimPrefix(string(fileURI), "file:")