From fffd2d9228cf5c59ea7fb601e7037dd38070b0ef Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Mon, 20 Mar 2023 15:48:56 +0100 Subject: [PATCH] Test updates, I guess --- appservice/appservice_test.go | 19 +++--- clientapi/admin_test.go | 51 +++++++++------- clientapi/routing/joinroom_test.go | 20 +++--- clientapi/routing/login_test.go | 26 +++++--- clientapi/routing/register_test.go | 68 ++++++++++++--------- federationapi/federationapi_test.go | 2 +- federationapi/storage/storage_test.go | 2 +- internal/sqlutil/connection_manager.go | 17 ++++-- internal/sqlutil/connection_manager_test.go | 4 +- mediaapi/routing/upload_test.go | 2 +- mediaapi/storage/storage_test.go | 2 +- relayapi/relayapi_test.go | 7 ++- roomserver/internal/helpers/helpers_test.go | 2 +- roomserver/internal/input/input_test.go | 2 +- roomserver/roomserver_test.go | 8 +-- setup/base/base.go | 12 +--- setup/config/config_global.go | 2 + setup/jetstream/nats.go | 5 +- setup/mscs/msc2836/msc2836_test.go | 2 +- syncapi/storage/storage_test.go | 2 +- syncapi/syncapi_test.go | 14 ++--- test/testrig/base.go | 64 +++++++++++++++++++ userapi/consumers/roomserver_test.go | 2 +- userapi/internal/device_list_update_test.go | 2 +- userapi/internal/key_api_test.go | 2 +- userapi/storage/storage_test.go | 2 +- userapi/userapi_test.go | 2 +- userapi/util/notify_test.go | 2 +- 28 files changed, 221 insertions(+), 124 deletions(-) diff --git a/appservice/appservice_test.go b/appservice/appservice_test.go index eef68492c..d2e5f903e 100644 --- a/appservice/appservice_test.go +++ b/appservice/appservice_test.go @@ -9,13 +9,16 @@ import ( "regexp" "strings" "testing" + "time" "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/userapi" @@ -106,11 +109,11 @@ func TestAppserviceInternalAPI(t *testing.T) { } test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, closeBase := testrig.CreateBaseDendrite(t, dbType) - defer closeBase() + cfg, closeDb := testrig.CreateConfig(t, dbType) + defer closeDb() // Create a dummy application service - base.Cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{ + cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{ { ID: "someID", URL: srv.URL, @@ -125,12 +128,14 @@ func TestAppserviceInternalAPI(t *testing.T) { }, } - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + ctx := process.NewProcessContext() + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) // Create required internal APIs natsInstance := jetstream.NATSInstance{} - rsAPI := roomserver.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, caches, base.EnableMetrics) - usrAPI := userapi.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, rsAPI, nil) - asAPI := appservice.NewInternalAPI(base.ProcessContext, base.Cfg, &natsInstance, usrAPI, rsAPI) + cm := sqlutil.NewConnectionManager(cfg.Global.DatabaseOptions) + rsAPI := roomserver.NewInternalAPI(ctx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + usrAPI := userapi.NewInternalAPI(ctx, cfg, cm, &natsInstance, rsAPI, nil) + asAPI := appservice.NewInternalAPI(ctx, cfg, &natsInstance, usrAPI, rsAPI) runCases(t, asAPI) }) diff --git a/clientapi/admin_test.go b/clientapi/admin_test.go index fe20a445a..ea0be64dc 100644 --- a/clientapi/admin_test.go +++ b/clientapi/admin_test.go @@ -5,14 +5,18 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/syncapi" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -31,20 +35,24 @@ func TestAdminResetPassword(t *testing.T) { ctx := context.Background() test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, baseClose := testrig.CreateBaseDendrite(t, dbType) - defer baseClose() + cfg, closeDB := testrig.CreateConfig(t, dbType) + defer closeDB() natsInstance := jetstream.NATSInstance{} // add a vhost - base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{ + cfg.Global.VirtualHosts = append(cfg.Global.VirtualHosts, &config.VirtualHost{ SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"}, }) + t.Logf("XXX: %#v", cfg.Global.VirtualHosts) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - rsAPI := roomserver.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, caches, base.EnableMetrics) + processCtx := process.NewProcessContext() + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) // Needed for changing the password/login - userAPI := userapi.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. - AddPublicRoutes(base.ProcessContext, base.Routers, base.Cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, base.EnableMetrics) + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) // Create the users in the userapi and login accessTokens := map[*test.User]string{ @@ -74,7 +82,7 @@ func TestAdminResetPassword(t *testing.T) { "password": password, })) rec := httptest.NewRecorder() - base.Routers.Client.ServeHTTP(rec, req) + routers.Client.ServeHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("failed to login: %s", rec.Body.String()) } @@ -127,7 +135,7 @@ func TestAdminResetPassword(t *testing.T) { } rec := httptest.NewRecorder() - base.Routers.DendriteAdmin.ServeHTTP(rec, req) + routers.DendriteAdmin.ServeHTTP(rec, req) t.Logf("%s", rec.Body.String()) if tc.wantOK && rec.Code != http.StatusOK { t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) @@ -150,18 +158,19 @@ func TestPurgeRoom(t *testing.T) { ctx := context.Background() test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, baseClose := testrig.CreateBaseDendrite(t, dbType) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + cfg, closeDB := testrig.CreateConfig(t, dbType) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) natsInstance := jetstream.NATSInstance{} - defer baseClose() - - fedClient := base.CreateFederationClient() - rsAPI := roomserver.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, caches, base.EnableMetrics) - userAPI := userapi.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, rsAPI, nil) + defer closeDB() + processCtx := process.NewProcessContext() + routers := httputil.NewRouters() + cm := sqlutil.NewConnectionManager(cfg.Global.DatabaseOptions) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // this starts the JetStream consumers - syncapi.AddPublicRoutes(base.ProcessContext, base.Routers, base.Cfg, base.ConnectionManager, &natsInstance, userAPI, rsAPI, caches, base.EnableMetrics) - federationapi.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, fedClient, rsAPI, caches, nil, true) + syncapi.AddPublicRoutes(processCtx, routers, cfg, cm, &natsInstance, userAPI, rsAPI, caches, caching.DisableMetrics) + federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true) rsAPI.SetFederationAPI(nil, nil) // Create the room @@ -170,7 +179,7 @@ func TestPurgeRoom(t *testing.T) { } // We mostly need the rsAPI for this test, so nil for other APIs/caches etc. - AddPublicRoutes(base.ProcessContext, base.Routers, base.Cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, base.EnableMetrics) + AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics) // Create the users in the userapi and login accessTokens := map[*test.User]string{ @@ -198,7 +207,7 @@ func TestPurgeRoom(t *testing.T) { "password": password, })) rec := httptest.NewRecorder() - base.Routers.Client.ServeHTTP(rec, req) + routers.Client.ServeHTTP(rec, req) if rec.Code != http.StatusOK { t.Fatalf("failed to login: %s", rec.Body.String()) } @@ -223,7 +232,7 @@ func TestPurgeRoom(t *testing.T) { req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin]) rec := httptest.NewRecorder() - base.Routers.DendriteAdmin.ServeHTTP(rec, req) + routers.DendriteAdmin.ServeHTTP(rec, req) t.Logf("%s", rec.Body.String()) if tc.wantOK && rec.Code != http.StatusOK { t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) diff --git a/clientapi/routing/joinroom_test.go b/clientapi/routing/joinroom_test.go index 4cac0cf78..bc7207450 100644 --- a/clientapi/routing/joinroom_test.go +++ b/clientapi/routing/joinroom_test.go @@ -8,7 +8,9 @@ import ( "time" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/appservice" @@ -26,14 +28,16 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { ctx := context.Background() test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, baseClose := testrig.CreateBaseDendrite(t, dbType) - defer baseClose() + cfg, dbClose := testrig.CreateConfig(t, dbType) + defer dbClose() - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + processCtx := process.NewProcessContext() + cm := sqlutil.NewConnectionManager(cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) natsInstance := jetstream.NATSInstance{} - rsAPI := roomserver.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, caches, base.EnableMetrics) - userAPI := userapi.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, rsAPI, nil) - asAPI := appservice.NewInternalAPI(base.ProcessContext, base.Cfg, &natsInstance, userAPI, rsAPI) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) + asAPI := appservice.NewInternalAPI(processCtx, cfg, &natsInstance, userAPI, rsAPI) rsAPI.SetFederationAPI(nil, nil) // creates the rs.Inputer etc // Create the users in the userapi @@ -65,7 +69,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { RoomAliasName: "alias", Invite: []string{bob.ID}, GuestCanJoin: false, - }, aliceDev, &base.Cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now()) + }, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now()) crResp, ok := resp.JSON.(createRoomResponse) if !ok { t.Fatalf("response is not a createRoomResponse: %+v", resp) @@ -80,7 +84,7 @@ func TestJoinRoomByIDOrAlias(t *testing.T) { Preset: presetPublicChat, Invite: []string{charlie.ID}, GuestCanJoin: true, - }, aliceDev, &base.Cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now()) + }, aliceDev, &cfg.ClientAPI, userAPI, rsAPI, asAPI, time.Now()) crRespWithGuestAccess, ok := resp.JSON.(createRoomResponse) if !ok { t.Fatalf("response is not a createRoomResponse: %+v", resp) diff --git a/clientapi/routing/login_test.go b/clientapi/routing/login_test.go index 44be87cb0..9481a9c04 100644 --- a/clientapi/routing/login_test.go +++ b/clientapi/routing/login_test.go @@ -7,12 +7,16 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -30,22 +34,24 @@ func TestLogin(t *testing.T) { ctx := context.Background() test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, baseClose := testrig.CreateBaseDendrite(t, dbType) - defer baseClose() - base.Cfg.ClientAPI.RateLimiting.Enabled = false + cfg, closeDB := testrig.CreateConfig(t, dbType) + defer closeDB() + cfg.ClientAPI.RateLimiting.Enabled = false natsInstance := jetstream.NATSInstance{} // add a vhost - base.Cfg.Global.VirtualHosts = append(base.Cfg.Global.VirtualHosts, &config.VirtualHost{ + cfg.Global.VirtualHosts = append(cfg.Global.VirtualHosts, &config.VirtualHost{ SigningIdentity: gomatrixserverlib.SigningIdentity{ServerName: "vh1"}, }) - - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - rsAPI := roomserver.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, caches, base.EnableMetrics) + processCtx := process.NewProcessContext() + cm := sqlutil.NewConnectionManager(cfg.Global.DatabaseOptions) + routers := httputil.NewRouters() + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) // Needed for /login - userAPI := userapi.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, rsAPI, nil) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) // We mostly need the userAPI for this test, so nil for other APIs/caches etc. - Setup(base.Routers, base.Cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, base.EnableMetrics) + Setup(routers, cfg, nil, nil, userAPI, nil, nil, nil, nil, nil, nil, nil, caching.DisableMetrics) // Create password password := util.RandomString(8) @@ -118,7 +124,7 @@ func TestLogin(t *testing.T) { "password": password, })) rec := httptest.NewRecorder() - base.Routers.Client.ServeHTTP(rec, req) + routers.Client.ServeHTTP(rec, req) if tc.wantOK && rec.Code != http.StatusOK { t.Fatalf("failed to login: %s", rec.Body.String()) } diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index a3d055a7e..51d0648c1 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -31,9 +31,11 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" + "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/userapi" @@ -406,13 +408,15 @@ func Test_register(t *testing.T) { } test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, baseClose := testrig.CreateBaseDendrite(t, dbType) - defer baseClose() + cfg, dbClose := testrig.CreateConfig(t, dbType) + defer dbClose() - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) natsInstance := jetstream.NATSInstance{} - rsAPI := roomserver.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, caches, base.EnableMetrics) - userAPI := userapi.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, rsAPI, nil) + processCtx := process.NewProcessContext() + cm := sqlutil.NewConnectionManager(cfg.Global.DatabaseOptions) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -434,16 +438,16 @@ func Test_register(t *testing.T) { } })) defer srv.Close() - base.Cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL + cfg.ClientAPI.RecaptchaSiteVerifyAPI = srv.URL } - if err := base.Cfg.Derive(); err != nil { + if err := cfg.Derive(); err != nil { t.Fatalf("failed to derive config: %s", err) } - base.Cfg.ClientAPI.RecaptchaEnabled = tc.enableRecaptcha - base.Cfg.ClientAPI.RegistrationDisabled = tc.registrationDisabled - base.Cfg.ClientAPI.GuestsDisabled = tc.guestsDisabled + cfg.ClientAPI.RecaptchaEnabled = tc.enableRecaptcha + cfg.ClientAPI.RegistrationDisabled = tc.registrationDisabled + cfg.ClientAPI.GuestsDisabled = tc.guestsDisabled if tc.kind == "" { tc.kind = "user" @@ -471,15 +475,15 @@ func Test_register(t *testing.T) { req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/?kind=%s", tc.kind), body) - resp := Register(req, userAPI, &base.Cfg.ClientAPI) + resp := Register(req, userAPI, &cfg.ClientAPI) t.Logf("Resp: %+v", resp) // The first request should return a userInteractiveResponse switch r := resp.JSON.(type) { case userInteractiveResponse: // Check that the flows are the ones we configured - if !reflect.DeepEqual(r.Flows, base.Cfg.Derived.Registration.Flows) { - t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, base.Cfg.Derived.Registration.Flows) + if !reflect.DeepEqual(r.Flows, cfg.Derived.Registration.Flows) { + t.Fatalf("unexpected registration flows: %+v, want %+v", r.Flows, cfg.Derived.Registration.Flows) } case *jsonerror.MatrixError: if !reflect.DeepEqual(tc.wantResponse, resp) { @@ -535,7 +539,7 @@ func Test_register(t *testing.T) { req = httptest.NewRequest(http.MethodPost, "/", body) - resp = Register(req, userAPI, &base.Cfg.ClientAPI) + resp = Register(req, userAPI, &cfg.ClientAPI) switch resp.JSON.(type) { case *jsonerror.MatrixError: @@ -578,18 +582,20 @@ func Test_register(t *testing.T) { func TestRegisterUserWithDisplayName(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, baseClose := testrig.CreateBaseDendrite(t, dbType) + cfg, baseClose := testrig.CreateConfig(t, dbType) defer baseClose() - base.Cfg.Global.ServerName = "server" + cfg.Global.ServerName = "server" - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + processCtx := process.NewProcessContext() + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) natsInstance := jetstream.NATSInstance{} - rsAPI := roomserver.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, caches, base.EnableMetrics) - userAPI := userapi.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, rsAPI, nil) + cm := sqlutil.NewConnectionManager(cfg.Global.DatabaseOptions) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) deviceName, deviceID := "deviceName", "deviceID" expectedDisplayName := "DisplayName" response := completeRegistration( - base.Context(), + processCtx.Context(), userAPI, "user", "server", @@ -609,7 +615,7 @@ func TestRegisterUserWithDisplayName(t *testing.T) { req := api.QueryProfileRequest{UserID: "@user:server"} var res api.QueryProfileResponse - err := userAPI.QueryProfile(base.Context(), &req, &res) + err := userAPI.QueryProfile(processCtx.Context(), &req, &res) assert.NoError(t, err) assert.Equal(t, expectedDisplayName, res.DisplayName) }) @@ -617,15 +623,17 @@ func TestRegisterUserWithDisplayName(t *testing.T) { func TestRegisterAdminUsingSharedSecret(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { - base, baseClose := testrig.CreateBaseDendrite(t, dbType) - defer baseClose() + cfg, closeDb := testrig.CreateConfig(t, dbType) + defer closeDb() natsInstance := jetstream.NATSInstance{} - base.Cfg.Global.ServerName = "server" + cfg.Global.ServerName = "server" sharedSecret := "dendritetest" - base.Cfg.ClientAPI.RegistrationSharedSecret = sharedSecret - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) - rsAPI := roomserver.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, caches, base.EnableMetrics) - userAPI := userapi.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, rsAPI, nil) + cfg.ClientAPI.RegistrationSharedSecret = sharedSecret + processCtx := process.NewProcessContext() + cm := sqlutil.NewConnectionManager(cfg.Global.DatabaseOptions) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) + rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics) + userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil) expectedDisplayName := "rabbit" jsonStr := []byte(`{"admin":true,"mac":"24dca3bba410e43fe64b9b5c28306693bf3baa9f","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice","displayname":"rabbit"}`) @@ -649,7 +657,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) { ssrr := httptest.NewRequest(http.MethodPost, "/", body) response := handleSharedSecretRegistration( - &base.Cfg.ClientAPI, + &cfg.ClientAPI, userAPI, r, ssrr, @@ -658,7 +666,7 @@ func TestRegisterAdminUsingSharedSecret(t *testing.T) { profilReq := api.QueryProfileRequest{UserID: "@alice:server"} var profileRes api.QueryProfileResponse - err = userAPI.QueryProfile(base.Context(), &profilReq, &profileRes) + err = userAPI.QueryProfile(processCtx.Context(), &profilReq, &profileRes) assert.NoError(t, err) assert.Equal(t, expectedDisplayName, profileRes.DisplayName) }) diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index 634699439..a5b6c5d4c 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -164,7 +164,7 @@ func TestFederationAPIJoinThenKeyUpdate(t *testing.T) { func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) { base, close := testrig.CreateBaseDendrite(t, dbType) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) natsInstance := jetstream.NATSInstance{} base.Cfg.FederationAPI.PreferDirectFetch = true base.Cfg.FederationAPI.KeyPerspectives = nil diff --git a/federationapi/storage/storage_test.go b/federationapi/storage/storage_test.go index a02b71b60..321f9b758 100644 --- a/federationapi/storage/storage_test.go +++ b/federationapi/storage/storage_test.go @@ -20,7 +20,7 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType) (storage.Dat caches := caching.NewRistrettoCache(8*1024*1024, time.Hour, false) connStr, dbClose := test.PrepareDBConnectionString(t, dbType) ctx := context.Background() - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(config.DatabaseOptions{}) db, err := storage.NewDatabase(ctx, cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, caches, func(server gomatrixserverlib.ServerName) bool { return server == "localhost" }) diff --git a/internal/sqlutil/connection_manager.go b/internal/sqlutil/connection_manager.go index cefd9f808..8176a7592 100644 --- a/internal/sqlutil/connection_manager.go +++ b/internal/sqlutil/connection_manager.go @@ -22,12 +22,15 @@ import ( ) type Connections struct { - db *sql.DB - writer Writer + db *sql.DB + writer Writer + globalConfig config.DatabaseOptions } -func NewConnectionManager() Connections { - return Connections{} +func NewConnectionManager(globalConfig config.DatabaseOptions) Connections { + return Connections{ + globalConfig: globalConfig, + } } func (c *Connections) Connection(dbProperties *config.DatabaseOptions) (*sql.DB, Writer, error) { @@ -35,8 +38,12 @@ func (c *Connections) Connection(dbProperties *config.DatabaseOptions) (*sql.DB, if dbProperties.ConnectionString.IsSQLite() { writer = NewExclusiveWriter() } + var err error + if dbProperties.ConnectionString == "" { + // if no connectionString was provided, try the global one + dbProperties = &c.globalConfig + } if dbProperties.ConnectionString != "" || c.db == nil { - var err error // Open a new database connection using the supplied config. c.db, err = Open(dbProperties, writer) if err != nil { diff --git a/internal/sqlutil/connection_manager_test.go b/internal/sqlutil/connection_manager_test.go index 610629d5e..1c26041dc 100644 --- a/internal/sqlutil/connection_manager_test.go +++ b/internal/sqlutil/connection_manager_test.go @@ -13,7 +13,7 @@ func TestConnectionManager(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { conStr, close := test.PrepareDBConnectionString(t, dbType) t.Cleanup(close) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(config.DatabaseOptions{}) dbProps := &config.DatabaseOptions{ConnectionString: config.DataSource(string(conStr))} db, writer, err := cm.Connection(dbProps) @@ -47,7 +47,7 @@ func TestConnectionManager(t *testing.T) { } // test invalid connection string configured - cm = sqlutil.NewConnectionManager() + cm = sqlutil.NewConnectionManager(config.DatabaseOptions{}) _, _, err = cm.Connection(&config.DatabaseOptions{ConnectionString: "http://"}) if err == nil { t.Fatal("expected an error but got none") diff --git a/mediaapi/routing/upload_test.go b/mediaapi/routing/upload_test.go index d4fb45d1b..bc58e5d1f 100644 --- a/mediaapi/routing/upload_test.go +++ b/mediaapi/routing/upload_test.go @@ -50,7 +50,7 @@ func Test_uploadRequest_doUpload(t *testing.T) { // create testdata folder and remove when done _ = os.Mkdir(testdataPath, os.ModePerm) defer fileutils.RemoveDir(types.Path(testdataPath), nil) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(config.DatabaseOptions{}) db, err := storage.NewMediaAPIDatasource(cm, &config.DatabaseOptions{ ConnectionString: "file::memory:?cache=shared", MaxOpenConnections: 100, diff --git a/mediaapi/storage/storage_test.go b/mediaapi/storage/storage_test.go index 11febd275..4731d7701 100644 --- a/mediaapi/storage/storage_test.go +++ b/mediaapi/storage/storage_test.go @@ -14,7 +14,7 @@ import ( func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { connStr, close := test.PrepareDBConnectionString(t, dbType) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(config.DatabaseOptions{}) db, err := storage.NewMediaAPIDatasource(cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }) diff --git a/relayapi/relayapi_test.go b/relayapi/relayapi_test.go index 62d2c8622..2c5241519 100644 --- a/relayapi/relayapi_test.go +++ b/relayapi/relayapi_test.go @@ -21,6 +21,7 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/gorilla/mux" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" @@ -35,7 +36,7 @@ import ( func TestCreateNewRelayInternalAPI(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { base, close := testrig.CreateBaseDendrite(t, dbType) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) defer close() relayAPI := relayapi.NewRelayInternalAPI(base.Cfg, base.ConnectionManager, nil, nil, nil, nil, true, caches) @@ -109,7 +110,7 @@ func TestCreateRelayPublicRoutes(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { base, close := testrig.CreateBaseDendrite(t, dbType) defer close() - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) relayAPI := relayapi.NewRelayInternalAPI(base.Cfg, base.ConnectionManager, nil, nil, nil, nil, true, caches) assert.NotNil(t, relayAPI) @@ -159,7 +160,7 @@ func TestDisableRelayPublicRoutes(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { base, close := testrig.CreateBaseDendrite(t, dbType) defer close() - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) relayAPI := relayapi.NewRelayInternalAPI(base.Cfg, base.ConnectionManager, nil, nil, nil, nil, false, caches) assert.NotNil(t, relayAPI) diff --git a/roomserver/internal/helpers/helpers_test.go b/roomserver/internal/helpers/helpers_test.go index 03a8bf575..0d1086359 100644 --- a/roomserver/internal/helpers/helpers_test.go +++ b/roomserver/internal/helpers/helpers_test.go @@ -19,7 +19,7 @@ import ( func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { conStr, close := test.PrepareDBConnectionString(t, dbType) caches := caching.NewRistrettoCache(8*1024*1024, time.Hour, caching.DisableMetrics) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(config.DatabaseOptions{}) db, err := storage.Open(context.Background(), cm, &config.DatabaseOptions{ConnectionString: config.DataSource(conStr)}, caches) if err != nil { t.Fatalf("failed to create Database: %v", err) diff --git a/roomserver/internal/input/input_test.go b/roomserver/internal/input/input_test.go index 555ec9c6d..1bfd8f50a 100644 --- a/roomserver/internal/input/input_test.go +++ b/roomserver/internal/input/input_test.go @@ -49,7 +49,7 @@ func TestSingleTransactionOnInput(t *testing.T) { Kind: api.KindOutlier, // don't panic if we generate an output event Event: event.Headered(gomatrixserverlib.RoomVersionV6), } - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(config.DatabaseOptions{}) db, err := storage.Open( context.Background(), cm, &config.DatabaseOptions{ diff --git a/roomserver/roomserver_test.go b/roomserver/roomserver_test.go index 6a4044559..c39c98565 100644 --- a/roomserver/roomserver_test.go +++ b/roomserver/roomserver_test.go @@ -33,7 +33,7 @@ import ( func mustCreateDatabase(t *testing.T, dbType test.DBType) (*base.BaseDendrite, storage.Database, func()) { t.Helper() base, close := testrig.CreateBaseDendrite(t, dbType) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) db, err := storage.Open(base.ProcessContext.Context(), base.ConnectionManager, &base.Cfg.RoomServer.Database, caches) if err != nil { t.Fatalf("failed to create Database: %v", err) @@ -45,7 +45,7 @@ func TestUsers(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { base, close := testrig.CreateBaseDendrite(t, dbType) defer close() - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) natsInstance := jetstream.NATSInstance{} rsAPI := roomserver.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, caches, base.EnableMetrics) // SetFederationAPI starts the room event input consumer @@ -185,7 +185,7 @@ func Test_QueryLeftUsers(t *testing.T) { base, _, close := mustCreateDatabase(t, dbType) defer close() - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) natsInstance := jetstream.NATSInstance{} rsAPI := roomserver.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, caches, base.EnableMetrics) // SetFederationAPI starts the room event input consumer @@ -239,7 +239,7 @@ func TestPurgeRoom(t *testing.T) { defer jetstream.DeleteAllStreams(jsCtx, &base.Cfg.Global.JetStream) fedClient := base.CreateFederationClient() - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, caches, base.EnableMetrics) userAPI := userapi.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, rsAPI, nil) diff --git a/setup/base/base.go b/setup/base/base.go index a0361b550..dbcdac6e4 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -145,17 +145,7 @@ func NewBaseDendrite(cfg *config.Dendrite, options ...BaseDendriteOptions) *Base // If we're in monolith mode, we'll set up a global pool of database // connections. A component is welcome to use this pool if they don't // have a separate database config of their own. - cm := sqlutil.NewConnectionManager() - if cfg.Global.DatabaseOptions.ConnectionString != "" { - if cfg.Global.DatabaseOptions.ConnectionString.IsSQLite() { - logrus.Panic("Using a global database connection pool is not supported with SQLite databases") - } - _, _, err := cm.Connection(&cfg.Global.DatabaseOptions) - if err != nil { - logrus.WithError(err).Panic("Failed to set up global database connections") - } - logrus.Debug("Using global database connection pool") - } + cm := sqlutil.NewConnectionManager(cfg.Global.DatabaseOptions) // Ideally we would only use SkipClean on routes which we know can allow '/' but due to // https://github.com/gorilla/mux/issues/460 we have to attach this at the top router. diff --git a/setup/config/config_global.go b/setup/config/config_global.go index 7d3ab6a40..4f461368f 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -8,6 +8,7 @@ import ( "time" "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" "golang.org/x/crypto/ed25519" ) @@ -125,6 +126,7 @@ func (c *Global) IsLocalServerName(serverName gomatrixserverlib.ServerName) bool if c.ServerName == serverName { return true } + logrus.Infof("XXX: %#v", c.VirtualHosts) for _, v := range c.VirtualHosts { if v.ServerName == serverName { return true diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go index 8b144436a..06a58d542 100644 --- a/setup/jetstream/nats.go +++ b/setup/jetstream/nats.go @@ -56,7 +56,9 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS if err != nil { panic(err) } - s.SetLogger(NewLogAdapter(), opts.Debug, opts.Trace) + if !cfg.NoLog { + s.SetLogger(NewLogAdapter(), opts.Debug, opts.Trace) + } go func() { process.ComponentStarted() s.Start() @@ -73,7 +75,6 @@ func (s *NATSInstance) Prepare(process *process.ProcessContext, cfg *config.JetS } // reuse existing connections if s.nc != nil { - logrus.Infof("XXX: reusing connection") return s.js, s.nc } nc, err := natsclient.Connect("", natsclient.InProcessServer(s)) diff --git a/setup/mscs/msc2836/msc2836_test.go b/setup/mscs/msc2836/msc2836_test.go index 24e96f931..fc2ff60d8 100644 --- a/setup/mscs/msc2836/msc2836_test.go +++ b/setup/mscs/msc2836/msc2836_test.go @@ -555,7 +555,7 @@ func injectEvents(t *testing.T, userAPI userapi.UserInternalAPI, rsAPI roomserve cfg.Global.ServerName = "localhost" cfg.MSCs.Database.ConnectionString = "file:msc2836_test.db" cfg.MSCs.MSCs = []string{"msc2836"} - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(config.DatabaseOptions{}) base := &base.BaseDendrite{ Cfg: cfg, Routers: httputil.NewRouters(), diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 9f0064907..f5aa68096 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -22,7 +22,7 @@ var ctx = context.Background() func MustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { connStr, close := test.PrepareDBConnectionString(t, dbType) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(config.DatabaseOptions{}) db, err := storage.NewSyncServerDatasource(context.Background(), cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }) diff --git a/syncapi/syncapi_test.go b/syncapi/syncapi_test.go index 1c4e2cc88..ed119a875 100644 --- a/syncapi/syncapi_test.go +++ b/syncapi/syncapi_test.go @@ -115,7 +115,7 @@ func testSyncAccessTokens(t *testing.T, dbType test.DBType) { } base, close := testrig.CreateBaseDendrite(t, dbType) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) natsInstance := jetstream.NATSInstance{} defer close() @@ -222,7 +222,7 @@ func testSyncAPICreateRoomSyncEarly(t *testing.T, dbType test.DBType) { // m.room.history_visibility msgs := toNATSMsgs(t, base, room.Events()...) sinceTokens := make([]string, len(msgs)) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) AddPublicRoutes(base.ProcessContext, base.Routers, base.Cfg, base.ConnectionManager, &natsInstance, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{rooms: []*test.Room{room}}, caches, base.EnableMetrics) for i, msg := range msgs { testrig.MustPublishMsgs(t, jsctx, msg) @@ -308,7 +308,7 @@ func testSyncAPIUpdatePresenceImmediately(t *testing.T, dbType test.DBType) { jsctx, _ := natsInstance.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) AddPublicRoutes(base.ProcessContext, base.Routers, base.Cfg, base.ConnectionManager, &natsInstance, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}, caches, base.EnableMetrics) w := httptest.NewRecorder() base.Routers.Client.ServeHTTP(w, test.NewRequest(t, "GET", "/_matrix/client/v3/sync", test.WithQueryParams(map[string]string{ @@ -425,7 +425,7 @@ func testHistoryVisibility(t *testing.T, dbType test.DBType) { defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) // Use the actual internal roomserver API - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, caches, base.EnableMetrics) rsAPI.SetFederationAPI(nil, nil) AddPublicRoutes(base.ProcessContext, base.Routers, base.Cfg, base.ConnectionManager, &natsInstance, &syncUserAPI{accounts: []userapi.Device{aliceDev, bobDev}}, rsAPI, caches, base.EnableMetrics) @@ -725,7 +725,7 @@ func TestGetMembership(t *testing.T) { defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) // Use an actual roomserver for this - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) rsAPI := roomserver.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, caches, base.EnableMetrics) rsAPI.SetFederationAPI(nil, nil) @@ -796,7 +796,7 @@ func testSendToDevice(t *testing.T, dbType test.DBType) { jsctx, _ := natsInstance.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) AddPublicRoutes(base.ProcessContext, base.Routers, base.Cfg, base.ConnectionManager, &natsInstance, &syncUserAPI{accounts: []userapi.Device{alice}}, &syncRoomserverAPI{}, caches, base.EnableMetrics) producer := producers.SyncAPIProducer{ @@ -1013,7 +1013,7 @@ func testContext(t *testing.T, dbType test.DBType) { defer baseClose() // Use an actual roomserver for this - caches := caching.NewRistrettoCache(base.Cfg.Global.Cache.EstimatedMaxSize, base.Cfg.Global.Cache.MaxAge, caching.DisableMetrics) + caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics) natsInstance := jetstream.NATSInstance{} rsAPI := roomserver.NewInternalAPI(base.ProcessContext, base.Cfg, base.ConnectionManager, &natsInstance, caches, base.EnableMetrics) rsAPI.SetFederationAPI(nil, nil) diff --git a/test/testrig/base.go b/test/testrig/base.go index d9ccfbde8..2a7bb8ba0 100644 --- a/test/testrig/base.go +++ b/test/testrig/base.go @@ -99,6 +99,70 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f return nil, nil } +func CreateConfig(t *testing.T, dbType test.DBType) (*config.Dendrite, func()) { + var cfg config.Dendrite + cfg.Defaults(config.DefaultOpts{ + Generate: false, + SingleDatabase: true, + }) + cfg.Global.JetStream.InMemory = true + cfg.FederationAPI.KeyPerspectives = nil + switch dbType { + case test.DBTypePostgres: + cfg.Global.Defaults(config.DefaultOpts{ // autogen a signing key + Generate: true, + SingleDatabase: true, + }) + cfg.MediaAPI.Defaults(config.DefaultOpts{ // autogen a media path + Generate: true, + SingleDatabase: true, + }) + cfg.SyncAPI.Fulltext.Defaults(config.DefaultOpts{ // use in memory fts + Generate: true, + SingleDatabase: true, + }) + cfg.Global.ServerName = "test" + // use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use + // the file system event with InMemory=true :( + cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType) + + connStr, closeDb := test.PrepareDBConnectionString(t, dbType) + cfg.Global.DatabaseOptions = config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + MaxOpenConnections: 10, + MaxIdleConnections: 2, + ConnMaxLifetimeSeconds: 60, + } + return &cfg, closeDb + case test.DBTypeSQLite: + cfg.Defaults(config.DefaultOpts{ + Generate: true, + SingleDatabase: false, + }) + cfg.Global.ServerName = "test" + + // use a distinct prefix else concurrent postgres/sqlite runs will clash since NATS will use + // the file system event with InMemory=true :( + cfg.Global.JetStream.TopicPrefix = fmt.Sprintf("Test_%d_", dbType) + + // Use a temp dir provided by go for tests, this will be cleanup by a call to t.CleanUp() + tempDir := t.TempDir() + cfg.FederationAPI.Database.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "federationapi.db")) + cfg.KeyServer.Database.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "keyserver.db")) + cfg.MSCs.Database.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "mscs.db")) + cfg.MediaAPI.Database.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "mediaapi.db")) + cfg.RoomServer.Database.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "roomserver.db")) + cfg.SyncAPI.Database.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "syncapi.db")) + cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "userapi.db")) + cfg.RelayAPI.Database.ConnectionString = config.DataSource(filepath.Join("file://", tempDir, "relayapi.db")) + + return &cfg, func() {} + default: + t.Fatalf("unknown db type: %v", dbType) + } + return &config.Dendrite{}, func() {} +} + func Base(cfg *config.Dendrite) (*base.BaseDendrite, nats.JetStreamContext, *nats.Conn) { if cfg == nil { cfg = &config.Dendrite{} diff --git a/userapi/consumers/roomserver_test.go b/userapi/consumers/roomserver_test.go index a54706dbb..c48821fca 100644 --- a/userapi/consumers/roomserver_test.go +++ b/userapi/consumers/roomserver_test.go @@ -21,7 +21,7 @@ import ( func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.UserDatabase, func()) { t.Helper() connStr, close := test.PrepareDBConnectionString(t, dbType) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(config.DatabaseOptions{}) db, err := storage.NewUserDatabase(context.Background(), cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, "", 4, 0, 0, "") diff --git a/userapi/internal/device_list_update_test.go b/userapi/internal/device_list_update_test.go index 6ccc4b5fb..9b826908b 100644 --- a/userapi/internal/device_list_update_test.go +++ b/userapi/internal/device_list_update_test.go @@ -364,7 +364,7 @@ func mustCreateKeyserverDB(t *testing.T, dbType test.DBType) (storage.KeyDatabas t.Helper() connStr, clearDB := test.PrepareDBConnectionString(t, dbType) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(config.DatabaseOptions{}) db, err := storage.NewKeyDatabase(cm, &config.DatabaseOptions{ConnectionString: config.DataSource(connStr)}) if err != nil { t.Fatal(err) diff --git a/userapi/internal/key_api_test.go b/userapi/internal/key_api_test.go index ec315c738..e2b01bd56 100644 --- a/userapi/internal/key_api_test.go +++ b/userapi/internal/key_api_test.go @@ -16,7 +16,7 @@ import ( func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.KeyDatabase, func()) { t.Helper() connStr, close := test.PrepareDBConnectionString(t, dbType) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(config.DatabaseOptions{}) db, err := storage.NewKeyDatabase(cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }) diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index c6369ef93..5fd9c0341 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -35,7 +35,7 @@ var ( func mustCreateUserDatabase(t *testing.T, dbType test.DBType) (storage.UserDatabase, func()) { connStr, close := test.PrepareDBConnectionString(t, dbType) - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(config.DatabaseOptions{}) db, err := storage.NewUserDatabase(context.Background(), cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, "localhost", bcrypt.MinCost, openIDLifetimeMS, loginTokenLifetime, "_server") diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index c2d4e5a27..9b9978776 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -74,7 +74,7 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts, dbType test.DBType, pub if opts.serverName != "" { sName = gomatrixserverlib.ServerName(opts.serverName) } - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(config.DatabaseOptions{}) ctx := context.Background() accountDB, err := storage.NewUserDatabase(ctx, cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), diff --git a/userapi/util/notify_test.go b/userapi/util/notify_test.go index c899e3a7c..ab39187b1 100644 --- a/userapi/util/notify_test.go +++ b/userapi/util/notify_test.go @@ -77,7 +77,7 @@ func TestNotifyUserCountsAsync(t *testing.T) { // Create DB and Dendrite base connStr, close := test.PrepareDBConnectionString(t, dbType) defer close() - cm := sqlutil.NewConnectionManager() + cm := sqlutil.NewConnectionManager(config.DatabaseOptions{}) db, err := storage.NewUserDatabase(ctx, cm, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, "test", bcrypt.MinCost, 0, 0, "")