diff --git a/CHANGES.md b/CHANGES.md index b13908f73..c058da6a1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,13 @@ # Changelog +## Dendrite 0.8.4 (2022-05-10) + +### Fixes + +* Fixes a regression introduced in the previous version where appservices, push and phone-home statistics would not work over plain HTTP +* Adds missing indexes to the sync API output events table, which should significantly improve `/sync` performance and reduce database CPU usage +* Building Dendrite with the `bimg` thumbnailer should now work again (contributed by [database64128](https://github.com/database64128)) + ## Dendrite 0.8.3 (2022-05-09) ### Features diff --git a/appservice/appservice.go b/appservice/appservice.go index c5ae9ceb2..8fe1b2fc4 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -16,6 +16,8 @@ package appservice import ( "context" + "crypto/tls" + "net/http" "sync" "time" @@ -33,7 +35,6 @@ import ( "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" ) // AddInternalRoutes registers HTTP handlers for internal API calls @@ -45,15 +46,19 @@ func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInte // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( base *base.BaseDendrite, - userAPI userapi.AppserviceUserAPI, - rsAPI roomserverAPI.AppserviceRoomserverAPI, + userAPI userapi.UserInternalAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, ) appserviceAPI.AppServiceInternalAPI { - client := gomatrixserverlib.NewClient( - gomatrixserverlib.WithTimeout(time.Second*30), - gomatrixserverlib.WithKeepAlives(false), - gomatrixserverlib.WithSkipVerify(base.Cfg.AppServiceAPI.DisableTLSValidation), - ) - + client := &http.Client{ + Timeout: time.Second * 30, + Transport: &http.Transport{ + DisableKeepAlives: true, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: base.Cfg.AppServiceAPI.DisableTLSValidation, + }, + Proxy: http.ProxyFromEnvironment, + }, + } js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) // Create a connection to the appservice postgres DB diff --git a/appservice/query/query.go b/appservice/query/query.go index b7b0b335a..dacd3caa8 100644 --- a/appservice/query/query.go +++ b/appservice/query/query.go @@ -23,7 +23,6 @@ import ( "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/gomatrixserverlib" opentracing "github.com/opentracing/opentracing-go" log "github.com/sirupsen/logrus" ) @@ -33,7 +32,7 @@ const userIDExistsPath = "/users/" // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI type AppServiceQueryAPI struct { - HTTPClient *gomatrixserverlib.Client + HTTPClient *http.Client Cfg *config.Dendrite } @@ -65,8 +64,9 @@ func (a *AppServiceQueryAPI) RoomAliasExists( if err != nil { return err } + req = req.WithContext(ctx) - resp, err := a.HTTPClient.DoHTTPRequest(ctx, req) + resp, err := a.HTTPClient.Do(req) if resp != nil { defer func() { err = resp.Body.Close() @@ -130,7 +130,7 @@ func (a *AppServiceQueryAPI) UserIDExists( if err != nil { return err } - resp, err := a.HTTPClient.DoHTTPRequest(ctx, req) + resp, err := a.HTTPClient.Do(req.WithContext(ctx)) if resp != nil { defer func() { err = resp.Body.Close() diff --git a/appservice/workers/transaction_scheduler.go b/appservice/workers/transaction_scheduler.go index 47d447c2c..4dab00bd7 100644 --- a/appservice/workers/transaction_scheduler.go +++ b/appservice/workers/transaction_scheduler.go @@ -42,7 +42,7 @@ var ( // size), then send that off to the AS's /transactions/{txnID} endpoint. It also // handles exponentially backing off in case the AS isn't currently available. func SetupTransactionWorkers( - client *gomatrixserverlib.Client, + client *http.Client, appserviceDB storage.Database, workerStates []types.ApplicationServiceWorkerState, ) error { @@ -58,7 +58,7 @@ func SetupTransactionWorkers( // worker is a goroutine that sends any queued events to the application service // it is given. -func worker(client *gomatrixserverlib.Client, db storage.Database, ws types.ApplicationServiceWorkerState) { +func worker(client *http.Client, db storage.Database, ws types.ApplicationServiceWorkerState) { log.WithFields(log.Fields{ "appservice": ws.AppService.ID, }).Info("Starting application service") @@ -200,7 +200,7 @@ func createTransaction( // send sends events to an application service. Returns an error if an OK was not // received back from the application service or the request timed out. func send( - client *gomatrixserverlib.Client, + client *http.Client, appservice config.ApplicationService, txnID int, transaction []byte, @@ -213,7 +213,7 @@ func send( return err } req.Header.Set("Content-Type", "application/json") - resp, err := client.DoHTTPRequest(context.TODO(), req) + resp, err := client.Do(req) if err != nil { return err } diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 6e7426a7f..9670fecad 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -231,7 +231,7 @@ func queryIDServerStoreInvite( profile = &authtypes.Profile{} } - client := gomatrixserverlib.NewClient() + client := http.Client{} data := url.Values{} data.Add("medium", body.Medium) @@ -253,7 +253,7 @@ func queryIDServerStoreInvite( } req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - resp, err := client.DoHTTPRequest(ctx, req) + resp, err := client.Do(req.WithContext(ctx)) if err != nil { return nil, err } diff --git a/internal/pushgateway/client.go b/internal/pushgateway/client.go index 231327a1e..95f5afd90 100644 --- a/internal/pushgateway/client.go +++ b/internal/pushgateway/client.go @@ -3,28 +3,32 @@ package pushgateway import ( "bytes" "context" + "crypto/tls" "encoding/json" "fmt" "net/http" "time" - "github.com/matrix-org/gomatrixserverlib" "github.com/opentracing/opentracing-go" ) type httpClient struct { - hc *gomatrixserverlib.Client + hc *http.Client } // NewHTTPClient creates a new Push Gateway client. func NewHTTPClient(disableTLSValidation bool) Client { - return &httpClient{ - hc: gomatrixserverlib.NewClient( - gomatrixserverlib.WithTimeout(time.Second*30), - gomatrixserverlib.WithKeepAlives(false), - gomatrixserverlib.WithSkipVerify(disableTLSValidation), - ), + hc := &http.Client{ + Timeout: 30 * time.Second, + Transport: &http.Transport{ + DisableKeepAlives: true, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: disableTLSValidation, + }, + Proxy: http.ProxyFromEnvironment, + }, } + return &httpClient{hc: hc} } func (h *httpClient) Notify(ctx context.Context, url string, req *NotifyRequest, resp *NotifyResponse) error { @@ -41,7 +45,7 @@ func (h *httpClient) Notify(ctx context.Context, url string, req *NotifyRequest, } hreq.Header.Set("Content-Type", "application/json") - hresp, err := h.hc.DoHTTPRequest(ctx, hreq) + hresp, err := h.hc.Do(hreq) if err != nil { return err } diff --git a/internal/version.go b/internal/version.go index 08c02cfcd..5097bb2a6 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 8 - VersionPatch = 3 + VersionPatch = 4 VersionTag = "" // example: "rc1" ) diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 17e2feab6..d84d0cfa2 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -69,6 +69,11 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( -- were emitted. exclude_from_sync BOOL DEFAULT FALSE ); + +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output_room_events (type); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_sender_idx ON syncapi_output_room_events (sender); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_output_room_events (room_id); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync); ` const insertEventSQL = "" + diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 188f7582b..f9961a9d1 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -49,6 +49,11 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( transaction_id TEXT, exclude_from_sync BOOL NOT NULL DEFAULT FALSE ); + +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output_room_events (type); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_sender_idx ON syncapi_output_room_events (sender); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_output_room_events (room_id); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync); ` const insertEventSQL = "" + diff --git a/userapi/util/phonehomestats.go b/userapi/util/phonehomestats.go index e24daba6b..ad93a50e3 100644 --- a/userapi/util/phonehomestats.go +++ b/userapi/util/phonehomestats.go @@ -39,7 +39,7 @@ type phoneHomeStats struct { cfg *config.Dendrite db storage.Statistics isMonolith bool - client *gomatrixserverlib.Client + client *http.Client } type timestampToRUUsage struct { @@ -55,9 +55,10 @@ func StartPhoneHomeCollector(startTime time.Time, cfg *config.Dendrite, statsDB cfg: cfg, db: statsDB, isMonolith: cfg.IsMonolith, - client: gomatrixserverlib.NewClient( - gomatrixserverlib.WithTimeout(time.Second * 30), - ), + client: &http.Client{ + Timeout: time.Second * 30, + Transport: http.DefaultTransport, + }, } // start initial run after 5min @@ -151,7 +152,8 @@ func (p *phoneHomeStats) collect() { } request.Header.Set("User-Agent", "Dendrite/"+internal.VersionString()) - if _, err = p.client.DoHTTPRequest(ctx, request); err != nil { + _, err = p.client.Do(request) + if err != nil { logrus.WithError(err).Error("unable to send anonymous stats") return }