Merge branch 'main' of github.com:matrix-org/dendrite into s7evink/roomservertests3

CHANGES.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## Dendrite 0.8.4 (2022-05-10)
+
+### Fixes
+
+* Fixes a regression introduced in the previous version where appservices, push and phone-home statistics would not work over plain HTTP
+* Adds missing indexes to the sync API output events table, which should significantly improve `/sync` performance and reduce database CPU usage
+* Building Dendrite with the `bimg` thumbnailer should now work again (contributed by [database64128](https://github.com/database64128))
+
 ## Dendrite 0.8.3 (2022-05-09)
 
 ### Features

appservice/appservice.go

@@ -16,6 +16,8 @@ package appservice
 
 import (
 	"context"
+	"crypto/tls"
+	"net/http"
 	"sync"
 	"time"
 
@@ -33,7 +35,6 @@ import (
 	"github.com/matrix-org/dendrite/setup/base"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
-	"github.com/matrix-org/gomatrixserverlib"
 )
 
 // AddInternalRoutes registers HTTP handlers for internal API calls
@@ -45,15 +46,19 @@ func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceInte
 // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes.
 func NewInternalAPI(
 	base *base.BaseDendrite,
-	userAPI userapi.AppserviceUserAPI,
+	userAPI userapi.UserInternalAPI,
-	rsAPI roomserverAPI.AppserviceRoomserverAPI,
+	rsAPI roomserverAPI.RoomserverInternalAPI,
 ) appserviceAPI.AppServiceInternalAPI {
-	client := gomatrixserverlib.NewClient(
+	client := &http.Client{
-		gomatrixserverlib.WithTimeout(time.Second*30),
+		Timeout: time.Second * 30,
-		gomatrixserverlib.WithKeepAlives(false),
+		Transport: &http.Transport{
-		gomatrixserverlib.WithSkipVerify(base.Cfg.AppServiceAPI.DisableTLSValidation),
+			DisableKeepAlives: true,
-	)
+			TLSClientConfig: &tls.Config{
-
+				InsecureSkipVerify: base.Cfg.AppServiceAPI.DisableTLSValidation,
+			},
+			Proxy: http.ProxyFromEnvironment,
+		},
+	}
 	js, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream)
 
 	// Create a connection to the appservice postgres DB

appservice/query/query.go

@@ -23,7 +23,6 @@ import (
 
 	"github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/setup/config"
-	"github.com/matrix-org/gomatrixserverlib"
 	opentracing "github.com/opentracing/opentracing-go"
 	log "github.com/sirupsen/logrus"
 )
@@ -33,7 +32,7 @@ const userIDExistsPath = "/users/"
 
 // AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI
 type AppServiceQueryAPI struct {
-	HTTPClient *gomatrixserverlib.Client
+	HTTPClient *http.Client
 	Cfg        *config.Dendrite
 }
 
@@ -65,8 +64,9 @@ func (a *AppServiceQueryAPI) RoomAliasExists(
 			if err != nil {
 				return err
 			}
+			req = req.WithContext(ctx)
 
-			resp, err := a.HTTPClient.DoHTTPRequest(ctx, req)
+			resp, err := a.HTTPClient.Do(req)
 			if resp != nil {
 				defer func() {
 					err = resp.Body.Close()
@@ -130,7 +130,7 @@ func (a *AppServiceQueryAPI) UserIDExists(
 			if err != nil {
 				return err
 			}
-			resp, err := a.HTTPClient.DoHTTPRequest(ctx, req)
+			resp, err := a.HTTPClient.Do(req.WithContext(ctx))
 			if resp != nil {
 				defer func() {
 					err = resp.Body.Close()

appservice/workers/transaction_scheduler.go

@@ -42,7 +42,7 @@ var (
 // size), then send that off to the AS's /transactions/{txnID} endpoint. It also
 // handles exponentially backing off in case the AS isn't currently available.
 func SetupTransactionWorkers(
-	client *gomatrixserverlib.Client,
+	client *http.Client,
 	appserviceDB storage.Database,
 	workerStates []types.ApplicationServiceWorkerState,
 ) error {
@@ -58,7 +58,7 @@ func SetupTransactionWorkers(
 
 // worker is a goroutine that sends any queued events to the application service
 // it is given.
-func worker(client *gomatrixserverlib.Client, db storage.Database, ws types.ApplicationServiceWorkerState) {
+func worker(client *http.Client, db storage.Database, ws types.ApplicationServiceWorkerState) {
 	log.WithFields(log.Fields{
 		"appservice": ws.AppService.ID,
 	}).Info("Starting application service")
@@ -200,7 +200,7 @@ func createTransaction(
 // send sends events to an application service. Returns an error if an OK was not
 // received back from the application service or the request timed out.
 func send(
-	client *gomatrixserverlib.Client,
+	client *http.Client,
 	appservice config.ApplicationService,
 	txnID int,
 	transaction []byte,
@@ -213,7 +213,7 @@ func send(
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
-	resp, err := client.DoHTTPRequest(context.TODO(), req)
+	resp, err := client.Do(req)
 	if err != nil {
 		return err
 	}

clientapi/threepid/invites.go

@@ -231,7 +231,7 @@ func queryIDServerStoreInvite(
 		profile = &authtypes.Profile{}
 	}
 
-	client := gomatrixserverlib.NewClient()
+	client := http.Client{}
 
 	data := url.Values{}
 	data.Add("medium", body.Medium)
@@ -253,7 +253,7 @@ func queryIDServerStoreInvite(
 	}
 
 	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
-	resp, err := client.DoHTTPRequest(ctx, req)
+	resp, err := client.Do(req.WithContext(ctx))
 	if err != nil {
 		return nil, err
 	}

internal/pushgateway/client.go

@@ -3,28 +3,32 @@ package pushgateway
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"net/http"
 	"time"
 
-	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/opentracing/opentracing-go"
 )
 
 type httpClient struct {
-	hc *gomatrixserverlib.Client
+	hc *http.Client
 }
 
 // NewHTTPClient creates a new Push Gateway client.
 func NewHTTPClient(disableTLSValidation bool) Client {
-	return &httpClient{
+	hc := &http.Client{
-		hc: gomatrixserverlib.NewClient(
+		Timeout: 30 * time.Second,
-			gomatrixserverlib.WithTimeout(time.Second*30),
+		Transport: &http.Transport{
-			gomatrixserverlib.WithKeepAlives(false),
+			DisableKeepAlives: true,
-			gomatrixserverlib.WithSkipVerify(disableTLSValidation),
+			TLSClientConfig: &tls.Config{
-		),
+				InsecureSkipVerify: disableTLSValidation,
+			},
+			Proxy: http.ProxyFromEnvironment,
+		},
 	}
+	return &httpClient{hc: hc}
 }
 
 func (h *httpClient) Notify(ctx context.Context, url string, req *NotifyRequest, resp *NotifyResponse) error {
@@ -41,7 +45,7 @@ func (h *httpClient) Notify(ctx context.Context, url string, req *NotifyRequest,
 	}
 	hreq.Header.Set("Content-Type", "application/json")
 
-	hresp, err := h.hc.DoHTTPRequest(ctx, hreq)
+	hresp, err := h.hc.Do(hreq)
 	if err != nil {
 		return err
 	}

internal/version.go

@@ -17,7 +17,7 @@ var build string
 const (
 	VersionMajor = 0
 	VersionMinor = 8
-	VersionPatch = 3
+	VersionPatch = 4
 	VersionTag   = "" // example: "rc1"
 )
 

syncapi/storage/postgres/output_room_events_table.go

@@ -69,6 +69,11 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events (
   -- were emitted.
   exclude_from_sync BOOL DEFAULT FALSE
 );
+
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output_room_events (type);
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_sender_idx ON syncapi_output_room_events (sender);
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_output_room_events (room_id);
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync);
 `
 
 const insertEventSQL = "" +

syncapi/storage/sqlite3/output_room_events_table.go

@@ -49,6 +49,11 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events (
   transaction_id TEXT,
   exclude_from_sync BOOL NOT NULL DEFAULT FALSE
 );
+
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output_room_events (type);
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_sender_idx ON syncapi_output_room_events (sender);
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_output_room_events (room_id);
+CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync);
 `
 
 const insertEventSQL = "" +

userapi/util/phonehomestats.go

@@ -39,7 +39,7 @@ type phoneHomeStats struct {
 	cfg        *config.Dendrite
 	db         storage.Statistics
 	isMonolith bool
-	client     *gomatrixserverlib.Client
+	client     *http.Client
 }
 
 type timestampToRUUsage struct {
@@ -55,9 +55,10 @@ func StartPhoneHomeCollector(startTime time.Time, cfg *config.Dendrite, statsDB
 		cfg:        cfg,
 		db:         statsDB,
 		isMonolith: cfg.IsMonolith,
-		client: gomatrixserverlib.NewClient(
+		client: &http.Client{
-			gomatrixserverlib.WithTimeout(time.Second * 30),
+			Timeout:   time.Second * 30,
-		),
+			Transport: http.DefaultTransport,
+		},
 	}
 
 	// start initial run after 5min
@@ -151,7 +152,8 @@ func (p *phoneHomeStats) collect() {
 	}
 	request.Header.Set("User-Agent", "Dendrite/"+internal.VersionString())
 
-	if _, err = p.client.DoHTTPRequest(ctx, request); err != nil {
+	_, err = p.client.Do(request)
+	if err != nil {
 		logrus.WithError(err).Error("unable to send anonymous stats")
 		return
 	}