sigb.us/dendrite
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases 1 Wiki Activity

Try to fix backfilling (#2548)

Browse source 
* Try to fix backfilling

* Return start/end to not confuse clients

* Update GMSL

* Update GMSL
This commit is contained in:
Till 2022-07-01 11:49:26 +02:00 committed by GitHub
parent 086f182e24
commit 89cd0e8fc1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 54 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -34,7 +34,7 @@ require (
github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a
github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f
github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
github.com/mattn/go-sqlite3 v1.14.13

6
go.sum
View file

@ -418,8 +418,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a h1:jOkrb6twViAGTHHadA51sQwdloHT0Vx1MCptk9InTHo=
github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f h1:XF2+J6sOq07yhK1I7ItwsgRwXorjj7gqiCvgZ4dn8W8=
github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
@ -482,8 +482,6 @@ github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJE
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f h1:Fc+TjdV1mOy0oISSzfoxNWdTqjg7tN/Vdgf+B2cwvdo=
github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f/go.mod h1:vIdpKz3OG+DCg4q/xVPdXHoztEyKDWRtykQ4N7hd7C4=
github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e h1:kNIzIzj2OvnlreA+sTJ12nWJzTP3OSLNKDL/Iq9mF6Y=
github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
github.com/neilalexander/nats.go v1.13.1-0.20220621084451-ac518c356673 h1:TcKfa3Tf0qwUotv63PQVu2d1bBoLi2iEA4RHVMGDh5M=
github.com/neilalexander/nats.go v1.13.1-0.20220621084451-ac518c356673/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ=

45
roomserver/internal/perform/perform_backfill.go
View file

@ -18,6 +18,7 @@ import (
"context"
"fmt"
"github.com/getsentry/sentry-go"
federationAPI "github.com/matrix-org/dendrite/federationapi/api"
"github.com/matrix-org/dendrite/internal/eventutil"
"github.com/matrix-org/dendrite/roomserver/api"
@ -206,8 +207,17 @@ func (r *Backfiller) fetchAndStoreMissingEvents(ctx context.Context, roomVer gom
}
logger.Infof("returned %d PDUs which made events %+v", len(res.PDUs), result)
for _, res := range result {
if res.Error != nil {
logger.WithError(res.Error).Warn("event failed PDU checks")
switch err := res.Error.(type) {
case nil:
case gomatrixserverlib.SignatureErr:
// The signature of the event might not be valid anymore, for example if
// the key ID was reused with a different signature.
logger.WithError(err).Errorf("event failed PDU checks, storing anyway")
case gomatrixserverlib.AuthChainErr, gomatrixserverlib.AuthRulesErr:
logger.WithError(err).Warn("event failed PDU checks")
continue
default:
logger.WithError(err).Warn("event failed PDU checks")
continue
}
missingMap[id] = res.Event
@ -306,6 +316,7 @@ FederationHit:
b.eventIDToBeforeStateIDs[targetEvent.EventID()] = res
return res, nil
}
sentry.CaptureException(lastErr) // temporary to see if we might need to raise the server limit
return nil, lastErr
}
@ -366,19 +377,25 @@ func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatr
}
}
c := gomatrixserverlib.FederatedStateProvider{
FedClient: b.fsAPI,
RememberAuthEvents: false,
Server: b.servers[0],
var lastErr error
for _, srv := range b.servers {
c := gomatrixserverlib.FederatedStateProvider{
FedClient: b.fsAPI,
RememberAuthEvents: false,
Server: srv,
}
result, err := c.StateBeforeEvent(ctx, roomVer, event, eventIDs)
if err != nil {
lastErr = err
continue
}
for eventID, ev := range result {
b.eventIDMap[eventID] = ev
}
return result, nil
}
result, err := c.StateBeforeEvent(ctx, roomVer, event, eventIDs)
if err != nil {
return nil, err
}
for eventID, ev := range result {
b.eventIDMap[eventID] = ev
}
return result, nil
sentry.CaptureException(lastErr) // temporary to see if we might need to raise the server limit
return nil, lastErr
}
// ServersAtEvent is called when trying to determine which server to request from.

21
syncapi/routing/context.go
View file

@ -15,6 +15,7 @@
package routing
import (
"context"
"database/sql"
"encoding/json"
"fmt"
@ -25,6 +26,7 @@ import (
"github.com/matrix-org/dendrite/internal/caching"
roomserver "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/syncapi/storage"
"github.com/matrix-org/dendrite/syncapi/types"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
@ -149,13 +151,30 @@ func Context(
if len(response.State) > filter.Limit {
response.State = response.State[len(response.State)-filter.Limit:]
}
start, end, err := getStartEnd(ctx, syncDB, eventsBefore, eventsAfter)
if err == nil {
response.End = end.String()
response.Start = start.String()
}
return util.JSONResponse{
Code: http.StatusOK,
JSON: response,
}
}
func getStartEnd(ctx context.Context, syncDB storage.Database, startEvents, endEvents []*gomatrixserverlib.HeaderedEvent) (start, end types.TopologyToken, err error) {
if len(startEvents) > 0 {
start, err = syncDB.EventPositionInTopology(ctx, startEvents[0].EventID())
if err != nil {
return
}
}
if len(endEvents) > 0 {
end, err = syncDB.EventPositionInTopology(ctx, endEvents[0].EventID())
}
return
}
func applyLazyLoadMembers(
device *userapi.Device,
filter *gomatrixserverlib.RoomEventFilter,