Try to fix backfilling (#2548)
* Try to fix backfilling * Return start/end to not confuse clients * Update GMSL * Update GMSL
This commit is contained in:
Till
GitHub
parent
086f182e24
commit
89cd0e8fc1
2
go.mod
2
go.mod
|
|
@ -34,7 +34,7 @@ require (
|
|||
github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
|
||||
github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
|
||||
github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
|
||||
github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a
|
||||
github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f
|
||||
github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48
|
||||
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
|
||||
github.com/mattn/go-sqlite3 v1.14.13
|
||||
|
|
|
|||
6
go.sum
6
go.sum
|
|
@ -418,8 +418,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1
|
|||
github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
|
||||
github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4=
|
||||
github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
|
||||
github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a h1:jOkrb6twViAGTHHadA51sQwdloHT0Vx1MCptk9InTHo=
|
||||
github.com/matrix-org/gomatrixserverlib v0.0.0-20220613132209-aedb3fbb511a/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
|
||||
github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f h1:XF2+J6sOq07yhK1I7ItwsgRwXorjj7gqiCvgZ4dn8W8=
|
||||
github.com/matrix-org/gomatrixserverlib v0.0.0-20220701090733-da53994b0c7f/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk=
|
||||
github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48 h1:W0sjjC6yjskHX4mb0nk3p0fXAlbU5bAFUFeEtlrPASE=
|
||||
github.com/matrix-org/pinecone v0.0.0-20220408153826-2999ea29ed48/go.mod h1:ulJzsVOTssIVp1j/m5eI//4VpAGDkMt5NrRuAVX7wpc=
|
||||
github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
|
||||
|
|
@ -482,8 +482,6 @@ github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJE
|
|||
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
|
||||
github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f h1:Fc+TjdV1mOy0oISSzfoxNWdTqjg7tN/Vdgf+B2cwvdo=
|
||||
github.com/neilalexander/nats-server/v2 v2.8.3-0.20220513095553-73a9a246d34f/go.mod h1:vIdpKz3OG+DCg4q/xVPdXHoztEyKDWRtykQ4N7hd7C4=
|
||||
github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e h1:kNIzIzj2OvnlreA+sTJ12nWJzTP3OSLNKDL/Iq9mF6Y=
|
||||
github.com/neilalexander/nats.go v1.13.1-0.20220419101051-b262d9f0be1e/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
|
||||
github.com/neilalexander/nats.go v1.13.1-0.20220621084451-ac518c356673 h1:TcKfa3Tf0qwUotv63PQVu2d1bBoLi2iEA4RHVMGDh5M=
|
||||
github.com/neilalexander/nats.go v1.13.1-0.20220621084451-ac518c356673/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
|
||||
github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 h1:lrVQzBtkeQEGGYUHwSX1XPe1E5GL6U3KYCNe2G4bncQ=
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
federationAPI "github.com/matrix-org/dendrite/federationapi/api"
|
||||
"github.com/matrix-org/dendrite/internal/eventutil"
|
||||
"github.com/matrix-org/dendrite/roomserver/api"
|
||||
|
|
@ -206,8 +207,17 @@ func (r *Backfiller) fetchAndStoreMissingEvents(ctx context.Context, roomVer gom
|
|||
}
|
||||
logger.Infof("returned %d PDUs which made events %+v", len(res.PDUs), result)
|
||||
for _, res := range result {
|
||||
if res.Error != nil {
|
||||
logger.WithError(res.Error).Warn("event failed PDU checks")
|
||||
switch err := res.Error.(type) {
|
||||
case nil:
|
||||
case gomatrixserverlib.SignatureErr:
|
||||
// The signature of the event might not be valid anymore, for example if
|
||||
// the key ID was reused with a different signature.
|
||||
logger.WithError(err).Errorf("event failed PDU checks, storing anyway")
|
||||
case gomatrixserverlib.AuthChainErr, gomatrixserverlib.AuthRulesErr:
|
||||
logger.WithError(err).Warn("event failed PDU checks")
|
||||
continue
|
||||
default:
|
||||
logger.WithError(err).Warn("event failed PDU checks")
|
||||
continue
|
||||
}
|
||||
missingMap[id] = res.Event
|
||||
|
|
@ -306,6 +316,7 @@ FederationHit:
|
|||
b.eventIDToBeforeStateIDs[targetEvent.EventID()] = res
|
||||
return res, nil
|
||||
}
|
||||
sentry.CaptureException(lastErr) // temporary to see if we might need to raise the server limit
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
|
|
@ -366,20 +377,26 @@ func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatr
|
|||
}
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for _, srv := range b.servers {
|
||||
c := gomatrixserverlib.FederatedStateProvider{
|
||||
FedClient: b.fsAPI,
|
||||
RememberAuthEvents: false,
|
||||
Server: b.servers[0],
|
||||
Server: srv,
|
||||
}
|
||||
result, err := c.StateBeforeEvent(ctx, roomVer, event, eventIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
for eventID, ev := range result {
|
||||
b.eventIDMap[eventID] = ev
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
sentry.CaptureException(lastErr) // temporary to see if we might need to raise the server limit
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
// ServersAtEvent is called when trying to determine which server to request from.
|
||||
// It returns a list of servers which can be queried for backfill requests. These servers
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
package routing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
|
@ -25,6 +26,7 @@ import (
|
|||
"github.com/matrix-org/dendrite/internal/caching"
|
||||
roomserver "github.com/matrix-org/dendrite/roomserver/api"
|
||||
"github.com/matrix-org/dendrite/syncapi/storage"
|
||||
"github.com/matrix-org/dendrite/syncapi/types"
|
||||
userapi "github.com/matrix-org/dendrite/userapi/api"
|
||||
"github.com/matrix-org/gomatrixserverlib"
|
||||
"github.com/matrix-org/util"
|
||||
|
|
@ -149,13 +151,30 @@ func Context(
|
|||
if len(response.State) > filter.Limit {
|
||||
response.State = response.State[len(response.State)-filter.Limit:]
|
||||
}
|
||||
|
||||
start, end, err := getStartEnd(ctx, syncDB, eventsBefore, eventsAfter)
|
||||
if err == nil {
|
||||
response.End = end.String()
|
||||
response.Start = start.String()
|
||||
}
|
||||
return util.JSONResponse{
|
||||
Code: http.StatusOK,
|
||||
JSON: response,
|
||||
}
|
||||
}
|
||||
|
||||
func getStartEnd(ctx context.Context, syncDB storage.Database, startEvents, endEvents []*gomatrixserverlib.HeaderedEvent) (start, end types.TopologyToken, err error) {
|
||||
if len(startEvents) > 0 {
|
||||
start, err = syncDB.EventPositionInTopology(ctx, startEvents[0].EventID())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
if len(endEvents) > 0 {
|
||||
end, err = syncDB.EventPositionInTopology(ctx, endEvents[0].EventID())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func applyLazyLoadMembers(
|
||||
device *userapi.Device,
|
||||
filter *gomatrixserverlib.RoomEventFilter,
|
||||
|
|
|
|||
Loading…
Reference in a new issue