diff --git a/.golangci.yml b/.golangci.yml index 0d0f51bd2..7fdd4d003 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -102,7 +102,7 @@ linters-settings: #local-prefixes: github.com/org/project gocyclo: # minimal code complexity to report, 30 by default (but we recommend 10-20) - min-complexity: 12 + min-complexity: 13 maligned: # print struct with more effective memory layout or not, false by default suggest-new: true diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 6d3ea808f..3e47dee56 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -101,11 +101,11 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { "type": ev.Type(), }).Info("appservice received an event from roomserver") - missingEvents, err := s.lookupMissingStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev) + missingEvents, err := s.lookupMissingStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev.Event) if err != nil { return err } - events := append(missingEvents, ev) + events := append(missingEvents, ev.Event) // Send event to any relevant application services return s.filterRoomserverEvents(context.TODO(), events) @@ -143,7 +143,9 @@ func (s *OutputRoomEventConsumer) lookupMissingStateEvents( return nil, err } - result = append(result, eventResp.Events...) + for _, headeredEvent := range eventResp.Events { + result = append(result, headeredEvent.Event) + } return result, nil } diff --git a/clientapi/auth/storage/accounts/interface.go b/clientapi/auth/storage/accounts/interface.go index 9f6e3e1ea..a5052b047 100644 --- a/clientapi/auth/storage/accounts/interface.go +++ b/clientapi/auth/storage/accounts/interface.go @@ -33,6 +33,7 @@ type Database interface { CreateGuestAccount(ctx context.Context) (*authtypes.Account, error) UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error) + GetRoomIDsByLocalPart(ctx context.Context, localpart string) ([]string, error) GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error) SaveAccountData(ctx context.Context, localpart, roomID, dataType, content string) error GetAccountData(ctx context.Context, localpart string) (global []gomatrixserverlib.ClientEvent, rooms map[string][]gomatrixserverlib.ClientEvent, err error) diff --git a/clientapi/auth/storage/accounts/postgres/account_data_table.go b/clientapi/auth/storage/accounts/postgres/account_data_table.go index 4573999b4..9198a7440 100644 --- a/clientapi/auth/storage/accounts/postgres/account_data_table.go +++ b/clientapi/auth/storage/accounts/postgres/account_data_table.go @@ -18,6 +18,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" ) @@ -90,7 +92,7 @@ func (s *accountDataStatements) selectAccountData( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectAccountData: rows.close() failed") global = []gomatrixserverlib.ClientEvent{} rooms = make(map[string][]gomatrixserverlib.ClientEvent) diff --git a/clientapi/auth/storage/accounts/postgres/membership_table.go b/clientapi/auth/storage/accounts/postgres/membership_table.go index 426c2d6ac..04e9095e9 100644 --- a/clientapi/auth/storage/accounts/postgres/membership_table.go +++ b/clientapi/auth/storage/accounts/postgres/membership_table.go @@ -18,6 +18,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/lib/pq" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" ) @@ -51,6 +53,9 @@ const selectMembershipsByLocalpartSQL = "" + const selectMembershipInRoomByLocalpartSQL = "" + "SELECT event_id FROM account_memberships WHERE localpart = $1 AND room_id = $2" +const selectRoomIDsByLocalPartSQL = "" + + "SELECT room_id FROM account_memberships WHERE localpart = $1" + const deleteMembershipsByEventIDsSQL = "" + "DELETE FROM account_memberships WHERE event_id = ANY($1)" @@ -59,6 +64,7 @@ type membershipStatements struct { insertMembershipStmt *sql.Stmt selectMembershipInRoomByLocalpartStmt *sql.Stmt selectMembershipsByLocalpartStmt *sql.Stmt + selectRoomIDsByLocalPartStmt *sql.Stmt } func (s *membershipStatements) prepare(db *sql.DB) (err error) { @@ -78,6 +84,9 @@ func (s *membershipStatements) prepare(db *sql.DB) (err error) { if s.selectMembershipsByLocalpartStmt, err = db.Prepare(selectMembershipsByLocalpartSQL); err != nil { return } + if s.selectRoomIDsByLocalPartStmt, err = db.Prepare(selectRoomIDsByLocalPartSQL); err != nil { + return + } return } @@ -118,7 +127,7 @@ func (s *membershipStatements) selectMembershipsByLocalpart( memberships = []authtypes.Membership{} - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsByLocalpart: rows.close() failed") for rows.Next() { var m authtypes.Membership m.Localpart = localpart @@ -129,3 +138,23 @@ func (s *membershipStatements) selectMembershipsByLocalpart( } return memberships, rows.Err() } + +func (s *membershipStatements) selectRoomIDsByLocalPart( + ctx context.Context, localPart string, +) ([]string, error) { + stmt := s.selectRoomIDsByLocalPartStmt + rows, err := stmt.QueryContext(ctx, localPart) + if err != nil { + return nil, err + } + roomIDs := []string{} + defer rows.Close() // nolint: errcheck + for rows.Next() { + var roomID string + if err = rows.Scan(&roomID); err != nil { + return nil, err + } + roomIDs = append(roomIDs, roomID) + } + return roomIDs, rows.Err() +} diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/clientapi/auth/storage/accounts/postgres/storage.go index 8115dca43..4a0a2060b 100644 --- a/clientapi/auth/storage/accounts/postgres/storage.go +++ b/clientapi/auth/storage/accounts/postgres/storage.go @@ -234,6 +234,16 @@ func (d *Database) GetMembershipInRoomByLocalpart( return d.memberships.selectMembershipInRoomByLocalpart(ctx, localpart, roomID) } +// GetRoomIDsByLocalPart returns an array containing the room ids of all +// the rooms a user matching a given localpart is a member of +// If no membership match the given localpart, returns an empty array +// If there was an issue during the retrieval, returns the SQL error +func (d *Database) GetRoomIDsByLocalPart( + ctx context.Context, localpart string, +) ([]string, error) { + return d.memberships.selectRoomIDsByLocalPart(ctx, localpart) +} + // GetMembershipsByLocalpart returns an array containing the memberships for all // the rooms a user matching a given localpart is a member of // If no membership match the given localpart, returns an empty array diff --git a/clientapi/auth/storage/accounts/sqlite3/membership_table.go b/clientapi/auth/storage/accounts/sqlite3/membership_table.go index 38f21b7f3..bd9838b6b 100644 --- a/clientapi/auth/storage/accounts/sqlite3/membership_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/membership_table.go @@ -51,6 +51,9 @@ const selectMembershipsByLocalpartSQL = "" + const selectMembershipInRoomByLocalpartSQL = "" + "SELECT event_id FROM account_memberships WHERE localpart = $1 AND room_id = $2" +const selectRoomIDsByLocalPartSQL = "" + + "SELECT room_id FROM account_memberships WHERE localpart = $1" + const deleteMembershipsByEventIDsSQL = "" + "DELETE FROM account_memberships WHERE event_id IN ($1)" @@ -58,6 +61,7 @@ type membershipStatements struct { insertMembershipStmt *sql.Stmt selectMembershipInRoomByLocalpartStmt *sql.Stmt selectMembershipsByLocalpartStmt *sql.Stmt + selectRoomIDsByLocalPartStmt *sql.Stmt } func (s *membershipStatements) prepare(db *sql.DB) (err error) { @@ -74,6 +78,9 @@ func (s *membershipStatements) prepare(db *sql.DB) (err error) { if s.selectMembershipsByLocalpartStmt, err = db.Prepare(selectMembershipsByLocalpartSQL); err != nil { return } + if s.selectRoomIDsByLocalPartStmt, err = db.Prepare(selectRoomIDsByLocalPartSQL); err != nil { + return + } return } @@ -118,7 +125,7 @@ func (s *membershipStatements) selectMembershipsByLocalpart( memberships = []authtypes.Membership{} - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsByLocalpart: rows.close() failed") for rows.Next() { var m authtypes.Membership m.Localpart = localpart @@ -130,3 +137,22 @@ func (s *membershipStatements) selectMembershipsByLocalpart( return } +func (s *membershipStatements) selectRoomIDsByLocalPart( + ctx context.Context, localPart string, +) ([]string, error) { + stmt := s.selectRoomIDsByLocalPartStmt + rows, err := stmt.QueryContext(ctx, localPart) + if err != nil { + return nil, err + } + roomIDs := []string{} + defer rows.Close() // nolint: errcheck + for rows.Next() { + var roomID string + if err = rows.Scan(&roomID); err != nil { + return nil, err + } + roomIDs = append(roomIDs, roomID) + } + return roomIDs, rows.Err() +} diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index 9124640c6..bfb7b4ea1 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -253,6 +253,16 @@ func (d *Database) GetMembershipsByLocalpart( return d.memberships.selectMembershipsByLocalpart(ctx, localpart) } +// GetRoomIDsByLocalPart returns an array containing the room ids of all +// the rooms a user matching a given localpart is a member of +// If no membership match the given localpart, returns an empty array +// If there was an issue during the retrieval, returns the SQL error +func (d *Database) GetRoomIDsByLocalPart( + ctx context.Context, localpart string, +) ([]string, error) { + return d.memberships.selectRoomIDsByLocalPart(ctx, localpart) +} + // newMembership saves a new membership in the database. // If the event isn't a valid m.room.member event with type `join`, does nothing. // If an error occurred, returns the SQL error diff --git a/clientapi/auth/storage/accounts/sqlite3/threepid_table.go b/clientapi/auth/storage/accounts/sqlite3/threepid_table.go index 53f6408d1..29ee4c3d0 100644 --- a/clientapi/auth/storage/accounts/sqlite3/threepid_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/threepid_table.go @@ -97,7 +97,7 @@ func (s *threepidStatements) selectThreePIDsForLocalpart( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectThreePIDsForLocalpart: rows.close() failed") threepids = []authtypes.ThreePID{} for rows.Next() { diff --git a/clientapi/auth/storage/devices/postgres/devices_table.go b/clientapi/auth/storage/devices/postgres/devices_table.go index 6cae3743e..ee5591706 100644 --- a/clientapi/auth/storage/devices/postgres/devices_table.go +++ b/clientapi/auth/storage/devices/postgres/devices_table.go @@ -226,7 +226,7 @@ func (s *devicesStatements) selectDevicesByLocalpart( if err != nil { return devices, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectDevicesByLocalpart: rows.close() failed") for rows.Next() { var dev authtypes.Device diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go index a65281514..6d5bb09a6 100644 --- a/clientapi/consumers/roomserver.go +++ b/clientapi/consumers/roomserver.go @@ -91,7 +91,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { "type": ev.Type(), }).Info("received event from roomserver") - events, err := s.lookupStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev) + events, err := s.lookupStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev.Event) if err != nil { return err } @@ -138,7 +138,9 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( return nil, err } - result = append(result, eventResp.Events...) + for _, headeredEvent := range eventResp.Events { + result = append(result, headeredEvent.Event) + } return result, nil } diff --git a/clientapi/jsonerror/jsonerror.go b/clientapi/jsonerror/jsonerror.go index 8df1fead2..735de5bea 100644 --- a/clientapi/jsonerror/jsonerror.go +++ b/clientapi/jsonerror/jsonerror.go @@ -124,6 +124,12 @@ func GuestAccessForbidden(msg string) *MatrixError { return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg} } +// UnsupportedRoomVersion is an error which is returned when the client +// requests a room with a version that is unsupported. +func UnsupportedRoomVersion(msg string) *MatrixError { + return &MatrixError{"M_UNSUPPORTED_ROOM_VERSION", msg} +} + // LimitExceededError is a rate-limiting error. type LimitExceededError struct { MatrixError diff --git a/clientapi/producers/roomserver.go b/clientapi/producers/roomserver.go index e50561a70..0fe2d556b 100644 --- a/clientapi/producers/roomserver.go +++ b/clientapi/producers/roomserver.go @@ -40,9 +40,11 @@ func (c *RoomserverProducer) SendEvents( ) (string, error) { ires := make([]api.InputRoomEvent, len(events)) for i, event := range events { + roomVersion := gomatrixserverlib.RoomVersionV1 + ires[i] = api.InputRoomEvent{ Kind: api.KindNew, - Event: event, + Event: event.Headered(roomVersion), AuthEventIDs: event.AuthEventIDs(), SendAsServer: string(sendAsServer), TransactionID: txnID, @@ -61,11 +63,14 @@ func (c *RoomserverProducer) SendEventWithState( return err } + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + ires := make([]api.InputRoomEvent, len(outliers)+1) for i, outlier := range outliers { ires[i] = api.InputRoomEvent{ Kind: api.KindOutlier, - Event: outlier, + Event: outlier.Headered(roomVersion), AuthEventIDs: outlier.AuthEventIDs(), } } @@ -77,7 +82,7 @@ func (c *RoomserverProducer) SendEventWithState( ires[len(outliers)] = api.InputRoomEvent{ Kind: api.KindNew, - Event: event, + Event: event.Headered(roomVersion), AuthEventIDs: event.AuthEventIDs(), HasState: true, StateEventIDs: stateEventIDs, @@ -104,8 +109,13 @@ func (c *RoomserverProducer) SendInputRoomEvents( func (c *RoomserverProducer) SendInvite( ctx context.Context, inviteEvent gomatrixserverlib.Event, ) error { + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + request := api.InputRoomEventsRequest{ - InputInviteEvents: []api.InputInviteEvent{{Event: inviteEvent}}, + InputInviteEvents: []api.InputInviteEvent{{ + Event: inviteEvent.Headered(roomVersion), + }}, } var response api.InputRoomEventsResponse return c.InputAPI.InputRoomEvents(ctx, &request, &response) diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index c9623acb9..964200760 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -23,6 +23,7 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" @@ -38,15 +39,16 @@ import ( // https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-createroom type createRoomRequest struct { - Invite []string `json:"invite"` - Name string `json:"name"` - Visibility string `json:"visibility"` - Topic string `json:"topic"` - Preset string `json:"preset"` - CreationContent map[string]interface{} `json:"creation_content"` - InitialState []fledglingEvent `json:"initial_state"` - RoomAliasName string `json:"room_alias_name"` - GuestCanJoin bool `json:"guest_can_join"` + Invite []string `json:"invite"` + Name string `json:"name"` + Visibility string `json:"visibility"` + Topic string `json:"topic"` + Preset string `json:"preset"` + CreationContent map[string]interface{} `json:"creation_content"` + InitialState []fledglingEvent `json:"initial_state"` + RoomAliasName string `json:"room_alias_name"` + GuestCanJoin bool `json:"guest_can_join"` + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` } const ( @@ -180,15 +182,28 @@ func createRoom( } r.CreationContent["creator"] = userID - r.CreationContent["room_version"] = "1" // TODO: We set this to 1 before we support Room versioning + roomVersion := roomserverVersion.DefaultRoomVersion() + if r.RoomVersion != "" { + candidateVersion := gomatrixserverlib.RoomVersion(r.RoomVersion) + _, roomVersionError := roomserverVersion.SupportedRoomVersion(candidateVersion) + if roomVersionError != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion(roomVersionError.Error()), + } + } + roomVersion = candidateVersion + } + r.CreationContent["room_version"] = roomVersion // TODO: visibility/presets/raw initial state // TODO: Create room alias association // Make sure this doesn't fall into an application service's namespace though! logger.WithFields(log.Fields{ - "userID": userID, - "roomID": roomID, + "userID": userID, + "roomID": roomID, + "roomVersion": r.CreationContent["room_version"], }).Info("Creating new room") profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) @@ -366,7 +381,7 @@ func buildEvent( eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), cfg.Matrix.ServerName) event, err := builder.Build(eventID, evTime, cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey) if err != nil { - return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %s", builder.Type, err) + return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %w", builder.Type, err) } return &event, nil } diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go index 1c1a7add9..2d3152510 100644 --- a/clientapi/routing/getevent.go +++ b/clientapi/routing/getevent.go @@ -66,7 +66,7 @@ func GetEvent( } } - requestedEvent := eventsResp.Events[0] + requestedEvent := eventsResp.Events[0].Event r := getEventRequest{ req: req, diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index a6899eeb8..0b846e5e3 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -17,6 +17,8 @@ package routing import ( "net/http" + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" @@ -25,10 +27,14 @@ import ( "github.com/matrix-org/util" ) -type response struct { +type getMembershipResponse struct { Chunk []gomatrixserverlib.ClientEvent `json:"chunk"` } +type getJoinedRoomsResponse struct { + JoinedRooms []string `json:"joined_rooms"` +} + // GetMemberships implements GET /rooms/{roomId}/members func GetMemberships( req *http.Request, device *authtypes.Device, roomID string, joinedOnly bool, @@ -55,6 +61,27 @@ func GetMemberships( return util.JSONResponse{ Code: http.StatusOK, - JSON: response{queryRes.JoinEvents}, + JSON: getMembershipResponse{queryRes.JoinEvents}, + } +} + +func GetJoinedRooms( + req *http.Request, + device *authtypes.Device, + accountsDB accounts.Database, +) util.JSONResponse { + localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() + } + joinedRooms, err := accountsDB.GetRoomIDsByLocalPart(req.Context(), localpart) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("accountsDB.GetRoomIDsByLocalPart failed") + return jsonerror.InternalServerError() + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: getJoinedRoomsResponse{joinedRooms}, } } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index f0841b796..22ff12b02 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -105,6 +105,12 @@ func Setup( ) }), ).Methods(http.MethodPost, http.MethodOptions) + r0mux.Handle("/joined_rooms", + common.MakeAuthAPI("joined_rooms", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + return GetJoinedRooms(req, device, accountDB) + }), + ).Methods(http.MethodGet, http.MethodOptions) + r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|leave|invite)}", common.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(req)) @@ -390,7 +396,7 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) - unstableMux.Handle("/thirdparty/protocols", + r0mux.Handle("/thirdparty/protocols", common.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse { // TODO: Return the third party protcols return util.JSONResponse{ diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index da96c5c65..dbfea09c5 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -144,7 +144,7 @@ func generateSendEvent( // check to see if this user can perform this operation stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents)) for i := range queryRes.StateEvents { - stateEvents[i] = &queryRes.StateEvents[i] + stateEvents[i] = &queryRes.StateEvents[i].Event } provider := gomatrixserverlib.NewAuthEvents(stateEvents) if err = gomatrixserverlib.Allowed(*e, &provider); err != nil { diff --git a/cmd/create-room-events/main.go b/cmd/create-room-events/main.go index 8475914f0..ef442a0c4 100644 --- a/cmd/create-room-events/main.go +++ b/cmd/create-room-events/main.go @@ -125,7 +125,7 @@ func writeEvent(event gomatrixserverlib.Event) { if *format == "InputRoomEvent" { var ire api.InputRoomEvent ire.Kind = api.KindNew - ire.Event = event + ire.Event = event.Headered(gomatrixserverlib.RoomVersionV1) authEventIDs := []string{} for _, ref := range b.AuthEvents { authEventIDs = append(authEventIDs, ref.EventID) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index b3de9adde..27c3054b8 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -69,7 +69,7 @@ func main() { ) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI) mediaapi.SetupMediaAPIComponent(base, deviceDB) - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, federation, nil) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) httpHandler := common.WrapHandlerInCORS(base.APIMux) diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index f8bd8b06a..6b7eac7d5 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -28,7 +28,7 @@ func main() { _, _, query := base.CreateHTTPRoomserverAPIs() - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, nil, nil) base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI)) diff --git a/cmd/dendritejs/keyfetcher.go b/cmd/dendritejs/keyfetcher.go index 2b0a54fd9..ee4905d4f 100644 --- a/cmd/dendritejs/keyfetcher.go +++ b/cmd/dendritejs/keyfetcher.go @@ -53,15 +53,15 @@ func (f *libp2pKeyFetcher) FetchKeys( peerIDStr := string(req.ServerName) peerID, err := peer.Decode(peerIDStr) if err != nil { - return nil, fmt.Errorf("Failed to decode peer ID from server name '%s': %s", peerIDStr, err) + return nil, fmt.Errorf("Failed to decode peer ID from server name '%s': %w", peerIDStr, err) } pubKey, err := peerID.ExtractPublicKey() if err != nil { - return nil, fmt.Errorf("Failed to extract public key from peer ID: %s", err) + return nil, fmt.Errorf("Failed to extract public key from peer ID: %w", err) } pubKeyBytes, err := pubKey.Raw() if err != nil { - return nil, fmt.Errorf("Failed to extract raw bytes from public key: %s", err) + return nil, fmt.Errorf("Failed to extract raw bytes from public key: %w", err) } util.GetLogger(ctx).Info("libp2pKeyFetcher.FetchKeys: Using public key %v for server name %s", pubKeyBytes, req.ServerName) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 458427a7b..7c8526715 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -69,7 +69,7 @@ func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLoc } func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http_js_libp2p.P2pLocalNode) { - hosted := "/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/" + hosted := "/dns4/rendezvous.matrix.org/tcp/8443/wss/p2p-websocket-star/" node = go_http_js_libp2p.NewP2pLocalNode("org.matrix.p2p.experiment", privKey.Seed(), []string{hosted}) serverName = node.Id fmt.Println("p2p assigned ServerName: ", serverName) @@ -119,6 +119,7 @@ func main() { }, KeyDatabase: keyDB, } + p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node) alias, input, query := roomserver.SetupRoomServerComponent(base) typingInputAPI := typingserver.SetupTypingServerComponent(base, cache.NewTypingCache()) @@ -134,7 +135,7 @@ func main() { ) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI) mediaapi.SetupMediaAPIComponent(base, deviceDB) - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, federation, p2pPublicRoomProvider) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) httpHandler := common.WrapHandlerInCORS(base.APIMux) @@ -145,11 +146,10 @@ func main() { if node != nil { go func() { logrus.Info("Listening on libp2p-js host ID ", node.Id) - - listener := go_http_js_libp2p.NewP2pListener(node) - defer listener.Close() - s := &http.Server{} - s.Serve(listener) + s := JSServer{ + Mux: http.DefaultServeMux, + } + s.ListenAndServe("p2p") }() } diff --git a/cmd/dendritejs/publicrooms.go b/cmd/dendritejs/publicrooms.go new file mode 100644 index 000000000..17822e7ad --- /dev/null +++ b/cmd/dendritejs/publicrooms.go @@ -0,0 +1,46 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build wasm + +package main + +import ( + "github.com/matrix-org/go-http-js-libp2p/go_http_js_libp2p" +) + +type libp2pPublicRoomsProvider struct { + node *go_http_js_libp2p.P2pLocalNode + providers []go_http_js_libp2p.PeerInfo +} + +func NewLibP2PPublicRoomsProvider(node *go_http_js_libp2p.P2pLocalNode) *libp2pPublicRoomsProvider { + p := &libp2pPublicRoomsProvider{ + node: node, + } + node.RegisterFoundProviders(p.foundProviders) + return p +} + +func (p *libp2pPublicRoomsProvider) foundProviders(peerInfos []go_http_js_libp2p.PeerInfo) { + p.providers = peerInfos +} + +func (p *libp2pPublicRoomsProvider) Homeservers() []string { + result := make([]string, len(p.providers)) + for i := range p.providers { + result[i] = p.providers[i].Id + } + return result +} diff --git a/cmd/syncserver-integration-tests/main.go b/cmd/syncserver-integration-tests/main.go index 7f315cd33..d14e854c3 100644 --- a/cmd/syncserver-integration-tests/main.go +++ b/cmd/syncserver-integration-tests/main.go @@ -104,7 +104,7 @@ func clientEventJSONForOutputRoomEvent(outputRoomEvent string) string { panic("failed to unmarshal output room event: " + err.Error()) } clientEvs := gomatrixserverlib.ToClientEvents([]gomatrixserverlib.Event{ - out.NewRoomEvent.Event, + out.NewRoomEvent.Event.Event, }, gomatrixserverlib.FormatSync) b, err := json.Marshal(clientEvs[0]) if err != nil { diff --git a/common/consumers.go b/common/consumers.go index 2e58a6241..f33993494 100644 --- a/common/consumers.go +++ b/common/consumers.go @@ -112,7 +112,7 @@ func (c *ContinualConsumer) consumePartition(pc sarama.PartitionConsumer) { msgErr := c.ProcessMessage(message) // Advance our position in the stream so that we will start at the right position after a restart. if err := c.PartitionStore.SetPartitionOffset(context.TODO(), c.Topic, message.Partition, message.Offset); err != nil { - panic(fmt.Errorf("the ContinualConsumer failed to SetPartitionOffset: %s", err)) + panic(fmt.Errorf("the ContinualConsumer failed to SetPartitionOffset: %w", err)) } // Shutdown if we were told to do so. if msgErr == ErrShutdown { diff --git a/common/events.go b/common/events.go index 3c060ee65..ff46e8e0b 100644 --- a/common/events.go +++ b/common/events.go @@ -89,7 +89,7 @@ func AddPrevEventsToEvent( authEvents := gomatrixserverlib.NewAuthEvents(nil) for i := range queryRes.StateEvents { - err = authEvents.AddEvent(&queryRes.StateEvents[i]) + err = authEvents.AddEvent(&queryRes.StateEvents[i].Event) if err != nil { return err } diff --git a/common/keydb/postgres/server_key_table.go b/common/keydb/postgres/server_key_table.go index 6b13cc3c2..0434eb8b1 100644 --- a/common/keydb/postgres/server_key_table.go +++ b/common/keydb/postgres/server_key_table.go @@ -19,6 +19,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/lib/pq" "github.com/matrix-org/gomatrixserverlib" ) @@ -91,7 +93,7 @@ func (s *serverKeyStatements) bulkSelectServerKeys( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectServerKeys: rows.close() failed") results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} for rows.Next() { var serverName string diff --git a/common/keydb/sqlite3/server_key_table.go b/common/keydb/sqlite3/server_key_table.go index 3caf4ea4c..ba1cc0606 100644 --- a/common/keydb/sqlite3/server_key_table.go +++ b/common/keydb/sqlite3/server_key_table.go @@ -124,7 +124,7 @@ func (s *serverKeyStatements) bulkSelectServerKeys( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectServerKeys: rows.close() failed") results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} for rows.Next() { var serverName string diff --git a/common/log.go b/common/log.go index f9ed84edb..11339ada4 100644 --- a/common/log.go +++ b/common/log.go @@ -15,13 +15,17 @@ package common import ( + "context" "fmt" + "io" "os" "path" "path/filepath" "runtime" "strings" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dugong" "github.com/sirupsen/logrus" @@ -156,3 +160,17 @@ func setupFileHook(hook config.LogrusHook, level logrus.Level, componentName str ), }) } + +//CloseAndLogIfError Closes io.Closer and logs the error if any +func CloseAndLogIfError(ctx context.Context, closer io.Closer, message string) { + if closer == nil { + return + } + err := closer.Close() + if ctx == nil { + ctx = context.TODO() + } + if err != nil { + util.GetLogger(ctx).WithError(err).Error(message) + } +} diff --git a/common/partition_offset_table.go b/common/partition_offset_table.go index 6bc066a69..aa799f8a0 100644 --- a/common/partition_offset_table.go +++ b/common/partition_offset_table.go @@ -90,7 +90,7 @@ func (s *PartitionOffsetStatements) selectPartitionOffsets( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer CloseAndLogIfError(ctx, rows, "selectPartitionOffsets: rows.close() failed") var results []PartitionOffset for rows.Next() { var offset PartitionOffset diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index b74bfe3ac..a4bc3c67d 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -84,10 +84,10 @@ func Backfill( // Filter any event that's not from the requested room out. evs := make([]gomatrixserverlib.Event, 0) - var ev gomatrixserverlib.Event + var ev gomatrixserverlib.HeaderedEvent for _, ev = range res.Events { if ev.RoomID() == roomID { - evs = append(evs, ev) + evs = append(evs, ev.Event) } } diff --git a/federationapi/routing/eventauth.go b/federationapi/routing/eventauth.go new file mode 100644 index 000000000..003165c85 --- /dev/null +++ b/federationapi/routing/eventauth.go @@ -0,0 +1,43 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "context" + "net/http" + + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +// GetEventAuth returns event auth for the roomID and eventID +func GetEventAuth( + ctx context.Context, + request *gomatrixserverlib.FederationRequest, + query api.RoomserverQueryAPI, + roomID string, + eventID string, +) util.JSONResponse { + // TODO: Optimisation: we shouldn't be querying all the room state + // that is in state.StateEvents - we just ignore it. + state, err := getState(ctx, request, query, roomID, eventID) + if err != nil { + return *err + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: gomatrixserverlib.RespEventAuth{AuthEvents: state.AuthEvents}, + } +} diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go index c4248022b..a91528b3d 100644 --- a/federationapi/routing/events.go +++ b/federationapi/routing/events.go @@ -80,5 +80,5 @@ func getEvent( return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} } - return &eventsResponse.Events[0], nil + return &eventsResponse.Events[0].Event, nil } diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 369e19aed..7d48c86dc 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -78,7 +78,7 @@ func MakeJoin( // Check that the join is allowed or not stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents)) for i := range queryRes.StateEvents { - stateEvents[i] = &queryRes.StateEvents[i] + stateEvents[i] = &queryRes.StateEvents[i].Event } provider := gomatrixserverlib.NewAuthEvents(stateEvents) if err = gomatrixserverlib.Allowed(*event, &provider); err != nil { diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index f27989ff7..3eceb6f25 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -76,7 +76,7 @@ func MakeLeave( // Check that the leave is allowed or not stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents)) for i := range queryRes.StateEvents { - stateEvents[i] = &queryRes.StateEvents[i] + stateEvents[i] = &queryRes.StateEvents[i].Event } provider := gomatrixserverlib.NewAuthEvents(stateEvents) if err = gomatrixserverlib.Allowed(*event, &provider); err != nil { diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go index 661ec33a4..069bff3dd 100644 --- a/federationapi/routing/missingevents.go +++ b/federationapi/routing/missingevents.go @@ -68,8 +68,8 @@ func GetMissingEvents( // filterEvents returns only those events with matching roomID and having depth greater than minDepth func filterEvents( - events []gomatrixserverlib.Event, minDepth int64, roomID string, -) []gomatrixserverlib.Event { + events []gomatrixserverlib.HeaderedEvent, minDepth int64, roomID string, +) []gomatrixserverlib.HeaderedEvent { ref := events[:0] for _, ev := range events { if ev.Depth() >= minDepth && ev.RoomID() == roomID { diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 3b119301a..b5c8e53de 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -131,7 +131,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/state/{roomID}", common.MakeFedAPI( - "federation_get_event_auth", cfg.Matrix.ServerName, keys, + "federation_get_state", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -144,7 +144,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/state_ids/{roomID}", common.MakeFedAPI( - "federation_get_event_auth", cfg.Matrix.ServerName, keys, + "federation_get_state_ids", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -156,6 +156,16 @@ func Setup( }, )).Methods(http.MethodGet) + v1fedmux.Handle("/event_auth/{roomID}/{eventID}", common.MakeFedAPI( + "federation_get_event_auth", cfg.Matrix.ServerName, keys, + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { + vars := mux.Vars(httpReq) + return GetEventAuth( + httpReq.Context(), request, query, vars["roomID"], vars["eventID"], + ) + }, + )).Methods(http.MethodGet) + v1fedmux.Handle("/query/directory", common.MakeFedAPI( "federation_query_room_alias", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 079e121f9..d3e060ac2 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -116,6 +116,7 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { results[e.EventID()] = gomatrixserverlib.PDUResult{ Error: err.Error(), } + util.GetLogger(t.context).WithError(err).WithField("event_id", e.EventID()).Warn("Failed to process incoming federation event, skipping it.") } else { results[e.EventID()] = gomatrixserverlib.PDUResult{} } @@ -162,7 +163,11 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event) error { } // Check that the event is allowed by the state at the event. - if err := checkAllowedByState(e, stateResp.StateEvents); err != nil { + var events []gomatrixserverlib.Event + for _, headeredEvent := range stateResp.StateEvents { + events = append(events, headeredEvent.Event) + } + if err := checkAllowedByState(e, events); err != nil { return err } diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 86cf1cf54..86a1e08d2 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -129,9 +129,17 @@ func getState( return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} } + var stateEvents, authEvents []gomatrixserverlib.Event + for _, headeredEvent := range response.StateEvents { + stateEvents = append(stateEvents, headeredEvent.Event) + } + for _, headeredEvent := range response.AuthChainEvents { + authEvents = append(authEvents, headeredEvent.Event) + } + return &gomatrixserverlib.RespState{ - StateEvents: response.StateEvents, - AuthEvents: response.AuthChainEvents, + StateEvents: stateEvents, + AuthEvents: authEvents, }, nil } diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 03f3c5bba..18ebc07e1 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -264,7 +264,7 @@ func buildMembershipEvent( authEvents := gomatrixserverlib.NewAuthEvents(nil) for i := range queryRes.StateEvents { - err = authEvents.AddEvent(&queryRes.StateEvents[i]) + err = authEvents.AddEvent(&queryRes.StateEvents[i].Event) if err != nil { return nil, err } diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 4568f44dc..2d4b1b2ef 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -109,7 +109,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { // processMessage updates the list of currently joined hosts in the room // and then sends the event to the hosts that were joined before the event. func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) error { - addsStateEvents, err := s.lookupStateEvents(ore.AddsStateEventIDs, ore.Event) + addsStateEvents, err := s.lookupStateEvents(ore.AddsStateEventIDs, ore.Event.Event) if err != nil { return err } @@ -155,7 +155,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) err // Send the event. return s.queues.SendEvent( - &ore.Event, gomatrixserverlib.ServerName(ore.SendAsServer), joinedHostsAtEvent, + &ore.Event.Event, gomatrixserverlib.ServerName(ore.SendAsServer), joinedHostsAtEvent, ) } @@ -178,7 +178,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent( ore.AddsStateEventIDs, ore.RemovesStateEventIDs, ore.StateBeforeAddsEventIDs, ore.StateBeforeRemovesEventIDs, ) - combinedAddsEvents, err := s.lookupStateEvents(combinedAdds, ore.Event) + combinedAddsEvents, err := s.lookupStateEvents(combinedAdds, ore.Event.Event) if err != nil { return nil, err } @@ -325,7 +325,10 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( return nil, err } - result = append(result, eventResp.Events...) + for _, headeredEvent := range eventResp.Events { + result = append(result, headeredEvent.Event) + } + missing = missingEventsFrom(result, addsStateEventIDs) if len(missing) != 0 { diff --git a/federationsender/storage/postgres/joined_hosts_table.go b/federationsender/storage/postgres/joined_hosts_table.go index e5c30a010..b3c45abda 100644 --- a/federationsender/storage/postgres/joined_hosts_table.go +++ b/federationsender/storage/postgres/joined_hosts_table.go @@ -118,7 +118,7 @@ func joinedHostsFromStmt( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "joinedHostsFromStmt: rows.close() failed") var result []types.JoinedHost for rows.Next() { diff --git a/federationsender/storage/sqlite3/joined_hosts_table.go b/federationsender/storage/sqlite3/joined_hosts_table.go index 1437a062b..466ae4991 100644 --- a/federationsender/storage/sqlite3/joined_hosts_table.go +++ b/federationsender/storage/sqlite3/joined_hosts_table.go @@ -121,7 +121,7 @@ func joinedHostsFromStmt( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "joinedHostsFromStmt: rows.close() failed") var result []types.JoinedHost for rows.Next() { diff --git a/go.mod b/go.mod index 92e0b7201..3ceaea528 100644 --- a/go.mod +++ b/go.mod @@ -2,15 +2,16 @@ module github.com/matrix-org/dendrite require ( github.com/btcsuite/btcutil v1.0.1 + github.com/golangci/golangci-lint v1.19.1 // indirect github.com/gorilla/mux v1.7.3 github.com/hashicorp/golang-lru v0.5.4 github.com/lib/pq v1.2.0 github.com/libp2p/go-libp2p-core v0.5.0 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 - github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c + github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200318180716-bc4ff56961e2 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible @@ -18,11 +19,13 @@ require ( github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.4.1 + github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/sirupsen/logrus v1.4.2 github.com/tidwall/gjson v1.6.0 // indirect github.com/tidwall/pretty v1.0.1 // indirect github.com/uber/jaeger-client-go v2.22.1+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible + go.uber.org/atomic v1.6.0 // indirect golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d gopkg.in/Shopify/sarama.v1 v1.20.1 gopkg.in/h2non/bimg.v1 v1.0.18 diff --git a/go.sum b/go.sum index c28254d46..0f4542b28 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,20 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us= +github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -24,37 +30,82 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db h1:GYXWx7Vr3+zv833u+8IoXbNnQY0AdXsxAgI0kX7xcwA= +github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0= +github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g= +github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= +github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8= +github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= +github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ= +github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= +github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k= +github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= +github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= +github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= +github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg= +github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= +github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= +github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= +github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4= +github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= +github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA= +github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -63,6 +114,37 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0= +github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= +github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM= +github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= +github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w= +github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= +github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw= +github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= +github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8= +github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= +github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8= +github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= +github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU= +github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= +github.com/golangci/golangci-lint v1.19.1 h1:g9xL8KW7UZDCkVlgHYJMA6F4Sj/sRVa0FoCeXI+Z3iM= +github.com/golangci/golangci-lint v1.19.1/go.mod h1:2CEc4Fxx3vxDv7g8DyXkHCBF73AOzAymcJAprs2vCps= +github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI= +github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= +github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217 h1:En/tZdwhAn0JNwLuXzP3k2RVtMqMmOEK7Yu/g3tmtJE= +github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= +github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= +github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= +github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770 h1:EL/O5HGrF7Jaq0yNhBLucz9hTuRzj2LdwGBOaENgxIk= +github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= +github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us= +github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= +github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1aiNg8JbBMO2+E1WIQ+j/gL4SQqGPg= +github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= +github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys= +github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= @@ -70,13 +152,24 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw= +github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU= github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= @@ -84,15 +177,22 @@ github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr1 github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -100,6 +200,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= @@ -110,7 +211,12 @@ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS github.com/libp2p/go-libp2p-core v0.5.0 h1:FBQ1fpq2Fo/ClyjojVJ5AKXlKhvNc/B6U0O+7AN1ffE= github.com/libp2p/go-libp2p-core v0.5.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86 h1:q6SrfsK4FojRnJ1j8+8OJzyq3g9Y1oSVyL6nYGJXXBk= +github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/matrix-org/dendrite v0.0.0-20200220135450-0352f250b857/go.mod h1:DZ35IoR+ViBNVPe9umdlOSnjvKl7wfyRmZg4QfWGvTo= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= @@ -124,6 +230,8 @@ github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437 h1:zc github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437/go.mod h1:/giSXVd8D6DZGSfTmhQrLEoZZwsfkC14kSqP9MiLqIY= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c h1:jj/LIZKMO7GK6O0UarpRwse9L3ZyzozpyMtdPA7ddSk= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f h1:5TOte9uk/epk8L+Pbp6qwaV8YsKYXKjyECPHUhJTWQc= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk= github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074 h1:UWz6vfhmQVshBuE67X1BCsdMhEDtd+uOz8CJ48Fc0F4= github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db h1:ERuFJq4DI8fakfBZlvXHltHZ0ix3K5YsLG0tQfQn6TI= @@ -138,6 +246,36 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:km github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424 h1:H61lT6ckUFIDl9qb636qNomfo0B52lFt73ecioiqF10= github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316100021-ac4a53d49690 h1:aQPPypOdoIsquJHhoalRYvKtDoiJfSyyJqOEn6R7yTY= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316100021-ac4a53d49690/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f h1:JXSvzG4movqXT1KcdX+XZ9HM61m1FW4rIMFKUM9j/Dk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316163031-36bb5f40ae9b h1:DdVa6Ledhi5dqwYMw8e9zG+kZmRlnWMgsYOxb3NaFOU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316163031-36bb5f40ae9b/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316173242-f66834855555 h1:7M7QwpGbc5fJ6Uxn62ECsagekKkV36hjvnbaDQGJhls= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316173242-f66834855555/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316191245-60f1fa78147c h1:hbu6d/7LOKwPpISyE62cUXapx6askaDtKnpb5dtqr1I= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316191245-60f1fa78147c/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316200447-0820dfcdff10 h1:O6EHrD7m9Ah9SjMoHuEHHWh4moKllCYo0HRVtSSGzYc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316200447-0820dfcdff10/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316202926-7ce8b3298af6 h1:ZeRF0DpsnaxwYL7b6U1J3XMyUBpB8GWiNCqz3bWdivs= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316202926-7ce8b3298af6/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316203446-64795a12215c h1:s7XUmnqbJTJmMHuZ9Ooad6vKlBsSpIe/C3aJc6G6cy8= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316203446-64795a12215c/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316222111-b66d3b63dd85 h1:I7ocq675X1B+RYDUQPvMcYcpfHkdTcreil48r3qs2KE= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316222111-b66d3b63dd85/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317103236-134415251e65 h1:llq2yETQhD75ImQhCwHbnivSKOgzOP6QdUqH9sse7XM= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317103236-134415251e65/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317114945-9a368ea4620d h1:0GYO2Jye1TNVzsn02IF5tqV80psDi0KIWC4+glH6+/Q= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317114945-9a368ea4620d/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd h1:n95A8YyiCZ8Nu2beqw4akCaPIRrZr/nesHYDZV8WkXI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318141718-859b9f256ffd h1:381JBgdNYOQwWelV/LOPqDoUkdSBUCkVZY8Of+n5jAM= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318141718-859b9f256ffd/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318145320-bc896516d72a h1:7+e7ManmyiGNqo06sQIRIoUtoV92XWzSbc0o3e4aTGY= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318145320-bc896516d72a/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318150006-bc27294f9203 h1:7HkL6bF7/M2cYteNFVtvGW5qjD4wHIiR0HsdCm2Rqao= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318150006-bc27294f9203/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= @@ -145,9 +283,14 @@ github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXL github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U= github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= @@ -160,10 +303,16 @@ github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8Rv github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -181,15 +330,22 @@ github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= +github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E= +github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -198,6 +354,7 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= @@ -207,29 +364,65 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/securego/gosec v0.0.0-20190912120752-140048b2a218 h1:O0yPHYL49quNL4Oj2wVq+zbGMu4dAM6iLoOQtm49TrQ= +github.com/securego/gosec v0.0.0-20190912120752-140048b2a218/go.mod h1:q6oYAujd2qyeU4cJqIri4LBIgdHXGvxWHZ1E29HNFRE= +github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs= +github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= @@ -245,6 +438,9 @@ github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.0.3 h1:DeF+0LZqvIt4fKYw41aPB29ZGlvwVkHKktoXJ1YW9Y8= github.com/tidwall/sjson v1.0.3/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= +github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec h1:AmoEvWAO3nDx1MEcMzPh+GzOOIA5Znpv6++c7bePPY0= +github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber-go/atomic v1.3.0 h1:ylWoWcs+jXihgo3Us1Sdsatf2R6+OlBGm8fexR3oFG4= github.com/uber-go/atomic v1.3.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= @@ -255,18 +451,38 @@ github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo= +github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= +github.com/ultraware/whitespace v0.0.3 h1:S5BCRRB5sttNy0bSOhbpw+0mb+cHiCmWfrvxpEzuUk0= +github.com/ultraware/whitespace v0.0.3/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= +github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010= go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI= @@ -275,48 +491,75 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc= +golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678 h1:rM1Udd0CgtYI3KUIhu9ROz0QCqjW+n/ODp/hH7c60Xc= +golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -325,6 +568,7 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE= gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -334,6 +578,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= @@ -341,9 +586,11 @@ gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI= gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -351,3 +598,13 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I= +mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= +mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo= +mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= +mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4= +mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 h1:JPJh2pk3+X4lXAkZIk2RuE/7/FoK9maXw+TNPJhVS/c= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/mediaapi/fileutils/fileutils.go b/mediaapi/fileutils/fileutils.go index 36b2c5b89..1de242a00 100644 --- a/mediaapi/fileutils/fileutils.go +++ b/mediaapi/fileutils/fileutils.go @@ -49,7 +49,7 @@ func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath config.Path) "file", )) if err != nil { - return "", fmt.Errorf("Unable to construct filePath: %q", err) + return "", fmt.Errorf("Unable to construct filePath: %w", err) } // check if the absolute absBasePath is a prefix of the absolute filePath @@ -73,7 +73,7 @@ func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata duplicate := false finalPath, err := GetPathFromBase64Hash(mediaMetadata.Base64Hash, absBasePath) if err != nil { - return "", duplicate, fmt.Errorf("failed to get file path from metadata: %q", err) + return "", duplicate, fmt.Errorf("failed to get file path from metadata: %w", err) } var stat os.FileInfo @@ -91,7 +91,7 @@ func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata types.Path(finalPath), ) if err != nil { - return "", duplicate, fmt.Errorf("failed to move file to final destination (%v): %q", finalPath, err) + return "", duplicate, fmt.Errorf("failed to move file to final destination (%v): %w", finalPath, err) } return types.Path(finalPath), duplicate, nil } @@ -143,11 +143,11 @@ func moveFile(src types.Path, dst types.Path) error { err := os.MkdirAll(dstDir, 0770) if err != nil { - return fmt.Errorf("Failed to make directory: %q", err) + return fmt.Errorf("Failed to make directory: %w", err) } err = os.Rename(string(src), string(dst)) if err != nil { - return fmt.Errorf("Failed to move directory: %q", err) + return fmt.Errorf("Failed to move directory: %w", err) } return nil } @@ -155,11 +155,11 @@ func moveFile(src types.Path, dst types.Path) error { func createTempFileWriter(absBasePath config.Path) (*bufio.Writer, *os.File, types.Path, error) { tmpDir, err := createTempDir(absBasePath) if err != nil { - return nil, nil, "", fmt.Errorf("Failed to create temp dir: %q", err) + return nil, nil, "", fmt.Errorf("Failed to create temp dir: %w", err) } writer, tmpFile, err := createFileWriter(tmpDir) if err != nil { - return nil, nil, "", fmt.Errorf("Failed to create file writer: %q", err) + return nil, nil, "", fmt.Errorf("Failed to create file writer: %w", err) } return writer, tmpFile, tmpDir, nil } @@ -168,11 +168,11 @@ func createTempFileWriter(absBasePath config.Path) (*bufio.Writer, *os.File, typ func createTempDir(baseDirectory config.Path) (types.Path, error) { baseTmpDir := filepath.Join(string(baseDirectory), "tmp") if err := os.MkdirAll(baseTmpDir, 0770); err != nil { - return "", fmt.Errorf("Failed to create base temp dir: %v", err) + return "", fmt.Errorf("Failed to create base temp dir: %w", err) } tmpDir, err := ioutil.TempDir(baseTmpDir, "") if err != nil { - return "", fmt.Errorf("Failed to create temp dir: %v", err) + return "", fmt.Errorf("Failed to create temp dir: %w", err) } return types.Path(tmpDir), nil } @@ -184,7 +184,7 @@ func createFileWriter(directory types.Path) (*bufio.Writer, *os.File, error) { filePath := filepath.Join(string(directory), "content") file, err := os.Create(filePath) if err != nil { - return nil, nil, fmt.Errorf("Failed to create file: %v", err) + return nil, nil, fmt.Errorf("Failed to create file: %w", err) } return bufio.NewWriter(file), file, nil diff --git a/mediaapi/storage/postgres/thumbnail_table.go b/mediaapi/storage/postgres/thumbnail_table.go index 127b86bb9..08bddc36f 100644 --- a/mediaapi/storage/postgres/thumbnail_table.go +++ b/mediaapi/storage/postgres/thumbnail_table.go @@ -20,6 +20,8 @@ import ( "database/sql" "time" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -144,7 +146,7 @@ func (s *thumbnailStatements) selectThumbnails( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectThumbnails: rows.close() failed") var thumbnails []*types.ThumbnailMetadata for rows.Next() { diff --git a/mediaapi/storage/sqlite3/thumbnail_table.go b/mediaapi/storage/sqlite3/thumbnail_table.go index 95332c9d4..280fafe8d 100644 --- a/mediaapi/storage/sqlite3/thumbnail_table.go +++ b/mediaapi/storage/sqlite3/thumbnail_table.go @@ -20,6 +20,8 @@ import ( "database/sql" "time" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -134,7 +136,7 @@ func (s *thumbnailStatements) selectThumbnails( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectThumbnails: rows.close() failed") var thumbnails []*types.ThumbnailMetadata for rows.Next() { diff --git a/publicroomsapi/consumers/roomserver.go b/publicroomsapi/consumers/roomserver.go index 9a817735a..2bbd92b72 100644 --- a/publicroomsapi/consumers/roomserver.go +++ b/publicroomsapi/consumers/roomserver.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" sarama "gopkg.in/Shopify/sarama.v1" ) @@ -98,5 +99,13 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { return err } - return s.db.UpdateRoomFromEvents(context.TODO(), addQueryRes.Events, remQueryRes.Events) + var addQueryEvents, remQueryEvents []gomatrixserverlib.Event + for _, headeredEvent := range addQueryRes.Events { + addQueryEvents = append(addQueryEvents, headeredEvent.Event) + } + for _, headeredEvent := range remQueryRes.Events { + remQueryEvents = append(remQueryEvents, headeredEvent.Event) + } + + return s.db.UpdateRoomFromEvents(context.TODO(), addQueryEvents, remQueryEvents) } diff --git a/publicroomsapi/directory/directory.go b/publicroomsapi/directory/directory.go index ea4c81696..de14967df 100644 --- a/publicroomsapi/directory/directory.go +++ b/publicroomsapi/directory/directory.go @@ -91,7 +91,7 @@ func SetVisibility( util.GetLogger(req.Context()).WithError(err).Error("could not query events from room") return jsonerror.InternalServerError() } - power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0]) + power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event) //Check if the user's power is greater than power required to change m.room.aliases event if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomAliases, true) { diff --git a/publicroomsapi/directory/public_rooms.go b/publicroomsapi/directory/public_rooms.go index fd327942e..7bd6740eb 100644 --- a/publicroomsapi/directory/public_rooms.go +++ b/publicroomsapi/directory/public_rooms.go @@ -15,17 +15,21 @@ package directory import ( + "context" "net/http" "strconv" + "sync" + "time" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/publicroomsapi/types" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) -type publicRoomReq struct { +type PublicRoomReq struct { Since string `json:"since,omitempty"` Limit int16 `json:"limit,omitempty"` Filter filter `json:"filter,omitempty"` @@ -35,65 +39,182 @@ type filter struct { SearchTerms string `json:"generic_search_term,omitempty"` } -type publicRoomRes struct { - Chunk []types.PublicRoom `json:"chunk"` - NextBatch string `json:"next_batch,omitempty"` - PrevBatch string `json:"prev_batch,omitempty"` - Estimate int64 `json:"total_room_count_estimate,omitempty"` -} - // GetPostPublicRooms implements GET and POST /publicRooms func GetPostPublicRooms( req *http.Request, publicRoomDatabase storage.Database, ) util.JSONResponse { - var limit int16 - var offset int64 - var request publicRoomReq - var response publicRoomRes - + var request PublicRoomReq if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil { return *fillErr } - - limit = request.Limit - offset, err := strconv.ParseInt(request.Since, 10, 64) - // ParseInt returns 0 and an error when trying to parse an empty string - // In that case, we want to assign 0 so we ignore the error - if err != nil && len(request.Since) > 0 { - util.GetLogger(req.Context()).WithError(err).Error("strconv.ParseInt failed") + response, err := publicRooms(req.Context(), request, publicRoomDatabase) + if err != nil { return jsonerror.InternalServerError() } - - if response.Estimate, err = publicRoomDatabase.CountPublicRooms(req.Context()); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("publicRoomDatabase.CountPublicRooms failed") - return jsonerror.InternalServerError() - } - - if offset > 0 { - response.PrevBatch = strconv.Itoa(int(offset) - 1) - } - nextIndex := int(offset) + int(limit) - if response.Estimate > int64(nextIndex) { - response.NextBatch = strconv.Itoa(nextIndex) - } - - if response.Chunk, err = publicRoomDatabase.GetPublicRooms( - req.Context(), offset, limit, request.Filter.SearchTerms, - ); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("publicRoomDatabase.GetPublicRooms failed") - return jsonerror.InternalServerError() - } - return util.JSONResponse{ Code: http.StatusOK, JSON: response, } } +// GetPostPublicRoomsWithExternal is the same as GetPostPublicRooms but also mixes in public rooms from the provider supplied. +func GetPostPublicRoomsWithExternal( + req *http.Request, publicRoomDatabase storage.Database, fedClient *gomatrixserverlib.FederationClient, + extRoomsProvider types.ExternalPublicRoomsProvider, +) util.JSONResponse { + var request PublicRoomReq + if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil { + return *fillErr + } + response, err := publicRooms(req.Context(), request, publicRoomDatabase) + if err != nil { + return jsonerror.InternalServerError() + } + + if request.Since != "" { + // TODO: handle pagination tokens sensibly rather than ignoring them. + // ignore paginated requests since we don't handle them yet over federation. + // Only the initial request will contain federated rooms. + return util.JSONResponse{ + Code: http.StatusOK, + JSON: response, + } + } + + // If we have already hit the limit on the number of rooms, bail. + var limit int + if request.Limit > 0 { + limit = int(request.Limit) - len(response.Chunk) + if limit <= 0 { + return util.JSONResponse{ + Code: http.StatusOK, + JSON: response, + } + } + } + + // downcasting `limit` is safe as we know it isn't bigger than request.Limit which is int16 + fedRooms := bulkFetchPublicRoomsFromServers(req.Context(), fedClient, extRoomsProvider.Homeservers(), int16(limit)) + response.Chunk = append(response.Chunk, fedRooms...) + return util.JSONResponse{ + Code: http.StatusOK, + JSON: response, + } +} + +// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers. +// Returns a list of public rooms up to the limit specified. +func bulkFetchPublicRoomsFromServers( + ctx context.Context, fedClient *gomatrixserverlib.FederationClient, homeservers []string, limit int16, +) (publicRooms []gomatrixserverlib.PublicRoom) { + // follow pipeline semantics, see https://blog.golang.org/pipelines for more info. + // goroutines send rooms to this channel + roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit)) + // signalling channel to tell goroutines to stop sending rooms and quit + done := make(chan bool) + // signalling to say when we can close the room channel + var wg sync.WaitGroup + wg.Add(len(homeservers)) + // concurrently query for public rooms + for _, hs := range homeservers { + go func(homeserverDomain string) { + defer wg.Done() + util.GetLogger(ctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms") + fres, err := fedClient.GetPublicRooms(ctx, gomatrixserverlib.ServerName(homeserverDomain), int(limit), "", false, "") + if err != nil { + util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Warn( + "bulkFetchPublicRoomsFromServers: failed to query hs", + ) + return + } + for _, room := range fres.Chunk { + // atomically send a room or stop + select { + case roomCh <- room: + case <-done: + util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending rooms") + return + } + } + }(hs) + } + + // Close the room channel when the goroutines have quit so we don't leak, but don't let it stop the in-flight request. + // This also allows the request to fail fast if all HSes experience errors as it will cause the room channel to be + // closed. + go func() { + wg.Wait() + util.GetLogger(ctx).Info("Cleaning up resources") + close(roomCh) + }() + + // fan-in results with timeout. We stop when we reach the limit. +FanIn: + for len(publicRooms) < int(limit) || limit == 0 { + // add a room or timeout + select { + case room, ok := <-roomCh: + if !ok { + util.GetLogger(ctx).Info("All homeservers have been queried, returning results.") + break FanIn + } + publicRooms = append(publicRooms, room) + case <-time.After(15 * time.Second): // we've waited long enough, let's tell the client what we got. + util.GetLogger(ctx).Info("Waited 15s for federated public rooms, returning early") + break FanIn + case <-ctx.Done(): // the client hung up on us, let's stop. + util.GetLogger(ctx).Info("Client hung up, returning early") + break FanIn + } + } + // tell goroutines to stop + close(done) + + return publicRooms +} + +func publicRooms(ctx context.Context, request PublicRoomReq, publicRoomDatabase storage.Database) (*gomatrixserverlib.RespPublicRooms, error) { + var response gomatrixserverlib.RespPublicRooms + var limit int16 + var offset int64 + limit = request.Limit + offset, err := strconv.ParseInt(request.Since, 10, 64) + // ParseInt returns 0 and an error when trying to parse an empty string + // In that case, we want to assign 0 so we ignore the error + if err != nil && len(request.Since) > 0 { + util.GetLogger(ctx).WithError(err).Error("strconv.ParseInt failed") + return nil, err + } + + est, err := publicRoomDatabase.CountPublicRooms(ctx) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("publicRoomDatabase.CountPublicRooms failed") + return nil, err + } + response.TotalRoomCountEstimate = int(est) + + if offset > 0 { + response.PrevBatch = strconv.Itoa(int(offset) - 1) + } + nextIndex := int(offset) + int(limit) + if response.TotalRoomCountEstimate > nextIndex { + response.NextBatch = strconv.Itoa(nextIndex) + } + + if response.Chunk, err = publicRoomDatabase.GetPublicRooms( + ctx, offset, limit, request.Filter.SearchTerms, + ); err != nil { + util.GetLogger(ctx).WithError(err).Error("publicRoomDatabase.GetPublicRooms failed") + return nil, err + } + + return &response, nil +} + // fillPublicRoomsReq fills the Limit, Since and Filter attributes of a GET or POST request // on /publicRooms by parsing the incoming HTTP request // Filter is only filled for POST requests -func fillPublicRoomsReq(httpReq *http.Request, request *publicRoomReq) *util.JSONResponse { +func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSONResponse { if httpReq.Method == http.MethodGet { limit, err := strconv.Atoi(httpReq.FormValue("limit")) // Atoi returns 0 and an error when trying to parse an empty string diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index fc411849f..4d8a26018 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -20,7 +20,9 @@ import ( "github.com/matrix-org/dendrite/publicroomsapi/consumers" "github.com/matrix-org/dendrite/publicroomsapi/routing" "github.com/matrix-org/dendrite/publicroomsapi/storage" + "github.com/matrix-org/dendrite/publicroomsapi/types" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) @@ -30,6 +32,8 @@ func SetupPublicRoomsAPIComponent( base *basecomponent.BaseDendrite, deviceDB devices.Database, rsQueryAPI roomserverAPI.RoomserverQueryAPI, + fedClient *gomatrixserverlib.FederationClient, + extRoomsProvider types.ExternalPublicRoomsProvider, ) { publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) if err != nil { @@ -43,5 +47,5 @@ func SetupPublicRoomsAPIComponent( logrus.WithError(err).Panic("failed to start public rooms server consumer") } - routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsQueryAPI) + routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsQueryAPI, fedClient, extRoomsProvider) } diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index 6927c5936..da5ea90d6 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -26,6 +26,8 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/publicroomsapi/directory" "github.com/matrix-org/dendrite/publicroomsapi/storage" + "github.com/matrix-org/dendrite/publicroomsapi/types" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -36,7 +38,10 @@ const pathPrefixR0 = "/_matrix/client/r0" // Due to Setup being used to call many other functions, a gocyclo nolint is // applied: // nolint: gocyclo -func Setup(apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, queryAPI api.RoomserverQueryAPI) { +func Setup( + apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, queryAPI api.RoomserverQueryAPI, + fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, +) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() authData := auth.Data{ @@ -66,7 +71,17 @@ func Setup(apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage. ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/publicRooms", common.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse { + if extRoomsProvider != nil { + return directory.GetPostPublicRoomsWithExternal(req, publicRoomsDB, fedClient, extRoomsProvider) + } return directory.GetPostPublicRooms(req, publicRoomsDB) }), ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) + + // Federation - TODO: should this live here or in federation API? It's sure easier if it's here so here it is. + apiMux.Handle("/_matrix/federation/v1/publicRooms", + common.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse { + return directory.GetPostPublicRooms(req, publicRoomsDB) + }), + ).Methods(http.MethodGet) } diff --git a/publicroomsapi/storage/interface.go b/publicroomsapi/storage/interface.go index 3f7e65892..0feca0e20 100644 --- a/publicroomsapi/storage/interface.go +++ b/publicroomsapi/storage/interface.go @@ -18,7 +18,6 @@ import ( "context" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/publicroomsapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -27,7 +26,7 @@ type Database interface { GetRoomVisibility(ctx context.Context, roomID string) (bool, error) SetRoomVisibility(ctx context.Context, visible bool, roomID string) error CountPublicRooms(ctx context.Context) (int64, error) - GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]types.PublicRoom, error) + GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error } diff --git a/publicroomsapi/storage/postgres/public_rooms_table.go b/publicroomsapi/storage/postgres/public_rooms_table.go index edf9ad2ab..7e31afd2a 100644 --- a/publicroomsapi/storage/postgres/public_rooms_table.go +++ b/publicroomsapi/storage/postgres/public_rooms_table.go @@ -21,8 +21,10 @@ import ( "errors" "fmt" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" + "github.com/lib/pq" - "github.com/matrix-org/dendrite/publicroomsapi/types" ) var editableAttributes = []string{ @@ -175,7 +177,7 @@ func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64, func (s *publicRoomsStatements) selectPublicRooms( ctx context.Context, offset int64, limit int16, filter string, -) ([]types.PublicRoom, error) { +) ([]gomatrixserverlib.PublicRoom, error) { var rows *sql.Rows var err error @@ -201,17 +203,17 @@ func (s *publicRoomsStatements) selectPublicRooms( } if err != nil { - return []types.PublicRoom{}, nil + return []gomatrixserverlib.PublicRoom{}, nil } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectPublicRooms: rows.close() failed") - rooms := []types.PublicRoom{} + rooms := []gomatrixserverlib.PublicRoom{} for rows.Next() { - var r types.PublicRoom + var r gomatrixserverlib.PublicRoom var aliases pq.StringArray err = rows.Scan( - &r.RoomID, &r.NumJoinedMembers, &aliases, &r.CanonicalAlias, + &r.RoomID, &r.JoinedMembersCount, &aliases, &r.CanonicalAlias, &r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL, ) if err != nil { diff --git a/publicroomsapi/storage/postgres/storage.go b/publicroomsapi/storage/postgres/storage.go index 5365c766f..5a4bc8f9a 100644 --- a/publicroomsapi/storage/postgres/storage.go +++ b/publicroomsapi/storage/postgres/storage.go @@ -21,7 +21,6 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/publicroomsapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -85,7 +84,7 @@ func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64 // Returns an error if the retrieval failed. func (d *PublicRoomsServerDatabase) GetPublicRooms( ctx context.Context, offset int64, limit int16, filter string, -) ([]types.PublicRoom, error) { +) ([]gomatrixserverlib.PublicRoom, error) { return d.statements.selectPublicRooms(ctx, offset, limit, filter) } diff --git a/publicroomsapi/storage/sqlite3/public_rooms_table.go b/publicroomsapi/storage/sqlite3/public_rooms_table.go index beba0d694..44679837f 100644 --- a/publicroomsapi/storage/sqlite3/public_rooms_table.go +++ b/publicroomsapi/storage/sqlite3/public_rooms_table.go @@ -22,7 +22,8 @@ import ( "errors" "fmt" - "github.com/matrix-org/dendrite/publicroomsapi/types" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" ) var editableAttributes = []string{ @@ -66,7 +67,7 @@ const selectPublicRoomsWithLimitSQL = "" + "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + " FROM publicroomsapi_public_rooms WHERE visibility = true" + " ORDER BY joined_members DESC" + - " LIMIT $2 OFFSET $1" + " LIMIT $1 OFFSET $2" const selectPublicRoomsWithFilterSQL = "" + "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + @@ -164,7 +165,7 @@ func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64, func (s *publicRoomsStatements) selectPublicRooms( ctx context.Context, offset int64, limit int16, filter string, -) ([]types.PublicRoom, error) { +) ([]gomatrixserverlib.PublicRoom, error) { var rows *sql.Rows var err error @@ -190,16 +191,17 @@ func (s *publicRoomsStatements) selectPublicRooms( } if err != nil { - return []types.PublicRoom{}, nil + return []gomatrixserverlib.PublicRoom{}, nil } + defer common.CloseAndLogIfError(ctx, rows, "selectPublicRooms failed to close rows") - rooms := []types.PublicRoom{} + rooms := []gomatrixserverlib.PublicRoom{} for rows.Next() { - var r types.PublicRoom + var r gomatrixserverlib.PublicRoom var aliasesJSON string err = rows.Scan( - &r.RoomID, &r.NumJoinedMembers, &aliasesJSON, &r.CanonicalAlias, + &r.RoomID, &r.JoinedMembersCount, &aliasesJSON, &r.CanonicalAlias, &r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL, ) if err != nil { diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go index f8ba71a89..80c04cab7 100644 --- a/publicroomsapi/storage/sqlite3/storage.go +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -23,7 +23,6 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/publicroomsapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -87,7 +86,7 @@ func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64 // Returns an error if the retrieval failed. func (d *PublicRoomsServerDatabase) GetPublicRooms( ctx context.Context, offset int64, limit int16, filter string, -) ([]types.PublicRoom, error) { +) ([]gomatrixserverlib.PublicRoom, error) { return d.statements.selectPublicRooms(ctx, offset, limit, filter) } diff --git a/publicroomsapi/types/types.go b/publicroomsapi/types/types.go index c284bcca4..11cb0d204 100644 --- a/publicroomsapi/types/types.go +++ b/publicroomsapi/types/types.go @@ -14,15 +14,11 @@ package types -// PublicRoom represents a local public room -type PublicRoom struct { - RoomID string `json:"room_id"` - Aliases []string `json:"aliases,omitempty"` - CanonicalAlias string `json:"canonical_alias,omitempty"` - Name string `json:"name,omitempty"` - Topic string `json:"topic,omitempty"` - AvatarURL string `json:"avatar_url,omitempty"` - NumJoinedMembers int64 `json:"num_joined_members"` - WorldReadable bool `json:"world_readable"` - GuestCanJoin bool `json:"guest_can_join"` +// ExternalPublicRoomsProvider provides a list of homeservers who should be queried +// periodically for a list of public rooms on their server. +type ExternalPublicRoomsProvider interface { + // The list of homeserver domains to query. These servers will receive a request + // via this API: https://matrix.org/docs/spec/server_server/latest#public-room-directory + // This will be called -on demand- by clients, so cache appropriately! + Homeservers() []string } diff --git a/roomserver/alias/alias.go b/roomserver/alias/alias.go index aeaf5ae94..dfb7e76a7 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/alias/alias.go @@ -46,6 +46,10 @@ type RoomserverAliasAPIDatabase interface { // Remove a given room alias. // Returns an error if there was a problem talking to the database. RemoveRoomAlias(ctx context.Context, alias string) error + // Look up the room version for a given room. + GetRoomVersionForRoom( + ctx context.Context, roomID string, + ) (gomatrixserverlib.RoomVersion, error) } // RoomserverAliasAPI is an implementation of alias.RoomserverAliasAPI @@ -229,7 +233,7 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( // Add auth events authEvents := gomatrixserverlib.NewAuthEvents(nil) for i := range res.StateEvents { - err = authEvents.AddEvent(&res.StateEvents[i]) + err = authEvents.AddEvent(&res.StateEvents[i].Event) if err != nil { return err } @@ -240,6 +244,11 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( } builder.AuthEvents = refs + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, roomID) + if err != nil { + return err + } + // Build the event eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), r.Cfg.Matrix.ServerName) now := time.Now() @@ -253,7 +262,7 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( // Create the request ire := roomserverAPI.InputRoomEvent{ Kind: roomserverAPI.KindNew, - Event: event, + Event: event.Headered(roomVersion), AuthEventIDs: event.AuthEventIDs(), SendAsServer: serverName, } diff --git a/roomserver/alias/alias_test.go b/roomserver/alias/alias_test.go index 6ddb63a73..0aefa19d9 100644 --- a/roomserver/alias/alias_test.go +++ b/roomserver/alias/alias_test.go @@ -22,6 +22,7 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" ) type MockRoomserverAliasAPIDatabase struct { @@ -49,6 +50,12 @@ func (db *MockRoomserverAliasAPIDatabase) GetCreatorIDForAlias( return "", nil } +func (db *MockRoomserverAliasAPIDatabase) GetRoomVersionForRoom( + ctx context.Context, roomID string, +) (gomatrixserverlib.RoomVersion, error) { + return gomatrixserverlib.RoomVersionV1, nil +} + // This method needs to change depending on test case func (db *MockRoomserverAliasAPIDatabase) GetRoomIDForAlias( ctx context.Context, diff --git a/roomserver/api/input.go b/roomserver/api/input.go index 9643a927c..f07cc0221 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -51,7 +51,7 @@ type InputRoomEvent struct { // This controls how the event is processed. Kind int `json:"kind"` // The event JSON for the event to add. - Event gomatrixserverlib.Event `json:"event"` + Event gomatrixserverlib.HeaderedEvent `json:"event"` // List of state event IDs that authenticate this event. // These are likely derived from the "auth_events" JSON key of the event. // But can be different because the "auth_events" key can be incomplete or wrong. @@ -85,7 +85,7 @@ type TransactionID struct { // the usual context a matrix room event would have. We usually do not have // access to the events needed to check the event auth rules for the invite. type InputInviteEvent struct { - Event gomatrixserverlib.Event `json:"event"` + Event gomatrixserverlib.HeaderedEvent `json:"event"` } // InputRoomEventsRequest is a request to InputRoomEvents diff --git a/roomserver/api/output.go b/roomserver/api/output.go index c09d5a1e5..4e7adff79 100644 --- a/roomserver/api/output.go +++ b/roomserver/api/output.go @@ -54,7 +54,7 @@ type OutputEvent struct { // prev_events. type OutputNewRoomEvent struct { // The Event. - Event gomatrixserverlib.Event `json:"event"` + Event gomatrixserverlib.HeaderedEvent `json:"event"` // The latest events in the room after this event. // This can be used to set the prev events for new events in the room. // This also can be used to get the full current state after this event. @@ -117,7 +117,7 @@ type OutputNewRoomEvent struct { // tracked separately from the room events themselves. type OutputNewInviteEvent struct { // The "m.room.member" invite event. - Event gomatrixserverlib.Event `json:"event"` + Event gomatrixserverlib.HeaderedEvent `json:"event"` } // An OutputRetireInviteEvent is written whenever an existing invite is no longer diff --git a/roomserver/api/query.go b/roomserver/api/query.go index e1850e723..9003a95a3 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -43,6 +43,8 @@ type QueryLatestEventsAndStateResponse struct { // Does the room exist? // If the room doesn't exist this will be false and LatestEvents will be empty. RoomExists bool `json:"room_exists"` + // The room version of the room. + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` // The latest events in the room. // These are used to set the prev_events when sending an event. LatestEvents []gomatrixserverlib.EventReference `json:"latest_events"` @@ -50,7 +52,7 @@ type QueryLatestEventsAndStateResponse struct { // This list will be in an arbitrary order. // These are used to set the auth_events when sending an event. // These are used to check whether the event is allowed. - StateEvents []gomatrixserverlib.Event `json:"state_events"` + StateEvents []gomatrixserverlib.HeaderedEvent `json:"state_events"` // The depth of the latest events. // This is one greater than the maximum depth of the latest events. // This is used to set the depth when sending an event. @@ -74,12 +76,14 @@ type QueryStateAfterEventsResponse struct { // Does the room exist on this roomserver? // If the room doesn't exist this will be false and StateEvents will be empty. RoomExists bool `json:"room_exists"` + // The room version of the room. + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` // Do all the previous events exist on this roomserver? // If some of previous events do not exist this will be false and StateEvents will be empty. PrevEventsExist bool `json:"prev_events_exist"` // The state events requested. // This list will be in an arbitrary order. - StateEvents []gomatrixserverlib.Event `json:"state_events"` + StateEvents []gomatrixserverlib.HeaderedEvent `json:"state_events"` } // QueryEventsByIDRequest is a request to QueryEventsByID @@ -99,7 +103,7 @@ type QueryEventsByIDResponse struct { // fails to read it from the database then it will fail // the entire request. // This list will be in an arbitrary order. - Events []gomatrixserverlib.Event `json:"events"` + Events []gomatrixserverlib.HeaderedEvent `json:"events"` } // QueryMembershipForUserRequest is a request to QueryMembership @@ -186,7 +190,7 @@ type QueryMissingEventsRequest struct { // QueryMissingEventsResponse is a response to QueryMissingEvents type QueryMissingEventsResponse struct { // Missing events, arbritrary order. - Events []gomatrixserverlib.Event `json:"events"` + Events []gomatrixserverlib.HeaderedEvent `json:"events"` } // QueryStateAndAuthChainRequest is a request to QueryStateAndAuthChain @@ -207,13 +211,15 @@ type QueryStateAndAuthChainResponse struct { // Does the room exist on this roomserver? // If the room doesn't exist this will be false and StateEvents will be empty. RoomExists bool `json:"room_exists"` + // The room version of the room. + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` // Do all the previous events exist on this roomserver? // If some of previous events do not exist this will be false and StateEvents will be empty. PrevEventsExist bool `json:"prev_events_exist"` // The state and auth chain events that were requested. // The lists will be in an arbitrary order. - StateEvents []gomatrixserverlib.Event `json:"state_events"` - AuthChainEvents []gomatrixserverlib.Event `json:"auth_chain_events"` + StateEvents []gomatrixserverlib.HeaderedEvent `json:"state_events"` + AuthChainEvents []gomatrixserverlib.HeaderedEvent `json:"auth_chain_events"` } // QueryBackfillRequest is a request to QueryBackfill. @@ -229,7 +235,7 @@ type QueryBackfillRequest struct { // QueryBackfillResponse is a response to QueryBackfill. type QueryBackfillResponse struct { // Missing events, arbritrary order. - Events []gomatrixserverlib.Event `json:"events"` + Events []gomatrixserverlib.HeaderedEvent `json:"events"` } // QueryServersInRoomAtEventRequest is a request to QueryServersInRoomAtEvent @@ -249,10 +255,10 @@ type QueryServersInRoomAtEventResponse struct { // QueryRoomVersionCapabilities asks for the default room version type QueryRoomVersionCapabilitiesRequest struct{} -// QueryRoomVersionCapabilitiesResponse is a response to QueryServersInRoomAtEventResponse +// QueryRoomVersionCapabilitiesResponse is a response to QueryRoomVersionCapabilitiesRequest type QueryRoomVersionCapabilitiesResponse struct { - DefaultRoomVersion string `json:"default"` - AvailableRoomVersions map[string]string `json:"available"` + DefaultRoomVersion gomatrixserverlib.RoomVersion `json:"default"` + AvailableRoomVersions map[gomatrixserverlib.RoomVersion]string `json:"available"` } // RoomserverQueryAPI is used to query information from the room server. @@ -536,7 +542,7 @@ func (h *httpRoomserverQueryAPI) QueryServersInRoomAtEvent( return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } -// QueryServersInRoomAtEvent implements RoomServerQueryAPI +// QueryRoomVersionCapabilities implements RoomServerQueryAPI func (h *httpRoomserverQueryAPI) QueryRoomVersionCapabilities( ctx context.Context, request *QueryRoomVersionCapabilitiesRequest, diff --git a/roomserver/input/events.go b/roomserver/input/events.go index a3b70753e..ab2bbe1cd 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -72,6 +72,10 @@ type RoomEventDatabase interface { ctx context.Context, transactionID string, sessionID int64, userID string, ) (string, error) + // Look up the room version for a given room. + GetRoomVersionForRoom( + ctx context.Context, roomID string, + ) (gomatrixserverlib.RoomVersion, error) } // OutputRoomEventWriter has the APIs needed to write an event to the output logs. @@ -95,7 +99,7 @@ func processRoomEvent( event := input.Event // Check that the event passes authentication checks and work out the numeric IDs for the auth events. - authEventNIDs, err := checkAuthEvents(ctx, db, event, input.AuthEventIDs) + authEventNIDs, err := checkAuthEvents(ctx, db, event.Event, input.AuthEventIDs) if err != nil { return } @@ -112,7 +116,7 @@ func processRoomEvent( } // Store the event - roomNID, stateAtEvent, err := db.StoreEvent(ctx, event, input.TransactionID, authEventNIDs) + roomNID, stateAtEvent, err := db.StoreEvent(ctx, event.Event, input.TransactionID, authEventNIDs) if err != nil { return } @@ -127,7 +131,7 @@ func processRoomEvent( if stateAtEvent.BeforeStateSnapshotNID == 0 { // We haven't calculated a state for this event yet. // Lets calculate one. - err = calculateAndSetState(ctx, db, input, roomNID, &stateAtEvent, event) + err = calculateAndSetState(ctx, db, input, roomNID, &stateAtEvent, event.Event) if err != nil { return } @@ -140,7 +144,7 @@ func processRoomEvent( // Update the extremities of the event graph for the room return event.EventID(), updateLatestEvents( - ctx, db, ow, roomNID, stateAtEvent, event, input.SendAsServer, input.TransactionID, + ctx, db, ow, roomNID, stateAtEvent, event.Event, input.SendAsServer, input.TransactionID, ) } @@ -152,11 +156,8 @@ func calculateAndSetState( stateAtEvent *types.StateAtEvent, event gomatrixserverlib.Event, ) error { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, db) - if err != nil { - return err - } + var err error + roomState := state.NewStateResolution(db) if input.HasState { // We've been told what the state at the event is so we don't need to calculate it. @@ -234,7 +235,7 @@ func processInviteEvent( return nil } - outputUpdates, err := updateToInviteMembership(updater, &input.Event, nil) + outputUpdates, err := updateToInviteMembership(updater, &input.Event.Event, nil) if err != nil { return err } diff --git a/roomserver/input/latest_events.go b/roomserver/input/latest_events.go index f9fd1d5d4..525a6f518 100644 --- a/roomserver/input/latest_events.go +++ b/roomserver/input/latest_events.go @@ -178,11 +178,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { func (u *latestEventsUpdater) latestState() error { var err error - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, u.db) - if err != nil { - return err - } + roomState := state.NewStateResolution(u.db) latestStateAtEvents := make([]types.StateAtEvent, len(u.latest)) for i := range u.latest { @@ -253,8 +249,13 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) latestEventIDs[i] = u.latest[i].EventID } + roomVersion, err := u.db.GetRoomVersionForRoom(u.ctx, u.event.RoomID()) + if err != nil { + return nil, err + } + ore := api.OutputNewRoomEvent{ - Event: u.event, + Event: u.event.Headered(roomVersion), LastSentEventID: u.lastEventIDSent, LatestEventIDs: latestEventIDs, TransactionID: u.transactionID, diff --git a/roomserver/input/membership.go b/roomserver/input/membership.go index 841c5fec6..f2ac3b510 100644 --- a/roomserver/input/membership.go +++ b/roomserver/input/membership.go @@ -136,13 +136,14 @@ func updateToInviteMembership( return nil, err } if needsSending { + roomVersion := gomatrixserverlib.RoomVersionV1 // We notify the consumers using a special event even though we will // notify them about the change in current state as part of the normal // room event stream. This ensures that the consumers only have to // consider a single stream of events when determining whether a user // is invited, rather than having to combine multiple streams themselves. onie := api.OutputNewInviteEvent{ - Event: *add, + Event: (*add).Headered(roomVersion), } updates = append(updates, api.OutputEvent{ Type: api.OutputTypeNewInviteEvent, diff --git a/roomserver/query/query.go b/roomserver/query/query.go index f138686b5..2638919ad 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -20,7 +20,6 @@ import ( "context" "encoding/json" "net/http" - "strconv" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" @@ -90,6 +89,10 @@ type RoomserverQueryAPIDatabase interface { EventStateKeys( context.Context, []types.EventStateKeyNID, ) (map[types.EventStateKeyNID]string, error) + // Look up the room version for a given room. + GetRoomVersionForRoom( + ctx context.Context, roomID string, + ) (gomatrixserverlib.RoomVersion, error) } // RoomserverQueryAPI is an implementation of api.RoomserverQueryAPI @@ -103,11 +106,14 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( request *api.QueryLatestEventsAndStateRequest, response *api.QueryLatestEventsAndStateResponse, ) error { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) if err != nil { - return err + response.RoomExists = false + return nil } + + roomState := state.NewStateResolution(r.DB) + response.QueryLatestEventsAndStateRequest = *request roomNID, err := r.DB.RoomNID(ctx, request.RoomID) if err != nil { @@ -117,6 +123,7 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( return nil } response.RoomExists = true + var currentStateSnapshotNID types.StateSnapshotNID response.LatestEvents, currentStateSnapshotNID, response.Depth, err = r.DB.LatestEventIDs(ctx, roomNID) @@ -137,7 +144,10 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( return err } - response.StateEvents = stateEvents + for _, event := range stateEvents { + response.StateEvents = append(response.StateEvents, event.Headered(roomVersion)) + } + return nil } @@ -147,11 +157,14 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( request *api.QueryStateAfterEventsRequest, response *api.QueryStateAfterEventsResponse, ) error { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) if err != nil { - return err + response.RoomExists = false + return nil } + + roomState := state.NewStateResolution(r.DB) + response.QueryStateAfterEventsRequest = *request roomNID, err := r.DB.RoomNID(ctx, request.RoomID) if err != nil { @@ -175,7 +188,7 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( // Look up the currrent state for the requested tuples. stateEntries, err := roomState.LoadStateAfterEventsForStringTuples( - ctx, prevStates, request.StateToFetch, + ctx, roomNID, prevStates, request.StateToFetch, ) if err != nil { return err @@ -186,7 +199,10 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( return err } - response.StateEvents = stateEvents + for _, event := range stateEvents { + response.StateEvents = append(response.StateEvents, event.Headered(roomVersion)) + } + return nil } @@ -213,7 +229,15 @@ func (r *RoomserverQueryAPI) QueryEventsByID( return err } - response.Events = events + for _, event := range events { + roomVersion, verr := r.DB.GetRoomVersionForRoom(ctx, event.RoomID()) + if verr != nil { + return verr + } + + response.Events = append(response.Events, event.Headered(roomVersion)) + } + return nil } @@ -330,11 +354,7 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( func (r *RoomserverQueryAPI) getMembershipsBeforeEventNID( ctx context.Context, eventNID types.EventNID, joinedOnly bool, ) ([]types.Event, error) { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) - if err != nil { - return []types.Event{}, err - } + roomState := state.NewStateResolution(r.DB) events := []types.Event{} // Lookup the event NID eIDs, err := r.DB.EventIDs(ctx, []types.EventNID{eventNID}) @@ -436,12 +456,7 @@ func (r *RoomserverQueryAPI) QueryServerAllowedToSeeEvent( func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent( ctx context.Context, eventID string, serverName gomatrixserverlib.ServerName, ) (bool, error) { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) - if err != nil { - return false, err - } - + roomState := state.NewStateResolution(r.DB) stateEntries, err := roomState.LoadStateAtEvent(ctx, eventID) if err != nil { return false, err @@ -487,10 +502,15 @@ func (r *RoomserverQueryAPI) QueryMissingEvents( return err } - response.Events = make([]gomatrixserverlib.Event, 0, len(loadedEvents)-len(eventsToFilter)) + response.Events = make([]gomatrixserverlib.HeaderedEvent, 0, len(loadedEvents)-len(eventsToFilter)) for _, event := range loadedEvents { if !eventsToFilter[event.EventID()] { - response.Events = append(response.Events, event) + roomVersion, verr := r.DB.GetRoomVersionForRoom(ctx, event.RoomID()) + if verr != nil { + return verr + } + + response.Events = append(response.Events, event.Headered(roomVersion)) } } @@ -526,7 +546,21 @@ func (r *RoomserverQueryAPI) QueryBackfill( } // Retrieve events from the list that was filled previously. - response.Events, err = r.loadEvents(ctx, resultNIDs) + var loadedEvents []gomatrixserverlib.Event + loadedEvents, err = r.loadEvents(ctx, resultNIDs) + if err != nil { + return err + } + + for _, event := range loadedEvents { + roomVersion, verr := r.DB.GetRoomVersionForRoom(ctx, event.RoomID()) + if verr != nil { + return verr + } + + response.Events = append(response.Events, event.Headered(roomVersion)) + } + return err } @@ -596,12 +630,6 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( request *api.QueryStateAndAuthChainRequest, response *api.QueryStateAndAuthChainResponse, ) error { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) - if err != nil { - return err - } - response.QueryStateAndAuthChainRequest = *request roomNID, err := r.DB.RoomNID(ctx, request.RoomID) if err != nil { @@ -612,33 +640,62 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( } response.RoomExists = true - prevStates, err := r.DB.StateAtEventIDs(ctx, request.PrevEventIDs) + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) + if err != nil { + return err + } + + stateEvents, err := r.loadStateAtEventIDs(ctx, request.PrevEventIDs) + if err != nil { + return err + } + response.PrevEventsExist = true + + // add the auth event IDs for the current state events too + var authEventIDs []string + authEventIDs = append(authEventIDs, request.AuthEventIDs...) + for _, se := range stateEvents { + authEventIDs = append(authEventIDs, se.AuthEventIDs()...) + } + authEventIDs = util.UniqueStrings(authEventIDs) // de-dupe + + authEvents, err := getAuthChain(ctx, r.DB, authEventIDs) + if err != nil { + return err + } + + for _, event := range stateEvents { + response.StateEvents = append(response.StateEvents, event.Headered(roomVersion)) + } + + for _, event := range authEvents { + response.AuthChainEvents = append(response.AuthChainEvents, event.Headered(roomVersion)) + } + + return err +} + +func (r *RoomserverQueryAPI) loadStateAtEventIDs(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { + roomState := state.NewStateResolution(r.DB) + prevStates, err := r.DB.StateAtEventIDs(ctx, eventIDs) if err != nil { switch err.(type) { case types.MissingEventError: - return nil + return nil, nil default: - return err + return nil, err } } - response.PrevEventsExist = true // Look up the currrent state for the requested tuples. stateEntries, err := roomState.LoadCombinedStateAfterEvents( ctx, prevStates, ) if err != nil { - return err + return nil, err } - stateEvents, err := r.loadStateEvents(ctx, stateEntries) - if err != nil { - return err - } - - response.StateEvents = stateEvents - response.AuthChainEvents, err = getAuthChain(ctx, r.DB, request.AuthEventIDs) - return err + return r.loadStateEvents(ctx, stateEntries) } // getAuthChain fetches the auth chain for the given auth events. An auth chain @@ -733,14 +790,13 @@ func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities( request *api.QueryRoomVersionCapabilitiesRequest, response *api.QueryRoomVersionCapabilitiesResponse, ) error { - response.DefaultRoomVersion = strconv.Itoa(int(version.GetDefaultRoomVersion())) - response.AvailableRoomVersions = make(map[string]string) - for v, desc := range version.GetSupportedRoomVersions() { - sv := strconv.Itoa(int(v)) + response.DefaultRoomVersion = version.DefaultRoomVersion() + response.AvailableRoomVersions = make(map[gomatrixserverlib.RoomVersion]string) + for v, desc := range version.SupportedRoomVersions() { if desc.Stable { - response.AvailableRoomVersions[sv] = "stable" + response.AvailableRoomVersions[v] = "stable" } else { - response.AvailableRoomVersions[sv] = "unstable" + response.AvailableRoomVersions[v] = "unstable" } } return nil diff --git a/roomserver/state/database/database.go b/roomserver/state/database/database.go index ede6c5ec3..80f1b14f4 100644 --- a/roomserver/state/database/database.go +++ b/roomserver/state/database/database.go @@ -20,6 +20,7 @@ import ( "context" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" ) // A RoomStateDatabase has the storage APIs needed to load state from the database @@ -61,4 +62,6 @@ type RoomStateDatabase interface { Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error) // Look up snapshot NID for an event ID string SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) + // Look up a room version from the room NID. + GetRoomVersionForRoomNID(ctx context.Context, roomNID types.RoomNID) (gomatrixserverlib.RoomVersion, error) } diff --git a/roomserver/state/shared/shared.go b/roomserver/state/shared/shared.go new file mode 100644 index 000000000..a29b5e403 --- /dev/null +++ b/roomserver/state/shared/shared.go @@ -0,0 +1 @@ +package shared diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 687a120e3..b8e3e18a1 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -19,63 +19,1069 @@ package state import ( "context" "errors" + "fmt" + "sort" + "time" "github.com/matrix-org/dendrite/roomserver/state/database" - v1 "github.com/matrix-org/dendrite/roomserver/state/v1" + "github.com/matrix-org/util" + "github.com/prometheus/client_golang/prometheus" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) -type StateResolutionVersion int +type StateResolution struct { + db database.RoomStateDatabase +} -const ( - StateResolutionAlgorithmV1 StateResolutionVersion = iota + 1 - StateResolutionAlgorithmV2 -) - -func GetStateResolutionAlgorithm( - version StateResolutionVersion, db database.RoomStateDatabase, -) (StateResolutionImpl, error) { - switch version { - case StateResolutionAlgorithmV1: - return v1.Prepare(db), nil - default: - return nil, errors.New("unsupported room version") +func NewStateResolution(db database.RoomStateDatabase) StateResolution { + return StateResolution{ + db: db, } } -type StateResolutionImpl interface { - LoadStateAtSnapshot( - ctx context.Context, stateNID types.StateSnapshotNID, - ) ([]types.StateEntry, error) - LoadStateAtEvent( - ctx context.Context, eventID string, - ) ([]types.StateEntry, error) - LoadCombinedStateAfterEvents( - ctx context.Context, prevStates []types.StateAtEvent, - ) ([]types.StateEntry, error) - DifferenceBetweeenStateSnapshots( - ctx context.Context, oldStateNID, newStateNID types.StateSnapshotNID, - ) (removed, added []types.StateEntry, err error) - LoadStateAtSnapshotForStringTuples( - ctx context.Context, - stateNID types.StateSnapshotNID, - stateKeyTuples []gomatrixserverlib.StateKeyTuple, - ) ([]types.StateEntry, error) - LoadStateAfterEventsForStringTuples( - ctx context.Context, - prevStates []types.StateAtEvent, - stateKeyTuples []gomatrixserverlib.StateKeyTuple, - ) ([]types.StateEntry, error) - CalculateAndStoreStateBeforeEvent( - ctx context.Context, - event gomatrixserverlib.Event, - roomNID types.RoomNID, - ) (types.StateSnapshotNID, error) - CalculateAndStoreStateAfterEvents( - ctx context.Context, - roomNID types.RoomNID, - prevStates []types.StateAtEvent, - ) (types.StateSnapshotNID, error) +// LoadStateAtSnapshot loads the full state of a room at a particular snapshot. +// This is typically the state before an event or the current state of a room. +// Returns a sorted list of state entries or an error if there was a problem talking to the database. +func (v StateResolution) LoadStateAtSnapshot( + ctx context.Context, stateNID types.StateSnapshotNID, +) ([]types.StateEntry, error) { + stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) + if err != nil { + return nil, err + } + // We've asked for exactly one snapshot from the db so we should have exactly one entry in the result. + stateBlockNIDList := stateBlockNIDLists[0] + + stateEntryLists, err := v.db.StateEntries(ctx, stateBlockNIDList.StateBlockNIDs) + if err != nil { + return nil, err + } + stateEntriesMap := stateEntryListMap(stateEntryLists) + + // Combine all the state entries for this snapshot. + // The order of state block NIDs in the list tells us the order to combine them in. + var fullState []types.StateEntry + for _, stateBlockNID := range stateBlockNIDList.StateBlockNIDs { + entries, ok := stateEntriesMap.lookup(stateBlockNID) + if !ok { + // This should only get hit if the database is corrupt. + // It should be impossible for an event to reference a NID that doesn't exist + panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) + } + fullState = append(fullState, entries...) + } + + // Stable sort so that the most recent entry for each state key stays + // remains later in the list than the older entries for the same state key. + sort.Stable(stateEntryByStateKeySorter(fullState)) + // Unique returns the last entry and hence the most recent entry for each state key. + fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] + return fullState, nil +} + +// LoadStateAtEvent loads the full state of a room before a particular event. +func (v StateResolution) LoadStateAtEvent( + ctx context.Context, eventID string, +) ([]types.StateEntry, error) { + snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID) + if err != nil { + return nil, err + } + + stateEntries, err := v.LoadStateAtSnapshot(ctx, snapshotNID) + if err != nil { + return nil, err + } + + return stateEntries, nil +} + +// LoadCombinedStateAfterEvents loads a snapshot of the state after each of the events +// and combines those snapshots together into a single list. At this point it is +// possible to run into duplicate (type, state key) tuples. +func (v StateResolution) LoadCombinedStateAfterEvents( + ctx context.Context, prevStates []types.StateAtEvent, +) ([]types.StateEntry, error) { + stateNIDs := make([]types.StateSnapshotNID, len(prevStates)) + for i, state := range prevStates { + stateNIDs[i] = state.BeforeStateSnapshotNID + } + // Fetch the state snapshots for the state before the each prev event from the database. + // Deduplicate the IDs before passing them to the database. + // There could be duplicates because the events could be state events where + // the snapshot of the room state before them was the same. + stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, uniqueStateSnapshotNIDs(stateNIDs)) + if err != nil { + return nil, err + } + + var stateBlockNIDs []types.StateBlockNID + for _, list := range stateBlockNIDLists { + stateBlockNIDs = append(stateBlockNIDs, list.StateBlockNIDs...) + } + // Fetch the state entries that will be combined to create the snapshots. + // Deduplicate the IDs before passing them to the database. + // There could be duplicates because a block of state entries could be reused by + // multiple snapshots. + stateEntryLists, err := v.db.StateEntries(ctx, uniqueStateBlockNIDs(stateBlockNIDs)) + if err != nil { + return nil, err + } + stateBlockNIDsMap := stateBlockNIDListMap(stateBlockNIDLists) + stateEntriesMap := stateEntryListMap(stateEntryLists) + + // Combine the entries from all the snapshots of state after each prev event into a single list. + var combined []types.StateEntry + for _, prevState := range prevStates { + // Grab the list of state data NIDs for this snapshot. + stateBlockNIDs, ok := stateBlockNIDsMap.lookup(prevState.BeforeStateSnapshotNID) + if !ok { + // This should only get hit if the database is corrupt. + // It should be impossible for an event to reference a NID that doesn't exist + panic(fmt.Errorf("Corrupt DB: Missing state snapshot numeric ID %d", prevState.BeforeStateSnapshotNID)) + } + + // Combine all the state entries for this snapshot. + // The order of state block NIDs in the list tells us the order to combine them in. + var fullState []types.StateEntry + for _, stateBlockNID := range stateBlockNIDs { + entries, ok := stateEntriesMap.lookup(stateBlockNID) + if !ok { + // This should only get hit if the database is corrupt. + // It should be impossible for an event to reference a NID that doesn't exist + panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) + } + fullState = append(fullState, entries...) + } + if prevState.IsStateEvent() { + // If the prev event was a state event then add an entry for the event itself + // so that we get the state after the event rather than the state before. + fullState = append(fullState, prevState.StateEntry) + } + + // Stable sort so that the most recent entry for each state key stays + // remains later in the list than the older entries for the same state key. + sort.Stable(stateEntryByStateKeySorter(fullState)) + // Unique returns the last entry and hence the most recent entry for each state key. + fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] + // Add the full state for this StateSnapshotNID. + combined = append(combined, fullState...) + } + return combined, nil +} + +// DifferenceBetweeenStateSnapshots works out which state entries have been added and removed between two snapshots. +func (v StateResolution) DifferenceBetweeenStateSnapshots( + ctx context.Context, oldStateNID, newStateNID types.StateSnapshotNID, +) (removed, added []types.StateEntry, err error) { + if oldStateNID == newStateNID { + // If the snapshot NIDs are the same then nothing has changed + return nil, nil, nil + } + + var oldEntries []types.StateEntry + var newEntries []types.StateEntry + if oldStateNID != 0 { + oldEntries, err = v.LoadStateAtSnapshot(ctx, oldStateNID) + if err != nil { + return nil, nil, err + } + } + if newStateNID != 0 { + newEntries, err = v.LoadStateAtSnapshot(ctx, newStateNID) + if err != nil { + return nil, nil, err + } + } + + var oldI int + var newI int + for { + switch { + case oldI == len(oldEntries): + // We've reached the end of the old entries. + // The rest of the new list must have been newly added. + added = append(added, newEntries[newI:]...) + return + case newI == len(newEntries): + // We've reached the end of the new entries. + // The rest of the old list must be have been removed. + removed = append(removed, oldEntries[oldI:]...) + return + case oldEntries[oldI] == newEntries[newI]: + // The entry is in both lists so skip over it. + oldI++ + newI++ + case oldEntries[oldI].LessThan(newEntries[newI]): + // The lists are sorted so the old entry being less than the new entry means that it only appears in the old list. + removed = append(removed, oldEntries[oldI]) + oldI++ + default: + // Reaching the default case implies that the new entry is less than the old entry. + // Since the lists are sorted this means that it only appears in the new list. + added = append(added, newEntries[newI]) + newI++ + } + } +} + +// LoadStateAtSnapshotForStringTuples loads the state for a list of event type and state key pairs at a snapshot. +// This is used when we only want to load a subset of the room state at a snapshot. +// If there is no entry for a given event type and state key pair then it will be discarded. +// This is typically the state before an event or the current state of a room. +// Returns a sorted list of state entries or an error if there was a problem talking to the database. +func (v StateResolution) LoadStateAtSnapshotForStringTuples( + ctx context.Context, + stateNID types.StateSnapshotNID, + stateKeyTuples []gomatrixserverlib.StateKeyTuple, +) ([]types.StateEntry, error) { + numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples) + if err != nil { + return nil, err + } + return v.loadStateAtSnapshotForNumericTuples(ctx, stateNID, numericTuples) +} + +// stringTuplesToNumericTuples converts the string state key tuples into numeric IDs +// If there isn't a numeric ID for either the event type or the event state key then the tuple is discarded. +// Returns an error if there was a problem talking to the database. +func (v StateResolution) stringTuplesToNumericTuples( + ctx context.Context, + stringTuples []gomatrixserverlib.StateKeyTuple, +) ([]types.StateKeyTuple, error) { + eventTypes := make([]string, len(stringTuples)) + stateKeys := make([]string, len(stringTuples)) + for i := range stringTuples { + eventTypes[i] = stringTuples[i].EventType + stateKeys[i] = stringTuples[i].StateKey + } + eventTypes = util.UniqueStrings(eventTypes) + eventTypeMap, err := v.db.EventTypeNIDs(ctx, eventTypes) + if err != nil { + return nil, err + } + stateKeys = util.UniqueStrings(stateKeys) + stateKeyMap, err := v.db.EventStateKeyNIDs(ctx, stateKeys) + if err != nil { + return nil, err + } + + var result []types.StateKeyTuple + for _, stringTuple := range stringTuples { + var numericTuple types.StateKeyTuple + var ok1, ok2 bool + numericTuple.EventTypeNID, ok1 = eventTypeMap[stringTuple.EventType] + numericTuple.EventStateKeyNID, ok2 = stateKeyMap[stringTuple.StateKey] + // Discard the tuple if there wasn't a numeric ID for either the event type or the state key. + if ok1 && ok2 { + result = append(result, numericTuple) + } + } + + return result, nil +} + +// loadStateAtSnapshotForNumericTuples loads the state for a list of event type and state key pairs at a snapshot. +// This is used when we only want to load a subset of the room state at a snapshot. +// If there is no entry for a given event type and state key pair then it will be discarded. +// This is typically the state before an event or the current state of a room. +// Returns a sorted list of state entries or an error if there was a problem talking to the database. +func (v StateResolution) loadStateAtSnapshotForNumericTuples( + ctx context.Context, + stateNID types.StateSnapshotNID, + stateKeyTuples []types.StateKeyTuple, +) ([]types.StateEntry, error) { + stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) + if err != nil { + return nil, err + } + // We've asked for exactly one snapshot from the db so we should have exactly one entry in the result. + stateBlockNIDList := stateBlockNIDLists[0] + + stateEntryLists, err := v.db.StateEntriesForTuples( + ctx, stateBlockNIDList.StateBlockNIDs, stateKeyTuples, + ) + if err != nil { + return nil, err + } + stateEntriesMap := stateEntryListMap(stateEntryLists) + + // Combine all the state entries for this snapshot. + // The order of state block NIDs in the list tells us the order to combine them in. + var fullState []types.StateEntry + for _, stateBlockNID := range stateBlockNIDList.StateBlockNIDs { + entries, ok := stateEntriesMap.lookup(stateBlockNID) + if !ok { + // If the block is missing from the map it means that none of its entries matched a requested tuple. + // This can happen if the block doesn't contain an update for one of the requested tuples. + // If none of the requested tuples are in the block then it can be safely skipped. + continue + } + fullState = append(fullState, entries...) + } + + // Stable sort so that the most recent entry for each state key stays + // remains later in the list than the older entries for the same state key. + sort.Stable(stateEntryByStateKeySorter(fullState)) + // Unique returns the last entry and hence the most recent entry for each state key. + fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] + return fullState, nil +} + +// LoadStateAfterEventsForStringTuples loads the state for a list of event type +// and state key pairs after list of events. +// This is used when we only want to load a subset of the room state after a list of events. +// If there is no entry for a given event type and state key pair then it will be discarded. +// This is typically the state before an event. +// Returns a sorted list of state entries or an error if there was a problem talking to the database. +func (v StateResolution) LoadStateAfterEventsForStringTuples( + ctx context.Context, roomNID types.RoomNID, + prevStates []types.StateAtEvent, + stateKeyTuples []gomatrixserverlib.StateKeyTuple, +) ([]types.StateEntry, error) { + numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples) + if err != nil { + return nil, err + } + return v.loadStateAfterEventsForNumericTuples(ctx, roomNID, prevStates, numericTuples) +} + +func (v StateResolution) loadStateAfterEventsForNumericTuples( + ctx context.Context, roomNID types.RoomNID, + prevStates []types.StateAtEvent, + stateKeyTuples []types.StateKeyTuple, +) ([]types.StateEntry, error) { + roomVersion, err := v.db.GetRoomVersionForRoomNID(ctx, roomNID) + if err != nil { + return nil, err + } + + if len(prevStates) == 1 { + // Fast path for a single event. + prevState := prevStates[0] + var result []types.StateEntry + result, err = v.loadStateAtSnapshotForNumericTuples( + ctx, prevState.BeforeStateSnapshotNID, stateKeyTuples, + ) + if err != nil { + return nil, err + } + if prevState.IsStateEvent() { + // The result is current the state before the requested event. + // We want the state after the requested event. + // If the requested event was a state event then we need to + // update that key in the result. + // If the requested event wasn't a state event then the state after + // it is the same as the state before it. + set := false + for i := range result { + if result[i].StateKeyTuple == prevState.StateKeyTuple { + result[i] = prevState.StateEntry + set = true + } + } + if !set { // no previous state exists for this event: add new state + result = append(result, prevState.StateEntry) + } + } + return result, nil + } + + // Slow path for more that one event. + // Load the entire state so that we can do conflict resolution if we need to. + // TODO: The are some optimistations we could do here: + // 1) We only need to do conflict resolution if there is a conflict in the + // requested tuples so we might try loading just those tuples and then + // checking for conflicts. + // 2) When there is a conflict we still only need to load the state + // needed to do conflict resolution which would save us having to load + // the full state. + + // TODO: Add metrics for this as it could take a long time for big rooms + // with large conflicts. + fullState, _, _, err := v.calculateStateAfterManyEvents(ctx, roomVersion, prevStates) + if err != nil { + return nil, err + } + + // Sort the full state so we can use it as a map. + sort.Sort(stateEntrySorter(fullState)) + + // Filter the full state down to the required tuples. + var result []types.StateEntry + for _, tuple := range stateKeyTuples { + eventNID, ok := stateEntryMap(fullState).lookup(tuple) + if ok { + result = append(result, types.StateEntry{ + StateKeyTuple: tuple, + EventNID: eventNID, + }) + } + } + sort.Sort(stateEntrySorter(result)) + return result, nil +} + +var calculateStateDurations = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "dendrite", + Subsystem: "roomserver", + Name: "calculate_state_duration_microseconds", + Help: "How long it takes to calculate the state after a list of events", + }, + // Takes two labels: + // algorithm: + // The algorithm used to calculate the state or the step it failed on if it failed. + // Labels starting with "_" are used to indicate when the algorithm fails halfway. + // outcome: + // Whether the state was successfully calculated. + // + // The possible values for algorithm are: + // empty_state -> The list of events was empty so the state is empty. + // no_change -> The state hasn't changed. + // single_delta -> There was a single event added to the state in a way that can be encoded as a single delta + // full_state_no_conflicts -> We created a new copy of the full room state, but didn't enounter any conflicts + // while doing so. + // full_state_with_conflicts -> We created a new copy of the full room state and had to resolve conflicts to do so. + // _load_state_block_nids -> Failed loading the state block nids for a single previous state. + // _load_combined_state -> Failed to load the combined state. + // _resolve_conflicts -> Failed to resolve conflicts. + []string{"algorithm", "outcome"}, +) + +var calculateStatePrevEventLength = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "dendrite", + Subsystem: "roomserver", + Name: "calculate_state_prev_event_length", + Help: "The length of the list of events to calculate the state after", + }, + []string{"algorithm", "outcome"}, +) + +var calculateStateFullStateLength = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "dendrite", + Subsystem: "roomserver", + Name: "calculate_state_full_state_length", + Help: "The length of the full room state.", + }, + []string{"algorithm", "outcome"}, +) + +var calculateStateConflictLength = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "dendrite", + Subsystem: "roomserver", + Name: "calculate_state_conflict_state_length", + Help: "The length of the conflicted room state.", + }, + []string{"algorithm", "outcome"}, +) + +type calculateStateMetrics struct { + algorithm string + startTime time.Time + prevEventLength int + fullStateLength int + conflictLength int +} + +func (c *calculateStateMetrics) stop(stateNID types.StateSnapshotNID, err error) (types.StateSnapshotNID, error) { + var outcome string + if err == nil { + outcome = "success" + } else { + outcome = "failure" + } + endTime := time.Now() + calculateStateDurations.WithLabelValues(c.algorithm, outcome).Observe( + float64(endTime.Sub(c.startTime).Nanoseconds()) / 1000., + ) + calculateStatePrevEventLength.WithLabelValues(c.algorithm, outcome).Observe( + float64(c.prevEventLength), + ) + calculateStateFullStateLength.WithLabelValues(c.algorithm, outcome).Observe( + float64(c.fullStateLength), + ) + calculateStateConflictLength.WithLabelValues(c.algorithm, outcome).Observe( + float64(c.conflictLength), + ) + return stateNID, err +} + +func init() { + prometheus.MustRegister( + calculateStateDurations, calculateStatePrevEventLength, + calculateStateFullStateLength, calculateStateConflictLength, + ) +} + +// CalculateAndStoreStateBeforeEvent calculates a snapshot of the state of a room before an event. +// Stores the snapshot of the state in the database. +// Returns a numeric ID for the snapshot of the state before the event. +func (v StateResolution) CalculateAndStoreStateBeforeEvent( + ctx context.Context, + event gomatrixserverlib.Event, + roomNID types.RoomNID, +) (types.StateSnapshotNID, error) { + // Load the state at the prev events. + prevEventRefs := event.PrevEvents() + prevEventIDs := make([]string, len(prevEventRefs)) + for i := range prevEventRefs { + prevEventIDs[i] = prevEventRefs[i].EventID + } + + prevStates, err := v.db.StateAtEventIDs(ctx, prevEventIDs) + if err != nil { + return 0, err + } + + // The state before this event will be the state after the events that came before it. + return v.CalculateAndStoreStateAfterEvents(ctx, roomNID, prevStates) +} + +// CalculateAndStoreStateAfterEvents finds the room state after the given events. +// Stores the resulting state in the database and returns a numeric ID for that snapshot. +func (v StateResolution) CalculateAndStoreStateAfterEvents( + ctx context.Context, + roomNID types.RoomNID, + prevStates []types.StateAtEvent, +) (types.StateSnapshotNID, error) { + metrics := calculateStateMetrics{startTime: time.Now(), prevEventLength: len(prevStates)} + + if len(prevStates) == 0 { + // 2) There weren't any prev_events for this event so the state is + // empty. + metrics.algorithm = "empty_state" + return metrics.stop(v.db.AddState(ctx, roomNID, nil, nil)) + } + + if len(prevStates) == 1 { + prevState := prevStates[0] + if prevState.EventStateKeyNID == 0 { + // 3) None of the previous events were state events and they all + // have the same state, so this event has exactly the same state + // as the previous events. + // This should be the common case. + metrics.algorithm = "no_change" + return metrics.stop(prevState.BeforeStateSnapshotNID, nil) + } + // The previous event was a state event so we need to store a copy + // of the previous state updated with that event. + stateBlockNIDLists, err := v.db.StateBlockNIDs( + ctx, []types.StateSnapshotNID{prevState.BeforeStateSnapshotNID}, + ) + if err != nil { + metrics.algorithm = "_load_state_blocks" + return metrics.stop(0, err) + } + stateBlockNIDs := stateBlockNIDLists[0].StateBlockNIDs + if len(stateBlockNIDs) < maxStateBlockNIDs { + // 4) The number of state data blocks is small enough that we can just + // add the state event as a block of size one to the end of the blocks. + metrics.algorithm = "single_delta" + return metrics.stop(v.db.AddState( + ctx, roomNID, stateBlockNIDs, []types.StateEntry{prevState.StateEntry}, + )) + } + // If there are too many deltas then we need to calculate the full state + // So fall through to calculateAndStoreStateAfterManyEvents + } + + return v.calculateAndStoreStateAfterManyEvents(ctx, roomNID, prevStates, metrics) +} + +// maxStateBlockNIDs is the maximum number of state data blocks to use to encode a snapshot of room state. +// Increasing this number means that we can encode more of the state changes as simple deltas which means that +// we need fewer entries in the state data table. However making this number bigger will increase the size of +// the rows in the state table itself and will require more index lookups when retrieving a snapshot. +// TODO: Tune this to get the right balance between size and lookup performance. +const maxStateBlockNIDs = 64 + +// calculateAndStoreStateAfterManyEvents finds the room state after the given events. +// This handles the slow path of calculateAndStoreStateAfterEvents for when there is more than one event. +// Stores the resulting state and returns a numeric ID for the snapshot. +func (v StateResolution) calculateAndStoreStateAfterManyEvents( + ctx context.Context, + roomNID types.RoomNID, + prevStates []types.StateAtEvent, + metrics calculateStateMetrics, +) (types.StateSnapshotNID, error) { + roomVersion, err := v.db.GetRoomVersionForRoomNID(ctx, roomNID) + if err != nil { + return metrics.stop(0, err) + } + + state, algorithm, conflictLength, err := + v.calculateStateAfterManyEvents(ctx, roomVersion, prevStates) + metrics.algorithm = algorithm + if err != nil { + return metrics.stop(0, err) + } + + // TODO: Check if we can encode the new state as a delta against the + // previous state. + metrics.conflictLength = conflictLength + metrics.fullStateLength = len(state) + return metrics.stop(v.db.AddState(ctx, roomNID, nil, state)) +} + +func (v StateResolution) calculateStateAfterManyEvents( + ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, + prevStates []types.StateAtEvent, +) (state []types.StateEntry, algorithm string, conflictLength int, err error) { + var combined []types.StateEntry + // Conflict resolution. + // First stage: load the state after each of the prev events. + combined, err = v.LoadCombinedStateAfterEvents(ctx, prevStates) + if err != nil { + algorithm = "_load_combined_state" + return + } + + // Collect all the entries with the same type and key together. + // We don't care about the order here because the conflict resolution + // algorithm doesn't depend on the order of the prev events. + // Remove duplicate entires. + combined = combined[:util.SortAndUnique(stateEntrySorter(combined))] + + // Find the conflicts + conflicts := findDuplicateStateKeys(combined) + + if len(conflicts) > 0 { + conflictLength = len(conflicts) + + // 5) There are conflicting state events, for each conflict workout + // what the appropriate state event is. + + // Work out which entries aren't conflicted. + var notConflicted []types.StateEntry + for _, entry := range combined { + if _, ok := stateEntryMap(conflicts).lookup(entry.StateKeyTuple); !ok { + notConflicted = append(notConflicted, entry) + } + } + + var resolved []types.StateEntry + resolved, err = v.resolveConflicts(ctx, roomVersion, notConflicted, conflicts) + if err != nil { + algorithm = "_resolve_conflicts" + return + } + algorithm = "full_state_with_conflicts" + state = resolved + } else { + algorithm = "full_state_no_conflicts" + // 6) There weren't any conflicts + state = combined + } + return +} + +func (v StateResolution) resolveConflicts( + ctx context.Context, version gomatrixserverlib.RoomVersion, + notConflicted, conflicted []types.StateEntry, +) ([]types.StateEntry, error) { + stateResAlgo, err := version.StateResAlgorithm() + if err != nil { + return nil, err + } + switch stateResAlgo { + case gomatrixserverlib.StateResV1: + return v.resolveConflictsV1(ctx, notConflicted, conflicted) + case gomatrixserverlib.StateResV2: + return v.resolveConflictsV2(ctx, notConflicted, conflicted) + } + return nil, errors.New("unsupported state resolution algorithm") +} + +// resolveConflicts resolves a list of conflicted state entries. It takes two lists. +// The first is a list of all state entries that are not conflicted. +// The second is a list of all state entries that are conflicted +// A state entry is conflicted when there is more than one numeric event ID for the same state key tuple. +// Returns a list that combines the entries without conflicts with the result of state resolution for the entries with conflicts. +// The returned list is sorted by state key tuple. +// Returns an error if there was a problem talking to the database. +func (v StateResolution) resolveConflictsV1( + ctx context.Context, + notConflicted, conflicted []types.StateEntry, +) ([]types.StateEntry, error) { + + // Load the conflicted events + conflictedEvents, eventIDMap, err := v.loadStateEvents(ctx, conflicted) + if err != nil { + return nil, err + } + + // Work out which auth events we need to load. + needed := gomatrixserverlib.StateNeededForAuth(conflictedEvents) + + // Find the numeric IDs for the necessary state keys. + var neededStateKeys []string + neededStateKeys = append(neededStateKeys, needed.Member...) + neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...) + stateKeyNIDMap, err := v.db.EventStateKeyNIDs(ctx, neededStateKeys) + if err != nil { + return nil, err + } + + // Load the necessary auth events. + tuplesNeeded := v.stateKeyTuplesNeeded(stateKeyNIDMap, needed) + var authEntries []types.StateEntry + for _, tuple := range tuplesNeeded { + if eventNID, ok := stateEntryMap(notConflicted).lookup(tuple); ok { + authEntries = append(authEntries, types.StateEntry{ + StateKeyTuple: tuple, + EventNID: eventNID, + }) + } + } + authEvents, _, err := v.loadStateEvents(ctx, authEntries) + if err != nil { + return nil, err + } + + // Resolve the conflicts. + resolvedEvents := gomatrixserverlib.ResolveStateConflicts(conflictedEvents, authEvents) + + // Map from the full events back to numeric state entries. + for _, resolvedEvent := range resolvedEvents { + entry, ok := eventIDMap[resolvedEvent.EventID()] + if !ok { + panic(fmt.Errorf("Missing state entry for event ID %q", resolvedEvent.EventID())) + } + notConflicted = append(notConflicted, entry) + } + + // Sort the result so it can be searched. + sort.Sort(stateEntrySorter(notConflicted)) + return notConflicted, nil +} + +// resolveConflicts resolves a list of conflicted state entries. It takes two lists. +// The first is a list of all state entries that are not conflicted. +// The second is a list of all state entries that are conflicted +// A state entry is conflicted when there is more than one numeric event ID for the same state key tuple. +// Returns a list that combines the entries without conflicts with the result of state resolution for the entries with conflicts. +// The returned list is sorted by state key tuple. +// Returns an error if there was a problem talking to the database. +// nolint:gocyclo +func (v StateResolution) resolveConflictsV2( + ctx context.Context, + notConflicted, conflicted []types.StateEntry, +) ([]types.StateEntry, error) { + eventIDMap := make(map[string]types.StateEntry) + + // Load the conflicted events + conflictedEvents, conflictedEventMap, err := v.loadStateEvents(ctx, conflicted) + if err != nil { + return nil, err + } + for k, v := range conflictedEventMap { + eventIDMap[k] = v + } + + // Load the non-conflicted events + nonConflictedEvents, nonConflictedEventMap, err := v.loadStateEvents(ctx, notConflicted) + if err != nil { + return nil, err + } + for k, v := range nonConflictedEventMap { + eventIDMap[k] = v + } + + // For each conflicted event, we will add a new set of auth events. Auth + // events may be duplicated across these sets but that's OK. + authSets := make(map[string][]gomatrixserverlib.Event) + var authEvents []gomatrixserverlib.Event + var authDifference []gomatrixserverlib.Event + + // For each conflicted event, let's try and get the needed auth events. + for _, conflictedEvent := range conflictedEvents { + // Work out which auth events we need to load. + key := conflictedEvent.EventID() + needed := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{conflictedEvent}) + + // Find the numeric IDs for the necessary state keys. + var neededStateKeys []string + neededStateKeys = append(neededStateKeys, needed.Member...) + neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...) + stateKeyNIDMap, err := v.db.EventStateKeyNIDs(ctx, neededStateKeys) + if err != nil { + return nil, err + } + + // Load the necessary auth events. + tuplesNeeded := v.stateKeyTuplesNeeded(stateKeyNIDMap, needed) + var authEntries []types.StateEntry + for _, tuple := range tuplesNeeded { + if eventNID, ok := stateEntryMap(notConflicted).lookup(tuple); ok { + authEntries = append(authEntries, types.StateEntry{ + StateKeyTuple: tuple, + EventNID: eventNID, + }) + } + } + + // Store the newly found auth events in the auth set for this event. + authSets[key], _, err = v.loadStateEvents(ctx, authEntries) + if err != nil { + return nil, err + } + authEvents = append(authEvents, authSets[key]...) + } + + // This function helps us to work out whether an event exists in one of the + // auth sets. + isInAuthList := func(k string, event gomatrixserverlib.Event) bool { + for _, e := range authSets[k] { + if e.EventID() == event.EventID() { + return true + } + } + return false + } + + // This function works out if an event exists in all of the auth sets. + isInAllAuthLists := func(event gomatrixserverlib.Event) bool { + found := true + for k := range authSets { + found = found && isInAuthList(k, event) + } + return found + } + + // Look through all of the auth events that we've been given and work out if + // there are any events which don't appear in all of the auth sets. If they + // don't then we add them to the auth difference. + for _, event := range authEvents { + if !isInAllAuthLists(event) { + authDifference = append(authDifference, event) + } + } + + // Resolve the conflicts. + resolvedEvents := gomatrixserverlib.ResolveStateConflictsV2( + conflictedEvents, + nonConflictedEvents, + authEvents, + authDifference, + ) + + // Map from the full events back to numeric state entries. + for _, resolvedEvent := range resolvedEvents { + entry, ok := eventIDMap[resolvedEvent.EventID()] + if !ok { + panic(fmt.Errorf("Missing state entry for event ID %q", resolvedEvent.EventID())) + } + notConflicted = append(notConflicted, entry) + } + + // Sort the result so it can be searched. + sort.Sort(stateEntrySorter(notConflicted)) + return notConflicted, nil +} + +// stateKeyTuplesNeeded works out which numeric state key tuples we need to authenticate some events. +func (v StateResolution) stateKeyTuplesNeeded(stateKeyNIDMap map[string]types.EventStateKeyNID, stateNeeded gomatrixserverlib.StateNeeded) []types.StateKeyTuple { + var keyTuples []types.StateKeyTuple + if stateNeeded.Create { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomCreateNID, + EventStateKeyNID: types.EmptyStateKeyNID, + }) + } + if stateNeeded.PowerLevels { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomPowerLevelsNID, + EventStateKeyNID: types.EmptyStateKeyNID, + }) + } + if stateNeeded.JoinRules { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomJoinRulesNID, + EventStateKeyNID: types.EmptyStateKeyNID, + }) + } + for _, member := range stateNeeded.Member { + stateKeyNID, ok := stateKeyNIDMap[member] + if ok { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomMemberNID, + EventStateKeyNID: stateKeyNID, + }) + } + } + for _, token := range stateNeeded.ThirdPartyInvite { + stateKeyNID, ok := stateKeyNIDMap[token] + if ok { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomThirdPartyInviteNID, + EventStateKeyNID: stateKeyNID, + }) + } + } + return keyTuples +} + +// loadStateEvents loads the matrix events for a list of state entries. +// Returns a list of state events in no particular order and a map from string event ID back to state entry. +// The map can be used to recover which numeric state entry a given event is for. +// Returns an error if there was a problem talking to the database. +func (v StateResolution) loadStateEvents( + ctx context.Context, entries []types.StateEntry, +) ([]gomatrixserverlib.Event, map[string]types.StateEntry, error) { + eventNIDs := make([]types.EventNID, len(entries)) + for i := range entries { + eventNIDs[i] = entries[i].EventNID + } + events, err := v.db.Events(ctx, eventNIDs) + if err != nil { + return nil, nil, err + } + eventIDMap := map[string]types.StateEntry{} + result := make([]gomatrixserverlib.Event, len(entries)) + for i := range entries { + event, ok := eventMap(events).lookup(entries[i].EventNID) + if !ok { + panic(fmt.Errorf("Corrupt DB: Missing event numeric ID %d", entries[i].EventNID)) + } + result[i] = event.Event + eventIDMap[event.Event.EventID()] = entries[i] + } + return result, eventIDMap, nil +} + +// findDuplicateStateKeys finds the state entries where the state key tuple appears more than once in a sorted list. +// Returns a sorted list of those state entries. +func findDuplicateStateKeys(a []types.StateEntry) []types.StateEntry { + var result []types.StateEntry + // j is the starting index of a block of entries with the same state key tuple. + j := 0 + for i := 1; i < len(a); i++ { + // Check if the state key tuple matches the start of the block + if a[j].StateKeyTuple != a[i].StateKeyTuple { + // If the state key tuple is different then we've reached the end of a block of duplicates. + // Check if the size of the block is bigger than one. + // If the size is one then there was only a single entry with that state key tuple so we don't add it to the result + if j+1 != i { + // Add the block to the result. + result = append(result, a[j:i]...) + } + // Start a new block for the next state key tuple. + j = i + } + } + // Check if the last block with the same state key tuple had more than one event in it. + if j+1 != len(a) { + result = append(result, a[j:]...) + } + return result +} + +type stateEntrySorter []types.StateEntry + +func (s stateEntrySorter) Len() int { return len(s) } +func (s stateEntrySorter) Less(i, j int) bool { return s[i].LessThan(s[j]) } +func (s stateEntrySorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +type stateBlockNIDListMap []types.StateBlockNIDList + +func (m stateBlockNIDListMap) lookup(stateNID types.StateSnapshotNID) (stateBlockNIDs []types.StateBlockNID, ok bool) { + list := []types.StateBlockNIDList(m) + i := sort.Search(len(list), func(i int) bool { + return list[i].StateSnapshotNID >= stateNID + }) + if i < len(list) && list[i].StateSnapshotNID == stateNID { + ok = true + stateBlockNIDs = list[i].StateBlockNIDs + } + return +} + +type stateEntryListMap []types.StateEntryList + +func (m stateEntryListMap) lookup(stateBlockNID types.StateBlockNID) (stateEntries []types.StateEntry, ok bool) { + list := []types.StateEntryList(m) + i := sort.Search(len(list), func(i int) bool { + return list[i].StateBlockNID >= stateBlockNID + }) + if i < len(list) && list[i].StateBlockNID == stateBlockNID { + ok = true + stateEntries = list[i].StateEntries + } + return +} + +type stateEntryByStateKeySorter []types.StateEntry + +func (s stateEntryByStateKeySorter) Len() int { return len(s) } +func (s stateEntryByStateKeySorter) Less(i, j int) bool { + return s[i].StateKeyTuple.LessThan(s[j].StateKeyTuple) +} +func (s stateEntryByStateKeySorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +type stateNIDSorter []types.StateSnapshotNID + +func (s stateNIDSorter) Len() int { return len(s) } +func (s stateNIDSorter) Less(i, j int) bool { return s[i] < s[j] } +func (s stateNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func uniqueStateSnapshotNIDs(nids []types.StateSnapshotNID) []types.StateSnapshotNID { + return nids[:util.SortAndUnique(stateNIDSorter(nids))] +} + +type stateBlockNIDSorter []types.StateBlockNID + +func (s stateBlockNIDSorter) Len() int { return len(s) } +func (s stateBlockNIDSorter) Less(i, j int) bool { return s[i] < s[j] } +func (s stateBlockNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func uniqueStateBlockNIDs(nids []types.StateBlockNID) []types.StateBlockNID { + return nids[:util.SortAndUnique(stateBlockNIDSorter(nids))] +} + +// Map from event type, state key tuple to numeric event ID. +// Implemented using binary search on a sorted array. +type stateEntryMap []types.StateEntry + +// lookup an entry in the event map. +func (m stateEntryMap) lookup(stateKey types.StateKeyTuple) (eventNID types.EventNID, ok bool) { + // Since the list is sorted we can implement this using binary search. + // This is faster than using a hash map. + // We don't have to worry about pathological cases because the keys are fixed + // size and are controlled by us. + list := []types.StateEntry(m) + i := sort.Search(len(list), func(i int) bool { + return !list[i].StateKeyTuple.LessThan(stateKey) + }) + if i < len(list) && list[i].StateKeyTuple == stateKey { + ok = true + eventNID = list[i].EventNID + } + return +} + +// Map from numeric event ID to event. +// Implemented using binary search on a sorted array. +type eventMap []types.Event + +// lookup an entry in the event map. +func (m eventMap) lookup(eventNID types.EventNID) (event *types.Event, ok bool) { + // Since the list is sorted we can implement this using binary search. + // This is faster than using a hash map. + // We don't have to worry about pathological cases because the keys are fixed + // size are controlled by us. + list := []types.Event(m) + i := sort.Search(len(list), func(i int) bool { + return list[i].EventNID >= eventNID + }) + if i < len(list) && list[i].EventNID == eventNID { + ok = true + event = &list[i] + } + return } diff --git a/roomserver/state/v1/state_test.go b/roomserver/state/state_test.go similarity index 99% rename from roomserver/state/v1/state_test.go rename to roomserver/state/state_test.go index 4dc7e52ec..c57056786 100644 --- a/roomserver/state/v1/state_test.go +++ b/roomserver/state/state_test.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package v1 +package state import ( "testing" diff --git a/roomserver/state/v1/state.go b/roomserver/state/v1/state.go deleted file mode 100644 index 5683745bf..000000000 --- a/roomserver/state/v1/state.go +++ /dev/null @@ -1,927 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package state provides functions for reading state from the database. -// The functions for writing state to the database are the input package. -package v1 - -import ( - "context" - "fmt" - "sort" - "time" - - "github.com/matrix-org/dendrite/roomserver/state/database" - "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - "github.com/prometheus/client_golang/prometheus" -) - -type StateResolutionV1 struct { - db database.RoomStateDatabase -} - -func Prepare(db database.RoomStateDatabase) StateResolutionV1 { - return StateResolutionV1{ - db: db, - } -} - -// LoadStateAtSnapshot loads the full state of a room at a particular snapshot. -// This is typically the state before an event or the current state of a room. -// Returns a sorted list of state entries or an error if there was a problem talking to the database. -func (v StateResolutionV1) LoadStateAtSnapshot( - ctx context.Context, stateNID types.StateSnapshotNID, -) ([]types.StateEntry, error) { - stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) - if err != nil { - return nil, err - } - // We've asked for exactly one snapshot from the db so we should have exactly one entry in the result. - stateBlockNIDList := stateBlockNIDLists[0] - - stateEntryLists, err := v.db.StateEntries(ctx, stateBlockNIDList.StateBlockNIDs) - if err != nil { - return nil, err - } - stateEntriesMap := stateEntryListMap(stateEntryLists) - - // Combine all the state entries for this snapshot. - // The order of state block NIDs in the list tells us the order to combine them in. - var fullState []types.StateEntry - for _, stateBlockNID := range stateBlockNIDList.StateBlockNIDs { - entries, ok := stateEntriesMap.lookup(stateBlockNID) - if !ok { - // This should only get hit if the database is corrupt. - // It should be impossible for an event to reference a NID that doesn't exist - panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) - } - fullState = append(fullState, entries...) - } - - // Stable sort so that the most recent entry for each state key stays - // remains later in the list than the older entries for the same state key. - sort.Stable(stateEntryByStateKeySorter(fullState)) - // Unique returns the last entry and hence the most recent entry for each state key. - fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] - return fullState, nil -} - -// LoadStateAtEvent loads the full state of a room at a particular event. -func (v StateResolutionV1) LoadStateAtEvent( - ctx context.Context, eventID string, -) ([]types.StateEntry, error) { - snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID) - if err != nil { - return nil, err - } - - stateEntries, err := v.LoadStateAtSnapshot(ctx, snapshotNID) - if err != nil { - return nil, err - } - - return stateEntries, nil -} - -// LoadCombinedStateAfterEvents loads a snapshot of the state after each of the events -// and combines those snapshots together into a single list. -func (v StateResolutionV1) LoadCombinedStateAfterEvents( - ctx context.Context, prevStates []types.StateAtEvent, -) ([]types.StateEntry, error) { - stateNIDs := make([]types.StateSnapshotNID, len(prevStates)) - for i, state := range prevStates { - stateNIDs[i] = state.BeforeStateSnapshotNID - } - // Fetch the state snapshots for the state before the each prev event from the database. - // Deduplicate the IDs before passing them to the database. - // There could be duplicates because the events could be state events where - // the snapshot of the room state before them was the same. - stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, uniqueStateSnapshotNIDs(stateNIDs)) - if err != nil { - return nil, err - } - - var stateBlockNIDs []types.StateBlockNID - for _, list := range stateBlockNIDLists { - stateBlockNIDs = append(stateBlockNIDs, list.StateBlockNIDs...) - } - // Fetch the state entries that will be combined to create the snapshots. - // Deduplicate the IDs before passing them to the database. - // There could be duplicates because a block of state entries could be reused by - // multiple snapshots. - stateEntryLists, err := v.db.StateEntries(ctx, uniqueStateBlockNIDs(stateBlockNIDs)) - if err != nil { - return nil, err - } - stateBlockNIDsMap := stateBlockNIDListMap(stateBlockNIDLists) - stateEntriesMap := stateEntryListMap(stateEntryLists) - - // Combine the entries from all the snapshots of state after each prev event into a single list. - var combined []types.StateEntry - for _, prevState := range prevStates { - // Grab the list of state data NIDs for this snapshot. - stateBlockNIDs, ok := stateBlockNIDsMap.lookup(prevState.BeforeStateSnapshotNID) - if !ok { - // This should only get hit if the database is corrupt. - // It should be impossible for an event to reference a NID that doesn't exist - panic(fmt.Errorf("Corrupt DB: Missing state snapshot numeric ID %d", prevState.BeforeStateSnapshotNID)) - } - - // Combine all the state entries for this snapshot. - // The order of state block NIDs in the list tells us the order to combine them in. - var fullState []types.StateEntry - for _, stateBlockNID := range stateBlockNIDs { - entries, ok := stateEntriesMap.lookup(stateBlockNID) - if !ok { - // This should only get hit if the database is corrupt. - // It should be impossible for an event to reference a NID that doesn't exist - panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) - } - fullState = append(fullState, entries...) - } - if prevState.IsStateEvent() { - // If the prev event was a state event then add an entry for the event itself - // so that we get the state after the event rather than the state before. - fullState = append(fullState, prevState.StateEntry) - } - - // Stable sort so that the most recent entry for each state key stays - // remains later in the list than the older entries for the same state key. - sort.Stable(stateEntryByStateKeySorter(fullState)) - // Unique returns the last entry and hence the most recent entry for each state key. - fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] - // Add the full state for this StateSnapshotNID. - combined = append(combined, fullState...) - } - return combined, nil -} - -// DifferenceBetweeenStateSnapshots works out which state entries have been added and removed between two snapshots. -func (v StateResolutionV1) DifferenceBetweeenStateSnapshots( - ctx context.Context, oldStateNID, newStateNID types.StateSnapshotNID, -) (removed, added []types.StateEntry, err error) { - if oldStateNID == newStateNID { - // If the snapshot NIDs are the same then nothing has changed - return nil, nil, nil - } - - var oldEntries []types.StateEntry - var newEntries []types.StateEntry - if oldStateNID != 0 { - oldEntries, err = v.LoadStateAtSnapshot(ctx, oldStateNID) - if err != nil { - return nil, nil, err - } - } - if newStateNID != 0 { - newEntries, err = v.LoadStateAtSnapshot(ctx, newStateNID) - if err != nil { - return nil, nil, err - } - } - - var oldI int - var newI int - for { - switch { - case oldI == len(oldEntries): - // We've reached the end of the old entries. - // The rest of the new list must have been newly added. - added = append(added, newEntries[newI:]...) - return - case newI == len(newEntries): - // We've reached the end of the new entries. - // The rest of the old list must be have been removed. - removed = append(removed, oldEntries[oldI:]...) - return - case oldEntries[oldI] == newEntries[newI]: - // The entry is in both lists so skip over it. - oldI++ - newI++ - case oldEntries[oldI].LessThan(newEntries[newI]): - // The lists are sorted so the old entry being less than the new entry means that it only appears in the old list. - removed = append(removed, oldEntries[oldI]) - oldI++ - default: - // Reaching the default case implies that the new entry is less than the old entry. - // Since the lists are sorted this means that it only appears in the new list. - added = append(added, newEntries[newI]) - newI++ - } - } -} - -// LoadStateAtSnapshotForStringTuples loads the state for a list of event type and state key pairs at a snapshot. -// This is used when we only want to load a subset of the room state at a snapshot. -// If there is no entry for a given event type and state key pair then it will be discarded. -// This is typically the state before an event or the current state of a room. -// Returns a sorted list of state entries or an error if there was a problem talking to the database. -func (v StateResolutionV1) LoadStateAtSnapshotForStringTuples( - ctx context.Context, - stateNID types.StateSnapshotNID, - stateKeyTuples []gomatrixserverlib.StateKeyTuple, -) ([]types.StateEntry, error) { - numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples) - if err != nil { - return nil, err - } - return v.loadStateAtSnapshotForNumericTuples(ctx, stateNID, numericTuples) -} - -// stringTuplesToNumericTuples converts the string state key tuples into numeric IDs -// If there isn't a numeric ID for either the event type or the event state key then the tuple is discarded. -// Returns an error if there was a problem talking to the database. -func (v StateResolutionV1) stringTuplesToNumericTuples( - ctx context.Context, - stringTuples []gomatrixserverlib.StateKeyTuple, -) ([]types.StateKeyTuple, error) { - eventTypes := make([]string, len(stringTuples)) - stateKeys := make([]string, len(stringTuples)) - for i := range stringTuples { - eventTypes[i] = stringTuples[i].EventType - stateKeys[i] = stringTuples[i].StateKey - } - eventTypes = util.UniqueStrings(eventTypes) - eventTypeMap, err := v.db.EventTypeNIDs(ctx, eventTypes) - if err != nil { - return nil, err - } - stateKeys = util.UniqueStrings(stateKeys) - stateKeyMap, err := v.db.EventStateKeyNIDs(ctx, stateKeys) - if err != nil { - return nil, err - } - - var result []types.StateKeyTuple - for _, stringTuple := range stringTuples { - var numericTuple types.StateKeyTuple - var ok1, ok2 bool - numericTuple.EventTypeNID, ok1 = eventTypeMap[stringTuple.EventType] - numericTuple.EventStateKeyNID, ok2 = stateKeyMap[stringTuple.StateKey] - // Discard the tuple if there wasn't a numeric ID for either the event type or the state key. - if ok1 && ok2 { - result = append(result, numericTuple) - } - } - - return result, nil -} - -// loadStateAtSnapshotForNumericTuples loads the state for a list of event type and state key pairs at a snapshot. -// This is used when we only want to load a subset of the room state at a snapshot. -// If there is no entry for a given event type and state key pair then it will be discarded. -// This is typically the state before an event or the current state of a room. -// Returns a sorted list of state entries or an error if there was a problem talking to the database. -func (v StateResolutionV1) loadStateAtSnapshotForNumericTuples( - ctx context.Context, - stateNID types.StateSnapshotNID, - stateKeyTuples []types.StateKeyTuple, -) ([]types.StateEntry, error) { - stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) - if err != nil { - return nil, err - } - // We've asked for exactly one snapshot from the db so we should have exactly one entry in the result. - stateBlockNIDList := stateBlockNIDLists[0] - - stateEntryLists, err := v.db.StateEntriesForTuples( - ctx, stateBlockNIDList.StateBlockNIDs, stateKeyTuples, - ) - if err != nil { - return nil, err - } - stateEntriesMap := stateEntryListMap(stateEntryLists) - - // Combine all the state entries for this snapshot. - // The order of state block NIDs in the list tells us the order to combine them in. - var fullState []types.StateEntry - for _, stateBlockNID := range stateBlockNIDList.StateBlockNIDs { - entries, ok := stateEntriesMap.lookup(stateBlockNID) - if !ok { - // If the block is missing from the map it means that none of its entries matched a requested tuple. - // This can happen if the block doesn't contain an update for one of the requested tuples. - // If none of the requested tuples are in the block then it can be safely skipped. - continue - } - fullState = append(fullState, entries...) - } - - // Stable sort so that the most recent entry for each state key stays - // remains later in the list than the older entries for the same state key. - sort.Stable(stateEntryByStateKeySorter(fullState)) - // Unique returns the last entry and hence the most recent entry for each state key. - fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] - return fullState, nil -} - -// LoadStateAfterEventsForStringTuples loads the state for a list of event type -// and state key pairs after list of events. -// This is used when we only want to load a subset of the room state after a list of events. -// If there is no entry for a given event type and state key pair then it will be discarded. -// This is typically the state before an event. -// Returns a sorted list of state entries or an error if there was a problem talking to the database. -func (v StateResolutionV1) LoadStateAfterEventsForStringTuples( - ctx context.Context, - prevStates []types.StateAtEvent, - stateKeyTuples []gomatrixserverlib.StateKeyTuple, -) ([]types.StateEntry, error) { - numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples) - if err != nil { - return nil, err - } - return v.loadStateAfterEventsForNumericTuples(ctx, prevStates, numericTuples) -} - -func (v StateResolutionV1) loadStateAfterEventsForNumericTuples( - ctx context.Context, - prevStates []types.StateAtEvent, - stateKeyTuples []types.StateKeyTuple, -) ([]types.StateEntry, error) { - if len(prevStates) == 1 { - // Fast path for a single event. - prevState := prevStates[0] - result, err := v.loadStateAtSnapshotForNumericTuples( - ctx, prevState.BeforeStateSnapshotNID, stateKeyTuples, - ) - if err != nil { - return nil, err - } - if prevState.IsStateEvent() { - // The result is current the state before the requested event. - // We want the state after the requested event. - // If the requested event was a state event then we need to - // update that key in the result. - // If the requested event wasn't a state event then the state after - // it is the same as the state before it. - for i := range result { - if result[i].StateKeyTuple == prevState.StateKeyTuple { - result[i] = prevState.StateEntry - } - } - } - return result, nil - } - - // Slow path for more that one event. - // Load the entire state so that we can do conflict resolution if we need to. - // TODO: The are some optimistations we could do here: - // 1) We only need to do conflict resolution if there is a conflict in the - // requested tuples so we might try loading just those tuples and then - // checking for conflicts. - // 2) When there is a conflict we still only need to load the state - // needed to do conflict resolution which would save us having to load - // the full state. - - // TODO: Add metrics for this as it could take a long time for big rooms - // with large conflicts. - fullState, _, _, err := v.calculateStateAfterManyEvents(ctx, prevStates) - if err != nil { - return nil, err - } - - // Sort the full state so we can use it as a map. - sort.Sort(stateEntrySorter(fullState)) - - // Filter the full state down to the required tuples. - var result []types.StateEntry - for _, tuple := range stateKeyTuples { - eventNID, ok := stateEntryMap(fullState).lookup(tuple) - if ok { - result = append(result, types.StateEntry{ - StateKeyTuple: tuple, - EventNID: eventNID, - }) - } - } - sort.Sort(stateEntrySorter(result)) - return result, nil -} - -var calculateStateDurations = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "dendrite", - Subsystem: "roomserver", - Name: "calculate_state_duration_microseconds", - Help: "How long it takes to calculate the state after a list of events", - }, - // Takes two labels: - // algorithm: - // The algorithm used to calculate the state or the step it failed on if it failed. - // Labels starting with "_" are used to indicate when the algorithm fails halfway. - // outcome: - // Whether the state was successfully calculated. - // - // The possible values for algorithm are: - // empty_state -> The list of events was empty so the state is empty. - // no_change -> The state hasn't changed. - // single_delta -> There was a single event added to the state in a way that can be encoded as a single delta - // full_state_no_conflicts -> We created a new copy of the full room state, but didn't enounter any conflicts - // while doing so. - // full_state_with_conflicts -> We created a new copy of the full room state and had to resolve conflicts to do so. - // _load_state_block_nids -> Failed loading the state block nids for a single previous state. - // _load_combined_state -> Failed to load the combined state. - // _resolve_conflicts -> Failed to resolve conflicts. - []string{"algorithm", "outcome"}, -) - -var calculateStatePrevEventLength = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "dendrite", - Subsystem: "roomserver", - Name: "calculate_state_prev_event_length", - Help: "The length of the list of events to calculate the state after", - }, - []string{"algorithm", "outcome"}, -) - -var calculateStateFullStateLength = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "dendrite", - Subsystem: "roomserver", - Name: "calculate_state_full_state_length", - Help: "The length of the full room state.", - }, - []string{"algorithm", "outcome"}, -) - -var calculateStateConflictLength = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "dendrite", - Subsystem: "roomserver", - Name: "calculate_state_conflict_state_length", - Help: "The length of the conflicted room state.", - }, - []string{"algorithm", "outcome"}, -) - -type calculateStateMetrics struct { - algorithm string - startTime time.Time - prevEventLength int - fullStateLength int - conflictLength int -} - -func (c *calculateStateMetrics) stop(stateNID types.StateSnapshotNID, err error) (types.StateSnapshotNID, error) { - var outcome string - if err == nil { - outcome = "success" - } else { - outcome = "failure" - } - endTime := time.Now() - calculateStateDurations.WithLabelValues(c.algorithm, outcome).Observe( - float64(endTime.Sub(c.startTime).Nanoseconds()) / 1000., - ) - calculateStatePrevEventLength.WithLabelValues(c.algorithm, outcome).Observe( - float64(c.prevEventLength), - ) - calculateStateFullStateLength.WithLabelValues(c.algorithm, outcome).Observe( - float64(c.fullStateLength), - ) - calculateStateConflictLength.WithLabelValues(c.algorithm, outcome).Observe( - float64(c.conflictLength), - ) - return stateNID, err -} - -func init() { - prometheus.MustRegister( - calculateStateDurations, calculateStatePrevEventLength, - calculateStateFullStateLength, calculateStateConflictLength, - ) -} - -// CalculateAndStoreStateBeforeEvent calculates a snapshot of the state of a room before an event. -// Stores the snapshot of the state in the database. -// Returns a numeric ID for the snapshot of the state before the event. -func (v StateResolutionV1) CalculateAndStoreStateBeforeEvent( - ctx context.Context, - event gomatrixserverlib.Event, - roomNID types.RoomNID, -) (types.StateSnapshotNID, error) { - // Load the state at the prev events. - prevEventRefs := event.PrevEvents() - prevEventIDs := make([]string, len(prevEventRefs)) - for i := range prevEventRefs { - prevEventIDs[i] = prevEventRefs[i].EventID - } - - prevStates, err := v.db.StateAtEventIDs(ctx, prevEventIDs) - if err != nil { - return 0, err - } - - // The state before this event will be the state after the events that came before it. - return v.CalculateAndStoreStateAfterEvents(ctx, roomNID, prevStates) -} - -// CalculateAndStoreStateAfterEvents finds the room state after the given events. -// Stores the resulting state in the database and returns a numeric ID for that snapshot. -func (v StateResolutionV1) CalculateAndStoreStateAfterEvents( - ctx context.Context, - roomNID types.RoomNID, - prevStates []types.StateAtEvent, -) (types.StateSnapshotNID, error) { - metrics := calculateStateMetrics{startTime: time.Now(), prevEventLength: len(prevStates)} - - if len(prevStates) == 0 { - // 2) There weren't any prev_events for this event so the state is - // empty. - metrics.algorithm = "empty_state" - return metrics.stop(v.db.AddState(ctx, roomNID, nil, nil)) - } - - if len(prevStates) == 1 { - prevState := prevStates[0] - if prevState.EventStateKeyNID == 0 { - // 3) None of the previous events were state events and they all - // have the same state, so this event has exactly the same state - // as the previous events. - // This should be the common case. - metrics.algorithm = "no_change" - return metrics.stop(prevState.BeforeStateSnapshotNID, nil) - } - // The previous event was a state event so we need to store a copy - // of the previous state updated with that event. - stateBlockNIDLists, err := v.db.StateBlockNIDs( - ctx, []types.StateSnapshotNID{prevState.BeforeStateSnapshotNID}, - ) - if err != nil { - metrics.algorithm = "_load_state_blocks" - return metrics.stop(0, err) - } - stateBlockNIDs := stateBlockNIDLists[0].StateBlockNIDs - if len(stateBlockNIDs) < maxStateBlockNIDs { - // 4) The number of state data blocks is small enough that we can just - // add the state event as a block of size one to the end of the blocks. - metrics.algorithm = "single_delta" - return metrics.stop(v.db.AddState( - ctx, roomNID, stateBlockNIDs, []types.StateEntry{prevState.StateEntry}, - )) - } - // If there are too many deltas then we need to calculate the full state - // So fall through to calculateAndStoreStateAfterManyEvents - } - - return v.calculateAndStoreStateAfterManyEvents(ctx, roomNID, prevStates, metrics) -} - -// maxStateBlockNIDs is the maximum number of state data blocks to use to encode a snapshot of room state. -// Increasing this number means that we can encode more of the state changes as simple deltas which means that -// we need fewer entries in the state data table. However making this number bigger will increase the size of -// the rows in the state table itself and will require more index lookups when retrieving a snapshot. -// TODO: Tune this to get the right balance between size and lookup performance. -const maxStateBlockNIDs = 64 - -// calculateAndStoreStateAfterManyEvents finds the room state after the given events. -// This handles the slow path of calculateAndStoreStateAfterEvents for when there is more than one event. -// Stores the resulting state and returns a numeric ID for the snapshot. -func (v StateResolutionV1) calculateAndStoreStateAfterManyEvents( - ctx context.Context, - roomNID types.RoomNID, - prevStates []types.StateAtEvent, - metrics calculateStateMetrics, -) (types.StateSnapshotNID, error) { - - state, algorithm, conflictLength, err := - v.calculateStateAfterManyEvents(ctx, prevStates) - metrics.algorithm = algorithm - if err != nil { - return metrics.stop(0, err) - } - - // TODO: Check if we can encode the new state as a delta against the - // previous state. - metrics.conflictLength = conflictLength - metrics.fullStateLength = len(state) - return metrics.stop(v.db.AddState(ctx, roomNID, nil, state)) -} - -func (v StateResolutionV1) calculateStateAfterManyEvents( - ctx context.Context, prevStates []types.StateAtEvent, -) (state []types.StateEntry, algorithm string, conflictLength int, err error) { - var combined []types.StateEntry - // Conflict resolution. - // First stage: load the state after each of the prev events. - combined, err = v.LoadCombinedStateAfterEvents(ctx, prevStates) - if err != nil { - algorithm = "_load_combined_state" - return - } - - // Collect all the entries with the same type and key together. - // We don't care about the order here because the conflict resolution - // algorithm doesn't depend on the order of the prev events. - // Remove duplicate entires. - combined = combined[:util.SortAndUnique(stateEntrySorter(combined))] - - // Find the conflicts - conflicts := findDuplicateStateKeys(combined) - - if len(conflicts) > 0 { - conflictLength = len(conflicts) - - // 5) There are conflicting state events, for each conflict workout - // what the appropriate state event is. - - // Work out which entries aren't conflicted. - var notConflicted []types.StateEntry - for _, entry := range combined { - if _, ok := stateEntryMap(conflicts).lookup(entry.StateKeyTuple); !ok { - notConflicted = append(notConflicted, entry) - } - } - - var resolved []types.StateEntry - resolved, err = v.resolveConflicts(ctx, notConflicted, conflicts) - if err != nil { - algorithm = "_resolve_conflicts" - return - } - algorithm = "full_state_with_conflicts" - state = resolved - } else { - algorithm = "full_state_no_conflicts" - // 6) There weren't any conflicts - state = combined - } - return -} - -// resolveConflicts resolves a list of conflicted state entries. It takes two lists. -// The first is a list of all state entries that are not conflicted. -// The second is a list of all state entries that are conflicted -// A state entry is conflicted when there is more than one numeric event ID for the same state key tuple. -// Returns a list that combines the entries without conflicts with the result of state resolution for the entries with conflicts. -// The returned list is sorted by state key tuple. -// Returns an error if there was a problem talking to the database. -func (v StateResolutionV1) resolveConflicts( - ctx context.Context, - notConflicted, conflicted []types.StateEntry, -) ([]types.StateEntry, error) { - - // Load the conflicted events - conflictedEvents, eventIDMap, err := v.loadStateEvents(ctx, conflicted) - if err != nil { - return nil, err - } - - // Work out which auth events we need to load. - needed := gomatrixserverlib.StateNeededForAuth(conflictedEvents) - - // Find the numeric IDs for the necessary state keys. - var neededStateKeys []string - neededStateKeys = append(neededStateKeys, needed.Member...) - neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...) - stateKeyNIDMap, err := v.db.EventStateKeyNIDs(ctx, neededStateKeys) - if err != nil { - return nil, err - } - - // Load the necessary auth events. - tuplesNeeded := v.stateKeyTuplesNeeded(stateKeyNIDMap, needed) - var authEntries []types.StateEntry - for _, tuple := range tuplesNeeded { - if eventNID, ok := stateEntryMap(notConflicted).lookup(tuple); ok { - authEntries = append(authEntries, types.StateEntry{ - StateKeyTuple: tuple, - EventNID: eventNID, - }) - } - } - authEvents, _, err := v.loadStateEvents(ctx, authEntries) - if err != nil { - return nil, err - } - - // Resolve the conflicts. - resolvedEvents := gomatrixserverlib.ResolveStateConflicts(conflictedEvents, authEvents) - - // Map from the full events back to numeric state entries. - for _, resolvedEvent := range resolvedEvents { - entry, ok := eventIDMap[resolvedEvent.EventID()] - if !ok { - panic(fmt.Errorf("Missing state entry for event ID %q", resolvedEvent.EventID())) - } - notConflicted = append(notConflicted, entry) - } - - // Sort the result so it can be searched. - sort.Sort(stateEntrySorter(notConflicted)) - return notConflicted, nil -} - -// stateKeyTuplesNeeded works out which numeric state key tuples we need to authenticate some events. -func (v StateResolutionV1) stateKeyTuplesNeeded(stateKeyNIDMap map[string]types.EventStateKeyNID, stateNeeded gomatrixserverlib.StateNeeded) []types.StateKeyTuple { - var keyTuples []types.StateKeyTuple - if stateNeeded.Create { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomCreateNID, - EventStateKeyNID: types.EmptyStateKeyNID, - }) - } - if stateNeeded.PowerLevels { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomPowerLevelsNID, - EventStateKeyNID: types.EmptyStateKeyNID, - }) - } - if stateNeeded.JoinRules { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomJoinRulesNID, - EventStateKeyNID: types.EmptyStateKeyNID, - }) - } - for _, member := range stateNeeded.Member { - stateKeyNID, ok := stateKeyNIDMap[member] - if ok { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomMemberNID, - EventStateKeyNID: stateKeyNID, - }) - } - } - for _, token := range stateNeeded.ThirdPartyInvite { - stateKeyNID, ok := stateKeyNIDMap[token] - if ok { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomThirdPartyInviteNID, - EventStateKeyNID: stateKeyNID, - }) - } - } - return keyTuples -} - -// loadStateEvents loads the matrix events for a list of state entries. -// Returns a list of state events in no particular order and a map from string event ID back to state entry. -// The map can be used to recover which numeric state entry a given event is for. -// Returns an error if there was a problem talking to the database. -func (v StateResolutionV1) loadStateEvents( - ctx context.Context, entries []types.StateEntry, -) ([]gomatrixserverlib.Event, map[string]types.StateEntry, error) { - eventNIDs := make([]types.EventNID, len(entries)) - for i := range entries { - eventNIDs[i] = entries[i].EventNID - } - events, err := v.db.Events(ctx, eventNIDs) - if err != nil { - return nil, nil, err - } - eventIDMap := map[string]types.StateEntry{} - result := make([]gomatrixserverlib.Event, len(entries)) - for i := range entries { - event, ok := eventMap(events).lookup(entries[i].EventNID) - if !ok { - panic(fmt.Errorf("Corrupt DB: Missing event numeric ID %d", entries[i].EventNID)) - } - result[i] = event.Event - eventIDMap[event.Event.EventID()] = entries[i] - } - return result, eventIDMap, nil -} - -// findDuplicateStateKeys finds the state entries where the state key tuple appears more than once in a sorted list. -// Returns a sorted list of those state entries. -func findDuplicateStateKeys(a []types.StateEntry) []types.StateEntry { - var result []types.StateEntry - // j is the starting index of a block of entries with the same state key tuple. - j := 0 - for i := 1; i < len(a); i++ { - // Check if the state key tuple matches the start of the block - if a[j].StateKeyTuple != a[i].StateKeyTuple { - // If the state key tuple is different then we've reached the end of a block of duplicates. - // Check if the size of the block is bigger than one. - // If the size is one then there was only a single entry with that state key tuple so we don't add it to the result - if j+1 != i { - // Add the block to the result. - result = append(result, a[j:i]...) - } - // Start a new block for the next state key tuple. - j = i - } - } - // Check if the last block with the same state key tuple had more than one event in it. - if j+1 != len(a) { - result = append(result, a[j:]...) - } - return result -} - -type stateEntrySorter []types.StateEntry - -func (s stateEntrySorter) Len() int { return len(s) } -func (s stateEntrySorter) Less(i, j int) bool { return s[i].LessThan(s[j]) } -func (s stateEntrySorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -type stateBlockNIDListMap []types.StateBlockNIDList - -func (m stateBlockNIDListMap) lookup(stateNID types.StateSnapshotNID) (stateBlockNIDs []types.StateBlockNID, ok bool) { - list := []types.StateBlockNIDList(m) - i := sort.Search(len(list), func(i int) bool { - return list[i].StateSnapshotNID >= stateNID - }) - if i < len(list) && list[i].StateSnapshotNID == stateNID { - ok = true - stateBlockNIDs = list[i].StateBlockNIDs - } - return -} - -type stateEntryListMap []types.StateEntryList - -func (m stateEntryListMap) lookup(stateBlockNID types.StateBlockNID) (stateEntries []types.StateEntry, ok bool) { - list := []types.StateEntryList(m) - i := sort.Search(len(list), func(i int) bool { - return list[i].StateBlockNID >= stateBlockNID - }) - if i < len(list) && list[i].StateBlockNID == stateBlockNID { - ok = true - stateEntries = list[i].StateEntries - } - return -} - -type stateEntryByStateKeySorter []types.StateEntry - -func (s stateEntryByStateKeySorter) Len() int { return len(s) } -func (s stateEntryByStateKeySorter) Less(i, j int) bool { - return s[i].StateKeyTuple.LessThan(s[j].StateKeyTuple) -} -func (s stateEntryByStateKeySorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -type stateNIDSorter []types.StateSnapshotNID - -func (s stateNIDSorter) Len() int { return len(s) } -func (s stateNIDSorter) Less(i, j int) bool { return s[i] < s[j] } -func (s stateNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -func uniqueStateSnapshotNIDs(nids []types.StateSnapshotNID) []types.StateSnapshotNID { - return nids[:util.SortAndUnique(stateNIDSorter(nids))] -} - -type stateBlockNIDSorter []types.StateBlockNID - -func (s stateBlockNIDSorter) Len() int { return len(s) } -func (s stateBlockNIDSorter) Less(i, j int) bool { return s[i] < s[j] } -func (s stateBlockNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -func uniqueStateBlockNIDs(nids []types.StateBlockNID) []types.StateBlockNID { - return nids[:util.SortAndUnique(stateBlockNIDSorter(nids))] -} - -// Map from event type, state key tuple to numeric event ID. -// Implemented using binary search on a sorted array. -type stateEntryMap []types.StateEntry - -// lookup an entry in the event map. -func (m stateEntryMap) lookup(stateKey types.StateKeyTuple) (eventNID types.EventNID, ok bool) { - // Since the list is sorted we can implement this using binary search. - // This is faster than using a hash map. - // We don't have to worry about pathological cases because the keys are fixed - // size and are controlled by us. - list := []types.StateEntry(m) - i := sort.Search(len(list), func(i int) bool { - return !list[i].StateKeyTuple.LessThan(stateKey) - }) - if i < len(list) && list[i].StateKeyTuple == stateKey { - ok = true - eventNID = list[i].EventNID - } - return -} - -// Map from numeric event ID to event. -// Implemented using binary search on a sorted array. -type eventMap []types.Event - -// lookup an entry in the event map. -func (m eventMap) lookup(eventNID types.EventNID) (event *types.Event, ok bool) { - // Since the list is sorted we can implement this using binary search. - // This is faster than using a hash map. - // We don't have to worry about pathological cases because the keys are fixed - // size are controlled by us. - list := []types.Event(m) - i := sort.Search(len(list), func(i int) bool { - return list[i].EventNID >= eventNID - }) - if i < len(list) && list[i].EventNID == eventNID { - ok = true - event = &list[i] - } - return -} diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index f03d6feab..20db7ef7f 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -45,5 +45,5 @@ type Database interface { GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error) GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool) ([]types.EventNID, error) EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) - GetRoomVersionForRoom(ctx context.Context, roomNID types.RoomNID) (int64, error) + GetRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) } diff --git a/roomserver/storage/postgres/event_json_table.go b/roomserver/storage/postgres/event_json_table.go index 0b7ef6aa7..616eaf318 100644 --- a/roomserver/storage/postgres/event_json_table.go +++ b/roomserver/storage/postgres/event_json_table.go @@ -19,6 +19,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/types" ) @@ -86,7 +88,7 @@ func (s *eventJSONStatements) bulkSelectEventJSON( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventJSON: rows.close() failed") // We know that we will only get as many results as event NIDs // because of the unique constraint on event NIDs. diff --git a/roomserver/storage/postgres/event_state_keys_table.go b/roomserver/storage/postgres/event_state_keys_table.go index cbc29a69d..4c3496d91 100644 --- a/roomserver/storage/postgres/event_state_keys_table.go +++ b/roomserver/storage/postgres/event_state_keys_table.go @@ -114,7 +114,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKeyNID: rows.close() failed") result := make(map[string]types.EventStateKeyNID, len(eventStateKeys)) for rows.Next() { @@ -139,7 +139,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKey( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKey: rows.close() failed") result := make(map[types.EventStateKeyNID]string, len(eventStateKeyNIDs)) for rows.Next() { diff --git a/roomserver/storage/postgres/event_types_table.go b/roomserver/storage/postgres/event_types_table.go index faa887545..6537a5457 100644 --- a/roomserver/storage/postgres/event_types_table.go +++ b/roomserver/storage/postgres/event_types_table.go @@ -19,6 +19,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/lib/pq" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -132,7 +134,7 @@ func (s *eventTypeStatements) bulkSelectEventTypeNID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventTypeNID: rows.close() failed") result := make(map[string]types.EventTypeNID, len(eventTypes)) for rows.Next() { diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index d9b269bc8..0caa8199a 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -192,7 +192,7 @@ func (s *eventStatements) bulkSelectStateEventByID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateEventByID: rows.close() failed") // We know that we will only get as many results as event IDs // because of the unique constraint on event IDs. // So we can allocate an array of the correct size now. @@ -235,7 +235,7 @@ func (s *eventStatements) bulkSelectStateAtEventByID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventByID: rows.close() failed") results := make([]types.StateAtEvent, len(eventIDs)) i := 0 for ; rows.Next(); i++ { @@ -302,7 +302,7 @@ func (s *eventStatements) bulkSelectStateAtEventAndReference( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventAndReference: rows.close() failed") results := make([]types.StateAtEventAndReference, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -343,7 +343,7 @@ func (s *eventStatements) bulkSelectEventReference( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventReference: rows.close() failed") results := make([]gomatrixserverlib.EventReference, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -367,7 +367,7 @@ func (s *eventStatements) bulkSelectEventID(ctx context.Context, eventNIDs []typ if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventID: rows.close() failed") results := make(map[types.EventNID]string, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -394,7 +394,7 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, eventIDs []str if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventNID: rows.close() failed") results := make(map[string]types.EventNID, len(eventIDs)) for rows.Next() { var eventID string diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go index 603fed31b..f764b1561 100644 --- a/roomserver/storage/postgres/invite_table.go +++ b/roomserver/storage/postgres/invite_table.go @@ -120,7 +120,7 @@ func (s *inviteStatements) updateInviteRetired( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "updateInviteRetired: rows.close() failed") var eventIDs []string for rows.Next() { @@ -144,7 +144,7 @@ func (s *inviteStatements) selectInviteActiveForUserInRoom( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") var result []types.EventStateKeyNID for rows.Next() { var senderUserNID int64 diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index 70032fd1e..9c8a4c259 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -151,7 +151,7 @@ func (s *membershipStatements) selectMembershipsFromRoom( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoom: rows.close() failed") for rows.Next() { var eNID types.EventNID @@ -172,7 +172,7 @@ func (s *membershipStatements) selectMembershipsFromRoomAndMembership( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoomAndMembership: rows.close() failed") for rows.Next() { var eNID types.EventNID diff --git a/roomserver/storage/postgres/room_aliases_table.go b/roomserver/storage/postgres/room_aliases_table.go index 6de898c41..c37f383c9 100644 --- a/roomserver/storage/postgres/room_aliases_table.go +++ b/roomserver/storage/postgres/room_aliases_table.go @@ -18,6 +18,8 @@ package postgres import ( "context" "database/sql" + + "github.com/matrix-org/dendrite/common" ) const roomAliasesSchema = ` @@ -95,7 +97,7 @@ func (s *roomAliasesStatements) selectAliasesFromRoomID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed") var aliases []string for rows.Next() { diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index edd15a338..0fd9d5b53 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" ) const roomsSchema = ` @@ -42,13 +43,13 @@ CREATE TABLE IF NOT EXISTS roomserver_rooms ( state_snapshot_nid BIGINT NOT NULL DEFAULT 0, -- The version of the room, which will assist in determining the state resolution -- algorithm, event ID format, etc. - room_version BIGINT NOT NULL DEFAULT 1 + room_version TEXT NOT NULL ); ` // Same as insertEventTypeNIDSQL const insertRoomNIDSQL = "" + - "INSERT INTO roomserver_rooms (room_id) VALUES ($1)" + + "INSERT INTO roomserver_rooms (room_id, room_version) VALUES ($1, $2)" + " ON CONFLICT ON CONSTRAINT roomserver_room_id_unique" + " DO NOTHING RETURNING (room_nid)" @@ -64,6 +65,9 @@ const selectLatestEventNIDsForUpdateSQL = "" + const updateLatestEventNIDsSQL = "" + "UPDATE roomserver_rooms SET latest_event_nids = $2, last_event_sent_nid = $3, state_snapshot_nid = $4 WHERE room_nid = $1" +const selectRoomVersionForRoomIDSQL = "" + + "SELECT room_version FROM roomserver_rooms WHERE room_id = $1" + const selectRoomVersionForRoomNIDSQL = "" + "SELECT room_version FROM roomserver_rooms WHERE room_nid = $1" @@ -73,6 +77,7 @@ type roomStatements struct { selectLatestEventNIDsStmt *sql.Stmt selectLatestEventNIDsForUpdateStmt *sql.Stmt updateLatestEventNIDsStmt *sql.Stmt + selectRoomVersionForRoomIDStmt *sql.Stmt selectRoomVersionForRoomNIDStmt *sql.Stmt } @@ -87,16 +92,18 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { {&s.selectLatestEventNIDsStmt, selectLatestEventNIDsSQL}, {&s.selectLatestEventNIDsForUpdateStmt, selectLatestEventNIDsForUpdateSQL}, {&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL}, + {&s.selectRoomVersionForRoomIDStmt, selectRoomVersionForRoomIDSQL}, {&s.selectRoomVersionForRoomNIDStmt, selectRoomVersionForRoomNIDSQL}, }.prepare(db) } func (s *roomStatements) insertRoomNID( - ctx context.Context, txn *sql.Tx, roomID string, + ctx context.Context, txn *sql.Tx, + roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { var roomNID int64 stmt := common.TxStmt(txn, s.insertRoomNIDStmt) - err := stmt.QueryRowContext(ctx, roomID).Scan(&roomNID) + err := stmt.QueryRowContext(ctx, roomID, roomVersion).Scan(&roomNID) return types.RoomNID(roomNID), err } @@ -163,10 +170,19 @@ func (s *roomStatements) updateLatestEventNIDs( return err } +func (s *roomStatements) selectRoomVersionForRoomID( + ctx context.Context, txn *sql.Tx, roomID string, +) (gomatrixserverlib.RoomVersion, error) { + var roomVersion gomatrixserverlib.RoomVersion + stmt := common.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) + err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) + return roomVersion, err +} + func (s *roomStatements) selectRoomVersionForRoomNID( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, -) (int64, error) { - var roomVersion int64 +) (gomatrixserverlib.RoomVersion, error) { + var roomVersion gomatrixserverlib.RoomVersion stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) return roomVersion, err diff --git a/roomserver/storage/postgres/state_block_table.go b/roomserver/storage/postgres/state_block_table.go index e6f4f7fe9..b9246b763 100644 --- a/roomserver/storage/postgres/state_block_table.go +++ b/roomserver/storage/postgres/state_block_table.go @@ -21,6 +21,8 @@ import ( "fmt" "sort" + "github.com/matrix-org/dendrite/common" + "github.com/lib/pq" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/util" @@ -138,7 +140,7 @@ func (s *stateBlockStatements) bulkSelectStateBlockEntries( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockEntries: rows.close() failed") results := make([]types.StateEntryList, len(stateBlockNIDs)) // current is a pointer to the StateEntryList to append the state entries to. @@ -197,7 +199,7 @@ func (s *stateBlockStatements) bulkSelectFilteredStateBlockEntries( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectFilteredStateBlockEntries: rows.close() failed") var results []types.StateEntryList var current types.StateEntryList diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 77a792d68..084b83ce6 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -18,6 +18,9 @@ package postgres import ( "context" "database/sql" + "encoding/json" + + roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" // Import the postgres database driver. _ "github.com/lib/pq" @@ -68,7 +71,21 @@ func (d *Database) StoreEvent( } } - if roomNID, err = d.assignRoomNID(ctx, nil, event.RoomID()); err != nil { + // TODO: Here we should aim to have two different code paths for new rooms + // vs existing ones. + + // Get the default room version. If the client doesn't supply a room_version + // then we will use our configured default to create the room. + // https://matrix.org/docs/spec/client_server/r0.6.0#post-matrix-client-r0-createroom + // Note that the below logic depends on the m.room.create event being the + // first event that is persisted to the database when creating or joining a + // room. + var roomVersion gomatrixserverlib.RoomVersion + if roomVersion, err = extractRoomVersionFromCreateEvent(event); err != nil { + return 0, types.StateAtEvent{}, err + } + + if roomNID, err = d.assignRoomNID(ctx, nil, event.RoomID(), roomVersion); err != nil { return 0, types.StateAtEvent{}, err } @@ -120,14 +137,38 @@ func (d *Database) StoreEvent( }, nil } +func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( + gomatrixserverlib.RoomVersion, error, +) { + var err error + var roomVersion gomatrixserverlib.RoomVersion + // Look for m.room.create events. + if event.Type() != gomatrixserverlib.MRoomCreate { + return gomatrixserverlib.RoomVersion(""), nil + } + roomVersion = roomserverVersion.DefaultRoomVersion() + var createContent gomatrixserverlib.CreateContent + // The m.room.create event contains an optional "room_version" key in + // the event content, so we need to unmarshal that first. + if err = json.Unmarshal(event.Content(), &createContent); err != nil { + return gomatrixserverlib.RoomVersion(""), err + } + // A room version was specified in the event content? + if createContent.RoomVersion != nil { + roomVersion = gomatrixserverlib.RoomVersion(*createContent.RoomVersion) + } + return roomVersion, err +} + func (d *Database) assignRoomNID( - ctx context.Context, txn *sql.Tx, roomID string, + ctx context.Context, txn *sql.Tx, + roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { // Check if we already have a numeric ID in the database. roomNID, err := d.statements.selectRoomNID(ctx, txn, roomID) if err == sql.ErrNoRows { // We don't have a numeric ID so insert one into the database. - roomNID, err = d.statements.insertRoomNID(ctx, txn, roomID) + roomNID, err = d.statements.insertRoomNID(ctx, txn, roomID, roomVersion) if err == sql.ErrNoRows { // We raced with another insert so run the select again. roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) @@ -494,7 +535,8 @@ func (d *Database) MembershipUpdater( } }() - roomNID, err := d.assignRoomNID(ctx, txn, roomID) + // TODO: Room version here + roomNID, err := d.assignRoomNID(ctx, txn, roomID, "1") if err != nil { return nil, err } @@ -698,8 +740,16 @@ func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]type } func (d *Database) GetRoomVersionForRoom( + ctx context.Context, roomID string, +) (gomatrixserverlib.RoomVersion, error) { + return d.statements.selectRoomVersionForRoomID( + ctx, nil, roomID, + ) +} + +func (d *Database) GetRoomVersionForRoomNID( ctx context.Context, roomNID types.RoomNID, -) (int64, error) { +) (gomatrixserverlib.RoomVersion, error) { return d.statements.selectRoomVersionForRoomNID( ctx, nil, roomNID, ) diff --git a/roomserver/storage/sqlite3/event_json_table.go b/roomserver/storage/sqlite3/event_json_table.go index f6c83906a..fc661c1da 100644 --- a/roomserver/storage/sqlite3/event_json_table.go +++ b/roomserver/storage/sqlite3/event_json_table.go @@ -88,7 +88,7 @@ func (s *eventJSONStatements) bulkSelectEventJSON( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventJSON: rows.close() failed") // We know that we will only get as many results as event NIDs // because of the unique constraint on event NIDs. diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go index b8bc6c02d..fa8fc57eb 100644 --- a/roomserver/storage/sqlite3/event_state_keys_table.go +++ b/roomserver/storage/sqlite3/event_state_keys_table.go @@ -116,7 +116,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKeyNID: rows.close() failed") result := make(map[string]types.EventStateKeyNID, len(eventStateKeys)) for rows.Next() { var stateKey string @@ -142,7 +142,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKey( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKey: rows.close() failed") result := make(map[types.EventStateKeyNID]string, len(eventStateKeyNIDs)) for rows.Next() { var stateKey string diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go index edc06d4c6..777f8be79 100644 --- a/roomserver/storage/sqlite3/event_types_table.go +++ b/roomserver/storage/sqlite3/event_types_table.go @@ -138,7 +138,7 @@ func (s *eventTypeStatements) bulkSelectEventTypeNID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventTypeNID: rows.close() failed") result := make(map[string]types.EventTypeNID, len(eventTypes)) for rows.Next() { diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index 4fa095913..1e4ed448f 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -194,7 +194,7 @@ func (s *eventStatements) bulkSelectStateEventByID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateEventByID: rows.close() failed") // We know that we will only get as many results as event IDs // because of the unique constraint on event IDs. // So we can allocate an array of the correct size now. @@ -247,7 +247,7 @@ func (s *eventStatements) bulkSelectStateAtEventByID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventByID: rows.close() failed") results := make([]types.StateAtEvent, len(eventIDs)) i := 0 for ; rows.Next(); i++ { @@ -323,7 +323,7 @@ func (s *eventStatements) bulkSelectStateAtEventAndReference( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventAndReference: rows.close() failed") results := make([]types.StateAtEventAndReference, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -374,7 +374,7 @@ func (s *eventStatements) bulkSelectEventReference( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventReference: rows.close() failed") results := make([]gomatrixserverlib.EventReference, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -408,7 +408,7 @@ func (s *eventStatements) bulkSelectEventID(ctx context.Context, txn *sql.Tx, ev if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventID: rows.close() failed") results := make(map[types.EventNID]string, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -445,7 +445,7 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, e if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventNID: rows.close() failed") results := make(map[string]types.EventNID, len(eventIDs)) for rows.Next() { var eventID string diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index 641f80156..0ab3e6f36 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -137,7 +137,7 @@ func (s *inviteStatements) selectInviteActiveForUserInRoom( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") var result []types.EventStateKeyNID for rows.Next() { var senderUserNID int64 diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index 978776738..7ae28e4b8 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -134,7 +134,7 @@ func (s *membershipStatements) selectMembershipsFromRoom( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoom: rows.close() failed") for rows.Next() { var eNID types.EventNID @@ -154,7 +154,7 @@ func (s *membershipStatements) selectMembershipsFromRoomAndMembership( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoomAndMembership: rows.close() failed") for rows.Next() { var eNID types.EventNID diff --git a/roomserver/storage/sqlite3/room_aliases_table.go b/roomserver/storage/sqlite3/room_aliases_table.go index 71238b0e4..d29833918 100644 --- a/roomserver/storage/sqlite3/room_aliases_table.go +++ b/roomserver/storage/sqlite3/room_aliases_table.go @@ -103,7 +103,7 @@ func (s *roomAliasesStatements) selectAliasesFromRoomID( return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed") for rows.Next() { var alias string diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index 99c03476f..512b98137 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" ) const roomsSchema = ` @@ -31,13 +32,13 @@ const roomsSchema = ` latest_event_nids TEXT NOT NULL DEFAULT '[]', last_event_sent_nid INTEGER NOT NULL DEFAULT 0, state_snapshot_nid INTEGER NOT NULL DEFAULT 0, - room_version INTEGER NOT NULL DEFAULT 1 + room_version TEXT NOT NULL ); ` // Same as insertEventTypeNIDSQL const insertRoomNIDSQL = ` - INSERT INTO roomserver_rooms (room_id) VALUES ($1) + INSERT INTO roomserver_rooms (room_id, room_version) VALUES ($1, $2) ON CONFLICT DO NOTHING; ` @@ -53,6 +54,9 @@ const selectLatestEventNIDsForUpdateSQL = "" + const updateLatestEventNIDsSQL = "" + "UPDATE roomserver_rooms SET latest_event_nids = $1, last_event_sent_nid = $2, state_snapshot_nid = $3 WHERE room_nid = $4" +const selectRoomVersionForRoomIDSQL = "" + + "SELECT room_version FROM roomserver_rooms WHERE room_id = $1" + const selectRoomVersionForRoomNIDSQL = "" + "SELECT room_version FROM roomserver_rooms WHERE room_nid = $1" @@ -62,6 +66,7 @@ type roomStatements struct { selectLatestEventNIDsStmt *sql.Stmt selectLatestEventNIDsForUpdateStmt *sql.Stmt updateLatestEventNIDsStmt *sql.Stmt + selectRoomVersionForRoomIDStmt *sql.Stmt selectRoomVersionForRoomNIDStmt *sql.Stmt } @@ -76,16 +81,18 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { {&s.selectLatestEventNIDsStmt, selectLatestEventNIDsSQL}, {&s.selectLatestEventNIDsForUpdateStmt, selectLatestEventNIDsForUpdateSQL}, {&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL}, + {&s.selectRoomVersionForRoomIDStmt, selectRoomVersionForRoomIDSQL}, {&s.selectRoomVersionForRoomNIDStmt, selectRoomVersionForRoomNIDSQL}, }.prepare(db) } func (s *roomStatements) insertRoomNID( - ctx context.Context, txn *sql.Tx, roomID string, + ctx context.Context, txn *sql.Tx, + roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { var err error insertStmt := common.TxStmt(txn, s.insertRoomNIDStmt) - if _, err = insertStmt.ExecContext(ctx, roomID); err == nil { + if _, err = insertStmt.ExecContext(ctx, roomID, roomVersion); err == nil { return s.selectRoomNID(ctx, txn, roomID) } else { return types.RoomNID(0), err @@ -155,10 +162,19 @@ func (s *roomStatements) updateLatestEventNIDs( return err } +func (s *roomStatements) selectRoomVersionForRoomID( + ctx context.Context, txn *sql.Tx, roomID string, +) (gomatrixserverlib.RoomVersion, error) { + var roomVersion gomatrixserverlib.RoomVersion + stmt := common.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) + err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) + return roomVersion, err +} + func (s *roomStatements) selectRoomVersionForRoomNID( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, -) (int64, error) { - var roomVersion int64 +) (gomatrixserverlib.RoomVersion, error) { + var roomVersion gomatrixserverlib.RoomVersion stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) return roomVersion, err diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go index c94ca2385..cc7c75733 100644 --- a/roomserver/storage/sqlite3/state_block_table.go +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -137,7 +137,7 @@ func (s *stateBlockStatements) bulkSelectStateBlockEntries( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockEntries: rows.close() failed") results := make([]types.StateEntryList, len(stateBlockNIDs)) // current is a pointer to the StateEntryList to append the state entries to. @@ -207,7 +207,7 @@ func (s *stateBlockStatements) bulkSelectFilteredStateBlockEntries( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectFilteredStateBlockEntries: rows.close() failed") var results []types.StateEntryList var current types.StateEntryList diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index 0e1786eae..f367a779b 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -99,7 +99,7 @@ func (s *stateSnapshotStatements) bulkSelectStateBlockNIDs( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockNIDs: rows.close() failed") results := make([]types.StateBlockNIDList, len(stateNIDs)) i := 0 for ; rows.Next(); i++ { diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 41d4c2a94..28d608ca5 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -18,9 +18,12 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "errors" "net/url" + roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" @@ -90,7 +93,21 @@ func (d *Database) StoreEvent( } } - if roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID()); err != nil { + // TODO: Here we should aim to have two different code paths for new rooms + // vs existing ones. + + // Get the default room version. If the client doesn't supply a room_version + // then we will use our configured default to create the room. + // https://matrix.org/docs/spec/client_server/r0.6.0#post-matrix-client-r0-createroom + // Note that the below logic depends on the m.room.create event being the + // first event that is persisted to the database when creating or joining a + // room. + var roomVersion gomatrixserverlib.RoomVersion + if roomVersion, err = extractRoomVersionFromCreateEvent(event); err != nil { + return err + } + + if roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID(), roomVersion); err != nil { return err } @@ -149,14 +166,38 @@ func (d *Database) StoreEvent( }, nil } +func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( + gomatrixserverlib.RoomVersion, error, +) { + var err error + var roomVersion gomatrixserverlib.RoomVersion + // Look for m.room.create events. + if event.Type() != gomatrixserverlib.MRoomCreate { + return gomatrixserverlib.RoomVersion(""), nil + } + roomVersion = roomserverVersion.DefaultRoomVersion() + var createContent gomatrixserverlib.CreateContent + // The m.room.create event contains an optional "room_version" key in + // the event content, so we need to unmarshal that first. + if err = json.Unmarshal(event.Content(), &createContent); err != nil { + return gomatrixserverlib.RoomVersion(""), err + } + // A room version was specified in the event content? + if createContent.RoomVersion != nil { + roomVersion = gomatrixserverlib.RoomVersion(*createContent.RoomVersion) + } + return roomVersion, err +} + func (d *Database) assignRoomNID( - ctx context.Context, txn *sql.Tx, roomID string, + ctx context.Context, txn *sql.Tx, + roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (roomNID types.RoomNID, err error) { // Check if we already have a numeric ID in the database. roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) if err == sql.ErrNoRows { // We don't have a numeric ID so insert one into the database. - roomNID, err = d.statements.insertRoomNID(ctx, txn, roomID) + roomNID, err = d.statements.insertRoomNID(ctx, txn, roomID, roomVersion) if err == nil { // Now get the numeric ID back out of the database roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) @@ -630,7 +671,8 @@ func (d *Database) MembershipUpdater( } }() - roomNID, err := d.assignRoomNID(ctx, txn, roomID) + // TODO: Room version here + roomNID, err := d.assignRoomNID(ctx, txn, roomID, "1") if err != nil { return nil, err } @@ -852,8 +894,16 @@ func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]type } func (d *Database) GetRoomVersionForRoom( + ctx context.Context, roomID string, +) (gomatrixserverlib.RoomVersion, error) { + return d.statements.selectRoomVersionForRoomID( + ctx, nil, roomID, + ) +} + +func (d *Database) GetRoomVersionForRoomNID( ctx context.Context, roomNID types.RoomNID, -) (int64, error) { +) (gomatrixserverlib.RoomVersion, error) { return d.statements.selectRoomVersionForRoomNID( ctx, nil, roomNID, ) diff --git a/roomserver/version/version.go b/roomserver/version/version.go index 0943e3843..b18df49f5 100644 --- a/roomserver/version/version.go +++ b/roomserver/version/version.go @@ -15,85 +15,64 @@ package version import ( - "errors" + "fmt" - "github.com/matrix-org/dendrite/roomserver/state" -) - -type RoomVersionID int -type EventFormatID int - -const ( - RoomVersionV1 RoomVersionID = iota + 1 - RoomVersionV2 - RoomVersionV3 - RoomVersionV4 - RoomVersionV5 -) - -const ( - EventFormatV1 EventFormatID = iota + 1 // original event ID formatting - EventFormatV2 // event ID is event hash - EventFormatV3 // event ID is URL-safe base64 event hash + "github.com/matrix-org/gomatrixserverlib" ) +// RoomVersionDescription contains information about a room version, +// namely whether it is marked as supported or stable in this server +// version. +// A version is supported if the server has some support for rooms +// that are this version. A version is marked as stable or unstable +// in order to hint whether the version should be used to clients +// calling the /capabilities endpoint. +// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-capabilities type RoomVersionDescription struct { - Supported bool - Stable bool - StateResolution state.StateResolutionVersion - EventFormat EventFormatID - EnforceSigningKeyValidity bool + Supported bool + Stable bool } -var roomVersions = map[RoomVersionID]RoomVersionDescription{ - RoomVersionV1: RoomVersionDescription{ - Supported: true, - Stable: true, - StateResolution: state.StateResolutionAlgorithmV1, - EventFormat: EventFormatV1, - EnforceSigningKeyValidity: false, +var roomVersions = map[gomatrixserverlib.RoomVersion]RoomVersionDescription{ + gomatrixserverlib.RoomVersionV1: RoomVersionDescription{ + Supported: true, + Stable: true, }, - RoomVersionV2: RoomVersionDescription{ - Supported: false, - Stable: true, - StateResolution: state.StateResolutionAlgorithmV2, - EventFormat: EventFormatV1, - EnforceSigningKeyValidity: false, + gomatrixserverlib.RoomVersionV2: RoomVersionDescription{ + Supported: false, + Stable: false, }, - RoomVersionV3: RoomVersionDescription{ - Supported: false, - Stable: true, - StateResolution: state.StateResolutionAlgorithmV2, - EventFormat: EventFormatV2, - EnforceSigningKeyValidity: false, + gomatrixserverlib.RoomVersionV3: RoomVersionDescription{ + Supported: false, + Stable: false, }, - RoomVersionV4: RoomVersionDescription{ - Supported: false, - Stable: true, - StateResolution: state.StateResolutionAlgorithmV2, - EventFormat: EventFormatV3, - EnforceSigningKeyValidity: false, + gomatrixserverlib.RoomVersionV4: RoomVersionDescription{ + Supported: false, + Stable: false, }, - RoomVersionV5: RoomVersionDescription{ - Supported: false, - Stable: true, - StateResolution: state.StateResolutionAlgorithmV2, - EventFormat: EventFormatV3, - EnforceSigningKeyValidity: true, + gomatrixserverlib.RoomVersionV5: RoomVersionDescription{ + Supported: false, + Stable: false, }, } -func GetDefaultRoomVersion() RoomVersionID { - return RoomVersionV1 +// DefaultRoomVersion contains the room version that will, by +// default, be used to create new rooms on this server. +func DefaultRoomVersion() gomatrixserverlib.RoomVersion { + return gomatrixserverlib.RoomVersionV1 } -func GetRoomVersions() map[RoomVersionID]RoomVersionDescription { +// RoomVersions returns a map of all known room versions to this +// server. +func RoomVersions() map[gomatrixserverlib.RoomVersion]RoomVersionDescription { return roomVersions } -func GetSupportedRoomVersions() map[RoomVersionID]RoomVersionDescription { - versions := make(map[RoomVersionID]RoomVersionDescription) - for id, version := range GetRoomVersions() { +// SupportedRoomVersions returns a map of descriptions for room +// versions that are supported by this homeserver. +func SupportedRoomVersions() map[gomatrixserverlib.RoomVersion]RoomVersionDescription { + versions := make(map[gomatrixserverlib.RoomVersion]RoomVersionDescription) + for id, version := range RoomVersions() { if version.Supported { versions[id] = version } @@ -101,12 +80,46 @@ func GetSupportedRoomVersions() map[RoomVersionID]RoomVersionDescription { return versions } -func GetSupportedRoomVersion(version RoomVersionID) (desc RoomVersionDescription, err error) { +// RoomVersion returns information about a specific room version. +// An UnknownVersionError is returned if the version is not known +// to the server. +func RoomVersion(version gomatrixserverlib.RoomVersion) (RoomVersionDescription, error) { if version, ok := roomVersions[version]; ok { - desc = version + return version, nil } - if !desc.Supported { - err = errors.New("unsupported room version") - } - return + return RoomVersionDescription{}, UnknownVersionError{version} +} + +// SupportedRoomVersion returns information about a specific room +// version. An UnknownVersionError is returned if the version is not +// known to the server, or an UnsupportedVersionError is returned if +// the version is known but specifically marked as unsupported. +func SupportedRoomVersion(version gomatrixserverlib.RoomVersion) (RoomVersionDescription, error) { + result, err := RoomVersion(version) + if err != nil { + return RoomVersionDescription{}, err + } + if !result.Supported { + return RoomVersionDescription{}, UnsupportedVersionError{version} + } + return result, nil +} + +// UnknownVersionError is caused when the room version is not known. +type UnknownVersionError struct { + Version gomatrixserverlib.RoomVersion +} + +func (e UnknownVersionError) Error() string { + return fmt.Sprintf("room version '%s' is not known", e.Version) +} + +// UnsupportedVersionError is caused when the room version is specifically +// marked as unsupported. +type UnsupportedVersionError struct { + Version gomatrixserverlib.RoomVersion +} + +func (e UnsupportedVersionError) Error() string { + return fmt.Sprintf("room version '%s' is marked as unsupported", e.Version) } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 5dbef4b7d..f1e68c262 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -100,8 +100,9 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( ) error { ev := msg.Event log.WithFields(log.Fields{ - "event_id": ev.EventID(), - "room_id": ev.RoomID(), + "event_id": ev.EventID(), + "room_id": ev.RoomID(), + "room_version": ev.RoomVersion, }).Info("received event from roomserver") addsStateEvents, err := s.lookupStateEvents(msg.AddsStateEventIDs, ev) @@ -186,8 +187,8 @@ func (s *OutputRoomEventConsumer) onRetireInviteEvent( // lookupStateEvents looks up the state events that are added by a new event. func (s *OutputRoomEventConsumer) lookupStateEvents( - addsStateEventIDs []string, event gomatrixserverlib.Event, -) ([]gomatrixserverlib.Event, error) { + addsStateEventIDs []string, event gomatrixserverlib.HeaderedEvent, +) ([]gomatrixserverlib.HeaderedEvent, error) { // Fast path if there aren't any new state events. if len(addsStateEventIDs) == 0 { return nil, nil @@ -195,7 +196,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // Fast path if the only state event added is the event itself. if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() { - return []gomatrixserverlib.Event{event}, nil + return []gomatrixserverlib.HeaderedEvent{event}, nil } // Check if this is re-adding a state events that we previously processed @@ -241,7 +242,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( return result, nil } -func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.Event) (gomatrixserverlib.Event, error) { +func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.HeaderedEvent) (gomatrixserverlib.HeaderedEvent, error) { var stateKey string if event.StateKey() == nil { stateKey = "" @@ -250,7 +251,7 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.Event } prevEvent, err := s.db.GetStateEvent( - context.TODO(), event.Type(), event.RoomID(), stateKey, + context.TODO(), event.RoomID(), event.Type(), stateKey, ) if err != nil { return event, err @@ -266,10 +267,11 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.Event PrevSender: prevEvent.Sender(), } - return event.SetUnsigned(prev) + event.Event, err = event.SetUnsigned(prev) + return event, err } -func missingEventsFrom(events []gomatrixserverlib.Event, required []string) []string { +func missingEventsFrom(events []gomatrixserverlib.HeaderedEvent, required []string) []string { have := map[string]bool{} for _, event := range events { have[event.EventID()] = true diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 83bf75b2e..cd21c3bd8 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -177,11 +177,11 @@ func (r *messagesReq) retrieveEvents() ( r.ctx, r.from, r.to, r.roomID, r.limit, r.backwardOrdering, ) if err != nil { - err = fmt.Errorf("GetEventsInRange: %s", err) + err = fmt.Errorf("GetEventsInRange: %w", err) return } - var events []gomatrixserverlib.Event + var events []gomatrixserverlib.HeaderedEvent // There can be two reasons for streamEvents to be empty: either we've // reached the oldest event in the room (or the most recent one, depending @@ -217,7 +217,7 @@ func (r *messagesReq) retrieveEvents() ( } // Convert all of the events into client events. - clientEvents = gomatrixserverlib.ToClientEvents(events, gomatrixserverlib.FormatAll) + clientEvents = gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatAll) // Get the position of the first and the last event in the room's topology. // This position is currently determined by the event's depth, so we could // also use it instead of retrieving from the database. However, if we ever @@ -228,14 +228,14 @@ func (r *messagesReq) retrieveEvents() ( r.ctx, events[0].EventID(), ) if err != nil { - err = fmt.Errorf("EventPositionInTopology: for start event %s: %s", events[0].EventID(), err) + err = fmt.Errorf("EventPositionInTopology: for start event %s: %w", events[0].EventID(), err) return } endPos, err := r.db.EventPositionInTopology( r.ctx, events[len(events)-1].EventID(), ) if err != nil { - err = fmt.Errorf("EventPositionInTopology: for end event %s: %s", events[len(events)-1].EventID(), err) + err = fmt.Errorf("EventPositionInTopology: for end event %s: %w", events[len(events)-1].EventID(), err) return } // Generate pagination tokens to send to the client using the positions @@ -273,7 +273,7 @@ func (r *messagesReq) retrieveEvents() ( // Returns an error if there was an issue talking with the database or // backfilling. func (r *messagesReq) handleEmptyEventsSlice() ( - events []gomatrixserverlib.Event, err error, + events []gomatrixserverlib.HeaderedEvent, err error, ) { backwardExtremities, err := r.db.BackwardExtremitiesForRoom(r.ctx, r.roomID) @@ -287,7 +287,7 @@ func (r *messagesReq) handleEmptyEventsSlice() ( } else { // If not, it means the slice was empty because we reached the room's // creation, so return an empty slice. - events = []gomatrixserverlib.Event{} + events = []gomatrixserverlib.HeaderedEvent{} } return @@ -299,7 +299,7 @@ func (r *messagesReq) handleEmptyEventsSlice() ( // through backfilling if needed. // Returns an error if there was an issue while backfilling. func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent) ( - events []gomatrixserverlib.Event, err error, + events []gomatrixserverlib.HeaderedEvent, err error, ) { // Check if we have enough events. isSetLargeEnough := true @@ -326,7 +326,7 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent // Backfill is needed if we've reached a backward extremity and need more // events. It's only needed if the direction is backward. if len(backwardExtremities) > 0 && !isSetLargeEnough && r.backwardOrdering { - var pdus []gomatrixserverlib.Event + var pdus []gomatrixserverlib.HeaderedEvent // Only ask the remote server for enough events to reach the limit. pdus, err = r.backfill(backwardExtremities, r.limit-len(streamEvents)) if err != nil { @@ -400,7 +400,7 @@ func (r *messagesReq) containsBackwardExtremity(events []types.StreamEvent) (boo // event, or if there is no remote homeserver to contact. // Returns an error if there was an issue with retrieving the list of servers in // the room or sending the request. -func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserverlib.Event, error) { +func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { // Query the list of servers in the room when one of the backward extremities // was sent. var serversResponse api.QueryServersInRoomAtEventResponse @@ -428,7 +428,7 @@ func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserv } } - pdus := make([]gomatrixserverlib.Event, 0) + pdus := make([]gomatrixserverlib.HeaderedEvent, 0) // If the roomserver responded with at least one server that isn't us, // send it a request for backfill. @@ -440,13 +440,20 @@ func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserv return nil, err } - pdus = txn.PDUs + for _, p := range txn.PDUs { + pdus = append(pdus, p.Headered(gomatrixserverlib.RoomVersionV1)) + } // Store the events in the database, while marking them as unfit to show // up in responses to sync requests. for _, pdu := range pdus { + headered := pdu.Headered(gomatrixserverlib.RoomVersionV1) if _, err = r.db.WriteEvent( - r.ctx, &pdu, []gomatrixserverlib.Event{}, []string{}, []string{}, + r.ctx, + &headered, + []gomatrixserverlib.HeaderedEvent{}, + []string{}, + []string{}, nil, true, ); err != nil { return nil, err @@ -486,7 +493,7 @@ func setToDefault( // timestamp of two Matrix events. // Returns true if the first event happened before the second one, false // otherwise. -func sortEvents(e1 *gomatrixserverlib.Event, e2 *gomatrixserverlib.Event) bool { +func sortEvents(e1 *gomatrixserverlib.HeaderedEvent, e2 *gomatrixserverlib.HeaderedEvent) bool { t := e1.OriginServerTS().Time() return e2.OriginServerTS().Time().After(t) } diff --git a/syncapi/routing/state.go b/syncapi/routing/state.go index 5688086cd..87b6396ac 100644 --- a/syncapi/routing/state.go +++ b/syncapi/routing/state.go @@ -56,7 +56,9 @@ func OnIncomingStateRequest(req *http.Request, db storage.Database, roomID strin // Fill the prev_content and replaces_state keys if necessary for _, event := range stateEvents { stateEvent := stateEventInStateResp{ - ClientEvent: gomatrixserverlib.ToClientEvent(event, gomatrixserverlib.FormatAll), + ClientEvent: gomatrixserverlib.HeaderedToClientEvents( + []gomatrixserverlib.HeaderedEvent{event}, gomatrixserverlib.FormatAll, + )[0], } var prevEventRef types.PrevEventRef if len(event.Unsigned()) > 0 { @@ -113,7 +115,7 @@ func OnIncomingStateTypeRequest(req *http.Request, db storage.Database, roomID s } stateEvent := stateEventInStateResp{ - ClientEvent: gomatrixserverlib.ToClientEvent(*event, gomatrixserverlib.FormatAll), + ClientEvent: gomatrixserverlib.HeaderedToClientEvent(*event, gomatrixserverlib.FormatAll), } return util.JSONResponse{ diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 0abeac34d..b6dc19696 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -29,16 +29,16 @@ import ( type Database interface { common.PartitionStorer AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) - Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) - WriteEvent(context.Context, *gomatrixserverlib.Event, []gomatrixserverlib.Event, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) - GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.Event, error) - GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.Event, err error) + Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) + WriteEvent(context.Context, *gomatrixserverlib.HeaderedEvent, []gomatrixserverlib.HeaderedEvent, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) + GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) + GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) SyncPosition(ctx context.Context) (types.PaginationToken, error) IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error) GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) UpsertAccountData(ctx context.Context, userID, roomID, dataType string) (types.StreamPosition, error) - AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.Event) (types.StreamPosition, error) + AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent) (types.StreamPosition, error) RetireInviteEvent(ctx context.Context, inviteEventID string) error SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) AddTypingUser(userID, roomID string, expireTime *time.Time) types.StreamPosition @@ -48,6 +48,6 @@ type Database interface { EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error) BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) MaxTopologicalPosition(ctx context.Context, roomID string) (types.StreamPosition, error) - StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.Event + StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) } diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index d1811aa66..d1e3b527f 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -118,7 +118,7 @@ func (s *accountDataStatements) selectAccountDataInRange( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectAccountDataInRange: rows.close() failed") for rows.Next() { var dataType string diff --git a/syncapi/storage/postgres/backward_extremities_table.go b/syncapi/storage/postgres/backward_extremities_table.go index d63c546e3..8286ca434 100644 --- a/syncapi/storage/postgres/backward_extremities_table.go +++ b/syncapi/storage/postgres/backward_extremities_table.go @@ -17,6 +17,8 @@ package postgres import ( "context" "database/sql" + + "github.com/matrix-org/dendrite/common" ) const backwardExtremitiesSchema = ` @@ -91,7 +93,7 @@ func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") for rows.Next() { var eID string diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 6f5c1e803..ab8f07b21 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( -- The state_key value for this state event e.g '' state_key TEXT NOT NULL, -- The JSON for the event. Stored as TEXT because this should be valid UTF-8. - event_json TEXT NOT NULL, + headered_event_json TEXT NOT NULL, -- The 'content.membership' value if this event is an m.room.member event. For other -- events, this will be NULL. membership TEXT, @@ -59,10 +59,10 @@ CREATE INDEX IF NOT EXISTS syncapi_membership_idx ON syncapi_current_room_state( ` const upsertRoomStateSQL = "" + - "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, event_json, membership, added_at)" + + "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, headered_event_json, membership, added_at)" + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + " ON CONFLICT ON CONSTRAINT syncapi_room_state_unique" + - " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, event_json = $7, membership = $8, added_at = $9" + " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, headered_event_json = $7, membership = $8, added_at = $9" const deleteRoomStateByEventIDSQL = "" + "DELETE FROM syncapi_current_room_state WHERE event_id = $1" @@ -71,7 +71,7 @@ const selectRoomIDsWithMembershipSQL = "" + "SELECT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2" const selectCurrentStateSQL = "" + - "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1" + + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" + " AND ( $2::text[] IS NULL OR sender = ANY($2) )" + " AND ( $3::text[] IS NULL OR NOT(sender = ANY($3)) )" + " AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" + @@ -83,14 +83,14 @@ const selectJoinedUsersSQL = "" + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" const selectStateEventSQL = "" + - "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" const selectEventsWithEventIDsSQL = "" + // TODO: The session_id and transaction_id blanks are here because otherwise // the rowsToStreamEvents expects there to be exactly five columns. We need to // figure out if these really need to be in the DB, and if so, we need a // better permanent fix for this. - neilalexander, 2 Jan 2020 - "SELECT added_at, event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + + "SELECT added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + " FROM syncapi_current_room_state WHERE event_id = ANY($1)" type currentRoomStateStatements struct { @@ -140,7 +140,7 @@ func (s *currentRoomStateStatements) selectJoinedUsers( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectJoinedUsers: rows.close() failed") result := make(map[string][]string) for rows.Next() { @@ -168,7 +168,7 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithMembership: rows.close() failed") var result []string for rows.Next() { @@ -185,7 +185,7 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( func (s *currentRoomStateStatements) selectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, stateFilter *gomatrixserverlib.StateFilter, -) ([]gomatrixserverlib.Event, error) { +) ([]gomatrixserverlib.HeaderedEvent, error) { stmt := common.TxStmt(txn, s.selectCurrentStateStmt) rows, err := stmt.QueryContext(ctx, roomID, pq.StringArray(stateFilter.Senders), @@ -198,7 +198,7 @@ func (s *currentRoomStateStatements) selectCurrentState( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectCurrentState: rows.close() failed") return rowsToEvents(rows) } @@ -213,7 +213,7 @@ func (s *currentRoomStateStatements) deleteRoomStateByEventID( func (s *currentRoomStateStatements) upsertRoomState( ctx context.Context, txn *sql.Tx, - event gomatrixserverlib.Event, membership *string, addedAt types.StreamPosition, + event gomatrixserverlib.HeaderedEvent, membership *string, addedAt types.StreamPosition, ) error { // Parse content as JSON and search for an "url" key containsURL := false @@ -223,9 +223,14 @@ func (s *currentRoomStateStatements) upsertRoomState( _, containsURL = content["url"] } + headeredJSON, err := json.Marshal(event) + if err != nil { + return err + } + // upsert state event stmt := common.TxStmt(txn, s.upsertRoomStateStmt) - _, err := stmt.ExecContext( + _, err = stmt.ExecContext( ctx, event.RoomID(), event.EventID(), @@ -233,7 +238,7 @@ func (s *currentRoomStateStatements) upsertRoomState( event.Sender(), containsURL, *event.StateKey(), - event.JSON(), + headeredJSON, membership, addedAt, ) @@ -248,20 +253,20 @@ func (s *currentRoomStateStatements) selectEventsWithEventIDs( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed") return rowsToStreamEvents(rows) } -func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.Event, error) { - result := []gomatrixserverlib.Event{} +func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.HeaderedEvent, error) { + result := []gomatrixserverlib.HeaderedEvent{} for rows.Next() { var eventBytes []byte if err := rows.Scan(&eventBytes); err != nil { return nil, err } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, err } result = append(result, ev) @@ -271,7 +276,7 @@ func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.Event, error) { func (s *currentRoomStateStatements) selectStateEvent( ctx context.Context, roomID, evType, stateKey string, -) (*gomatrixserverlib.Event, error) { +) (*gomatrixserverlib.HeaderedEvent, error) { stmt := s.selectStateEventStmt var res []byte err := stmt.QueryRowContext(ctx, roomID, evType, stateKey).Scan(&res) @@ -281,6 +286,9 @@ func (s *currentRoomStateStatements) selectStateEvent( if err != nil { return nil, err } - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(res, false) + var ev gomatrixserverlib.HeaderedEvent + if err = json.Unmarshal(res, &ev); err != nil { + return nil, err + } return &ev, err } diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go index 2cb8fb199..ca0c64fb9 100644 --- a/syncapi/storage/postgres/invites_table.go +++ b/syncapi/storage/postgres/invites_table.go @@ -18,6 +18,7 @@ package postgres import ( "context" "database/sql" + "encoding/json" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" @@ -30,7 +31,7 @@ CREATE TABLE IF NOT EXISTS syncapi_invite_events ( event_id TEXT NOT NULL, room_id TEXT NOT NULL, target_user_id TEXT NOT NULL, - event_json TEXT NOT NULL + headered_event_json TEXT NOT NULL ); -- For looking up the invites for a given user. @@ -44,14 +45,14 @@ CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx const insertInviteEventSQL = "" + "INSERT INTO syncapi_invite_events (" + - " room_id, event_id, target_user_id, event_json" + + " room_id, event_id, target_user_id, headered_event_json" + ") VALUES ($1, $2, $3, $4) RETURNING id" const deleteInviteEventSQL = "" + "DELETE FROM syncapi_invite_events WHERE event_id = $1" const selectInviteEventsInRangeSQL = "" + - "SELECT room_id, event_json FROM syncapi_invite_events" + + "SELECT room_id, headered_event_json FROM syncapi_invite_events" + " WHERE target_user_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC" @@ -86,14 +87,20 @@ func (s *inviteEventsStatements) prepare(db *sql.DB) (err error) { } func (s *inviteEventsStatements) insertInviteEvent( - ctx context.Context, inviteEvent gomatrixserverlib.Event, + ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (streamPos types.StreamPosition, err error) { + var headeredJSON []byte + headeredJSON, err = json.Marshal(inviteEvent) + if err != nil { + return + } + err = s.insertInviteEventStmt.QueryRowContext( ctx, inviteEvent.RoomID(), inviteEvent.EventID(), *inviteEvent.StateKey(), - inviteEvent.JSON(), + headeredJSON, ).Scan(&streamPos) return } @@ -109,14 +116,14 @@ func (s *inviteEventsStatements) deleteInviteEvent( // active invites for the target user ID in the supplied range. func (s *inviteEventsStatements) selectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, startPos, endPos types.StreamPosition, -) (map[string]gomatrixserverlib.Event, error) { +) (map[string]gomatrixserverlib.HeaderedEvent, error) { stmt := common.TxStmt(txn, s.selectInviteEventsInRangeStmt) rows, err := stmt.QueryContext(ctx, targetUserID, startPos, endPos) if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck - result := map[string]gomatrixserverlib.Event{} + defer common.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") + result := map[string]gomatrixserverlib.HeaderedEvent{} for rows.Next() { var ( roomID string @@ -126,8 +133,8 @@ func (s *inviteEventsStatements) selectInviteEventsInRange( return nil, err } - event, err := gomatrixserverlib.NewEventFromTrustedJSON(eventJSON, false) - if err != nil { + var event gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventJSON, &event); err != nil { return nil, err } diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 2db46c5db..5f9a1d0c6 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -44,8 +44,9 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( event_id TEXT NOT NULL CONSTRAINT syncapi_event_id_idx UNIQUE, -- The 'room_id' key for the event. room_id TEXT NOT NULL, - -- The JSON for the event. Stored as TEXT because this should be valid UTF-8. - event_json TEXT NOT NULL, + -- The headered JSON for the event, containing potentially additional metadata such as + -- the room version. Stored as TEXT because this should be valid UTF-8. + headered_event_json TEXT NOT NULL, -- The event type e.g 'm.room.member'. type TEXT NOT NULL, -- The 'sender' property of the event. @@ -70,26 +71,26 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( const insertEventSQL = "" + "INSERT INTO syncapi_output_room_events (" + - "room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + + "room_id, event_id, headered_event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) " + "ON CONFLICT ON CONSTRAINT syncapi_event_id_idx DO UPDATE SET exclude_from_sync = $11 " + "RETURNING id" const selectEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = ANY($1)" + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = ANY($1)" const selectRecentEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC LIMIT $4" const selectRecentEventsForSyncSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" + " ORDER BY id DESC LIMIT $4" const selectEarlyEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id ASC LIMIT $4" @@ -98,7 +99,7 @@ const selectMaxEventIDSQL = "" + // In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). const selectStateInRangeSQL = "" + - "SELECT id, event_json, exclude_from_sync, add_state_ids, remove_state_ids" + + "SELECT id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2) AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + " AND ( $3::text[] IS NULL OR sender = ANY($3) )" + @@ -169,7 +170,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( if err != nil { return nil, nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectStateInRange: rows.close() failed") // Fetch all the state change events for all rooms between the two positions then loop each event and: // - Keep a cache of the event by ID (99% of state change events are for the event itself) // - For each room ID, build up an array of event IDs which represents cumulative adds/removes @@ -203,8 +204,8 @@ func (s *outputRoomEventsStatements) selectStateInRange( } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, nil, err } needSet := stateNeeded[ev.RoomID()] @@ -220,7 +221,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( stateNeeded[ev.RoomID()] = needSet eventIDToEvent[ev.EventID()] = types.StreamEvent{ - Event: ev, + HeaderedEvent: ev, StreamPosition: streamPos, ExcludeFromSync: excludeFromSync, } @@ -248,7 +249,7 @@ func (s *outputRoomEventsStatements) selectMaxEventID( // of the inserted event. func (s *outputRoomEventsStatements) insertEvent( ctx context.Context, txn *sql.Tx, - event *gomatrixserverlib.Event, addState, removeState []string, + event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, transactionID *api.TransactionID, excludeFromSync bool, ) (streamPos types.StreamPosition, err error) { var txnID *string @@ -266,12 +267,18 @@ func (s *outputRoomEventsStatements) insertEvent( _, containsURL = content["url"] } + var headeredJSON []byte + headeredJSON, err = json.Marshal(event) + if err != nil { + return + } + stmt := common.TxStmt(txn, s.insertEventStmt) err = stmt.QueryRowContext( ctx, event.RoomID(), event.EventID(), - event.JSON(), + headeredJSON, event.Type(), event.Sender(), containsURL, @@ -303,7 +310,7 @@ func (s *outputRoomEventsStatements) selectRecentEvents( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { return nil, err @@ -330,7 +337,7 @@ func (s *outputRoomEventsStatements) selectEarlyEvents( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEarlyEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { return nil, err @@ -354,7 +361,7 @@ func (s *outputRoomEventsStatements) selectEvents( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEvents: rows.close() failed") return rowsToStreamEvents(rows) } @@ -373,8 +380,8 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { return nil, err } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, err } @@ -386,7 +393,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { } result = append(result, types.StreamEvent{ - Event: ev, + HeaderedEvent: ev, StreamPosition: streamPos, TransactionID: transactionID, ExcludeFromSync: excludeFromSync, diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index 78a381da9..280d4ec39 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -18,6 +18,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -102,7 +104,7 @@ func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { // insertEventInTopology inserts the given event in the room's topology, based // on the event's depth. func (s *outputRoomEventsTopologyStatements) insertEventInTopology( - ctx context.Context, event *gomatrixserverlib.Event, + ctx context.Context, event *gomatrixserverlib.HeaderedEvent, ) (err error) { _, err = s.insertEventInTopologyStmt.ExecContext( ctx, event.EventID(), event.Depth(), event.RoomID(), @@ -134,7 +136,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( } else if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEventIDsInRange: rows.close() failed") // Return the IDs. var eventID string @@ -177,7 +179,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsFromPosition( } else if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEventIDsFromPosition: rows.close() failed") // Return the IDs. var eventID string for rows.Next() { diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index abb5b4a4c..ccf1c5656 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -37,7 +37,7 @@ import ( type stateDelta struct { roomID string - stateEvents []gomatrixserverlib.Event + stateEvents []gomatrixserverlib.HeaderedEvent membership string // The PDU stream position of the latest membership event for this user, if applicable. // Can be 0 if there is no membership event in this delta. @@ -100,7 +100,7 @@ func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[s // If an event is not found in the database then it will be omitted from the list. // Returns an error if there was a problem talking with the database. // Does not include any transaction IDs in the returned events. -func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { +func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) { streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs) if err != nil { return nil, err @@ -111,7 +111,7 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([ return d.StreamEventsToEvents(nil, streamEvents), nil } -func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, ev *gomatrixserverlib.Event) error { +func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent) error { // If the event is already known as a backward extremity, don't consider // it as such anymore now that we have it. isBackwardExtremity, err := d.backwardExtremities.isBackwardExtremity(ctx, ev.RoomID(), ev.EventID()) @@ -155,8 +155,8 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, ev // Returns an error if there was a problem inserting this event. func (d *SyncServerDatasource) WriteEvent( ctx context.Context, - ev *gomatrixserverlib.Event, - addStateEvents []gomatrixserverlib.Event, + ev *gomatrixserverlib.HeaderedEvent, + addStateEvents []gomatrixserverlib.HeaderedEvent, addStateEventIDs, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool, ) (pduPosition types.StreamPosition, returnErr error) { @@ -192,7 +192,7 @@ func (d *SyncServerDatasource) WriteEvent( func (d *SyncServerDatasource) updateRoomState( ctx context.Context, txn *sql.Tx, removedEventIDs []string, - addedEvents []gomatrixserverlib.Event, + addedEvents []gomatrixserverlib.HeaderedEvent, pduPosition types.StreamPosition, ) error { // remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add. @@ -228,7 +228,7 @@ func (d *SyncServerDatasource) updateRoomState( // If there was an issue during the retrieval, returns an error func (d *SyncServerDatasource) GetStateEvent( ctx context.Context, roomID, evType, stateKey string, -) (*gomatrixserverlib.Event, error) { +) (*gomatrixserverlib.HeaderedEvent, error) { return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey) } @@ -237,7 +237,7 @@ func (d *SyncServerDatasource) GetStateEvent( // Returns an error if there was an issue with the retrieval. func (d *SyncServerDatasource) GetStateEventsForRoom( ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter, -) (stateEvents []gomatrixserverlib.Event, err error) { +) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilter) return err @@ -599,7 +599,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // Build up a /sync response. Add joined rooms. for _, roomID := range joinedRoomIDs { - var stateEvents []gomatrixserverlib.Event + var stateEvents []gomatrixserverlib.HeaderedEvent stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, &stateFilter) if err != nil { return @@ -633,9 +633,9 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true - jr.State.Events = gomatrixserverlib.ToClientEvents(stateEvents, gomatrixserverlib.FormatSync) + jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[roomID] = *jr } @@ -707,7 +707,7 @@ func (d *SyncServerDatasource) UpsertAccountData( // If the invite was successfully stored this returns the stream ID it was stored at. // Returns an error if there was a problem communicating with the database. func (d *SyncServerDatasource) AddInviteEvent( - ctx context.Context, inviteEvent gomatrixserverlib.Event, + ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (types.StreamPosition, error) { return d.invites.insertInviteEvent(ctx, inviteEvent) } @@ -758,7 +758,7 @@ func (d *SyncServerDatasource) addInvitesToResponse( for roomID, inviteEvent := range invites { ir := types.NewInviteResponse() ir.InviteState.Events = gomatrixserverlib.ToClientEvents( - []gomatrixserverlib.Event{inviteEvent}, gomatrixserverlib.FormatSync, + []gomatrixserverlib.Event{inviteEvent.Event}, gomatrixserverlib.FormatSync, ) // TODO: add the invite state from the invite event. res.Rooms.Invite[roomID] = *ir @@ -821,9 +821,9 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - jr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[delta.roomID] = *jr case gomatrixserverlib.Leave: fallthrough // transitions to leave are the same as ban @@ -834,9 +834,9 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( lr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - lr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Leave[delta.roomID] = *lr } @@ -1074,7 +1074,7 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( } s := make([]types.StreamEvent, len(allState)) for i := 0; i < len(s); i++ { - s[i] = types.StreamEvent{Event: allState[i], StreamPosition: 0} + s[i] = types.StreamEvent{HeaderedEvent: allState[i], StreamPosition: 0} } return s, nil } @@ -1082,10 +1082,10 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and // matches the streamevent.transactionID device then the transaction ID gets // added to the unsigned section of the output event. -func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.Event { - out := make([]gomatrixserverlib.Event, len(in)) +func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent { + out := make([]gomatrixserverlib.HeaderedEvent, len(in)) for i := 0; i < len(in); i++ { - out[i] = in[i].Event + out[i] = in[i].HeaderedEvent if device != nil && in[i].TransactionID != nil { if device.UserID == in[i].Sender() && device.SessionID == in[i].TransactionID.SessionID { err := out[i].SetUnsignedField( @@ -1105,7 +1105,7 @@ func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in // There may be some overlap where events in stateEvents are already in recentEvents, so filter // them out so we don't include them twice in the /sync response. They should be in recentEvents // only, so clients get to the correct state once they have rolled forward. -func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.Event) []gomatrixserverlib.Event { +func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { for _, recentEv := range recentEvents { if recentEv.StateKey() == nil { continue // not a state event diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index 71105d0c3..3dbf961b4 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -19,6 +19,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -102,7 +104,7 @@ func (s *accountDataStatements) selectAccountDataInRange( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectAccountDataInRange: rows.close() failed") var entries int diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index ed76177be..9fafdbede 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( sender TEXT NOT NULL, contains_url BOOL NOT NULL DEFAULT false, state_key TEXT NOT NULL, - event_json TEXT NOT NULL, + headered_event_json TEXT NOT NULL, membership TEXT, added_at BIGINT, UNIQUE (room_id, type, state_key) @@ -47,10 +47,10 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_s ` const upsertRoomStateSQL = "" + - "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, event_json, membership, added_at)" + + "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, headered_event_json, membership, added_at)" + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + " ON CONFLICT (event_id, room_id, type, sender, contains_url)" + - " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, event_json = $7, membership = $8, added_at = $9" + " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, headered_event_json = $7, membership = $8, added_at = $9" const deleteRoomStateByEventIDSQL = "" + "DELETE FROM syncapi_current_room_state WHERE event_id = $1" @@ -59,7 +59,7 @@ const selectRoomIDsWithMembershipSQL = "" + "SELECT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2" const selectCurrentStateSQL = "" + - "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1" + + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" + " AND ( $2 IS NULL OR sender IN ($2) )" + " AND ( $3 IS NULL OR NOT(sender IN ($3)) )" + " AND ( $4 IS NULL OR type IN ($4) )" + @@ -71,14 +71,14 @@ const selectJoinedUsersSQL = "" + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" const selectStateEventSQL = "" + - "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" const selectEventsWithEventIDsSQL = "" + // TODO: The session_id and transaction_id blanks are here because otherwise // the rowsToStreamEvents expects there to be exactly five columns. We need to // figure out if these really need to be in the DB, and if so, we need a // better permanent fix for this. - neilalexander, 2 Jan 2020 - "SELECT added_at, event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + + "SELECT added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + " FROM syncapi_current_room_state WHERE event_id IN ($1)" type currentRoomStateStatements struct { @@ -126,7 +126,7 @@ func (s *currentRoomStateStatements) selectJoinedUsers( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectJoinedUsers: rows.close() failed") result := make(map[string][]string) for rows.Next() { @@ -154,7 +154,7 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithMembership: rows.close() failed") var result []string for rows.Next() { @@ -171,7 +171,7 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( func (s *currentRoomStateStatements) selectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, -) ([]gomatrixserverlib.Event, error) { +) ([]gomatrixserverlib.HeaderedEvent, error) { stmt := common.TxStmt(txn, s.selectCurrentStateStmt) rows, err := stmt.QueryContext(ctx, roomID, nil, // FIXME: pq.StringArray(stateFilterPart.Senders), @@ -184,7 +184,7 @@ func (s *currentRoomStateStatements) selectCurrentState( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectCurrentState: rows.close() failed") return rowsToEvents(rows) } @@ -199,7 +199,7 @@ func (s *currentRoomStateStatements) deleteRoomStateByEventID( func (s *currentRoomStateStatements) upsertRoomState( ctx context.Context, txn *sql.Tx, - event gomatrixserverlib.Event, membership *string, addedAt types.StreamPosition, + event gomatrixserverlib.HeaderedEvent, membership *string, addedAt types.StreamPosition, ) error { // Parse content as JSON and search for an "url" key containsURL := false @@ -209,9 +209,14 @@ func (s *currentRoomStateStatements) upsertRoomState( _, containsURL = content["url"] } + headeredJSON, err := json.Marshal(event) + if err != nil { + return err + } + // upsert state event stmt := common.TxStmt(txn, s.upsertRoomStateStmt) - _, err := stmt.ExecContext( + _, err = stmt.ExecContext( ctx, event.RoomID(), event.EventID(), @@ -219,7 +224,7 @@ func (s *currentRoomStateStatements) upsertRoomState( event.Sender(), containsURL, *event.StateKey(), - event.JSON(), + headeredJSON, membership, addedAt, ) @@ -238,20 +243,20 @@ func (s *currentRoomStateStatements) selectEventsWithEventIDs( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed") return rowsToStreamEvents(rows) } -func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.Event, error) { - result := []gomatrixserverlib.Event{} +func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.HeaderedEvent, error) { + result := []gomatrixserverlib.HeaderedEvent{} for rows.Next() { var eventBytes []byte if err := rows.Scan(&eventBytes); err != nil { return nil, err } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, err } result = append(result, ev) @@ -261,7 +266,7 @@ func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.Event, error) { func (s *currentRoomStateStatements) selectStateEvent( ctx context.Context, roomID, evType, stateKey string, -) (*gomatrixserverlib.Event, error) { +) (*gomatrixserverlib.HeaderedEvent, error) { stmt := s.selectStateEventStmt var res []byte err := stmt.QueryRowContext(ctx, roomID, evType, stateKey).Scan(&res) @@ -271,6 +276,9 @@ func (s *currentRoomStateStatements) selectStateEvent( if err != nil { return nil, err } - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(res, false) + var ev gomatrixserverlib.HeaderedEvent + if err = json.Unmarshal(res, &ev); err != nil { + return nil, err + } return &ev, err } diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index baf8871bd..22efeaeb0 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -18,6 +18,7 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" @@ -30,7 +31,7 @@ CREATE TABLE IF NOT EXISTS syncapi_invite_events ( event_id TEXT NOT NULL, room_id TEXT NOT NULL, target_user_id TEXT NOT NULL, - event_json TEXT NOT NULL + headered_event_json TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS syncapi_invites_target_user_id_idx ON syncapi_invite_events (target_user_id, id); @@ -39,14 +40,14 @@ CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx ON syncapi_invite_events const insertInviteEventSQL = "" + "INSERT INTO syncapi_invite_events" + - " (id, room_id, event_id, target_user_id, event_json)" + + " (id, room_id, event_id, target_user_id, headered_event_json)" + " VALUES ($1, $2, $3, $4, $5)" const deleteInviteEventSQL = "" + "DELETE FROM syncapi_invite_events WHERE event_id = $1" const selectInviteEventsInRangeSQL = "" + - "SELECT room_id, event_json FROM syncapi_invite_events" + + "SELECT room_id, headered_event_json FROM syncapi_invite_events" + " WHERE target_user_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC" @@ -83,15 +84,21 @@ func (s *inviteEventsStatements) prepare(db *sql.DB, streamID *streamIDStatement } func (s *inviteEventsStatements) insertInviteEvent( - ctx context.Context, txn *sql.Tx, inviteEvent gomatrixserverlib.Event, streamPos types.StreamPosition, + ctx context.Context, txn *sql.Tx, inviteEvent gomatrixserverlib.HeaderedEvent, streamPos types.StreamPosition, ) (err error) { + var headeredJSON []byte + headeredJSON, err = json.Marshal(inviteEvent) + if err != nil { + return + } + _, err = txn.Stmt(s.insertInviteEventStmt).ExecContext( ctx, streamPos, inviteEvent.RoomID(), inviteEvent.EventID(), *inviteEvent.StateKey(), - inviteEvent.JSON(), + headeredJSON, ) return } @@ -107,14 +114,14 @@ func (s *inviteEventsStatements) deleteInviteEvent( // active invites for the target user ID in the supplied range. func (s *inviteEventsStatements) selectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, startPos, endPos types.StreamPosition, -) (map[string]gomatrixserverlib.Event, error) { +) (map[string]gomatrixserverlib.HeaderedEvent, error) { stmt := common.TxStmt(txn, s.selectInviteEventsInRangeStmt) rows, err := stmt.QueryContext(ctx, targetUserID, startPos, endPos) if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck - result := map[string]gomatrixserverlib.Event{} + defer common.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") + result := map[string]gomatrixserverlib.HeaderedEvent{} for rows.Next() { var ( roomID string @@ -124,8 +131,8 @@ func (s *inviteEventsStatements) selectInviteEventsInRange( return nil, err } - event, err := gomatrixserverlib.NewEventFromTrustedJSON(eventJSON, false) - if err != nil { + var event gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventJSON, &event); err != nil { return nil, err } diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index be8937435..08299f64b 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( id INTEGER PRIMARY KEY AUTOINCREMENT, event_id TEXT NOT NULL UNIQUE, room_id TEXT NOT NULL, - event_json TEXT NOT NULL, + headered_event_json TEXT NOT NULL, type TEXT NOT NULL, sender TEXT NOT NULL, contains_url BOOL NOT NULL, @@ -49,25 +49,25 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( const insertEventSQL = "" + "INSERT INTO syncapi_output_room_events (" + - "id, room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + + "id, room_id, event_id, headered_event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) " + "ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = $13" const selectEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = $1" + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = $1" const selectRecentEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC LIMIT $4" const selectRecentEventsForSyncSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" + " ORDER BY id DESC LIMIT $4" const selectEarlyEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id ASC LIMIT $4" @@ -86,7 +86,7 @@ const selectMaxEventIDSQL = "" + $8 = stateFilterPart.Limit, */ const selectStateInRangeSQL = "" + - "SELECT id, event_json, exclude_from_sync, add_state_ids, remove_state_ids" + + "SELECT id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2)" + // old/new pos " AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + @@ -160,6 +160,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( if err != nil { return nil, nil, err } + defer rows.Close() // nolint: errcheck // Fetch all the state change events for all rooms between the two positions then loop each event and: // - Keep a cache of the event by ID (99% of state change events are for the event itself) // - For each room ID, build up an array of event IDs which represents cumulative adds/removes @@ -199,8 +200,8 @@ func (s *outputRoomEventsStatements) selectStateInRange( } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, nil, err } needSet := stateNeeded[ev.RoomID()] @@ -216,7 +217,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( stateNeeded[ev.RoomID()] = needSet eventIDToEvent[ev.EventID()] = types.StreamEvent{ - Event: ev, + HeaderedEvent: ev, StreamPosition: streamPos, ExcludeFromSync: excludeFromSync, } @@ -244,7 +245,7 @@ func (s *outputRoomEventsStatements) selectMaxEventID( // of the inserted event. func (s *outputRoomEventsStatements) insertEvent( ctx context.Context, txn *sql.Tx, - event *gomatrixserverlib.Event, addState, removeState []string, + event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, transactionID *api.TransactionID, excludeFromSync bool, ) (streamPos types.StreamPosition, err error) { var txnID *string @@ -262,6 +263,12 @@ func (s *outputRoomEventsStatements) insertEvent( _, containsURL = content["url"] } + var headeredJSON []byte + headeredJSON, err = json.Marshal(event) + if err != nil { + return + } + streamPos, err = s.streamIDStatements.nextStreamID(ctx, txn) if err != nil { return @@ -282,7 +289,7 @@ func (s *outputRoomEventsStatements) insertEvent( streamPos, event.RoomID(), event.EventID(), - event.JSON(), + headeredJSON, event.Type(), event.Sender(), containsURL, @@ -315,7 +322,7 @@ func (s *outputRoomEventsStatements) selectRecentEvents( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { return nil, err @@ -342,7 +349,7 @@ func (s *outputRoomEventsStatements) selectEarlyEvents( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEarlyEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { return nil, err @@ -371,7 +378,7 @@ func (s *outputRoomEventsStatements) selectEvents( if streamEvents, err := rowsToStreamEvents(rows); err == nil { returnEvents = append(returnEvents, streamEvents...) } - rows.Close() // nolint: errcheck + common.CloseAndLogIfError(ctx, rows, "selectEvents: rows.close() failed") } return returnEvents, nil } @@ -391,8 +398,8 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { return nil, err } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, err } @@ -404,7 +411,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { } result = append(result, types.StreamEvent{ - Event: ev, + HeaderedEvent: ev, StreamPosition: streamPos, TransactionID: transactionID, ExcludeFromSync: excludeFromSync, diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index f7075bd6f..a2944c2f9 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -101,7 +101,7 @@ func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { // insertEventInTopology inserts the given event in the room's topology, based // on the event's depth. func (s *outputRoomEventsTopologyStatements) insertEventInTopology( - ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.Event, + ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, ) (err error) { stmt := common.TxStmt(txn, s.insertEventInTopologyStmt) _, err = stmt.ExecContext( diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 0e84c8c86..a06fc91f5 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -40,7 +40,7 @@ import ( type stateDelta struct { roomID string - stateEvents []gomatrixserverlib.Event + stateEvents []gomatrixserverlib.HeaderedEvent membership string // The PDU stream position of the latest membership event for this user, if applicable. // Can be 0 if there is no membership event in this delta. @@ -126,7 +126,7 @@ func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[s // If an event is not found in the database then it will be omitted from the list. // Returns an error if there was a problem talking with the database. // Does not include any transaction IDs in the returned events. -func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { +func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) { streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs) if err != nil { return nil, err @@ -137,7 +137,7 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([ return d.StreamEventsToEvents(nil, streamEvents), nil } -func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.Event) error { +func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { // If the event is already known as a backward extremity, don't consider // it as such anymore now that we have it. isBackwardExtremity, err := d.backwardExtremities.isBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()) @@ -181,8 +181,8 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx // Returns an error if there was a problem inserting this event. func (d *SyncServerDatasource) WriteEvent( ctx context.Context, - ev *gomatrixserverlib.Event, - addStateEvents []gomatrixserverlib.Event, + ev *gomatrixserverlib.HeaderedEvent, + addStateEvents []gomatrixserverlib.HeaderedEvent, addStateEventIDs, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool, ) (pduPosition types.StreamPosition, returnErr error) { @@ -218,7 +218,7 @@ func (d *SyncServerDatasource) WriteEvent( func (d *SyncServerDatasource) updateRoomState( ctx context.Context, txn *sql.Tx, removedEventIDs []string, - addedEvents []gomatrixserverlib.Event, + addedEvents []gomatrixserverlib.HeaderedEvent, pduPosition types.StreamPosition, ) error { // remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add. @@ -254,7 +254,7 @@ func (d *SyncServerDatasource) updateRoomState( // If there was an issue during the retrieval, returns an error func (d *SyncServerDatasource) GetStateEvent( ctx context.Context, roomID, evType, stateKey string, -) (*gomatrixserverlib.Event, error) { +) (*gomatrixserverlib.HeaderedEvent, error) { return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey) } @@ -263,7 +263,7 @@ func (d *SyncServerDatasource) GetStateEvent( // Returns an error if there was an issue with the retrieval. func (d *SyncServerDatasource) GetStateEventsForRoom( ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, -) (stateEvents []gomatrixserverlib.Event, err error) { +) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilterPart) return err @@ -633,8 +633,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // Build up a /sync response. Add joined rooms. for _, roomID := range joinedRoomIDs { - - var stateEvents []gomatrixserverlib.Event + var stateEvents []gomatrixserverlib.HeaderedEvent stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, &stateFilterPart) if err != nil { return @@ -670,9 +669,9 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true - jr.State.Events = gomatrixserverlib.ToClientEvents(stateEvents, gomatrixserverlib.FormatSync) + jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[roomID] = *jr } @@ -748,7 +747,7 @@ func (d *SyncServerDatasource) UpsertAccountData( // If the invite was successfully stored this returns the stream ID it was stored at. // Returns an error if there was a problem communicating with the database. func (d *SyncServerDatasource) AddInviteEvent( - ctx context.Context, inviteEvent gomatrixserverlib.Event, + ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (streamPos types.StreamPosition, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { streamPos, err = d.streamID.nextStreamID(ctx, txn) @@ -805,8 +804,8 @@ func (d *SyncServerDatasource) addInvitesToResponse( } for roomID, inviteEvent := range invites { ir := types.NewInviteResponse() - ir.InviteState.Events = gomatrixserverlib.ToClientEvents( - []gomatrixserverlib.Event{inviteEvent}, gomatrixserverlib.FormatSync, + ir.InviteState.Events = gomatrixserverlib.HeaderedToClientEvents( + []gomatrixserverlib.HeaderedEvent{inviteEvent}, gomatrixserverlib.FormatSync, ) // TODO: add the invite state from the invite event. res.Rooms.Invite[roomID] = *ir @@ -859,7 +858,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( return err } recentEvents := d.StreamEventsToEvents(device, recentStreamEvents) - delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) // roll back + delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) backwardTopologyPos := d.getBackwardTopologyPos(ctx, txn, recentStreamEvents) switch delta.membership { @@ -869,9 +868,9 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - jr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[delta.roomID] = *jr case gomatrixserverlib.Leave: fallthrough // transitions to leave are the same as ban @@ -882,9 +881,9 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( lr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - lr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Leave[delta.roomID] = *lr } @@ -1013,7 +1012,7 @@ func (d *SyncServerDatasource) getStateDeltas( // dupe join events will result in the entire room state coming down to the client again. This is added in // the 'state' part of the response though, so is transparent modulo bandwidth concerns as it is not added to // the timeline. - if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { + if membership := getMembershipFromEvent(&ev.HeaderedEvent, userID); membership != "" { if membership == gomatrixserverlib.Join { // send full room state down instead of a delta var s []types.StreamEvent @@ -1094,7 +1093,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( for roomID, stateStreamEvents := range state { for _, ev := range stateStreamEvents { - if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { + if membership := getMembershipFromEvent(&ev.HeaderedEvent, userID); membership != "" { if membership != gomatrixserverlib.Join { // We've already added full state for all joined rooms above. deltas = append(deltas, stateDelta{ membership: membership, @@ -1122,7 +1121,7 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( } s := make([]types.StreamEvent, len(allState)) for i := 0; i < len(s); i++ { - s[i] = types.StreamEvent{Event: allState[i], StreamPosition: 0} + s[i] = types.StreamEvent{HeaderedEvent: allState[i], StreamPosition: 0} } return s, nil } @@ -1130,10 +1129,10 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and // matches the streamevent.transactionID device then the transaction ID gets // added to the unsigned section of the output event. -func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.Event { - out := make([]gomatrixserverlib.Event, len(in)) +func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent { + out := make([]gomatrixserverlib.HeaderedEvent, len(in)) for i := 0; i < len(in); i++ { - out[i] = in[i].Event + out[i] = in[i].HeaderedEvent if device != nil && in[i].TransactionID != nil { if device.UserID == in[i].Sender() && device.SessionID == in[i].TransactionID.SessionID { err := out[i].SetUnsignedField( @@ -1153,7 +1152,7 @@ func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in // There may be some overlap where events in stateEvents are already in recentEvents, so filter // them out so we don't include them twice in the /sync response. They should be in recentEvents // only, so clients get to the correct state once they have rolled forward. -func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.Event) []gomatrixserverlib.Event { +func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { for _, recentEv := range recentEvents { if recentEv.StateKey() == nil { continue // not a state event @@ -1177,7 +1176,7 @@ func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.Event) []gom // getMembershipFromEvent returns the value of content.membership iff the event is a state event // with type 'm.room.member' and state_key of userID. Otherwise, an empty string is returned. -func getMembershipFromEvent(ev *gomatrixserverlib.Event, userID string) string { +func getMembershipFromEvent(ev *gomatrixserverlib.HeaderedEvent, userID string) string { if ev.Type() == "m.room.member" && ev.StateKeyEquals(userID) { membership, err := ev.Membership() if err != nil { diff --git a/syncapi/sync/notifier.go b/syncapi/sync/notifier.go index aaee49d34..0d8050112 100644 --- a/syncapi/sync/notifier.go +++ b/syncapi/sync/notifier.go @@ -67,7 +67,7 @@ func NewNotifier(pos types.PaginationToken) *Notifier { // Typically a consumer supplies a posUpdate with the latest sync position for the // event type it handles, leaving other fields as 0. func (n *Notifier) OnNewEvent( - ev *gomatrixserverlib.Event, roomID string, userIDs []string, + ev *gomatrixserverlib.HeaderedEvent, roomID string, userIDs []string, posUpdate types.PaginationToken, ) { // update the current position then notify relevant /sync streams. diff --git a/syncapi/sync/notifier_test.go b/syncapi/sync/notifier_test.go index 02da0f7e6..350d757c6 100644 --- a/syncapi/sync/notifier_test.go +++ b/syncapi/sync/notifier_test.go @@ -16,6 +16,7 @@ package sync import ( "context" + "encoding/json" "fmt" "sync" "testing" @@ -29,9 +30,9 @@ import ( ) var ( - randomMessageEvent gomatrixserverlib.Event - aliceInviteBobEvent gomatrixserverlib.Event - bobLeaveEvent gomatrixserverlib.Event + randomMessageEvent gomatrixserverlib.HeaderedEvent + aliceInviteBobEvent gomatrixserverlib.HeaderedEvent + bobLeaveEvent gomatrixserverlib.HeaderedEvent syncPositionVeryOld types.PaginationToken syncPositionBefore types.PaginationToken syncPositionAfter types.PaginationToken @@ -67,7 +68,8 @@ func init() { syncPositionAfter2.PDUPosition = 13 var err error - randomMessageEvent, err = gomatrixserverlib.NewEventFromTrustedJSON([]byte(`{ + err = json.Unmarshal([]byte(`{ + "_room_version": "1", "type": "m.room.message", "content": { "body": "Hello World", @@ -75,13 +77,15 @@ func init() { }, "sender": "@noone:localhost", "room_id": "`+roomID+`", + "origin": "localhost", "origin_server_ts": 12345, "event_id": "$randomMessageEvent:localhost" - }`), false) + }`), &randomMessageEvent) if err != nil { panic(err) } - aliceInviteBobEvent, err = gomatrixserverlib.NewEventFromTrustedJSON([]byte(`{ + err = json.Unmarshal([]byte(`{ + "_room_version": "1", "type": "m.room.member", "state_key": "`+bob+`", "content": { @@ -89,13 +93,15 @@ func init() { }, "sender": "`+alice+`", "room_id": "`+roomID+`", + "origin": "localhost", "origin_server_ts": 12345, "event_id": "$aliceInviteBobEvent:localhost" - }`), false) + }`), &aliceInviteBobEvent) if err != nil { panic(err) } - bobLeaveEvent, err = gomatrixserverlib.NewEventFromTrustedJSON([]byte(`{ + err = json.Unmarshal([]byte(`{ + "_room_version": "1", "type": "m.room.member", "state_key": "`+bob+`", "content": { @@ -103,9 +109,10 @@ func init() { }, "sender": "`+bob+`", "room_id": "`+roomID+`", + "origin": "localhost", "origin_server_ts": 12345, "event_id": "$bobLeaveEvent:localhost" - }`), false) + }`), &bobLeaveEvent) if err != nil { panic(err) } @@ -135,7 +142,7 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { go func() { pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { - t.Errorf("TestNewEventAndJoinedToRoom error: %s", err) + t.Errorf("TestNewEventAndJoinedToRoom error: %w", err) } if pos != syncPositionAfter { t.Errorf("TestNewEventAndJoinedToRoom want %v, got %v", syncPositionAfter, pos) @@ -163,7 +170,7 @@ func TestNewInviteEventForUser(t *testing.T) { go func() { pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { - t.Errorf("TestNewInviteEventForUser error: %s", err) + t.Errorf("TestNewInviteEventForUser error: %w", err) } if pos != syncPositionAfter { t.Errorf("TestNewInviteEventForUser want %v, got %v", syncPositionAfter, pos) @@ -191,7 +198,7 @@ func TestEDUWakeup(t *testing.T) { go func() { pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionAfter)) if err != nil { - t.Errorf("TestNewInviteEventForUser error: %s", err) + t.Errorf("TestNewInviteEventForUser error: %w", err) } if pos != syncPositionNewEDU { t.Errorf("TestNewInviteEventForUser want %v, got %v", syncPositionNewEDU, pos) @@ -219,7 +226,7 @@ func TestMultipleRequestWakeup(t *testing.T) { poll := func() { pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { - t.Errorf("TestMultipleRequestWakeup error: %s", err) + t.Errorf("TestMultipleRequestWakeup error: %w", err) } if pos != syncPositionAfter { t.Errorf("TestMultipleRequestWakeup want %v, got %v", syncPositionAfter, pos) @@ -259,7 +266,7 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { go func() { pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { - t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %s", err) + t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %w", err) } if pos != syncPositionAfter { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %v, got %v", syncPositionAfter, pos) @@ -278,7 +285,7 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { go func() { pos, err := waitForEvents(n, newTestSyncRequest(alice, syncPositionAfter)) if err != nil { - t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %s", err) + t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %w", err) } if pos != syncPositionAfter2 { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %v, got %v", syncPositionAfter2, pos) diff --git a/syncapi/types/types.go b/syncapi/types/types.go index c25a38cdf..718906ecd 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -40,7 +40,7 @@ type StreamPosition int64 // Same as gomatrixserverlib.Event but also has the PDU stream position for this event. type StreamEvent struct { - gomatrixserverlib.Event + gomatrixserverlib.HeaderedEvent StreamPosition StreamPosition TransactionID *api.TransactionID ExcludeFromSync bool diff --git a/sytest-whitelist b/sytest-whitelist index eb91b3f0c..03e11b83b 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -104,12 +104,6 @@ Newly banned rooms appear in the leave section of incremental sync Newly banned rooms appear in the leave section of incremental sync local user can join room with version 1 User can invite local user to room with version 1 -local user can join room with version 2 -User can invite local user to room with version 2 -local user can join room with version 3 -User can invite local user to room with version 3 -local user can join room with version 4 -User can invite local user to room with version 4 Should reject keys claiming to belong to a different user Can add account data Can add account data to room @@ -140,8 +134,6 @@ Changing the actions of an unknown rule fails with 404 Enabling an unknown default rule fails with 404 Trying to get push rules with unknown rule_id fails with 404 Events come down the correct room -local user can join room with version 5 -User can invite local user to room with version 5 # SyTest currently only implements the v1 endpoints for /send_join and /send_leave, # whereas Dendrite only supports the v2 endpoints for those, so let's ignore this # test for now. @@ -198,10 +190,6 @@ Local non-members don't see posted message events Remote room members also see posted message events Lazy loading parameters in the filter are strictly boolean remote user can join room with version 1 -remote user can join room with version 2 -remote user can join room with version 3 -remote user can join room with version 4 -remote user can join room with version 5 Inbound federation can query room alias directory Outbound federation can query v2 /send_join Inbound federation can receive v2 /send_join @@ -230,3 +218,5 @@ Push rules come down in an initial /sync Regular users can add and delete aliases in the default room configuration Regular users can add and delete aliases when m.room.aliases is restricted GET /r0/capabilities is not public +GET /joined_rooms lists newly-created room +/joined_rooms returns only joined rooms