From 7353de7dbb146a16696cf2e9683753c3b08291f2 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 13 Mar 2020 10:15:01 +0000 Subject: [PATCH 01/17] Make P2P use JSServer --- cmd/dendritejs/main.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 458427a7b..25492b166 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -69,7 +69,7 @@ func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLoc } func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http_js_libp2p.P2pLocalNode) { - hosted := "/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/" + hosted := "/dns4/rendezvous.matrix.org/tcp/8443/wss/p2p-websocket-star/" node = go_http_js_libp2p.NewP2pLocalNode("org.matrix.p2p.experiment", privKey.Seed(), []string{hosted}) serverName = node.Id fmt.Println("p2p assigned ServerName: ", serverName) @@ -145,11 +145,10 @@ func main() { if node != nil { go func() { logrus.Info("Listening on libp2p-js host ID ", node.Id) - - listener := go_http_js_libp2p.NewP2pListener(node) - defer listener.Close() - s := &http.Server{} - s.Serve(listener) + s := JSServer{ + Mux: http.DefaultServeMux, + } + s.ListenAndServe("p2p") }() } From dfd8b93d93a586f0c4400b3a317c978a1c0e04d9 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Sat, 14 Mar 2020 20:08:54 +0000 Subject: [PATCH 02/17] Federation: Add event_auth endpoint (#905) * Federation: Add event_auth endpoint Signed-off-by: Sudhanshu Jaiswal * Update metric names Signed-off-by: Sudhanshu Jaiswal * Rebase * Add comment from @APwhitehat Co-authored-by: Sudhanshu Jaiswal --- federationapi/routing/eventauth.go | 43 ++++++++++++++++++++++++++++++ federationapi/routing/routing.go | 14 ++++++++-- 2 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 federationapi/routing/eventauth.go diff --git a/federationapi/routing/eventauth.go b/federationapi/routing/eventauth.go new file mode 100644 index 000000000..003165c85 --- /dev/null +++ b/federationapi/routing/eventauth.go @@ -0,0 +1,43 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "context" + "net/http" + + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +// GetEventAuth returns event auth for the roomID and eventID +func GetEventAuth( + ctx context.Context, + request *gomatrixserverlib.FederationRequest, + query api.RoomserverQueryAPI, + roomID string, + eventID string, +) util.JSONResponse { + // TODO: Optimisation: we shouldn't be querying all the room state + // that is in state.StateEvents - we just ignore it. + state, err := getState(ctx, request, query, roomID, eventID) + if err != nil { + return *err + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: gomatrixserverlib.RespEventAuth{AuthEvents: state.AuthEvents}, + } +} diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 3b119301a..b5c8e53de 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -131,7 +131,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/state/{roomID}", common.MakeFedAPI( - "federation_get_event_auth", cfg.Matrix.ServerName, keys, + "federation_get_state", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -144,7 +144,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/state_ids/{roomID}", common.MakeFedAPI( - "federation_get_event_auth", cfg.Matrix.ServerName, keys, + "federation_get_state_ids", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -156,6 +156,16 @@ func Setup( }, )).Methods(http.MethodGet) + v1fedmux.Handle("/event_auth/{roomID}/{eventID}", common.MakeFedAPI( + "federation_get_event_auth", cfg.Matrix.ServerName, keys, + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { + vars := mux.Vars(httpReq) + return GetEventAuth( + httpReq.Context(), request, query, vars["roomID"], vars["eventID"], + ) + }, + )).Methods(http.MethodGet) + v1fedmux.Handle("/query/directory", common.MakeFedAPI( "federation_query_room_alias", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { From 452f393dd7de5203c5cd3ebe5697af6ea0f4b009 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 16 Mar 2020 16:05:29 +0000 Subject: [PATCH 03/17] Update room version descriptors, add error handling (#906) * Update room version descriptors, add error handling * Fix database queries * Drop Get from version package * Fix database wrapping, add comments for version descriptions * Don't set default room_version value in SQL --- go.mod | 2 +- go.sum | 4 + roomserver/query/query.go | 7 +- roomserver/storage/interface.go | 2 +- roomserver/storage/postgres/rooms_table.go | 14 +- roomserver/storage/postgres/storage.go | 13 +- roomserver/storage/sqlite3/rooms_table.go | 14 +- roomserver/storage/sqlite3/storage.go | 13 +- roomserver/version/version.go | 147 +++++++++++---------- 9 files changed, 121 insertions(+), 95 deletions(-) diff --git a/go.mod b/go.mod index 92e0b7201..0728a03cc 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index c28254d46..af0876489 100644 --- a/go.sum +++ b/go.sum @@ -138,6 +138,10 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:km github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424 h1:H61lT6ckUFIDl9qb636qNomfo0B52lFt73ecioiqF10= github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316100021-ac4a53d49690 h1:aQPPypOdoIsquJHhoalRYvKtDoiJfSyyJqOEn6R7yTY= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316100021-ac4a53d49690/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f h1:JXSvzG4movqXT1KcdX+XZ9HM61m1FW4rIMFKUM9j/Dk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= diff --git a/roomserver/query/query.go b/roomserver/query/query.go index f138686b5..83f89777f 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -20,7 +20,6 @@ import ( "context" "encoding/json" "net/http" - "strconv" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" @@ -733,10 +732,10 @@ func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities( request *api.QueryRoomVersionCapabilitiesRequest, response *api.QueryRoomVersionCapabilitiesResponse, ) error { - response.DefaultRoomVersion = strconv.Itoa(int(version.GetDefaultRoomVersion())) + response.DefaultRoomVersion = string(version.DefaultRoomVersion()) response.AvailableRoomVersions = make(map[string]string) - for v, desc := range version.GetSupportedRoomVersions() { - sv := strconv.Itoa(int(v)) + for v, desc := range version.SupportedRoomVersions() { + sv := string(v) if desc.Stable { response.AvailableRoomVersions[sv] = "stable" } else { diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index f03d6feab..7f32b53f8 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -45,5 +45,5 @@ type Database interface { GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error) GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool) ([]types.EventNID, error) EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) - GetRoomVersionForRoom(ctx context.Context, roomNID types.RoomNID) (int64, error) + GetRoomVersionForRoom(ctx context.Context, roomNID types.RoomNID) (gomatrixserverlib.RoomVersion, error) } diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index edd15a338..ef8b8ecef 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" ) const roomsSchema = ` @@ -42,13 +43,13 @@ CREATE TABLE IF NOT EXISTS roomserver_rooms ( state_snapshot_nid BIGINT NOT NULL DEFAULT 0, -- The version of the room, which will assist in determining the state resolution -- algorithm, event ID format, etc. - room_version BIGINT NOT NULL DEFAULT 1 + room_version TEXT NOT NULL ); ` // Same as insertEventTypeNIDSQL const insertRoomNIDSQL = "" + - "INSERT INTO roomserver_rooms (room_id) VALUES ($1)" + + "INSERT INTO roomserver_rooms (room_id, room_version) VALUES ($1, $2)" + " ON CONFLICT ON CONSTRAINT roomserver_room_id_unique" + " DO NOTHING RETURNING (room_nid)" @@ -92,11 +93,12 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { } func (s *roomStatements) insertRoomNID( - ctx context.Context, txn *sql.Tx, roomID string, + ctx context.Context, txn *sql.Tx, + roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { var roomNID int64 stmt := common.TxStmt(txn, s.insertRoomNIDStmt) - err := stmt.QueryRowContext(ctx, roomID).Scan(&roomNID) + err := stmt.QueryRowContext(ctx, roomID, roomVersion).Scan(&roomNID) return types.RoomNID(roomNID), err } @@ -165,8 +167,8 @@ func (s *roomStatements) updateLatestEventNIDs( func (s *roomStatements) selectRoomVersionForRoomNID( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, -) (int64, error) { - var roomVersion int64 +) (gomatrixserverlib.RoomVersion, error) { + var roomVersion gomatrixserverlib.RoomVersion stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) return roomVersion, err diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 77a792d68..ee1a1cd4d 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -68,7 +68,8 @@ func (d *Database) StoreEvent( } } - if roomNID, err = d.assignRoomNID(ctx, nil, event.RoomID()); err != nil { + // TODO: Room version here + if roomNID, err = d.assignRoomNID(ctx, nil, event.RoomID(), "1"); err != nil { return 0, types.StateAtEvent{}, err } @@ -121,13 +122,14 @@ func (d *Database) StoreEvent( } func (d *Database) assignRoomNID( - ctx context.Context, txn *sql.Tx, roomID string, + ctx context.Context, txn *sql.Tx, + roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { // Check if we already have a numeric ID in the database. roomNID, err := d.statements.selectRoomNID(ctx, txn, roomID) if err == sql.ErrNoRows { // We don't have a numeric ID so insert one into the database. - roomNID, err = d.statements.insertRoomNID(ctx, txn, roomID) + roomNID, err = d.statements.insertRoomNID(ctx, txn, roomID, roomVersion) if err == sql.ErrNoRows { // We raced with another insert so run the select again. roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) @@ -494,7 +496,8 @@ func (d *Database) MembershipUpdater( } }() - roomNID, err := d.assignRoomNID(ctx, txn, roomID) + // TODO: Room version here + roomNID, err := d.assignRoomNID(ctx, txn, roomID, "1") if err != nil { return nil, err } @@ -699,7 +702,7 @@ func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]type func (d *Database) GetRoomVersionForRoom( ctx context.Context, roomNID types.RoomNID, -) (int64, error) { +) (gomatrixserverlib.RoomVersion, error) { return d.statements.selectRoomVersionForRoomNID( ctx, nil, roomNID, ) diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index 99c03476f..b750f63ea 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" ) const roomsSchema = ` @@ -31,13 +32,13 @@ const roomsSchema = ` latest_event_nids TEXT NOT NULL DEFAULT '[]', last_event_sent_nid INTEGER NOT NULL DEFAULT 0, state_snapshot_nid INTEGER NOT NULL DEFAULT 0, - room_version INTEGER NOT NULL DEFAULT 1 + room_version TEXT NOT NULL ); ` // Same as insertEventTypeNIDSQL const insertRoomNIDSQL = ` - INSERT INTO roomserver_rooms (room_id) VALUES ($1) + INSERT INTO roomserver_rooms (room_id, room_version) VALUES ($1, $2) ON CONFLICT DO NOTHING; ` @@ -81,11 +82,12 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { } func (s *roomStatements) insertRoomNID( - ctx context.Context, txn *sql.Tx, roomID string, + ctx context.Context, txn *sql.Tx, + roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { var err error insertStmt := common.TxStmt(txn, s.insertRoomNIDStmt) - if _, err = insertStmt.ExecContext(ctx, roomID); err == nil { + if _, err = insertStmt.ExecContext(ctx, roomID, roomVersion); err == nil { return s.selectRoomNID(ctx, txn, roomID) } else { return types.RoomNID(0), err @@ -157,8 +159,8 @@ func (s *roomStatements) updateLatestEventNIDs( func (s *roomStatements) selectRoomVersionForRoomNID( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, -) (int64, error) { - var roomVersion int64 +) (gomatrixserverlib.RoomVersion, error) { + var roomVersion gomatrixserverlib.RoomVersion stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) return roomVersion, err diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 41d4c2a94..d9513611e 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -90,7 +90,8 @@ func (d *Database) StoreEvent( } } - if roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID()); err != nil { + // TODO: Room version here + if roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID(), "1"); err != nil { return err } @@ -150,13 +151,14 @@ func (d *Database) StoreEvent( } func (d *Database) assignRoomNID( - ctx context.Context, txn *sql.Tx, roomID string, + ctx context.Context, txn *sql.Tx, + roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (roomNID types.RoomNID, err error) { // Check if we already have a numeric ID in the database. roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) if err == sql.ErrNoRows { // We don't have a numeric ID so insert one into the database. - roomNID, err = d.statements.insertRoomNID(ctx, txn, roomID) + roomNID, err = d.statements.insertRoomNID(ctx, txn, roomID, roomVersion) if err == nil { // Now get the numeric ID back out of the database roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) @@ -630,7 +632,8 @@ func (d *Database) MembershipUpdater( } }() - roomNID, err := d.assignRoomNID(ctx, txn, roomID) + // TODO: Room version here + roomNID, err := d.assignRoomNID(ctx, txn, roomID, "1") if err != nil { return nil, err } @@ -853,7 +856,7 @@ func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]type func (d *Database) GetRoomVersionForRoom( ctx context.Context, roomNID types.RoomNID, -) (int64, error) { +) (gomatrixserverlib.RoomVersion, error) { return d.statements.selectRoomVersionForRoomNID( ctx, nil, roomNID, ) diff --git a/roomserver/version/version.go b/roomserver/version/version.go index 0943e3843..b18df49f5 100644 --- a/roomserver/version/version.go +++ b/roomserver/version/version.go @@ -15,85 +15,64 @@ package version import ( - "errors" + "fmt" - "github.com/matrix-org/dendrite/roomserver/state" -) - -type RoomVersionID int -type EventFormatID int - -const ( - RoomVersionV1 RoomVersionID = iota + 1 - RoomVersionV2 - RoomVersionV3 - RoomVersionV4 - RoomVersionV5 -) - -const ( - EventFormatV1 EventFormatID = iota + 1 // original event ID formatting - EventFormatV2 // event ID is event hash - EventFormatV3 // event ID is URL-safe base64 event hash + "github.com/matrix-org/gomatrixserverlib" ) +// RoomVersionDescription contains information about a room version, +// namely whether it is marked as supported or stable in this server +// version. +// A version is supported if the server has some support for rooms +// that are this version. A version is marked as stable or unstable +// in order to hint whether the version should be used to clients +// calling the /capabilities endpoint. +// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-capabilities type RoomVersionDescription struct { - Supported bool - Stable bool - StateResolution state.StateResolutionVersion - EventFormat EventFormatID - EnforceSigningKeyValidity bool + Supported bool + Stable bool } -var roomVersions = map[RoomVersionID]RoomVersionDescription{ - RoomVersionV1: RoomVersionDescription{ - Supported: true, - Stable: true, - StateResolution: state.StateResolutionAlgorithmV1, - EventFormat: EventFormatV1, - EnforceSigningKeyValidity: false, +var roomVersions = map[gomatrixserverlib.RoomVersion]RoomVersionDescription{ + gomatrixserverlib.RoomVersionV1: RoomVersionDescription{ + Supported: true, + Stable: true, }, - RoomVersionV2: RoomVersionDescription{ - Supported: false, - Stable: true, - StateResolution: state.StateResolutionAlgorithmV2, - EventFormat: EventFormatV1, - EnforceSigningKeyValidity: false, + gomatrixserverlib.RoomVersionV2: RoomVersionDescription{ + Supported: false, + Stable: false, }, - RoomVersionV3: RoomVersionDescription{ - Supported: false, - Stable: true, - StateResolution: state.StateResolutionAlgorithmV2, - EventFormat: EventFormatV2, - EnforceSigningKeyValidity: false, + gomatrixserverlib.RoomVersionV3: RoomVersionDescription{ + Supported: false, + Stable: false, }, - RoomVersionV4: RoomVersionDescription{ - Supported: false, - Stable: true, - StateResolution: state.StateResolutionAlgorithmV2, - EventFormat: EventFormatV3, - EnforceSigningKeyValidity: false, + gomatrixserverlib.RoomVersionV4: RoomVersionDescription{ + Supported: false, + Stable: false, }, - RoomVersionV5: RoomVersionDescription{ - Supported: false, - Stable: true, - StateResolution: state.StateResolutionAlgorithmV2, - EventFormat: EventFormatV3, - EnforceSigningKeyValidity: true, + gomatrixserverlib.RoomVersionV5: RoomVersionDescription{ + Supported: false, + Stable: false, }, } -func GetDefaultRoomVersion() RoomVersionID { - return RoomVersionV1 +// DefaultRoomVersion contains the room version that will, by +// default, be used to create new rooms on this server. +func DefaultRoomVersion() gomatrixserverlib.RoomVersion { + return gomatrixserverlib.RoomVersionV1 } -func GetRoomVersions() map[RoomVersionID]RoomVersionDescription { +// RoomVersions returns a map of all known room versions to this +// server. +func RoomVersions() map[gomatrixserverlib.RoomVersion]RoomVersionDescription { return roomVersions } -func GetSupportedRoomVersions() map[RoomVersionID]RoomVersionDescription { - versions := make(map[RoomVersionID]RoomVersionDescription) - for id, version := range GetRoomVersions() { +// SupportedRoomVersions returns a map of descriptions for room +// versions that are supported by this homeserver. +func SupportedRoomVersions() map[gomatrixserverlib.RoomVersion]RoomVersionDescription { + versions := make(map[gomatrixserverlib.RoomVersion]RoomVersionDescription) + for id, version := range RoomVersions() { if version.Supported { versions[id] = version } @@ -101,12 +80,46 @@ func GetSupportedRoomVersions() map[RoomVersionID]RoomVersionDescription { return versions } -func GetSupportedRoomVersion(version RoomVersionID) (desc RoomVersionDescription, err error) { +// RoomVersion returns information about a specific room version. +// An UnknownVersionError is returned if the version is not known +// to the server. +func RoomVersion(version gomatrixserverlib.RoomVersion) (RoomVersionDescription, error) { if version, ok := roomVersions[version]; ok { - desc = version + return version, nil } - if !desc.Supported { - err = errors.New("unsupported room version") - } - return + return RoomVersionDescription{}, UnknownVersionError{version} +} + +// SupportedRoomVersion returns information about a specific room +// version. An UnknownVersionError is returned if the version is not +// known to the server, or an UnsupportedVersionError is returned if +// the version is known but specifically marked as unsupported. +func SupportedRoomVersion(version gomatrixserverlib.RoomVersion) (RoomVersionDescription, error) { + result, err := RoomVersion(version) + if err != nil { + return RoomVersionDescription{}, err + } + if !result.Supported { + return RoomVersionDescription{}, UnsupportedVersionError{version} + } + return result, nil +} + +// UnknownVersionError is caused when the room version is not known. +type UnknownVersionError struct { + Version gomatrixserverlib.RoomVersion +} + +func (e UnknownVersionError) Error() string { + return fmt.Sprintf("room version '%s' is not known", e.Version) +} + +// UnsupportedVersionError is caused when the room version is specifically +// marked as unsupported. +type UnsupportedVersionError struct { + Version gomatrixserverlib.RoomVersion +} + +func (e UnsupportedVersionError) Error() string { + return fmt.Sprintf("room version '%s' is marked as unsupported", e.Version) } From acb505b71751f637531ba9113d4b4daae589bcbc Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 16 Mar 2020 17:29:52 +0000 Subject: [PATCH 04/17] Implement gomatrixserverlib.HeaderedEvent in roomserver query API (#912) * Implement gomatrixserverlib.HeaderedEvent, which should allow us to store room version headers along with the event across API boundaries and consumers/producers, and intercept unmarshalling to get the event structure right * Add federationsender to previous --- appservice/consumers/roomserver.go | 4 +- clientapi/consumers/roomserver.go | 4 +- clientapi/routing/getevent.go | 2 +- clientapi/routing/sendevent.go | 2 +- common/events.go | 2 +- federationapi/routing/backfill.go | 4 +- federationapi/routing/events.go | 2 +- federationapi/routing/join.go | 2 +- federationapi/routing/leave.go | 2 +- federationapi/routing/missingevents.go | 4 +- federationapi/routing/send.go | 6 +- federationapi/routing/state.go | 12 ++- federationapi/routing/threepid.go | 2 +- federationsender/consumers/roomserver.go | 5 +- go.mod | 2 +- go.sum | 2 + publicroomsapi/consumers/roomserver.go | 11 ++- roomserver/alias/alias.go | 2 +- roomserver/api/query.go | 14 +-- roomserver/query/query.go | 110 +++++++++++++++++++++-- syncapi/consumers/roomserver.go | 5 +- 21 files changed, 163 insertions(+), 36 deletions(-) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 6d3ea808f..04eb3d992 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -143,7 +143,9 @@ func (s *OutputRoomEventConsumer) lookupMissingStateEvents( return nil, err } - result = append(result, eventResp.Events...) + for _, headeredEvent := range eventResp.Events { + result = append(result, headeredEvent.Event) + } return result, nil } diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go index a65281514..e90d56693 100644 --- a/clientapi/consumers/roomserver.go +++ b/clientapi/consumers/roomserver.go @@ -138,7 +138,9 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( return nil, err } - result = append(result, eventResp.Events...) + for _, headeredEvent := range eventResp.Events { + result = append(result, headeredEvent.Event) + } return result, nil } diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go index 1c1a7add9..2d3152510 100644 --- a/clientapi/routing/getevent.go +++ b/clientapi/routing/getevent.go @@ -66,7 +66,7 @@ func GetEvent( } } - requestedEvent := eventsResp.Events[0] + requestedEvent := eventsResp.Events[0].Event r := getEventRequest{ req: req, diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index da96c5c65..dbfea09c5 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -144,7 +144,7 @@ func generateSendEvent( // check to see if this user can perform this operation stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents)) for i := range queryRes.StateEvents { - stateEvents[i] = &queryRes.StateEvents[i] + stateEvents[i] = &queryRes.StateEvents[i].Event } provider := gomatrixserverlib.NewAuthEvents(stateEvents) if err = gomatrixserverlib.Allowed(*e, &provider); err != nil { diff --git a/common/events.go b/common/events.go index 3c060ee65..ff46e8e0b 100644 --- a/common/events.go +++ b/common/events.go @@ -89,7 +89,7 @@ func AddPrevEventsToEvent( authEvents := gomatrixserverlib.NewAuthEvents(nil) for i := range queryRes.StateEvents { - err = authEvents.AddEvent(&queryRes.StateEvents[i]) + err = authEvents.AddEvent(&queryRes.StateEvents[i].Event) if err != nil { return err } diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index b74bfe3ac..a4bc3c67d 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -84,10 +84,10 @@ func Backfill( // Filter any event that's not from the requested room out. evs := make([]gomatrixserverlib.Event, 0) - var ev gomatrixserverlib.Event + var ev gomatrixserverlib.HeaderedEvent for _, ev = range res.Events { if ev.RoomID() == roomID { - evs = append(evs, ev) + evs = append(evs, ev.Event) } } diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go index c4248022b..a91528b3d 100644 --- a/federationapi/routing/events.go +++ b/federationapi/routing/events.go @@ -80,5 +80,5 @@ func getEvent( return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} } - return &eventsResponse.Events[0], nil + return &eventsResponse.Events[0].Event, nil } diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 369e19aed..7d48c86dc 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -78,7 +78,7 @@ func MakeJoin( // Check that the join is allowed or not stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents)) for i := range queryRes.StateEvents { - stateEvents[i] = &queryRes.StateEvents[i] + stateEvents[i] = &queryRes.StateEvents[i].Event } provider := gomatrixserverlib.NewAuthEvents(stateEvents) if err = gomatrixserverlib.Allowed(*event, &provider); err != nil { diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index f27989ff7..3eceb6f25 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -76,7 +76,7 @@ func MakeLeave( // Check that the leave is allowed or not stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents)) for i := range queryRes.StateEvents { - stateEvents[i] = &queryRes.StateEvents[i] + stateEvents[i] = &queryRes.StateEvents[i].Event } provider := gomatrixserverlib.NewAuthEvents(stateEvents) if err = gomatrixserverlib.Allowed(*event, &provider); err != nil { diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go index 661ec33a4..069bff3dd 100644 --- a/federationapi/routing/missingevents.go +++ b/federationapi/routing/missingevents.go @@ -68,8 +68,8 @@ func GetMissingEvents( // filterEvents returns only those events with matching roomID and having depth greater than minDepth func filterEvents( - events []gomatrixserverlib.Event, minDepth int64, roomID string, -) []gomatrixserverlib.Event { + events []gomatrixserverlib.HeaderedEvent, minDepth int64, roomID string, +) []gomatrixserverlib.HeaderedEvent { ref := events[:0] for _, ev := range events { if ev.Depth() >= minDepth && ev.RoomID() == roomID { diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 079e121f9..ce9ad8b13 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -162,7 +162,11 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event) error { } // Check that the event is allowed by the state at the event. - if err := checkAllowedByState(e, stateResp.StateEvents); err != nil { + var events []gomatrixserverlib.Event + for _, headeredEvent := range stateResp.StateEvents { + events = append(events, headeredEvent.Event) + } + if err := checkAllowedByState(e, events); err != nil { return err } diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 86cf1cf54..86a1e08d2 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -129,9 +129,17 @@ func getState( return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} } + var stateEvents, authEvents []gomatrixserverlib.Event + for _, headeredEvent := range response.StateEvents { + stateEvents = append(stateEvents, headeredEvent.Event) + } + for _, headeredEvent := range response.AuthChainEvents { + authEvents = append(authEvents, headeredEvent.Event) + } + return &gomatrixserverlib.RespState{ - StateEvents: response.StateEvents, - AuthEvents: response.AuthChainEvents, + StateEvents: stateEvents, + AuthEvents: authEvents, }, nil } diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 03f3c5bba..18ebc07e1 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -264,7 +264,7 @@ func buildMembershipEvent( authEvents := gomatrixserverlib.NewAuthEvents(nil) for i := range queryRes.StateEvents { - err = authEvents.AddEvent(&queryRes.StateEvents[i]) + err = authEvents.AddEvent(&queryRes.StateEvents[i].Event) if err != nil { return nil, err } diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 4568f44dc..39d8d62db 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -325,7 +325,10 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( return nil, err } - result = append(result, eventResp.Events...) + for _, headeredEvent := range eventResp.Events { + result = append(result, headeredEvent.Event) + } + missing = missingEventsFrom(result, addsStateEventIDs) if len(missing) != 0 { diff --git a/go.mod b/go.mod index 0728a03cc..9b5aa1414 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f + github.com/matrix-org/gomatrixserverlib v0.0.0-20200316163031-36bb5f40ae9b github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index af0876489..95d39e5b0 100644 --- a/go.sum +++ b/go.sum @@ -142,6 +142,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200316100021-ac4a53d49690 h1:aQ github.com/matrix-org/gomatrixserverlib v0.0.0-20200316100021-ac4a53d49690/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f h1:JXSvzG4movqXT1KcdX+XZ9HM61m1FW4rIMFKUM9j/Dk= github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316163031-36bb5f40ae9b h1:DdVa6Ledhi5dqwYMw8e9zG+kZmRlnWMgsYOxb3NaFOU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316163031-36bb5f40ae9b/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= diff --git a/publicroomsapi/consumers/roomserver.go b/publicroomsapi/consumers/roomserver.go index 9a817735a..2bbd92b72 100644 --- a/publicroomsapi/consumers/roomserver.go +++ b/publicroomsapi/consumers/roomserver.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" sarama "gopkg.in/Shopify/sarama.v1" ) @@ -98,5 +99,13 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { return err } - return s.db.UpdateRoomFromEvents(context.TODO(), addQueryRes.Events, remQueryRes.Events) + var addQueryEvents, remQueryEvents []gomatrixserverlib.Event + for _, headeredEvent := range addQueryRes.Events { + addQueryEvents = append(addQueryEvents, headeredEvent.Event) + } + for _, headeredEvent := range remQueryRes.Events { + remQueryEvents = append(remQueryEvents, headeredEvent.Event) + } + + return s.db.UpdateRoomFromEvents(context.TODO(), addQueryEvents, remQueryEvents) } diff --git a/roomserver/alias/alias.go b/roomserver/alias/alias.go index aeaf5ae94..59da35b3f 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/alias/alias.go @@ -229,7 +229,7 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( // Add auth events authEvents := gomatrixserverlib.NewAuthEvents(nil) for i := range res.StateEvents { - err = authEvents.AddEvent(&res.StateEvents[i]) + err = authEvents.AddEvent(&res.StateEvents[i].Event) if err != nil { return err } diff --git a/roomserver/api/query.go b/roomserver/api/query.go index e1850e723..24cca03ec 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -50,7 +50,7 @@ type QueryLatestEventsAndStateResponse struct { // This list will be in an arbitrary order. // These are used to set the auth_events when sending an event. // These are used to check whether the event is allowed. - StateEvents []gomatrixserverlib.Event `json:"state_events"` + StateEvents []gomatrixserverlib.HeaderedEvent `json:"state_events"` // The depth of the latest events. // This is one greater than the maximum depth of the latest events. // This is used to set the depth when sending an event. @@ -79,7 +79,7 @@ type QueryStateAfterEventsResponse struct { PrevEventsExist bool `json:"prev_events_exist"` // The state events requested. // This list will be in an arbitrary order. - StateEvents []gomatrixserverlib.Event `json:"state_events"` + StateEvents []gomatrixserverlib.HeaderedEvent `json:"state_events"` } // QueryEventsByIDRequest is a request to QueryEventsByID @@ -99,7 +99,7 @@ type QueryEventsByIDResponse struct { // fails to read it from the database then it will fail // the entire request. // This list will be in an arbitrary order. - Events []gomatrixserverlib.Event `json:"events"` + Events []gomatrixserverlib.HeaderedEvent `json:"events"` } // QueryMembershipForUserRequest is a request to QueryMembership @@ -186,7 +186,7 @@ type QueryMissingEventsRequest struct { // QueryMissingEventsResponse is a response to QueryMissingEvents type QueryMissingEventsResponse struct { // Missing events, arbritrary order. - Events []gomatrixserverlib.Event `json:"events"` + Events []gomatrixserverlib.HeaderedEvent `json:"events"` } // QueryStateAndAuthChainRequest is a request to QueryStateAndAuthChain @@ -212,8 +212,8 @@ type QueryStateAndAuthChainResponse struct { PrevEventsExist bool `json:"prev_events_exist"` // The state and auth chain events that were requested. // The lists will be in an arbitrary order. - StateEvents []gomatrixserverlib.Event `json:"state_events"` - AuthChainEvents []gomatrixserverlib.Event `json:"auth_chain_events"` + StateEvents []gomatrixserverlib.HeaderedEvent `json:"state_events"` + AuthChainEvents []gomatrixserverlib.HeaderedEvent `json:"auth_chain_events"` } // QueryBackfillRequest is a request to QueryBackfill. @@ -229,7 +229,7 @@ type QueryBackfillRequest struct { // QueryBackfillResponse is a response to QueryBackfill. type QueryBackfillResponse struct { // Missing events, arbritrary order. - Events []gomatrixserverlib.Event `json:"events"` + Events []gomatrixserverlib.HeaderedEvent `json:"events"` } // QueryServersInRoomAtEventRequest is a request to QueryServersInRoomAtEvent diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 83f89777f..7f69699f7 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -89,6 +89,10 @@ type RoomserverQueryAPIDatabase interface { EventStateKeys( context.Context, []types.EventStateKeyNID, ) (map[types.EventStateKeyNID]string, error) + // Look up the room version for a given room. + GetRoomVersionForRoom( + ctx context.Context, roomNID types.RoomNID, + ) (gomatrixserverlib.RoomVersion, error) } // RoomserverQueryAPI is an implementation of api.RoomserverQueryAPI @@ -116,6 +120,12 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( return nil } response.RoomExists = true + + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, roomNID) + if err != nil { + return err + } + var currentStateSnapshotNID types.StateSnapshotNID response.LatestEvents, currentStateSnapshotNID, response.Depth, err = r.DB.LatestEventIDs(ctx, roomNID) @@ -136,7 +146,15 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( return err } - response.StateEvents = stateEvents + for _, event := range stateEvents { + response.StateEvents = append(response.StateEvents, gomatrixserverlib.HeaderedEvent{ + EventHeader: gomatrixserverlib.EventHeader{ + RoomVersion: roomVersion, + }, + Event: event, + }) + } + return nil } @@ -161,6 +179,11 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( } response.RoomExists = true + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, roomNID) + if err != nil { + return err + } + prevStates, err := r.DB.StateAtEventIDs(ctx, request.PrevEventIDs) if err != nil { switch err.(type) { @@ -185,7 +208,15 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( return err } - response.StateEvents = stateEvents + for _, event := range stateEvents { + response.StateEvents = append(response.StateEvents, gomatrixserverlib.HeaderedEvent{ + EventHeader: gomatrixserverlib.EventHeader{ + RoomVersion: roomVersion, + }, + Event: event, + }) + } + return nil } @@ -212,7 +243,18 @@ func (r *RoomserverQueryAPI) QueryEventsByID( return err } - response.Events = events + for _, event := range events { + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + + response.Events = append(response.Events, gomatrixserverlib.HeaderedEvent{ + EventHeader: gomatrixserverlib.EventHeader{ + RoomVersion: roomVersion, + }, + Event: event, + }) + } + return nil } @@ -486,10 +528,18 @@ func (r *RoomserverQueryAPI) QueryMissingEvents( return err } - response.Events = make([]gomatrixserverlib.Event, 0, len(loadedEvents)-len(eventsToFilter)) + response.Events = make([]gomatrixserverlib.HeaderedEvent, 0, len(loadedEvents)-len(eventsToFilter)) for _, event := range loadedEvents { if !eventsToFilter[event.EventID()] { - response.Events = append(response.Events, event) + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + + response.Events = append(response.Events, gomatrixserverlib.HeaderedEvent{ + EventHeader: gomatrixserverlib.EventHeader{ + RoomVersion: roomVersion, + }, + Event: event, + }) } } @@ -525,7 +575,24 @@ func (r *RoomserverQueryAPI) QueryBackfill( } // Retrieve events from the list that was filled previously. - response.Events, err = r.loadEvents(ctx, resultNIDs) + var loadedEvents []gomatrixserverlib.Event + loadedEvents, err = r.loadEvents(ctx, resultNIDs) + if err != nil { + return err + } + + for _, event := range loadedEvents { + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + + response.Events = append(response.Events, gomatrixserverlib.HeaderedEvent{ + EventHeader: gomatrixserverlib.EventHeader{ + RoomVersion: roomVersion, + }, + Event: event, + }) + } + return err } @@ -635,8 +702,35 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( return err } - response.StateEvents = stateEvents - response.AuthChainEvents, err = getAuthChain(ctx, r.DB, request.AuthEventIDs) + authEvents, err := getAuthChain(ctx, r.DB, request.AuthEventIDs) + if err != nil { + return err + } + + for _, event := range stateEvents { + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + + response.StateEvents = append(response.StateEvents, gomatrixserverlib.HeaderedEvent{ + EventHeader: gomatrixserverlib.EventHeader{ + RoomVersion: roomVersion, + }, + Event: event, + }) + } + + for _, event := range authEvents { + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + + response.AuthChainEvents = append(response.AuthChainEvents, gomatrixserverlib.HeaderedEvent{ + EventHeader: gomatrixserverlib.EventHeader{ + RoomVersion: roomVersion, + }, + Event: event, + }) + } + return err } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 5dbef4b7d..976dd8e8f 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -229,7 +229,10 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( return nil, err } - result = append(result, eventResp.Events...) + for _, headeredEvent := range eventResp.Events { + result = append(result, headeredEvent.Event) + } + missing = missingEventsFrom(result, addsStateEventIDs) if len(missing) != 0 { From 9f74a8798e26d9482f1964ff79a28acfe61d6dd1 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 16 Mar 2020 17:51:58 +0000 Subject: [PATCH 05/17] bugfix: Fix #908 by setting the correct state after the event (#913) * bugfix: Fix #908 by setting the correct state after the event Previously, this would only happen if the state already existed previously! * Structured logging --- federationapi/routing/send.go | 1 + roomserver/state/v1/state.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index ce9ad8b13..d3e060ac2 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -116,6 +116,7 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { results[e.EventID()] = gomatrixserverlib.PDUResult{ Error: err.Error(), } + util.GetLogger(t.context).WithError(err).WithField("event_id", e.EventID()).Warn("Failed to process incoming federation event, skipping it.") } else { results[e.EventID()] = gomatrixserverlib.PDUResult{} } diff --git a/roomserver/state/v1/state.go b/roomserver/state/v1/state.go index 5683745bf..3eb601925 100644 --- a/roomserver/state/v1/state.go +++ b/roomserver/state/v1/state.go @@ -366,11 +366,16 @@ func (v StateResolutionV1) loadStateAfterEventsForNumericTuples( // update that key in the result. // If the requested event wasn't a state event then the state after // it is the same as the state before it. + set := false for i := range result { if result[i].StateKeyTuple == prevState.StateKeyTuple { result[i] = prevState.StateEntry + set = true } } + if !set { // no previous state exists for this event: add new state + result = append(result, prevState.StateEntry) + } } return result, nil } From aebf347a79102597a359dcd59bb45593c8352780 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 17 Mar 2020 11:01:25 +0000 Subject: [PATCH 06/17] Implement gomatrixserverlib.HeaderedEvent in roomserver Kafka output (#914) * Use Event.Headered * Use HeaderedEvent in roomserver kafka output * Fix syncserver-integration-tests * Update producers to roomserver inputs * Update gomatrixserverlib * Update gomatrixserverlib * Update gomatrixserverlib * Update gomatrixserverlib * Update gomatrixserverlib * Update gomatrixserverlib --- appservice/consumers/roomserver.go | 4 +- clientapi/consumers/roomserver.go | 2 +- clientapi/producers/roomserver.go | 18 +- clientapi/routing/createroom.go | 3 +- cmd/create-room-events/main.go | 2 +- cmd/syncserver-integration-tests/main.go | 2 +- federationsender/consumers/roomserver.go | 6 +- go.mod | 3 +- go.sum | 234 +++++++++++++++++++++++ roomserver/alias/alias.go | 5 +- roomserver/api/input.go | 4 +- roomserver/api/output.go | 4 +- roomserver/input/events.go | 10 +- roomserver/input/latest_events.go | 5 +- roomserver/input/membership.go | 3 +- roomserver/query/query.go | 49 +---- syncapi/consumers/roomserver.go | 6 +- 17 files changed, 289 insertions(+), 71 deletions(-) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 04eb3d992..3e47dee56 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -101,11 +101,11 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { "type": ev.Type(), }).Info("appservice received an event from roomserver") - missingEvents, err := s.lookupMissingStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev) + missingEvents, err := s.lookupMissingStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev.Event) if err != nil { return err } - events := append(missingEvents, ev) + events := append(missingEvents, ev.Event) // Send event to any relevant application services return s.filterRoomserverEvents(context.TODO(), events) diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go index e90d56693..6d5bb09a6 100644 --- a/clientapi/consumers/roomserver.go +++ b/clientapi/consumers/roomserver.go @@ -91,7 +91,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { "type": ev.Type(), }).Info("received event from roomserver") - events, err := s.lookupStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev) + events, err := s.lookupStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev.Event) if err != nil { return err } diff --git a/clientapi/producers/roomserver.go b/clientapi/producers/roomserver.go index e50561a70..0fe2d556b 100644 --- a/clientapi/producers/roomserver.go +++ b/clientapi/producers/roomserver.go @@ -40,9 +40,11 @@ func (c *RoomserverProducer) SendEvents( ) (string, error) { ires := make([]api.InputRoomEvent, len(events)) for i, event := range events { + roomVersion := gomatrixserverlib.RoomVersionV1 + ires[i] = api.InputRoomEvent{ Kind: api.KindNew, - Event: event, + Event: event.Headered(roomVersion), AuthEventIDs: event.AuthEventIDs(), SendAsServer: string(sendAsServer), TransactionID: txnID, @@ -61,11 +63,14 @@ func (c *RoomserverProducer) SendEventWithState( return err } + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + ires := make([]api.InputRoomEvent, len(outliers)+1) for i, outlier := range outliers { ires[i] = api.InputRoomEvent{ Kind: api.KindOutlier, - Event: outlier, + Event: outlier.Headered(roomVersion), AuthEventIDs: outlier.AuthEventIDs(), } } @@ -77,7 +82,7 @@ func (c *RoomserverProducer) SendEventWithState( ires[len(outliers)] = api.InputRoomEvent{ Kind: api.KindNew, - Event: event, + Event: event.Headered(roomVersion), AuthEventIDs: event.AuthEventIDs(), HasState: true, StateEventIDs: stateEventIDs, @@ -104,8 +109,13 @@ func (c *RoomserverProducer) SendInputRoomEvents( func (c *RoomserverProducer) SendInvite( ctx context.Context, inviteEvent gomatrixserverlib.Event, ) error { + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + request := api.InputRoomEventsRequest{ - InputInviteEvents: []api.InputInviteEvent{{Event: inviteEvent}}, + InputInviteEvents: []api.InputInviteEvent{{ + Event: inviteEvent.Headered(roomVersion), + }}, } var response api.InputRoomEventsResponse return c.InputAPI.InputRoomEvents(ctx, &request, &response) diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index c9623acb9..3465d9854 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -180,7 +180,8 @@ func createRoom( } r.CreationContent["creator"] = userID - r.CreationContent["room_version"] = "1" // TODO: We set this to 1 before we support Room versioning + // TODO: Room version here + r.CreationContent["room_version"] = gomatrixserverlib.RoomVersionV1 // TODO: visibility/presets/raw initial state // TODO: Create room alias association diff --git a/cmd/create-room-events/main.go b/cmd/create-room-events/main.go index 8475914f0..ef442a0c4 100644 --- a/cmd/create-room-events/main.go +++ b/cmd/create-room-events/main.go @@ -125,7 +125,7 @@ func writeEvent(event gomatrixserverlib.Event) { if *format == "InputRoomEvent" { var ire api.InputRoomEvent ire.Kind = api.KindNew - ire.Event = event + ire.Event = event.Headered(gomatrixserverlib.RoomVersionV1) authEventIDs := []string{} for _, ref := range b.AuthEvents { authEventIDs = append(authEventIDs, ref.EventID) diff --git a/cmd/syncserver-integration-tests/main.go b/cmd/syncserver-integration-tests/main.go index 7f315cd33..d14e854c3 100644 --- a/cmd/syncserver-integration-tests/main.go +++ b/cmd/syncserver-integration-tests/main.go @@ -104,7 +104,7 @@ func clientEventJSONForOutputRoomEvent(outputRoomEvent string) string { panic("failed to unmarshal output room event: " + err.Error()) } clientEvs := gomatrixserverlib.ToClientEvents([]gomatrixserverlib.Event{ - out.NewRoomEvent.Event, + out.NewRoomEvent.Event.Event, }, gomatrixserverlib.FormatSync) b, err := json.Marshal(clientEvs[0]) if err != nil { diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 39d8d62db..2d4b1b2ef 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -109,7 +109,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { // processMessage updates the list of currently joined hosts in the room // and then sends the event to the hosts that were joined before the event. func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) error { - addsStateEvents, err := s.lookupStateEvents(ore.AddsStateEventIDs, ore.Event) + addsStateEvents, err := s.lookupStateEvents(ore.AddsStateEventIDs, ore.Event.Event) if err != nil { return err } @@ -155,7 +155,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) err // Send the event. return s.queues.SendEvent( - &ore.Event, gomatrixserverlib.ServerName(ore.SendAsServer), joinedHostsAtEvent, + &ore.Event.Event, gomatrixserverlib.ServerName(ore.SendAsServer), joinedHostsAtEvent, ) } @@ -178,7 +178,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent( ore.AddsStateEventIDs, ore.RemovesStateEventIDs, ore.StateBeforeAddsEventIDs, ore.StateBeforeRemovesEventIDs, ) - combinedAddsEvents, err := s.lookupStateEvents(combinedAdds, ore.Event) + combinedAddsEvents, err := s.lookupStateEvents(combinedAdds, ore.Event.Event) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 9b5aa1414..114a548e5 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,7 @@ module github.com/matrix-org/dendrite require ( github.com/btcsuite/btcutil v1.0.1 + github.com/golangci/golangci-lint v1.19.1 // indirect github.com/gorilla/mux v1.7.3 github.com/hashicorp/golang-lru v0.5.4 github.com/lib/pq v1.2.0 @@ -10,7 +11,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200316163031-36bb5f40ae9b + github.com/matrix-org/gomatrixserverlib v0.0.0-20200317103236-134415251e65 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 95d39e5b0..a02e29542 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,20 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us= +github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -24,37 +30,81 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db h1:GYXWx7Vr3+zv833u+8IoXbNnQY0AdXsxAgI0kX7xcwA= +github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0= +github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g= +github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= +github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8= +github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= +github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ= +github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= +github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k= +github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= +github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= +github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= +github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg= +github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= +github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= +github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= +github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4= +github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= +github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA= +github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -63,6 +113,37 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0= +github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= +github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM= +github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= +github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w= +github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= +github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw= +github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= +github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8= +github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= +github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8= +github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= +github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU= +github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= +github.com/golangci/golangci-lint v1.19.1 h1:g9xL8KW7UZDCkVlgHYJMA6F4Sj/sRVa0FoCeXI+Z3iM= +github.com/golangci/golangci-lint v1.19.1/go.mod h1:2CEc4Fxx3vxDv7g8DyXkHCBF73AOzAymcJAprs2vCps= +github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI= +github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= +github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217 h1:En/tZdwhAn0JNwLuXzP3k2RVtMqMmOEK7Yu/g3tmtJE= +github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= +github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= +github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= +github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770 h1:EL/O5HGrF7Jaq0yNhBLucz9hTuRzj2LdwGBOaENgxIk= +github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= +github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us= +github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= +github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1aiNg8JbBMO2+E1WIQ+j/gL4SQqGPg= +github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= +github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys= +github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= @@ -70,13 +151,24 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw= +github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU= github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= @@ -84,15 +176,22 @@ github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr1 github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -100,6 +199,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= @@ -110,7 +210,12 @@ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS github.com/libp2p/go-libp2p-core v0.5.0 h1:FBQ1fpq2Fo/ClyjojVJ5AKXlKhvNc/B6U0O+7AN1ffE= github.com/libp2p/go-libp2p-core v0.5.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86 h1:q6SrfsK4FojRnJ1j8+8OJzyq3g9Y1oSVyL6nYGJXXBk= +github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/matrix-org/dendrite v0.0.0-20200220135450-0352f250b857/go.mod h1:DZ35IoR+ViBNVPe9umdlOSnjvKl7wfyRmZg4QfWGvTo= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= @@ -144,6 +249,20 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f h1:JX github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/gomatrixserverlib v0.0.0-20200316163031-36bb5f40ae9b h1:DdVa6Ledhi5dqwYMw8e9zG+kZmRlnWMgsYOxb3NaFOU= github.com/matrix-org/gomatrixserverlib v0.0.0-20200316163031-36bb5f40ae9b/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316173242-f66834855555 h1:7M7QwpGbc5fJ6Uxn62ECsagekKkV36hjvnbaDQGJhls= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316173242-f66834855555/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316191245-60f1fa78147c h1:hbu6d/7LOKwPpISyE62cUXapx6askaDtKnpb5dtqr1I= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316191245-60f1fa78147c/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316200447-0820dfcdff10 h1:O6EHrD7m9Ah9SjMoHuEHHWh4moKllCYo0HRVtSSGzYc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316200447-0820dfcdff10/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316202926-7ce8b3298af6 h1:ZeRF0DpsnaxwYL7b6U1J3XMyUBpB8GWiNCqz3bWdivs= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316202926-7ce8b3298af6/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316203446-64795a12215c h1:s7XUmnqbJTJmMHuZ9Ooad6vKlBsSpIe/C3aJc6G6cy8= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316203446-64795a12215c/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316222111-b66d3b63dd85 h1:I7ocq675X1B+RYDUQPvMcYcpfHkdTcreil48r3qs2KE= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316222111-b66d3b63dd85/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317103236-134415251e65 h1:llq2yETQhD75ImQhCwHbnivSKOgzOP6QdUqH9sse7XM= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317103236-134415251e65/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= @@ -151,9 +270,14 @@ github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXL github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U= github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= @@ -166,10 +290,16 @@ github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8Rv github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -187,15 +317,22 @@ github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= +github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E= +github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -204,6 +341,7 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= @@ -213,29 +351,63 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/securego/gosec v0.0.0-20190912120752-140048b2a218 h1:O0yPHYL49quNL4Oj2wVq+zbGMu4dAM6iLoOQtm49TrQ= +github.com/securego/gosec v0.0.0-20190912120752-140048b2a218/go.mod h1:q6oYAujd2qyeU4cJqIri4LBIgdHXGvxWHZ1E29HNFRE= +github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs= +github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= @@ -251,6 +423,9 @@ github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.0.3 h1:DeF+0LZqvIt4fKYw41aPB29ZGlvwVkHKktoXJ1YW9Y8= github.com/tidwall/sjson v1.0.3/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= +github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec h1:AmoEvWAO3nDx1MEcMzPh+GzOOIA5Znpv6++c7bePPY0= +github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber-go/atomic v1.3.0 h1:ylWoWcs+jXihgo3Us1Sdsatf2R6+OlBGm8fexR3oFG4= github.com/uber-go/atomic v1.3.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= @@ -261,18 +436,37 @@ github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo= +github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= +github.com/ultraware/whitespace v0.0.3 h1:S5BCRRB5sttNy0bSOhbpw+0mb+cHiCmWfrvxpEzuUk0= +github.com/ultraware/whitespace v0.0.3/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= +github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010= go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI= @@ -281,48 +475,74 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678 h1:rM1Udd0CgtYI3KUIhu9ROz0QCqjW+n/ODp/hH7c60Xc= +golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -331,6 +551,7 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE= gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -340,6 +561,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= @@ -347,9 +569,11 @@ gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI= gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -357,3 +581,13 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I= +mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= +mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo= +mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= +mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4= +mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 h1:JPJh2pk3+X4lXAkZIk2RuE/7/FoK9maXw+TNPJhVS/c= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/roomserver/alias/alias.go b/roomserver/alias/alias.go index 59da35b3f..f4f5c20ce 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/alias/alias.go @@ -250,10 +250,13 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( return err } + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + // Create the request ire := roomserverAPI.InputRoomEvent{ Kind: roomserverAPI.KindNew, - Event: event, + Event: event.Headered(roomVersion), AuthEventIDs: event.AuthEventIDs(), SendAsServer: serverName, } diff --git a/roomserver/api/input.go b/roomserver/api/input.go index 9643a927c..f07cc0221 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -51,7 +51,7 @@ type InputRoomEvent struct { // This controls how the event is processed. Kind int `json:"kind"` // The event JSON for the event to add. - Event gomatrixserverlib.Event `json:"event"` + Event gomatrixserverlib.HeaderedEvent `json:"event"` // List of state event IDs that authenticate this event. // These are likely derived from the "auth_events" JSON key of the event. // But can be different because the "auth_events" key can be incomplete or wrong. @@ -85,7 +85,7 @@ type TransactionID struct { // the usual context a matrix room event would have. We usually do not have // access to the events needed to check the event auth rules for the invite. type InputInviteEvent struct { - Event gomatrixserverlib.Event `json:"event"` + Event gomatrixserverlib.HeaderedEvent `json:"event"` } // InputRoomEventsRequest is a request to InputRoomEvents diff --git a/roomserver/api/output.go b/roomserver/api/output.go index c09d5a1e5..4e7adff79 100644 --- a/roomserver/api/output.go +++ b/roomserver/api/output.go @@ -54,7 +54,7 @@ type OutputEvent struct { // prev_events. type OutputNewRoomEvent struct { // The Event. - Event gomatrixserverlib.Event `json:"event"` + Event gomatrixserverlib.HeaderedEvent `json:"event"` // The latest events in the room after this event. // This can be used to set the prev events for new events in the room. // This also can be used to get the full current state after this event. @@ -117,7 +117,7 @@ type OutputNewRoomEvent struct { // tracked separately from the room events themselves. type OutputNewInviteEvent struct { // The "m.room.member" invite event. - Event gomatrixserverlib.Event `json:"event"` + Event gomatrixserverlib.HeaderedEvent `json:"event"` } // An OutputRetireInviteEvent is written whenever an existing invite is no longer diff --git a/roomserver/input/events.go b/roomserver/input/events.go index a3b70753e..7fbc5d8a9 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -95,7 +95,7 @@ func processRoomEvent( event := input.Event // Check that the event passes authentication checks and work out the numeric IDs for the auth events. - authEventNIDs, err := checkAuthEvents(ctx, db, event, input.AuthEventIDs) + authEventNIDs, err := checkAuthEvents(ctx, db, event.Event, input.AuthEventIDs) if err != nil { return } @@ -112,7 +112,7 @@ func processRoomEvent( } // Store the event - roomNID, stateAtEvent, err := db.StoreEvent(ctx, event, input.TransactionID, authEventNIDs) + roomNID, stateAtEvent, err := db.StoreEvent(ctx, event.Event, input.TransactionID, authEventNIDs) if err != nil { return } @@ -127,7 +127,7 @@ func processRoomEvent( if stateAtEvent.BeforeStateSnapshotNID == 0 { // We haven't calculated a state for this event yet. // Lets calculate one. - err = calculateAndSetState(ctx, db, input, roomNID, &stateAtEvent, event) + err = calculateAndSetState(ctx, db, input, roomNID, &stateAtEvent, event.Event) if err != nil { return } @@ -140,7 +140,7 @@ func processRoomEvent( // Update the extremities of the event graph for the room return event.EventID(), updateLatestEvents( - ctx, db, ow, roomNID, stateAtEvent, event, input.SendAsServer, input.TransactionID, + ctx, db, ow, roomNID, stateAtEvent, event.Event, input.SendAsServer, input.TransactionID, ) } @@ -234,7 +234,7 @@ func processInviteEvent( return nil } - outputUpdates, err := updateToInviteMembership(updater, &input.Event, nil) + outputUpdates, err := updateToInviteMembership(updater, &input.Event.Event, nil) if err != nil { return err } diff --git a/roomserver/input/latest_events.go b/roomserver/input/latest_events.go index f9fd1d5d4..9a99ad76f 100644 --- a/roomserver/input/latest_events.go +++ b/roomserver/input/latest_events.go @@ -253,8 +253,11 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) latestEventIDs[i] = u.latest[i].EventID } + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + ore := api.OutputNewRoomEvent{ - Event: u.event, + Event: u.event.Headered(roomVersion), LastSentEventID: u.lastEventIDSent, LatestEventIDs: latestEventIDs, TransactionID: u.transactionID, diff --git a/roomserver/input/membership.go b/roomserver/input/membership.go index 841c5fec6..f2ac3b510 100644 --- a/roomserver/input/membership.go +++ b/roomserver/input/membership.go @@ -136,13 +136,14 @@ func updateToInviteMembership( return nil, err } if needsSending { + roomVersion := gomatrixserverlib.RoomVersionV1 // We notify the consumers using a special event even though we will // notify them about the change in current state as part of the normal // room event stream. This ensures that the consumers only have to // consider a single stream of events when determining whether a user // is invited, rather than having to combine multiple streams themselves. onie := api.OutputNewInviteEvent{ - Event: *add, + Event: (*add).Headered(roomVersion), } updates = append(updates, api.OutputEvent{ Type: api.OutputTypeNewInviteEvent, diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 7f69699f7..52b678ac3 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -147,12 +147,7 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( } for _, event := range stateEvents { - response.StateEvents = append(response.StateEvents, gomatrixserverlib.HeaderedEvent{ - EventHeader: gomatrixserverlib.EventHeader{ - RoomVersion: roomVersion, - }, - Event: event, - }) + response.StateEvents = append(response.StateEvents, event.Headered(roomVersion)) } return nil @@ -209,12 +204,7 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( } for _, event := range stateEvents { - response.StateEvents = append(response.StateEvents, gomatrixserverlib.HeaderedEvent{ - EventHeader: gomatrixserverlib.EventHeader{ - RoomVersion: roomVersion, - }, - Event: event, - }) + response.StateEvents = append(response.StateEvents, event.Headered(roomVersion)) } return nil @@ -247,12 +237,7 @@ func (r *RoomserverQueryAPI) QueryEventsByID( // TODO: Room version here roomVersion := gomatrixserverlib.RoomVersionV1 - response.Events = append(response.Events, gomatrixserverlib.HeaderedEvent{ - EventHeader: gomatrixserverlib.EventHeader{ - RoomVersion: roomVersion, - }, - Event: event, - }) + response.Events = append(response.Events, event.Headered(roomVersion)) } return nil @@ -534,12 +519,7 @@ func (r *RoomserverQueryAPI) QueryMissingEvents( // TODO: Room version here roomVersion := gomatrixserverlib.RoomVersionV1 - response.Events = append(response.Events, gomatrixserverlib.HeaderedEvent{ - EventHeader: gomatrixserverlib.EventHeader{ - RoomVersion: roomVersion, - }, - Event: event, - }) + response.Events = append(response.Events, event.Headered(roomVersion)) } } @@ -585,12 +565,7 @@ func (r *RoomserverQueryAPI) QueryBackfill( // TODO: Room version here roomVersion := gomatrixserverlib.RoomVersionV1 - response.Events = append(response.Events, gomatrixserverlib.HeaderedEvent{ - EventHeader: gomatrixserverlib.EventHeader{ - RoomVersion: roomVersion, - }, - Event: event, - }) + response.Events = append(response.Events, event.Headered(roomVersion)) } return err @@ -711,24 +686,14 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( // TODO: Room version here roomVersion := gomatrixserverlib.RoomVersionV1 - response.StateEvents = append(response.StateEvents, gomatrixserverlib.HeaderedEvent{ - EventHeader: gomatrixserverlib.EventHeader{ - RoomVersion: roomVersion, - }, - Event: event, - }) + response.StateEvents = append(response.StateEvents, event.Headered(roomVersion)) } for _, event := range authEvents { // TODO: Room version here roomVersion := gomatrixserverlib.RoomVersionV1 - response.AuthChainEvents = append(response.AuthChainEvents, gomatrixserverlib.HeaderedEvent{ - EventHeader: gomatrixserverlib.EventHeader{ - RoomVersion: roomVersion, - }, - Event: event, - }) + response.AuthChainEvents = append(response.AuthChainEvents, event.Headered(roomVersion)) } return err diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 976dd8e8f..5aaff5627 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -98,7 +98,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { func (s *OutputRoomEventConsumer) onNewRoomEvent( ctx context.Context, msg api.OutputNewRoomEvent, ) error { - ev := msg.Event + ev := msg.Event.Event log.WithFields(log.Fields{ "event_id": ev.EventID(), "room_id": ev.RoomID(), @@ -153,7 +153,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( func (s *OutputRoomEventConsumer) onNewInviteEvent( ctx context.Context, msg api.OutputNewInviteEvent, ) error { - pduPos, err := s.db.AddInviteEvent(ctx, msg.Event) + pduPos, err := s.db.AddInviteEvent(ctx, msg.Event.Event) if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ @@ -163,7 +163,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent( }).Panicf("roomserver output log: write invite failure") return nil } - s.notifier.OnNewEvent(&msg.Event, "", nil, types.PaginationToken{PDUPosition: pduPos}) + s.notifier.OnNewEvent(&msg.Event.Event, "", nil, types.PaginationToken{PDUPosition: pduPos}) return nil } From a66c701b29255a5cb7ec68b970948fb77f1f31af Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 17 Mar 2020 15:12:01 +0000 Subject: [PATCH 07/17] Add room version to room create request, persist in storage (#915) * Add room version into createRoomReq * Extract room version from m.room.create event when persisting * Reduce cyclomatic complexity * Update whitelist, gomatrixserverlib, tweaks to roomserver * Update sytest-whitelist again --- clientapi/jsonerror/jsonerror.go | 6 ++++ clientapi/routing/createroom.go | 40 +++++++++++++++-------- go.mod | 2 +- go.sum | 6 ++++ roomserver/storage/postgres/storage.go | 43 +++++++++++++++++++++++-- roomserver/storage/sqlite3/storage.go | 44 ++++++++++++++++++++++++-- sytest-whitelist | 12 ------- 7 files changed, 123 insertions(+), 30 deletions(-) diff --git a/clientapi/jsonerror/jsonerror.go b/clientapi/jsonerror/jsonerror.go index 8df1fead2..735de5bea 100644 --- a/clientapi/jsonerror/jsonerror.go +++ b/clientapi/jsonerror/jsonerror.go @@ -124,6 +124,12 @@ func GuestAccessForbidden(msg string) *MatrixError { return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg} } +// UnsupportedRoomVersion is an error which is returned when the client +// requests a room with a version that is unsupported. +func UnsupportedRoomVersion(msg string) *MatrixError { + return &MatrixError{"M_UNSUPPORTED_ROOM_VERSION", msg} +} + // LimitExceededError is a rate-limiting error. type LimitExceededError struct { MatrixError diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 3465d9854..92b1ae887 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -23,6 +23,7 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" @@ -38,15 +39,16 @@ import ( // https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-createroom type createRoomRequest struct { - Invite []string `json:"invite"` - Name string `json:"name"` - Visibility string `json:"visibility"` - Topic string `json:"topic"` - Preset string `json:"preset"` - CreationContent map[string]interface{} `json:"creation_content"` - InitialState []fledglingEvent `json:"initial_state"` - RoomAliasName string `json:"room_alias_name"` - GuestCanJoin bool `json:"guest_can_join"` + Invite []string `json:"invite"` + Name string `json:"name"` + Visibility string `json:"visibility"` + Topic string `json:"topic"` + Preset string `json:"preset"` + CreationContent map[string]interface{} `json:"creation_content"` + InitialState []fledglingEvent `json:"initial_state"` + RoomAliasName string `json:"room_alias_name"` + GuestCanJoin bool `json:"guest_can_join"` + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` } const ( @@ -180,16 +182,28 @@ func createRoom( } r.CreationContent["creator"] = userID - // TODO: Room version here - r.CreationContent["room_version"] = gomatrixserverlib.RoomVersionV1 + roomVersion := roomserverVersion.DefaultRoomVersion() + if r.RoomVersion != "" { + candidateVersion := gomatrixserverlib.RoomVersion(r.RoomVersion) + _, roomVersionError := roomserverVersion.SupportedRoomVersion(candidateVersion) + if roomVersionError != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion(roomVersionError.Error()), + } + } + roomVersion = candidateVersion + } + r.CreationContent["room_version"] = roomVersion // TODO: visibility/presets/raw initial state // TODO: Create room alias association // Make sure this doesn't fall into an application service's namespace though! logger.WithFields(log.Fields{ - "userID": userID, - "roomID": roomID, + "userID": userID, + "roomID": roomID, + "roomVersion": r.CreationContent["room_version"], }).Info("Creating new room") profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) diff --git a/go.mod b/go.mod index 114a548e5..958706db4 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200317103236-134415251e65 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index a02e29542..517d0bf1c 100644 --- a/go.sum +++ b/go.sum @@ -30,6 +30,7 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= @@ -263,6 +264,10 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200316222111-b66d3b63dd85 h1:I7 github.com/matrix-org/gomatrixserverlib v0.0.0-20200316222111-b66d3b63dd85/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/gomatrixserverlib v0.0.0-20200317103236-134415251e65 h1:llq2yETQhD75ImQhCwHbnivSKOgzOP6QdUqH9sse7XM= github.com/matrix-org/gomatrixserverlib v0.0.0-20200317103236-134415251e65/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317114945-9a368ea4620d h1:0GYO2Jye1TNVzsn02IF5tqV80psDi0KIWC4+glH6+/Q= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317114945-9a368ea4620d/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd h1:n95A8YyiCZ8Nu2beqw4akCaPIRrZr/nesHYDZV8WkXI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= @@ -453,6 +458,7 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010= go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index ee1a1cd4d..b2b4159c9 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -18,6 +18,9 @@ package postgres import ( "context" "database/sql" + "encoding/json" + + roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" // Import the postgres database driver. _ "github.com/lib/pq" @@ -68,8 +71,21 @@ func (d *Database) StoreEvent( } } - // TODO: Room version here - if roomNID, err = d.assignRoomNID(ctx, nil, event.RoomID(), "1"); err != nil { + // TODO: Here we should aim to have two different code paths for new rooms + // vs existing ones. + + // Get the default room version. If the client doesn't supply a room_version + // then we will use our configured default to create the room. + // https://matrix.org/docs/spec/client_server/r0.6.0#post-matrix-client-r0-createroom + // Note that the below logic depends on the m.room.create event being the + // first event that is persisted to the database when creating or joining a + // room. + var roomVersion gomatrixserverlib.RoomVersion + if roomVersion, err = extractRoomVersionFromCreateEvent(event); err != nil { + return 0, types.StateAtEvent{}, err + } + + if roomNID, err = d.assignRoomNID(ctx, nil, event.RoomID(), roomVersion); err != nil { return 0, types.StateAtEvent{}, err } @@ -121,6 +137,29 @@ func (d *Database) StoreEvent( }, nil } +func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( + gomatrixserverlib.RoomVersion, error, +) { + var err error + var roomVersion gomatrixserverlib.RoomVersion + // Look for m.room.create events. + if event.Type() != gomatrixserverlib.MRoomCreate { + return gomatrixserverlib.RoomVersion(""), nil + } + roomVersion = roomserverVersion.DefaultRoomVersion() + var createContent gomatrixserverlib.CreateContent + // The m.room.create event contains an optional "room_version" key in + // the event content, so we need to unmarshal that first. + if err = json.Unmarshal(event.Content(), &createContent); err != nil { + return gomatrixserverlib.RoomVersion(""), err + } + // A room version was specified in the event content? + if createContent.RoomVersion != nil { + roomVersion = *createContent.RoomVersion + } + return roomVersion, err +} + func (d *Database) assignRoomNID( ctx context.Context, txn *sql.Tx, roomID string, roomVersion gomatrixserverlib.RoomVersion, diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index d9513611e..b912b1c0e 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -18,9 +18,12 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "errors" "net/url" + roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" @@ -67,6 +70,7 @@ func Open(dataSourceName string) (*Database, error) { } // StoreEvent implements input.EventDatabase +// nolint:gocyclo func (d *Database) StoreEvent( ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID, @@ -90,8 +94,21 @@ func (d *Database) StoreEvent( } } - // TODO: Room version here - if roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID(), "1"); err != nil { + // TODO: Here we should aim to have two different code paths for new rooms + // vs existing ones. + + // Get the default room version. If the client doesn't supply a room_version + // then we will use our configured default to create the room. + // https://matrix.org/docs/spec/client_server/r0.6.0#post-matrix-client-r0-createroom + // Note that the below logic depends on the m.room.create event being the + // first event that is persisted to the database when creating or joining a + // room. + var roomVersion gomatrixserverlib.RoomVersion + if roomVersion, err = extractRoomVersionFromCreateEvent(event); err != nil { + return err + } + + if roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID(), roomVersion); err != nil { return err } @@ -150,6 +167,29 @@ func (d *Database) StoreEvent( }, nil } +func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( + gomatrixserverlib.RoomVersion, error, +) { + var err error + var roomVersion gomatrixserverlib.RoomVersion + // Look for m.room.create events. + if event.Type() != gomatrixserverlib.MRoomCreate { + return gomatrixserverlib.RoomVersion(""), nil + } + roomVersion = roomserverVersion.DefaultRoomVersion() + var createContent gomatrixserverlib.CreateContent + // The m.room.create event contains an optional "room_version" key in + // the event content, so we need to unmarshal that first. + if err = json.Unmarshal(event.Content(), &createContent); err != nil { + return gomatrixserverlib.RoomVersion(""), err + } + // A room version was specified in the event content? + if createContent.RoomVersion != nil { + roomVersion = *createContent.RoomVersion + } + return roomVersion, err +} + func (d *Database) assignRoomNID( ctx context.Context, txn *sql.Tx, roomID string, roomVersion gomatrixserverlib.RoomVersion, diff --git a/sytest-whitelist b/sytest-whitelist index eb91b3f0c..cac718288 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -104,12 +104,6 @@ Newly banned rooms appear in the leave section of incremental sync Newly banned rooms appear in the leave section of incremental sync local user can join room with version 1 User can invite local user to room with version 1 -local user can join room with version 2 -User can invite local user to room with version 2 -local user can join room with version 3 -User can invite local user to room with version 3 -local user can join room with version 4 -User can invite local user to room with version 4 Should reject keys claiming to belong to a different user Can add account data Can add account data to room @@ -140,8 +134,6 @@ Changing the actions of an unknown rule fails with 404 Enabling an unknown default rule fails with 404 Trying to get push rules with unknown rule_id fails with 404 Events come down the correct room -local user can join room with version 5 -User can invite local user to room with version 5 # SyTest currently only implements the v1 endpoints for /send_join and /send_leave, # whereas Dendrite only supports the v2 endpoints for those, so let's ignore this # test for now. @@ -198,10 +190,6 @@ Local non-members don't see posted message events Remote room members also see posted message events Lazy loading parameters in the filter are strictly boolean remote user can join room with version 1 -remote user can join room with version 2 -remote user can join room with version 3 -remote user can join room with version 4 -remote user can join room with version 5 Inbound federation can query room alias directory Outbound federation can query v2 /send_join Inbound federation can receive v2 /send_join From 4af8323df3af61e2f5e705de9ab0f6d0b84a978b Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 17 Mar 2020 16:45:40 +0000 Subject: [PATCH 08/17] bugfix: Fix a bug which caused prev_content not to be sent to clients (#919) I don't know how this ever passed QA... also fix a missing rows.Close() --- go.sum | 1 + syncapi/consumers/roomserver.go | 2 +- syncapi/storage/sqlite3/output_room_events_table.go | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.sum b/go.sum index 517d0bf1c..63423760c 100644 --- a/go.sum +++ b/go.sum @@ -497,6 +497,7 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc= golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 5aaff5627..332ac6205 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -253,7 +253,7 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.Event } prevEvent, err := s.db.GetStateEvent( - context.TODO(), event.Type(), event.RoomID(), stateKey, + context.TODO(), event.RoomID(), event.Type(), stateKey, ) if err != nil { return event, err diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index be8937435..05a33c067 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -160,6 +160,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( if err != nil { return nil, nil, err } + defer rows.Close() // nolint: errcheck // Fetch all the state change events for all rooms between the two positions then loop each event and: // - Keep a cache of the event by ID (99% of state change events are for the event itself) // - For each room ID, build up an array of event IDs which represents cumulative adds/removes From 1467cc10d8bd40b9ea85bc4f0aa4644b16cb37bb Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 17 Mar 2020 17:18:48 +0000 Subject: [PATCH 09/17] bugfix: Fix a bug which caused failures to join rooms over federation (#917) * bugfix: Fix a bug which caused failures to join rooms over federation The cause of this was the semantics of `/send_join`'s `auth_chain` response. Previously, we would only send back the auth chain *for the join event* and not the entire room state. However, we would then try to check that the room state is valid, and then be missing auth events. Now, we send back the entire auth chain for all room state in `/send_join`. The spec needs to be clarified that this is what the chain should be. * refactor: split out grabbing state to reduce cyclo complexity --- roomserver/query/query.go | 62 +++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 52b678ac3..2de8e0d08 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -637,12 +637,6 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( request *api.QueryStateAndAuthChainRequest, response *api.QueryStateAndAuthChainResponse, ) error { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) - if err != nil { - return err - } - response.QueryStateAndAuthChainRequest = *request roomNID, err := r.DB.RoomNID(ctx, request.RoomID) if err != nil { @@ -653,31 +647,21 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( } response.RoomExists = true - prevStates, err := r.DB.StateAtEventIDs(ctx, request.PrevEventIDs) + stateEvents, err := r.loadStateAtEventIDs(ctx, request.PrevEventIDs) if err != nil { - switch err.(type) { - case types.MissingEventError: - return nil - default: - return err - } + return err } response.PrevEventsExist = true - // Look up the currrent state for the requested tuples. - stateEntries, err := roomState.LoadCombinedStateAfterEvents( - ctx, prevStates, - ) - if err != nil { - return err + // add the auth event IDs for the current state events too + var authEventIDs []string + authEventIDs = append(authEventIDs, request.AuthEventIDs...) + for _, se := range stateEvents { + authEventIDs = append(authEventIDs, se.AuthEventIDs()...) } + authEventIDs = util.UniqueStrings(authEventIDs) // de-dupe - stateEvents, err := r.loadStateEvents(ctx, stateEntries) - if err != nil { - return err - } - - authEvents, err := getAuthChain(ctx, r.DB, request.AuthEventIDs) + authEvents, err := getAuthChain(ctx, r.DB, authEventIDs) if err != nil { return err } @@ -699,6 +683,34 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( return err } +func (r *RoomserverQueryAPI) loadStateAtEventIDs(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { + // TODO: get the correct room version + roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + if err != nil { + return nil, err + } + + prevStates, err := r.DB.StateAtEventIDs(ctx, eventIDs) + if err != nil { + switch err.(type) { + case types.MissingEventError: + return nil, nil + default: + return nil, err + } + } + + // Look up the currrent state for the requested tuples. + stateEntries, err := roomState.LoadCombinedStateAfterEvents( + ctx, prevStates, + ) + if err != nil { + return nil, err + } + + return r.loadStateEvents(ctx, stateEntries) +} + // getAuthChain fetches the auth chain for the given auth events. An auth chain // is the list of all events that are referenced in the auth_events section, and // all their auth_events, recursively. The returned set of events contain the From c2bd0b97b34c9040325bf31c5c4a2a06579239d9 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 17 Mar 2020 18:00:10 +0000 Subject: [PATCH 10/17] Get room versions from database (#918) * Retrieve room version where known in roomserver * Get room versions in alias code * Increase gocyclothreshold to 13, since we hit that number a lot * Remove gocyclo nolint from StoreEvent * Update interface to get room version from room ID instead of NID * Remove new API * Fixed this query for SQLite but not for Postgres --- .golangci.yml | 2 +- roomserver/alias/alias.go | 12 ++++++-- roomserver/alias/alias_test.go | 7 +++++ roomserver/input/events.go | 4 +++ roomserver/input/latest_events.go | 6 ++-- roomserver/query/query.go | 35 ++++++++++++---------- roomserver/storage/interface.go | 2 +- roomserver/storage/postgres/rooms_table.go | 16 +++++----- roomserver/storage/postgres/storage.go | 6 ++-- roomserver/storage/sqlite3/rooms_table.go | 16 +++++----- roomserver/storage/sqlite3/storage.go | 7 ++--- 11 files changed, 68 insertions(+), 45 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 0d0f51bd2..7fdd4d003 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -102,7 +102,7 @@ linters-settings: #local-prefixes: github.com/org/project gocyclo: # minimal code complexity to report, 30 by default (but we recommend 10-20) - min-complexity: 12 + min-complexity: 13 maligned: # print struct with more effective memory layout or not, false by default suggest-new: true diff --git a/roomserver/alias/alias.go b/roomserver/alias/alias.go index f4f5c20ce..dfb7e76a7 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/alias/alias.go @@ -46,6 +46,10 @@ type RoomserverAliasAPIDatabase interface { // Remove a given room alias. // Returns an error if there was a problem talking to the database. RemoveRoomAlias(ctx context.Context, alias string) error + // Look up the room version for a given room. + GetRoomVersionForRoom( + ctx context.Context, roomID string, + ) (gomatrixserverlib.RoomVersion, error) } // RoomserverAliasAPI is an implementation of alias.RoomserverAliasAPI @@ -240,6 +244,11 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( } builder.AuthEvents = refs + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, roomID) + if err != nil { + return err + } + // Build the event eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), r.Cfg.Matrix.ServerName) now := time.Now() @@ -250,9 +259,6 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( return err } - // TODO: Room version here - roomVersion := gomatrixserverlib.RoomVersionV1 - // Create the request ire := roomserverAPI.InputRoomEvent{ Kind: roomserverAPI.KindNew, diff --git a/roomserver/alias/alias_test.go b/roomserver/alias/alias_test.go index 6ddb63a73..0aefa19d9 100644 --- a/roomserver/alias/alias_test.go +++ b/roomserver/alias/alias_test.go @@ -22,6 +22,7 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" ) type MockRoomserverAliasAPIDatabase struct { @@ -49,6 +50,12 @@ func (db *MockRoomserverAliasAPIDatabase) GetCreatorIDForAlias( return "", nil } +func (db *MockRoomserverAliasAPIDatabase) GetRoomVersionForRoom( + ctx context.Context, roomID string, +) (gomatrixserverlib.RoomVersion, error) { + return gomatrixserverlib.RoomVersionV1, nil +} + // This method needs to change depending on test case func (db *MockRoomserverAliasAPIDatabase) GetRoomIDForAlias( ctx context.Context, diff --git a/roomserver/input/events.go b/roomserver/input/events.go index 7fbc5d8a9..034b06c11 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -72,6 +72,10 @@ type RoomEventDatabase interface { ctx context.Context, transactionID string, sessionID int64, userID string, ) (string, error) + // Look up the room version for a given room. + GetRoomVersionForRoom( + ctx context.Context, roomID string, + ) (gomatrixserverlib.RoomVersion, error) } // OutputRoomEventWriter has the APIs needed to write an event to the output logs. diff --git a/roomserver/input/latest_events.go b/roomserver/input/latest_events.go index 9a99ad76f..4d75daae9 100644 --- a/roomserver/input/latest_events.go +++ b/roomserver/input/latest_events.go @@ -253,8 +253,10 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) latestEventIDs[i] = u.latest[i].EventID } - // TODO: Room version here - roomVersion := gomatrixserverlib.RoomVersionV1 + roomVersion, err := u.db.GetRoomVersionForRoom(u.ctx, u.event.RoomID()) + if err != nil { + return nil, err + } ore := api.OutputNewRoomEvent{ Event: u.event.Headered(roomVersion), diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 2de8e0d08..3ab12d60a 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -91,7 +91,7 @@ type RoomserverQueryAPIDatabase interface { ) (map[types.EventStateKeyNID]string, error) // Look up the room version for a given room. GetRoomVersionForRoom( - ctx context.Context, roomNID types.RoomNID, + ctx context.Context, roomID string, ) (gomatrixserverlib.RoomVersion, error) } @@ -121,7 +121,7 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( } response.RoomExists = true - roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, roomNID) + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) if err != nil { return err } @@ -174,7 +174,7 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( } response.RoomExists = true - roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, roomNID) + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) if err != nil { return err } @@ -234,8 +234,10 @@ func (r *RoomserverQueryAPI) QueryEventsByID( } for _, event := range events { - // TODO: Room version here - roomVersion := gomatrixserverlib.RoomVersionV1 + roomVersion, verr := r.DB.GetRoomVersionForRoom(ctx, event.RoomID()) + if verr != nil { + return verr + } response.Events = append(response.Events, event.Headered(roomVersion)) } @@ -516,8 +518,10 @@ func (r *RoomserverQueryAPI) QueryMissingEvents( response.Events = make([]gomatrixserverlib.HeaderedEvent, 0, len(loadedEvents)-len(eventsToFilter)) for _, event := range loadedEvents { if !eventsToFilter[event.EventID()] { - // TODO: Room version here - roomVersion := gomatrixserverlib.RoomVersionV1 + roomVersion, verr := r.DB.GetRoomVersionForRoom(ctx, event.RoomID()) + if verr != nil { + return verr + } response.Events = append(response.Events, event.Headered(roomVersion)) } @@ -562,8 +566,10 @@ func (r *RoomserverQueryAPI) QueryBackfill( } for _, event := range loadedEvents { - // TODO: Room version here - roomVersion := gomatrixserverlib.RoomVersionV1 + roomVersion, verr := r.DB.GetRoomVersionForRoom(ctx, event.RoomID()) + if verr != nil { + return verr + } response.Events = append(response.Events, event.Headered(roomVersion)) } @@ -647,6 +653,11 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( } response.RoomExists = true + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) + if err != nil { + return err + } + stateEvents, err := r.loadStateAtEventIDs(ctx, request.PrevEventIDs) if err != nil { return err @@ -667,16 +678,10 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( } for _, event := range stateEvents { - // TODO: Room version here - roomVersion := gomatrixserverlib.RoomVersionV1 - response.StateEvents = append(response.StateEvents, event.Headered(roomVersion)) } for _, event := range authEvents { - // TODO: Room version here - roomVersion := gomatrixserverlib.RoomVersionV1 - response.AuthChainEvents = append(response.AuthChainEvents, event.Headered(roomVersion)) } diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 7f32b53f8..20db7ef7f 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -45,5 +45,5 @@ type Database interface { GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error) GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool) ([]types.EventNID, error) EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) - GetRoomVersionForRoom(ctx context.Context, roomNID types.RoomNID) (gomatrixserverlib.RoomVersion, error) + GetRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) } diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index ef8b8ecef..6bb96f1de 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -65,8 +65,8 @@ const selectLatestEventNIDsForUpdateSQL = "" + const updateLatestEventNIDsSQL = "" + "UPDATE roomserver_rooms SET latest_event_nids = $2, last_event_sent_nid = $3, state_snapshot_nid = $4 WHERE room_nid = $1" -const selectRoomVersionForRoomNIDSQL = "" + - "SELECT room_version FROM roomserver_rooms WHERE room_nid = $1" +const selectRoomVersionForRoomIDSQL = "" + + "SELECT room_version FROM roomserver_rooms WHERE room_id = $1" type roomStatements struct { insertRoomNIDStmt *sql.Stmt @@ -74,7 +74,7 @@ type roomStatements struct { selectLatestEventNIDsStmt *sql.Stmt selectLatestEventNIDsForUpdateStmt *sql.Stmt updateLatestEventNIDsStmt *sql.Stmt - selectRoomVersionForRoomNIDStmt *sql.Stmt + selectRoomVersionForRoomIDStmt *sql.Stmt } func (s *roomStatements) prepare(db *sql.DB) (err error) { @@ -88,7 +88,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { {&s.selectLatestEventNIDsStmt, selectLatestEventNIDsSQL}, {&s.selectLatestEventNIDsForUpdateStmt, selectLatestEventNIDsForUpdateSQL}, {&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL}, - {&s.selectRoomVersionForRoomNIDStmt, selectRoomVersionForRoomNIDSQL}, + {&s.selectRoomVersionForRoomIDStmt, selectRoomVersionForRoomIDSQL}, }.prepare(db) } @@ -165,11 +165,11 @@ func (s *roomStatements) updateLatestEventNIDs( return err } -func (s *roomStatements) selectRoomVersionForRoomNID( - ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, +func (s *roomStatements) selectRoomVersionForRoomID( + ctx context.Context, txn *sql.Tx, roomID string, ) (gomatrixserverlib.RoomVersion, error) { var roomVersion gomatrixserverlib.RoomVersion - stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) - err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) + stmt := common.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) + err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) return roomVersion, err } diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index b2b4159c9..af6afe5c1 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -740,10 +740,10 @@ func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]type } func (d *Database) GetRoomVersionForRoom( - ctx context.Context, roomNID types.RoomNID, + ctx context.Context, roomID string, ) (gomatrixserverlib.RoomVersion, error) { - return d.statements.selectRoomVersionForRoomNID( - ctx, nil, roomNID, + return d.statements.selectRoomVersionForRoomID( + ctx, nil, roomID, ) } diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index b750f63ea..49fa07ea8 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -54,8 +54,8 @@ const selectLatestEventNIDsForUpdateSQL = "" + const updateLatestEventNIDsSQL = "" + "UPDATE roomserver_rooms SET latest_event_nids = $1, last_event_sent_nid = $2, state_snapshot_nid = $3 WHERE room_nid = $4" -const selectRoomVersionForRoomNIDSQL = "" + - "SELECT room_version FROM roomserver_rooms WHERE room_nid = $1" +const selectRoomVersionForRoomIDSQL = "" + + "SELECT room_version FROM roomserver_rooms WHERE room_id = $1" type roomStatements struct { insertRoomNIDStmt *sql.Stmt @@ -63,7 +63,7 @@ type roomStatements struct { selectLatestEventNIDsStmt *sql.Stmt selectLatestEventNIDsForUpdateStmt *sql.Stmt updateLatestEventNIDsStmt *sql.Stmt - selectRoomVersionForRoomNIDStmt *sql.Stmt + selectRoomVersionForRoomIDStmt *sql.Stmt } func (s *roomStatements) prepare(db *sql.DB) (err error) { @@ -77,7 +77,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { {&s.selectLatestEventNIDsStmt, selectLatestEventNIDsSQL}, {&s.selectLatestEventNIDsForUpdateStmt, selectLatestEventNIDsForUpdateSQL}, {&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL}, - {&s.selectRoomVersionForRoomNIDStmt, selectRoomVersionForRoomNIDSQL}, + {&s.selectRoomVersionForRoomIDStmt, selectRoomVersionForRoomIDSQL}, }.prepare(db) } @@ -157,11 +157,11 @@ func (s *roomStatements) updateLatestEventNIDs( return err } -func (s *roomStatements) selectRoomVersionForRoomNID( - ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, +func (s *roomStatements) selectRoomVersionForRoomID( + ctx context.Context, txn *sql.Tx, roomID string, ) (gomatrixserverlib.RoomVersion, error) { var roomVersion gomatrixserverlib.RoomVersion - stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) - err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) + stmt := common.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) + err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) return roomVersion, err } diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index b912b1c0e..ea926ca66 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -70,7 +70,6 @@ func Open(dataSourceName string) (*Database, error) { } // StoreEvent implements input.EventDatabase -// nolint:gocyclo func (d *Database) StoreEvent( ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID, @@ -895,10 +894,10 @@ func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]type } func (d *Database) GetRoomVersionForRoom( - ctx context.Context, roomNID types.RoomNID, + ctx context.Context, roomID string, ) (gomatrixserverlib.RoomVersion, error) { - return d.statements.selectRoomVersionForRoomNID( - ctx, nil, roomNID, + return d.statements.selectRoomVersionForRoomID( + ctx, nil, roomID, ) } From c019ad708669a4c269072bea933f0a520a78ca7c Mon Sep 17 00:00:00 2001 From: Prateek Sachan <42961174+prateek2211@users.noreply.github.com> Date: Wed, 18 Mar 2020 15:47:18 +0530 Subject: [PATCH 11/17] Log errors from rows.Close (#920) * Log errors from rows.Close * fixed imports * Added contextual messages * fixed review changes --- .../accounts/postgres/account_data_table.go | 4 +++- .../accounts/postgres/membership_table.go | 4 +++- .../accounts/sqlite3/membership_table.go | 2 +- .../storage/accounts/sqlite3/threepid_table.go | 2 +- .../storage/devices/postgres/devices_table.go | 2 +- common/keydb/postgres/server_key_table.go | 4 +++- common/keydb/sqlite3/server_key_table.go | 2 +- common/log.go | 18 ++++++++++++++++++ common/partition_offset_table.go | 2 +- .../storage/postgres/joined_hosts_table.go | 2 +- .../storage/sqlite3/joined_hosts_table.go | 2 +- go.mod | 3 +++ go.sum | 2 ++ mediaapi/storage/postgres/thumbnail_table.go | 4 +++- mediaapi/storage/sqlite3/thumbnail_table.go | 4 +++- .../storage/postgres/public_rooms_table.go | 4 +++- .../storage/postgres/event_json_table.go | 4 +++- .../storage/postgres/event_state_keys_table.go | 4 ++-- .../storage/postgres/event_types_table.go | 4 +++- roomserver/storage/postgres/events_table.go | 12 ++++++------ roomserver/storage/postgres/invite_table.go | 4 ++-- .../storage/postgres/membership_table.go | 4 ++-- .../storage/postgres/room_aliases_table.go | 4 +++- .../storage/postgres/state_block_table.go | 6 ++++-- roomserver/storage/sqlite3/event_json_table.go | 2 +- .../storage/sqlite3/event_state_keys_table.go | 4 ++-- .../storage/sqlite3/event_types_table.go | 2 +- roomserver/storage/sqlite3/events_table.go | 12 ++++++------ roomserver/storage/sqlite3/invite_table.go | 2 +- roomserver/storage/sqlite3/membership_table.go | 4 ++-- .../storage/sqlite3/room_aliases_table.go | 2 +- .../storage/sqlite3/state_block_table.go | 4 ++-- .../storage/sqlite3/state_snapshot_table.go | 2 +- syncapi/storage/postgres/account_data_table.go | 2 +- .../postgres/backward_extremities_table.go | 4 +++- .../postgres/current_room_state_table.go | 8 ++++---- syncapi/storage/postgres/invites_table.go | 2 +- .../postgres/output_room_events_table.go | 8 ++++---- .../output_room_events_topology_table.go | 6 ++++-- syncapi/storage/sqlite3/account_data_table.go | 4 +++- .../sqlite3/current_room_state_table.go | 8 ++++---- syncapi/storage/sqlite3/invites_table.go | 2 +- .../sqlite3/output_room_events_table.go | 6 +++--- 43 files changed, 118 insertions(+), 69 deletions(-) diff --git a/clientapi/auth/storage/accounts/postgres/account_data_table.go b/clientapi/auth/storage/accounts/postgres/account_data_table.go index 4573999b4..9198a7440 100644 --- a/clientapi/auth/storage/accounts/postgres/account_data_table.go +++ b/clientapi/auth/storage/accounts/postgres/account_data_table.go @@ -18,6 +18,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" ) @@ -90,7 +92,7 @@ func (s *accountDataStatements) selectAccountData( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectAccountData: rows.close() failed") global = []gomatrixserverlib.ClientEvent{} rooms = make(map[string][]gomatrixserverlib.ClientEvent) diff --git a/clientapi/auth/storage/accounts/postgres/membership_table.go b/clientapi/auth/storage/accounts/postgres/membership_table.go index 426c2d6ac..27570b67d 100644 --- a/clientapi/auth/storage/accounts/postgres/membership_table.go +++ b/clientapi/auth/storage/accounts/postgres/membership_table.go @@ -18,6 +18,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/lib/pq" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" ) @@ -118,7 +120,7 @@ func (s *membershipStatements) selectMembershipsByLocalpart( memberships = []authtypes.Membership{} - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsByLocalpart: rows.close() failed") for rows.Next() { var m authtypes.Membership m.Localpart = localpart diff --git a/clientapi/auth/storage/accounts/sqlite3/membership_table.go b/clientapi/auth/storage/accounts/sqlite3/membership_table.go index 38f21b7f3..b4bff6334 100644 --- a/clientapi/auth/storage/accounts/sqlite3/membership_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/membership_table.go @@ -118,7 +118,7 @@ func (s *membershipStatements) selectMembershipsByLocalpart( memberships = []authtypes.Membership{} - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsByLocalpart: rows.close() failed") for rows.Next() { var m authtypes.Membership m.Localpart = localpart diff --git a/clientapi/auth/storage/accounts/sqlite3/threepid_table.go b/clientapi/auth/storage/accounts/sqlite3/threepid_table.go index 53f6408d1..29ee4c3d0 100644 --- a/clientapi/auth/storage/accounts/sqlite3/threepid_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/threepid_table.go @@ -97,7 +97,7 @@ func (s *threepidStatements) selectThreePIDsForLocalpart( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectThreePIDsForLocalpart: rows.close() failed") threepids = []authtypes.ThreePID{} for rows.Next() { diff --git a/clientapi/auth/storage/devices/postgres/devices_table.go b/clientapi/auth/storage/devices/postgres/devices_table.go index 6cae3743e..ee5591706 100644 --- a/clientapi/auth/storage/devices/postgres/devices_table.go +++ b/clientapi/auth/storage/devices/postgres/devices_table.go @@ -226,7 +226,7 @@ func (s *devicesStatements) selectDevicesByLocalpart( if err != nil { return devices, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectDevicesByLocalpart: rows.close() failed") for rows.Next() { var dev authtypes.Device diff --git a/common/keydb/postgres/server_key_table.go b/common/keydb/postgres/server_key_table.go index 6b13cc3c2..0434eb8b1 100644 --- a/common/keydb/postgres/server_key_table.go +++ b/common/keydb/postgres/server_key_table.go @@ -19,6 +19,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/lib/pq" "github.com/matrix-org/gomatrixserverlib" ) @@ -91,7 +93,7 @@ func (s *serverKeyStatements) bulkSelectServerKeys( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectServerKeys: rows.close() failed") results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} for rows.Next() { var serverName string diff --git a/common/keydb/sqlite3/server_key_table.go b/common/keydb/sqlite3/server_key_table.go index 3caf4ea4c..ba1cc0606 100644 --- a/common/keydb/sqlite3/server_key_table.go +++ b/common/keydb/sqlite3/server_key_table.go @@ -124,7 +124,7 @@ func (s *serverKeyStatements) bulkSelectServerKeys( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectServerKeys: rows.close() failed") results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} for rows.Next() { var serverName string diff --git a/common/log.go b/common/log.go index f9ed84edb..11339ada4 100644 --- a/common/log.go +++ b/common/log.go @@ -15,13 +15,17 @@ package common import ( + "context" "fmt" + "io" "os" "path" "path/filepath" "runtime" "strings" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dugong" "github.com/sirupsen/logrus" @@ -156,3 +160,17 @@ func setupFileHook(hook config.LogrusHook, level logrus.Level, componentName str ), }) } + +//CloseAndLogIfError Closes io.Closer and logs the error if any +func CloseAndLogIfError(ctx context.Context, closer io.Closer, message string) { + if closer == nil { + return + } + err := closer.Close() + if ctx == nil { + ctx = context.TODO() + } + if err != nil { + util.GetLogger(ctx).WithError(err).Error(message) + } +} diff --git a/common/partition_offset_table.go b/common/partition_offset_table.go index 6bc066a69..aa799f8a0 100644 --- a/common/partition_offset_table.go +++ b/common/partition_offset_table.go @@ -90,7 +90,7 @@ func (s *PartitionOffsetStatements) selectPartitionOffsets( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer CloseAndLogIfError(ctx, rows, "selectPartitionOffsets: rows.close() failed") var results []PartitionOffset for rows.Next() { var offset PartitionOffset diff --git a/federationsender/storage/postgres/joined_hosts_table.go b/federationsender/storage/postgres/joined_hosts_table.go index e5c30a010..b3c45abda 100644 --- a/federationsender/storage/postgres/joined_hosts_table.go +++ b/federationsender/storage/postgres/joined_hosts_table.go @@ -118,7 +118,7 @@ func joinedHostsFromStmt( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "joinedHostsFromStmt: rows.close() failed") var result []types.JoinedHost for rows.Next() { diff --git a/federationsender/storage/sqlite3/joined_hosts_table.go b/federationsender/storage/sqlite3/joined_hosts_table.go index 1437a062b..466ae4991 100644 --- a/federationsender/storage/sqlite3/joined_hosts_table.go +++ b/federationsender/storage/sqlite3/joined_hosts_table.go @@ -121,7 +121,7 @@ func joinedHostsFromStmt( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "joinedHostsFromStmt: rows.close() failed") var result []types.JoinedHost for rows.Next() { diff --git a/go.mod b/go.mod index 958706db4..82474944f 100644 --- a/go.mod +++ b/go.mod @@ -15,10 +15,13 @@ require ( github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.4.1 + github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/sirupsen/logrus v1.4.2 github.com/tidwall/gjson v1.6.0 // indirect github.com/tidwall/pretty v1.0.1 // indirect diff --git a/go.sum b/go.sum index 63423760c..3c7cf281e 100644 --- a/go.sum +++ b/go.sum @@ -372,6 +372,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= diff --git a/mediaapi/storage/postgres/thumbnail_table.go b/mediaapi/storage/postgres/thumbnail_table.go index 127b86bb9..08bddc36f 100644 --- a/mediaapi/storage/postgres/thumbnail_table.go +++ b/mediaapi/storage/postgres/thumbnail_table.go @@ -20,6 +20,8 @@ import ( "database/sql" "time" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -144,7 +146,7 @@ func (s *thumbnailStatements) selectThumbnails( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectThumbnails: rows.close() failed") var thumbnails []*types.ThumbnailMetadata for rows.Next() { diff --git a/mediaapi/storage/sqlite3/thumbnail_table.go b/mediaapi/storage/sqlite3/thumbnail_table.go index 95332c9d4..280fafe8d 100644 --- a/mediaapi/storage/sqlite3/thumbnail_table.go +++ b/mediaapi/storage/sqlite3/thumbnail_table.go @@ -20,6 +20,8 @@ import ( "database/sql" "time" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -134,7 +136,7 @@ func (s *thumbnailStatements) selectThumbnails( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectThumbnails: rows.close() failed") var thumbnails []*types.ThumbnailMetadata for rows.Next() { diff --git a/publicroomsapi/storage/postgres/public_rooms_table.go b/publicroomsapi/storage/postgres/public_rooms_table.go index edf9ad2ab..7e606e939 100644 --- a/publicroomsapi/storage/postgres/public_rooms_table.go +++ b/publicroomsapi/storage/postgres/public_rooms_table.go @@ -21,6 +21,8 @@ import ( "errors" "fmt" + "github.com/matrix-org/dendrite/common" + "github.com/lib/pq" "github.com/matrix-org/dendrite/publicroomsapi/types" ) @@ -203,7 +205,7 @@ func (s *publicRoomsStatements) selectPublicRooms( if err != nil { return []types.PublicRoom{}, nil } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectPublicRooms: rows.close() failed") rooms := []types.PublicRoom{} for rows.Next() { diff --git a/roomserver/storage/postgres/event_json_table.go b/roomserver/storage/postgres/event_json_table.go index 0b7ef6aa7..616eaf318 100644 --- a/roomserver/storage/postgres/event_json_table.go +++ b/roomserver/storage/postgres/event_json_table.go @@ -19,6 +19,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/types" ) @@ -86,7 +88,7 @@ func (s *eventJSONStatements) bulkSelectEventJSON( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventJSON: rows.close() failed") // We know that we will only get as many results as event NIDs // because of the unique constraint on event NIDs. diff --git a/roomserver/storage/postgres/event_state_keys_table.go b/roomserver/storage/postgres/event_state_keys_table.go index cbc29a69d..4c3496d91 100644 --- a/roomserver/storage/postgres/event_state_keys_table.go +++ b/roomserver/storage/postgres/event_state_keys_table.go @@ -114,7 +114,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKeyNID: rows.close() failed") result := make(map[string]types.EventStateKeyNID, len(eventStateKeys)) for rows.Next() { @@ -139,7 +139,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKey( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKey: rows.close() failed") result := make(map[types.EventStateKeyNID]string, len(eventStateKeyNIDs)) for rows.Next() { diff --git a/roomserver/storage/postgres/event_types_table.go b/roomserver/storage/postgres/event_types_table.go index faa887545..6537a5457 100644 --- a/roomserver/storage/postgres/event_types_table.go +++ b/roomserver/storage/postgres/event_types_table.go @@ -19,6 +19,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/lib/pq" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -132,7 +134,7 @@ func (s *eventTypeStatements) bulkSelectEventTypeNID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventTypeNID: rows.close() failed") result := make(map[string]types.EventTypeNID, len(eventTypes)) for rows.Next() { diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index d9b269bc8..0caa8199a 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -192,7 +192,7 @@ func (s *eventStatements) bulkSelectStateEventByID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateEventByID: rows.close() failed") // We know that we will only get as many results as event IDs // because of the unique constraint on event IDs. // So we can allocate an array of the correct size now. @@ -235,7 +235,7 @@ func (s *eventStatements) bulkSelectStateAtEventByID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventByID: rows.close() failed") results := make([]types.StateAtEvent, len(eventIDs)) i := 0 for ; rows.Next(); i++ { @@ -302,7 +302,7 @@ func (s *eventStatements) bulkSelectStateAtEventAndReference( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventAndReference: rows.close() failed") results := make([]types.StateAtEventAndReference, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -343,7 +343,7 @@ func (s *eventStatements) bulkSelectEventReference( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventReference: rows.close() failed") results := make([]gomatrixserverlib.EventReference, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -367,7 +367,7 @@ func (s *eventStatements) bulkSelectEventID(ctx context.Context, eventNIDs []typ if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventID: rows.close() failed") results := make(map[types.EventNID]string, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -394,7 +394,7 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, eventIDs []str if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventNID: rows.close() failed") results := make(map[string]types.EventNID, len(eventIDs)) for rows.Next() { var eventID string diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go index 603fed31b..f764b1561 100644 --- a/roomserver/storage/postgres/invite_table.go +++ b/roomserver/storage/postgres/invite_table.go @@ -120,7 +120,7 @@ func (s *inviteStatements) updateInviteRetired( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "updateInviteRetired: rows.close() failed") var eventIDs []string for rows.Next() { @@ -144,7 +144,7 @@ func (s *inviteStatements) selectInviteActiveForUserInRoom( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") var result []types.EventStateKeyNID for rows.Next() { var senderUserNID int64 diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index 70032fd1e..9c8a4c259 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -151,7 +151,7 @@ func (s *membershipStatements) selectMembershipsFromRoom( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoom: rows.close() failed") for rows.Next() { var eNID types.EventNID @@ -172,7 +172,7 @@ func (s *membershipStatements) selectMembershipsFromRoomAndMembership( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoomAndMembership: rows.close() failed") for rows.Next() { var eNID types.EventNID diff --git a/roomserver/storage/postgres/room_aliases_table.go b/roomserver/storage/postgres/room_aliases_table.go index 6de898c41..c37f383c9 100644 --- a/roomserver/storage/postgres/room_aliases_table.go +++ b/roomserver/storage/postgres/room_aliases_table.go @@ -18,6 +18,8 @@ package postgres import ( "context" "database/sql" + + "github.com/matrix-org/dendrite/common" ) const roomAliasesSchema = ` @@ -95,7 +97,7 @@ func (s *roomAliasesStatements) selectAliasesFromRoomID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed") var aliases []string for rows.Next() { diff --git a/roomserver/storage/postgres/state_block_table.go b/roomserver/storage/postgres/state_block_table.go index e6f4f7fe9..b9246b763 100644 --- a/roomserver/storage/postgres/state_block_table.go +++ b/roomserver/storage/postgres/state_block_table.go @@ -21,6 +21,8 @@ import ( "fmt" "sort" + "github.com/matrix-org/dendrite/common" + "github.com/lib/pq" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/util" @@ -138,7 +140,7 @@ func (s *stateBlockStatements) bulkSelectStateBlockEntries( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockEntries: rows.close() failed") results := make([]types.StateEntryList, len(stateBlockNIDs)) // current is a pointer to the StateEntryList to append the state entries to. @@ -197,7 +199,7 @@ func (s *stateBlockStatements) bulkSelectFilteredStateBlockEntries( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectFilteredStateBlockEntries: rows.close() failed") var results []types.StateEntryList var current types.StateEntryList diff --git a/roomserver/storage/sqlite3/event_json_table.go b/roomserver/storage/sqlite3/event_json_table.go index f6c83906a..fc661c1da 100644 --- a/roomserver/storage/sqlite3/event_json_table.go +++ b/roomserver/storage/sqlite3/event_json_table.go @@ -88,7 +88,7 @@ func (s *eventJSONStatements) bulkSelectEventJSON( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventJSON: rows.close() failed") // We know that we will only get as many results as event NIDs // because of the unique constraint on event NIDs. diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go index b8bc6c02d..fa8fc57eb 100644 --- a/roomserver/storage/sqlite3/event_state_keys_table.go +++ b/roomserver/storage/sqlite3/event_state_keys_table.go @@ -116,7 +116,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKeyNID: rows.close() failed") result := make(map[string]types.EventStateKeyNID, len(eventStateKeys)) for rows.Next() { var stateKey string @@ -142,7 +142,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKey( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKey: rows.close() failed") result := make(map[types.EventStateKeyNID]string, len(eventStateKeyNIDs)) for rows.Next() { var stateKey string diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go index edc06d4c6..777f8be79 100644 --- a/roomserver/storage/sqlite3/event_types_table.go +++ b/roomserver/storage/sqlite3/event_types_table.go @@ -138,7 +138,7 @@ func (s *eventTypeStatements) bulkSelectEventTypeNID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventTypeNID: rows.close() failed") result := make(map[string]types.EventTypeNID, len(eventTypes)) for rows.Next() { diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index 4fa095913..1e4ed448f 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -194,7 +194,7 @@ func (s *eventStatements) bulkSelectStateEventByID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateEventByID: rows.close() failed") // We know that we will only get as many results as event IDs // because of the unique constraint on event IDs. // So we can allocate an array of the correct size now. @@ -247,7 +247,7 @@ func (s *eventStatements) bulkSelectStateAtEventByID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventByID: rows.close() failed") results := make([]types.StateAtEvent, len(eventIDs)) i := 0 for ; rows.Next(); i++ { @@ -323,7 +323,7 @@ func (s *eventStatements) bulkSelectStateAtEventAndReference( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventAndReference: rows.close() failed") results := make([]types.StateAtEventAndReference, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -374,7 +374,7 @@ func (s *eventStatements) bulkSelectEventReference( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventReference: rows.close() failed") results := make([]gomatrixserverlib.EventReference, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -408,7 +408,7 @@ func (s *eventStatements) bulkSelectEventID(ctx context.Context, txn *sql.Tx, ev if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventID: rows.close() failed") results := make(map[types.EventNID]string, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -445,7 +445,7 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, e if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventNID: rows.close() failed") results := make(map[string]types.EventNID, len(eventIDs)) for rows.Next() { var eventID string diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index 641f80156..0ab3e6f36 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -137,7 +137,7 @@ func (s *inviteStatements) selectInviteActiveForUserInRoom( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") var result []types.EventStateKeyNID for rows.Next() { var senderUserNID int64 diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index 978776738..7ae28e4b8 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -134,7 +134,7 @@ func (s *membershipStatements) selectMembershipsFromRoom( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoom: rows.close() failed") for rows.Next() { var eNID types.EventNID @@ -154,7 +154,7 @@ func (s *membershipStatements) selectMembershipsFromRoomAndMembership( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoomAndMembership: rows.close() failed") for rows.Next() { var eNID types.EventNID diff --git a/roomserver/storage/sqlite3/room_aliases_table.go b/roomserver/storage/sqlite3/room_aliases_table.go index 71238b0e4..d29833918 100644 --- a/roomserver/storage/sqlite3/room_aliases_table.go +++ b/roomserver/storage/sqlite3/room_aliases_table.go @@ -103,7 +103,7 @@ func (s *roomAliasesStatements) selectAliasesFromRoomID( return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed") for rows.Next() { var alias string diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go index c94ca2385..cc7c75733 100644 --- a/roomserver/storage/sqlite3/state_block_table.go +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -137,7 +137,7 @@ func (s *stateBlockStatements) bulkSelectStateBlockEntries( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockEntries: rows.close() failed") results := make([]types.StateEntryList, len(stateBlockNIDs)) // current is a pointer to the StateEntryList to append the state entries to. @@ -207,7 +207,7 @@ func (s *stateBlockStatements) bulkSelectFilteredStateBlockEntries( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectFilteredStateBlockEntries: rows.close() failed") var results []types.StateEntryList var current types.StateEntryList diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index 0e1786eae..f367a779b 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -99,7 +99,7 @@ func (s *stateSnapshotStatements) bulkSelectStateBlockNIDs( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockNIDs: rows.close() failed") results := make([]types.StateBlockNIDList, len(stateNIDs)) i := 0 for ; rows.Next(); i++ { diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index d1811aa66..d1e3b527f 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -118,7 +118,7 @@ func (s *accountDataStatements) selectAccountDataInRange( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectAccountDataInRange: rows.close() failed") for rows.Next() { var dataType string diff --git a/syncapi/storage/postgres/backward_extremities_table.go b/syncapi/storage/postgres/backward_extremities_table.go index d63c546e3..8286ca434 100644 --- a/syncapi/storage/postgres/backward_extremities_table.go +++ b/syncapi/storage/postgres/backward_extremities_table.go @@ -17,6 +17,8 @@ package postgres import ( "context" "database/sql" + + "github.com/matrix-org/dendrite/common" ) const backwardExtremitiesSchema = ` @@ -91,7 +93,7 @@ func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") for rows.Next() { var eID string diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 6f5c1e803..48cc2b5fd 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -140,7 +140,7 @@ func (s *currentRoomStateStatements) selectJoinedUsers( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectJoinedUsers: rows.close() failed") result := make(map[string][]string) for rows.Next() { @@ -168,7 +168,7 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithMembership: rows.close() failed") var result []string for rows.Next() { @@ -198,7 +198,7 @@ func (s *currentRoomStateStatements) selectCurrentState( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectCurrentState: rows.close() failed") return rowsToEvents(rows) } @@ -248,7 +248,7 @@ func (s *currentRoomStateStatements) selectEventsWithEventIDs( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed") return rowsToStreamEvents(rows) } diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go index 2cb8fb199..6a7f9ef28 100644 --- a/syncapi/storage/postgres/invites_table.go +++ b/syncapi/storage/postgres/invites_table.go @@ -115,7 +115,7 @@ func (s *inviteEventsStatements) selectInviteEventsInRange( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") result := map[string]gomatrixserverlib.Event{} for rows.Next() { var ( diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 2db46c5db..e7cf19354 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -169,7 +169,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( if err != nil { return nil, nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectStateInRange: rows.close() failed") // Fetch all the state change events for all rooms between the two positions then loop each event and: // - Keep a cache of the event by ID (99% of state change events are for the event itself) // - For each room ID, build up an array of event IDs which represents cumulative adds/removes @@ -303,7 +303,7 @@ func (s *outputRoomEventsStatements) selectRecentEvents( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { return nil, err @@ -330,7 +330,7 @@ func (s *outputRoomEventsStatements) selectEarlyEvents( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEarlyEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { return nil, err @@ -354,7 +354,7 @@ func (s *outputRoomEventsStatements) selectEvents( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEvents: rows.close() failed") return rowsToStreamEvents(rows) } diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index 78a381da9..b114fd43d 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -18,6 +18,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -134,7 +136,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( } else if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEventIDsInRange: rows.close() failed") // Return the IDs. var eventID string @@ -177,7 +179,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsFromPosition( } else if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEventIDsFromPosition: rows.close() failed") // Return the IDs. var eventID string for rows.Next() { diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index 71105d0c3..3dbf961b4 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -19,6 +19,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -102,7 +104,7 @@ func (s *accountDataStatements) selectAccountDataInRange( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectAccountDataInRange: rows.close() failed") var entries int diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index ed76177be..87007f2b0 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -126,7 +126,7 @@ func (s *currentRoomStateStatements) selectJoinedUsers( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectJoinedUsers: rows.close() failed") result := make(map[string][]string) for rows.Next() { @@ -154,7 +154,7 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithMembership: rows.close() failed") var result []string for rows.Next() { @@ -184,7 +184,7 @@ func (s *currentRoomStateStatements) selectCurrentState( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectCurrentState: rows.close() failed") return rowsToEvents(rows) } @@ -238,7 +238,7 @@ func (s *currentRoomStateStatements) selectEventsWithEventIDs( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed") return rowsToStreamEvents(rows) } diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index baf8871bd..df6f1c9a5 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -113,7 +113,7 @@ func (s *inviteEventsStatements) selectInviteEventsInRange( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") result := map[string]gomatrixserverlib.Event{} for rows.Next() { var ( diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 05a33c067..5ae28ca3f 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -316,7 +316,7 @@ func (s *outputRoomEventsStatements) selectRecentEvents( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { return nil, err @@ -343,7 +343,7 @@ func (s *outputRoomEventsStatements) selectEarlyEvents( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEarlyEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { return nil, err @@ -372,7 +372,7 @@ func (s *outputRoomEventsStatements) selectEvents( if streamEvents, err := rowsToStreamEvents(rows); err == nil { returnEvents = append(returnEvents, streamEvents...) } - rows.Close() // nolint: errcheck + common.CloseAndLogIfError(ctx, rows, "selectEvents: rows.close() failed") } return returnEvents, nil } From ec38783192dd551718902719c8446c6cc3942dfa Mon Sep 17 00:00:00 2001 From: Abhinav Krishna C K Date: Wed, 18 Mar 2020 18:18:51 +0530 Subject: [PATCH 12/17] maintenance: Fix matrix-org#896 use %w format verb to wrap errors (#916) * maintenance: Fix matrix-org#896 use %w format verb to wrap errors * In Go version 1.13 a new formatting verb introduced for fmt.Errorf %w https://blog.golang.org/go1.13-errors * update %s to %w to wrap errors. * Update all instances of error type to use %w Signed-off-by: Abhinav Krishna C K Co-authored-by: Kegsay --- clientapi/routing/createroom.go | 2 +- cmd/dendritejs/keyfetcher.go | 6 +++--- common/consumers.go | 2 +- mediaapi/fileutils/fileutils.go | 20 ++++++++++---------- syncapi/routing/messages.go | 6 +++--- syncapi/sync/notifier_test.go | 12 ++++++------ 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 92b1ae887..964200760 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -381,7 +381,7 @@ func buildEvent( eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), cfg.Matrix.ServerName) event, err := builder.Build(eventID, evTime, cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey) if err != nil { - return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %s", builder.Type, err) + return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %w", builder.Type, err) } return &event, nil } diff --git a/cmd/dendritejs/keyfetcher.go b/cmd/dendritejs/keyfetcher.go index 2b0a54fd9..ee4905d4f 100644 --- a/cmd/dendritejs/keyfetcher.go +++ b/cmd/dendritejs/keyfetcher.go @@ -53,15 +53,15 @@ func (f *libp2pKeyFetcher) FetchKeys( peerIDStr := string(req.ServerName) peerID, err := peer.Decode(peerIDStr) if err != nil { - return nil, fmt.Errorf("Failed to decode peer ID from server name '%s': %s", peerIDStr, err) + return nil, fmt.Errorf("Failed to decode peer ID from server name '%s': %w", peerIDStr, err) } pubKey, err := peerID.ExtractPublicKey() if err != nil { - return nil, fmt.Errorf("Failed to extract public key from peer ID: %s", err) + return nil, fmt.Errorf("Failed to extract public key from peer ID: %w", err) } pubKeyBytes, err := pubKey.Raw() if err != nil { - return nil, fmt.Errorf("Failed to extract raw bytes from public key: %s", err) + return nil, fmt.Errorf("Failed to extract raw bytes from public key: %w", err) } util.GetLogger(ctx).Info("libp2pKeyFetcher.FetchKeys: Using public key %v for server name %s", pubKeyBytes, req.ServerName) diff --git a/common/consumers.go b/common/consumers.go index 2e58a6241..f33993494 100644 --- a/common/consumers.go +++ b/common/consumers.go @@ -112,7 +112,7 @@ func (c *ContinualConsumer) consumePartition(pc sarama.PartitionConsumer) { msgErr := c.ProcessMessage(message) // Advance our position in the stream so that we will start at the right position after a restart. if err := c.PartitionStore.SetPartitionOffset(context.TODO(), c.Topic, message.Partition, message.Offset); err != nil { - panic(fmt.Errorf("the ContinualConsumer failed to SetPartitionOffset: %s", err)) + panic(fmt.Errorf("the ContinualConsumer failed to SetPartitionOffset: %w", err)) } // Shutdown if we were told to do so. if msgErr == ErrShutdown { diff --git a/mediaapi/fileutils/fileutils.go b/mediaapi/fileutils/fileutils.go index 36b2c5b89..1de242a00 100644 --- a/mediaapi/fileutils/fileutils.go +++ b/mediaapi/fileutils/fileutils.go @@ -49,7 +49,7 @@ func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath config.Path) "file", )) if err != nil { - return "", fmt.Errorf("Unable to construct filePath: %q", err) + return "", fmt.Errorf("Unable to construct filePath: %w", err) } // check if the absolute absBasePath is a prefix of the absolute filePath @@ -73,7 +73,7 @@ func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata duplicate := false finalPath, err := GetPathFromBase64Hash(mediaMetadata.Base64Hash, absBasePath) if err != nil { - return "", duplicate, fmt.Errorf("failed to get file path from metadata: %q", err) + return "", duplicate, fmt.Errorf("failed to get file path from metadata: %w", err) } var stat os.FileInfo @@ -91,7 +91,7 @@ func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata types.Path(finalPath), ) if err != nil { - return "", duplicate, fmt.Errorf("failed to move file to final destination (%v): %q", finalPath, err) + return "", duplicate, fmt.Errorf("failed to move file to final destination (%v): %w", finalPath, err) } return types.Path(finalPath), duplicate, nil } @@ -143,11 +143,11 @@ func moveFile(src types.Path, dst types.Path) error { err := os.MkdirAll(dstDir, 0770) if err != nil { - return fmt.Errorf("Failed to make directory: %q", err) + return fmt.Errorf("Failed to make directory: %w", err) } err = os.Rename(string(src), string(dst)) if err != nil { - return fmt.Errorf("Failed to move directory: %q", err) + return fmt.Errorf("Failed to move directory: %w", err) } return nil } @@ -155,11 +155,11 @@ func moveFile(src types.Path, dst types.Path) error { func createTempFileWriter(absBasePath config.Path) (*bufio.Writer, *os.File, types.Path, error) { tmpDir, err := createTempDir(absBasePath) if err != nil { - return nil, nil, "", fmt.Errorf("Failed to create temp dir: %q", err) + return nil, nil, "", fmt.Errorf("Failed to create temp dir: %w", err) } writer, tmpFile, err := createFileWriter(tmpDir) if err != nil { - return nil, nil, "", fmt.Errorf("Failed to create file writer: %q", err) + return nil, nil, "", fmt.Errorf("Failed to create file writer: %w", err) } return writer, tmpFile, tmpDir, nil } @@ -168,11 +168,11 @@ func createTempFileWriter(absBasePath config.Path) (*bufio.Writer, *os.File, typ func createTempDir(baseDirectory config.Path) (types.Path, error) { baseTmpDir := filepath.Join(string(baseDirectory), "tmp") if err := os.MkdirAll(baseTmpDir, 0770); err != nil { - return "", fmt.Errorf("Failed to create base temp dir: %v", err) + return "", fmt.Errorf("Failed to create base temp dir: %w", err) } tmpDir, err := ioutil.TempDir(baseTmpDir, "") if err != nil { - return "", fmt.Errorf("Failed to create temp dir: %v", err) + return "", fmt.Errorf("Failed to create temp dir: %w", err) } return types.Path(tmpDir), nil } @@ -184,7 +184,7 @@ func createFileWriter(directory types.Path) (*bufio.Writer, *os.File, error) { filePath := filepath.Join(string(directory), "content") file, err := os.Create(filePath) if err != nil { - return nil, nil, fmt.Errorf("Failed to create file: %v", err) + return nil, nil, fmt.Errorf("Failed to create file: %w", err) } return bufio.NewWriter(file), file, nil diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 83bf75b2e..e0901a780 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -177,7 +177,7 @@ func (r *messagesReq) retrieveEvents() ( r.ctx, r.from, r.to, r.roomID, r.limit, r.backwardOrdering, ) if err != nil { - err = fmt.Errorf("GetEventsInRange: %s", err) + err = fmt.Errorf("GetEventsInRange: %w", err) return } @@ -228,14 +228,14 @@ func (r *messagesReq) retrieveEvents() ( r.ctx, events[0].EventID(), ) if err != nil { - err = fmt.Errorf("EventPositionInTopology: for start event %s: %s", events[0].EventID(), err) + err = fmt.Errorf("EventPositionInTopology: for start event %s: %w", events[0].EventID(), err) return } endPos, err := r.db.EventPositionInTopology( r.ctx, events[len(events)-1].EventID(), ) if err != nil { - err = fmt.Errorf("EventPositionInTopology: for end event %s: %s", events[len(events)-1].EventID(), err) + err = fmt.Errorf("EventPositionInTopology: for end event %s: %w", events[len(events)-1].EventID(), err) return } // Generate pagination tokens to send to the client using the positions diff --git a/syncapi/sync/notifier_test.go b/syncapi/sync/notifier_test.go index 02da0f7e6..350f85594 100644 --- a/syncapi/sync/notifier_test.go +++ b/syncapi/sync/notifier_test.go @@ -135,7 +135,7 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { go func() { pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { - t.Errorf("TestNewEventAndJoinedToRoom error: %s", err) + t.Errorf("TestNewEventAndJoinedToRoom error: %w", err) } if pos != syncPositionAfter { t.Errorf("TestNewEventAndJoinedToRoom want %v, got %v", syncPositionAfter, pos) @@ -163,7 +163,7 @@ func TestNewInviteEventForUser(t *testing.T) { go func() { pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { - t.Errorf("TestNewInviteEventForUser error: %s", err) + t.Errorf("TestNewInviteEventForUser error: %w", err) } if pos != syncPositionAfter { t.Errorf("TestNewInviteEventForUser want %v, got %v", syncPositionAfter, pos) @@ -191,7 +191,7 @@ func TestEDUWakeup(t *testing.T) { go func() { pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionAfter)) if err != nil { - t.Errorf("TestNewInviteEventForUser error: %s", err) + t.Errorf("TestNewInviteEventForUser error: %w", err) } if pos != syncPositionNewEDU { t.Errorf("TestNewInviteEventForUser want %v, got %v", syncPositionNewEDU, pos) @@ -219,7 +219,7 @@ func TestMultipleRequestWakeup(t *testing.T) { poll := func() { pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { - t.Errorf("TestMultipleRequestWakeup error: %s", err) + t.Errorf("TestMultipleRequestWakeup error: %w", err) } if pos != syncPositionAfter { t.Errorf("TestMultipleRequestWakeup want %v, got %v", syncPositionAfter, pos) @@ -259,7 +259,7 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { go func() { pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { - t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %s", err) + t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %w", err) } if pos != syncPositionAfter { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %v, got %v", syncPositionAfter, pos) @@ -278,7 +278,7 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { go func() { pos, err := waitForEvents(n, newTestSyncRequest(alice, syncPositionAfter)) if err != nil { - t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %s", err) + t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %w", err) } if pos != syncPositionAfter2 { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %v, got %v", syncPositionAfter2, pos) From dc06c69887f7061411868b3a7a0ee8221a7053e1 Mon Sep 17 00:00:00 2001 From: Prateek Sachan <42961174+prateek2211@users.noreply.github.com> Date: Thu, 19 Mar 2020 15:55:36 +0530 Subject: [PATCH 13/17] Implement /joined_rooms (#911) * Implemented /joined_rooms * Removed account endpoint added by mistake * trigger ci --- clientapi/auth/storage/accounts/interface.go | 1 + .../accounts/postgres/membership_table.go | 27 ++++++++++++++++ .../auth/storage/accounts/postgres/storage.go | 10 ++++++ .../accounts/sqlite3/membership_table.go | 26 ++++++++++++++++ .../auth/storage/accounts/sqlite3/storage.go | 10 ++++++ clientapi/routing/memberships.go | 31 +++++++++++++++++-- clientapi/routing/routing.go | 6 ++++ go.mod | 1 + sytest-whitelist | 2 ++ 9 files changed, 112 insertions(+), 2 deletions(-) diff --git a/clientapi/auth/storage/accounts/interface.go b/clientapi/auth/storage/accounts/interface.go index 9f6e3e1ea..a5052b047 100644 --- a/clientapi/auth/storage/accounts/interface.go +++ b/clientapi/auth/storage/accounts/interface.go @@ -33,6 +33,7 @@ type Database interface { CreateGuestAccount(ctx context.Context) (*authtypes.Account, error) UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error) + GetRoomIDsByLocalPart(ctx context.Context, localpart string) ([]string, error) GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error) SaveAccountData(ctx context.Context, localpart, roomID, dataType, content string) error GetAccountData(ctx context.Context, localpart string) (global []gomatrixserverlib.ClientEvent, rooms map[string][]gomatrixserverlib.ClientEvent, err error) diff --git a/clientapi/auth/storage/accounts/postgres/membership_table.go b/clientapi/auth/storage/accounts/postgres/membership_table.go index 27570b67d..04e9095e9 100644 --- a/clientapi/auth/storage/accounts/postgres/membership_table.go +++ b/clientapi/auth/storage/accounts/postgres/membership_table.go @@ -53,6 +53,9 @@ const selectMembershipsByLocalpartSQL = "" + const selectMembershipInRoomByLocalpartSQL = "" + "SELECT event_id FROM account_memberships WHERE localpart = $1 AND room_id = $2" +const selectRoomIDsByLocalPartSQL = "" + + "SELECT room_id FROM account_memberships WHERE localpart = $1" + const deleteMembershipsByEventIDsSQL = "" + "DELETE FROM account_memberships WHERE event_id = ANY($1)" @@ -61,6 +64,7 @@ type membershipStatements struct { insertMembershipStmt *sql.Stmt selectMembershipInRoomByLocalpartStmt *sql.Stmt selectMembershipsByLocalpartStmt *sql.Stmt + selectRoomIDsByLocalPartStmt *sql.Stmt } func (s *membershipStatements) prepare(db *sql.DB) (err error) { @@ -80,6 +84,9 @@ func (s *membershipStatements) prepare(db *sql.DB) (err error) { if s.selectMembershipsByLocalpartStmt, err = db.Prepare(selectMembershipsByLocalpartSQL); err != nil { return } + if s.selectRoomIDsByLocalPartStmt, err = db.Prepare(selectRoomIDsByLocalPartSQL); err != nil { + return + } return } @@ -131,3 +138,23 @@ func (s *membershipStatements) selectMembershipsByLocalpart( } return memberships, rows.Err() } + +func (s *membershipStatements) selectRoomIDsByLocalPart( + ctx context.Context, localPart string, +) ([]string, error) { + stmt := s.selectRoomIDsByLocalPartStmt + rows, err := stmt.QueryContext(ctx, localPart) + if err != nil { + return nil, err + } + roomIDs := []string{} + defer rows.Close() // nolint: errcheck + for rows.Next() { + var roomID string + if err = rows.Scan(&roomID); err != nil { + return nil, err + } + roomIDs = append(roomIDs, roomID) + } + return roomIDs, rows.Err() +} diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/clientapi/auth/storage/accounts/postgres/storage.go index 8115dca43..4a0a2060b 100644 --- a/clientapi/auth/storage/accounts/postgres/storage.go +++ b/clientapi/auth/storage/accounts/postgres/storage.go @@ -234,6 +234,16 @@ func (d *Database) GetMembershipInRoomByLocalpart( return d.memberships.selectMembershipInRoomByLocalpart(ctx, localpart, roomID) } +// GetRoomIDsByLocalPart returns an array containing the room ids of all +// the rooms a user matching a given localpart is a member of +// If no membership match the given localpart, returns an empty array +// If there was an issue during the retrieval, returns the SQL error +func (d *Database) GetRoomIDsByLocalPart( + ctx context.Context, localpart string, +) ([]string, error) { + return d.memberships.selectRoomIDsByLocalPart(ctx, localpart) +} + // GetMembershipsByLocalpart returns an array containing the memberships for all // the rooms a user matching a given localpart is a member of // If no membership match the given localpart, returns an empty array diff --git a/clientapi/auth/storage/accounts/sqlite3/membership_table.go b/clientapi/auth/storage/accounts/sqlite3/membership_table.go index b4bff6334..bd9838b6b 100644 --- a/clientapi/auth/storage/accounts/sqlite3/membership_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/membership_table.go @@ -51,6 +51,9 @@ const selectMembershipsByLocalpartSQL = "" + const selectMembershipInRoomByLocalpartSQL = "" + "SELECT event_id FROM account_memberships WHERE localpart = $1 AND room_id = $2" +const selectRoomIDsByLocalPartSQL = "" + + "SELECT room_id FROM account_memberships WHERE localpart = $1" + const deleteMembershipsByEventIDsSQL = "" + "DELETE FROM account_memberships WHERE event_id IN ($1)" @@ -58,6 +61,7 @@ type membershipStatements struct { insertMembershipStmt *sql.Stmt selectMembershipInRoomByLocalpartStmt *sql.Stmt selectMembershipsByLocalpartStmt *sql.Stmt + selectRoomIDsByLocalPartStmt *sql.Stmt } func (s *membershipStatements) prepare(db *sql.DB) (err error) { @@ -74,6 +78,9 @@ func (s *membershipStatements) prepare(db *sql.DB) (err error) { if s.selectMembershipsByLocalpartStmt, err = db.Prepare(selectMembershipsByLocalpartSQL); err != nil { return } + if s.selectRoomIDsByLocalPartStmt, err = db.Prepare(selectRoomIDsByLocalPartSQL); err != nil { + return + } return } @@ -130,3 +137,22 @@ func (s *membershipStatements) selectMembershipsByLocalpart( return } +func (s *membershipStatements) selectRoomIDsByLocalPart( + ctx context.Context, localPart string, +) ([]string, error) { + stmt := s.selectRoomIDsByLocalPartStmt + rows, err := stmt.QueryContext(ctx, localPart) + if err != nil { + return nil, err + } + roomIDs := []string{} + defer rows.Close() // nolint: errcheck + for rows.Next() { + var roomID string + if err = rows.Scan(&roomID); err != nil { + return nil, err + } + roomIDs = append(roomIDs, roomID) + } + return roomIDs, rows.Err() +} diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index 9124640c6..bfb7b4ea1 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -253,6 +253,16 @@ func (d *Database) GetMembershipsByLocalpart( return d.memberships.selectMembershipsByLocalpart(ctx, localpart) } +// GetRoomIDsByLocalPart returns an array containing the room ids of all +// the rooms a user matching a given localpart is a member of +// If no membership match the given localpart, returns an empty array +// If there was an issue during the retrieval, returns the SQL error +func (d *Database) GetRoomIDsByLocalPart( + ctx context.Context, localpart string, +) ([]string, error) { + return d.memberships.selectRoomIDsByLocalPart(ctx, localpart) +} + // newMembership saves a new membership in the database. // If the event isn't a valid m.room.member event with type `join`, does nothing. // If an error occurred, returns the SQL error diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index a6899eeb8..0b846e5e3 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -17,6 +17,8 @@ package routing import ( "net/http" + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" @@ -25,10 +27,14 @@ import ( "github.com/matrix-org/util" ) -type response struct { +type getMembershipResponse struct { Chunk []gomatrixserverlib.ClientEvent `json:"chunk"` } +type getJoinedRoomsResponse struct { + JoinedRooms []string `json:"joined_rooms"` +} + // GetMemberships implements GET /rooms/{roomId}/members func GetMemberships( req *http.Request, device *authtypes.Device, roomID string, joinedOnly bool, @@ -55,6 +61,27 @@ func GetMemberships( return util.JSONResponse{ Code: http.StatusOK, - JSON: response{queryRes.JoinEvents}, + JSON: getMembershipResponse{queryRes.JoinEvents}, + } +} + +func GetJoinedRooms( + req *http.Request, + device *authtypes.Device, + accountsDB accounts.Database, +) util.JSONResponse { + localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() + } + joinedRooms, err := accountsDB.GetRoomIDsByLocalPart(req.Context(), localpart) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("accountsDB.GetRoomIDsByLocalPart failed") + return jsonerror.InternalServerError() + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: getJoinedRoomsResponse{joinedRooms}, } } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index f0841b796..47b7b2675 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -105,6 +105,12 @@ func Setup( ) }), ).Methods(http.MethodPost, http.MethodOptions) + r0mux.Handle("/joined_rooms", + common.MakeAuthAPI("joined_rooms", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + return GetJoinedRooms(req, device, accountDB) + }), + ).Methods(http.MethodGet, http.MethodOptions) + r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|leave|invite)}", common.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(req)) diff --git a/go.mod b/go.mod index 82474944f..25debd748 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/tidwall/pretty v1.0.1 // indirect github.com/uber/jaeger-client-go v2.22.1+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible + go.uber.org/atomic v1.6.0 // indirect golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d gopkg.in/Shopify/sarama.v1 v1.20.1 gopkg.in/h2non/bimg.v1 v1.0.18 diff --git a/sytest-whitelist b/sytest-whitelist index cac718288..03e11b83b 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -218,3 +218,5 @@ Push rules come down in an initial /sync Regular users can add and delete aliases in the default room configuration Regular users can add and delete aliases when m.room.aliases is restricted GET /r0/capabilities is not public +GET /joined_rooms lists newly-created room +/joined_rooms returns only joined rooms From bfbf96eec9152f61cb3e54154f1ed82148d82a8a Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 19 Mar 2020 11:04:08 +0000 Subject: [PATCH 14/17] p2p: Implement published rooms (#923) * Create and glue ExternalPublicRoomsProvider into the public rooms component This is how we will link p2p stuff to dendrite proper. * Use gmsl structs rather than our own * Implement federated public rooms - Make thirdparty endpoint r0 so riot-web loads the public room list * Typo * Missing callsites --- clientapi/routing/routing.go | 2 +- cmd/dendrite-monolith-server/main.go | 2 +- cmd/dendrite-public-rooms-api-server/main.go | 2 +- cmd/dendritejs/main.go | 3 +- cmd/dendritejs/publicrooms.go | 46 ++++ go.mod | 4 +- go.sum | 2 + publicroomsapi/directory/public_rooms.go | 205 ++++++++++++++---- publicroomsapi/publicroomsapi.go | 6 +- publicroomsapi/routing/routing.go | 17 +- publicroomsapi/storage/interface.go | 3 +- .../storage/postgres/public_rooms_table.go | 12 +- publicroomsapi/storage/postgres/storage.go | 3 +- .../storage/sqlite3/public_rooms_table.go | 16 +- publicroomsapi/storage/sqlite3/storage.go | 3 +- publicroomsapi/types/types.go | 18 +- 16 files changed, 263 insertions(+), 81 deletions(-) create mode 100644 cmd/dendritejs/publicrooms.go diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 47b7b2675..22ff12b02 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -396,7 +396,7 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) - unstableMux.Handle("/thirdparty/protocols", + r0mux.Handle("/thirdparty/protocols", common.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse { // TODO: Return the third party protcols return util.JSONResponse{ diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index b3de9adde..27c3054b8 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -69,7 +69,7 @@ func main() { ) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI) mediaapi.SetupMediaAPIComponent(base, deviceDB) - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, federation, nil) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) httpHandler := common.WrapHandlerInCORS(base.APIMux) diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index f8bd8b06a..6b7eac7d5 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -28,7 +28,7 @@ func main() { _, _, query := base.CreateHTTPRoomserverAPIs() - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, nil, nil) base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI)) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 25492b166..7c8526715 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -119,6 +119,7 @@ func main() { }, KeyDatabase: keyDB, } + p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node) alias, input, query := roomserver.SetupRoomServerComponent(base) typingInputAPI := typingserver.SetupTypingServerComponent(base, cache.NewTypingCache()) @@ -134,7 +135,7 @@ func main() { ) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI) mediaapi.SetupMediaAPIComponent(base, deviceDB) - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, federation, p2pPublicRoomProvider) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) httpHandler := common.WrapHandlerInCORS(base.APIMux) diff --git a/cmd/dendritejs/publicrooms.go b/cmd/dendritejs/publicrooms.go new file mode 100644 index 000000000..17822e7ad --- /dev/null +++ b/cmd/dendritejs/publicrooms.go @@ -0,0 +1,46 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build wasm + +package main + +import ( + "github.com/matrix-org/go-http-js-libp2p/go_http_js_libp2p" +) + +type libp2pPublicRoomsProvider struct { + node *go_http_js_libp2p.P2pLocalNode + providers []go_http_js_libp2p.PeerInfo +} + +func NewLibP2PPublicRoomsProvider(node *go_http_js_libp2p.P2pLocalNode) *libp2pPublicRoomsProvider { + p := &libp2pPublicRoomsProvider{ + node: node, + } + node.RegisterFoundProviders(p.foundProviders) + return p +} + +func (p *libp2pPublicRoomsProvider) foundProviders(peerInfos []go_http_js_libp2p.PeerInfo) { + p.providers = peerInfos +} + +func (p *libp2pPublicRoomsProvider) Homeservers() []string { + result := make([]string, len(p.providers)) + for i := range p.providers { + result[i] = p.providers[i].Id + } + return result +} diff --git a/go.mod b/go.mod index 25debd748..556a6b8f1 100644 --- a/go.mod +++ b/go.mod @@ -8,15 +8,13 @@ require ( github.com/lib/pq v1.2.0 github.com/libp2p/go-libp2p-core v0.5.0 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 - github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c + github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 diff --git a/go.sum b/go.sum index 3c7cf281e..2c51915bb 100644 --- a/go.sum +++ b/go.sum @@ -230,6 +230,8 @@ github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437 h1:zc github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437/go.mod h1:/giSXVd8D6DZGSfTmhQrLEoZZwsfkC14kSqP9MiLqIY= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c h1:jj/LIZKMO7GK6O0UarpRwse9L3ZyzozpyMtdPA7ddSk= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f h1:5TOte9uk/epk8L+Pbp6qwaV8YsKYXKjyECPHUhJTWQc= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk= github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074 h1:UWz6vfhmQVshBuE67X1BCsdMhEDtd+uOz8CJ48Fc0F4= github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db h1:ERuFJq4DI8fakfBZlvXHltHZ0ix3K5YsLG0tQfQn6TI= diff --git a/publicroomsapi/directory/public_rooms.go b/publicroomsapi/directory/public_rooms.go index fd327942e..7bd6740eb 100644 --- a/publicroomsapi/directory/public_rooms.go +++ b/publicroomsapi/directory/public_rooms.go @@ -15,17 +15,21 @@ package directory import ( + "context" "net/http" "strconv" + "sync" + "time" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/publicroomsapi/types" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) -type publicRoomReq struct { +type PublicRoomReq struct { Since string `json:"since,omitempty"` Limit int16 `json:"limit,omitempty"` Filter filter `json:"filter,omitempty"` @@ -35,65 +39,182 @@ type filter struct { SearchTerms string `json:"generic_search_term,omitempty"` } -type publicRoomRes struct { - Chunk []types.PublicRoom `json:"chunk"` - NextBatch string `json:"next_batch,omitempty"` - PrevBatch string `json:"prev_batch,omitempty"` - Estimate int64 `json:"total_room_count_estimate,omitempty"` -} - // GetPostPublicRooms implements GET and POST /publicRooms func GetPostPublicRooms( req *http.Request, publicRoomDatabase storage.Database, ) util.JSONResponse { - var limit int16 - var offset int64 - var request publicRoomReq - var response publicRoomRes - + var request PublicRoomReq if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil { return *fillErr } - - limit = request.Limit - offset, err := strconv.ParseInt(request.Since, 10, 64) - // ParseInt returns 0 and an error when trying to parse an empty string - // In that case, we want to assign 0 so we ignore the error - if err != nil && len(request.Since) > 0 { - util.GetLogger(req.Context()).WithError(err).Error("strconv.ParseInt failed") + response, err := publicRooms(req.Context(), request, publicRoomDatabase) + if err != nil { return jsonerror.InternalServerError() } - - if response.Estimate, err = publicRoomDatabase.CountPublicRooms(req.Context()); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("publicRoomDatabase.CountPublicRooms failed") - return jsonerror.InternalServerError() - } - - if offset > 0 { - response.PrevBatch = strconv.Itoa(int(offset) - 1) - } - nextIndex := int(offset) + int(limit) - if response.Estimate > int64(nextIndex) { - response.NextBatch = strconv.Itoa(nextIndex) - } - - if response.Chunk, err = publicRoomDatabase.GetPublicRooms( - req.Context(), offset, limit, request.Filter.SearchTerms, - ); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("publicRoomDatabase.GetPublicRooms failed") - return jsonerror.InternalServerError() - } - return util.JSONResponse{ Code: http.StatusOK, JSON: response, } } +// GetPostPublicRoomsWithExternal is the same as GetPostPublicRooms but also mixes in public rooms from the provider supplied. +func GetPostPublicRoomsWithExternal( + req *http.Request, publicRoomDatabase storage.Database, fedClient *gomatrixserverlib.FederationClient, + extRoomsProvider types.ExternalPublicRoomsProvider, +) util.JSONResponse { + var request PublicRoomReq + if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil { + return *fillErr + } + response, err := publicRooms(req.Context(), request, publicRoomDatabase) + if err != nil { + return jsonerror.InternalServerError() + } + + if request.Since != "" { + // TODO: handle pagination tokens sensibly rather than ignoring them. + // ignore paginated requests since we don't handle them yet over federation. + // Only the initial request will contain federated rooms. + return util.JSONResponse{ + Code: http.StatusOK, + JSON: response, + } + } + + // If we have already hit the limit on the number of rooms, bail. + var limit int + if request.Limit > 0 { + limit = int(request.Limit) - len(response.Chunk) + if limit <= 0 { + return util.JSONResponse{ + Code: http.StatusOK, + JSON: response, + } + } + } + + // downcasting `limit` is safe as we know it isn't bigger than request.Limit which is int16 + fedRooms := bulkFetchPublicRoomsFromServers(req.Context(), fedClient, extRoomsProvider.Homeservers(), int16(limit)) + response.Chunk = append(response.Chunk, fedRooms...) + return util.JSONResponse{ + Code: http.StatusOK, + JSON: response, + } +} + +// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers. +// Returns a list of public rooms up to the limit specified. +func bulkFetchPublicRoomsFromServers( + ctx context.Context, fedClient *gomatrixserverlib.FederationClient, homeservers []string, limit int16, +) (publicRooms []gomatrixserverlib.PublicRoom) { + // follow pipeline semantics, see https://blog.golang.org/pipelines for more info. + // goroutines send rooms to this channel + roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit)) + // signalling channel to tell goroutines to stop sending rooms and quit + done := make(chan bool) + // signalling to say when we can close the room channel + var wg sync.WaitGroup + wg.Add(len(homeservers)) + // concurrently query for public rooms + for _, hs := range homeservers { + go func(homeserverDomain string) { + defer wg.Done() + util.GetLogger(ctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms") + fres, err := fedClient.GetPublicRooms(ctx, gomatrixserverlib.ServerName(homeserverDomain), int(limit), "", false, "") + if err != nil { + util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Warn( + "bulkFetchPublicRoomsFromServers: failed to query hs", + ) + return + } + for _, room := range fres.Chunk { + // atomically send a room or stop + select { + case roomCh <- room: + case <-done: + util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending rooms") + return + } + } + }(hs) + } + + // Close the room channel when the goroutines have quit so we don't leak, but don't let it stop the in-flight request. + // This also allows the request to fail fast if all HSes experience errors as it will cause the room channel to be + // closed. + go func() { + wg.Wait() + util.GetLogger(ctx).Info("Cleaning up resources") + close(roomCh) + }() + + // fan-in results with timeout. We stop when we reach the limit. +FanIn: + for len(publicRooms) < int(limit) || limit == 0 { + // add a room or timeout + select { + case room, ok := <-roomCh: + if !ok { + util.GetLogger(ctx).Info("All homeservers have been queried, returning results.") + break FanIn + } + publicRooms = append(publicRooms, room) + case <-time.After(15 * time.Second): // we've waited long enough, let's tell the client what we got. + util.GetLogger(ctx).Info("Waited 15s for federated public rooms, returning early") + break FanIn + case <-ctx.Done(): // the client hung up on us, let's stop. + util.GetLogger(ctx).Info("Client hung up, returning early") + break FanIn + } + } + // tell goroutines to stop + close(done) + + return publicRooms +} + +func publicRooms(ctx context.Context, request PublicRoomReq, publicRoomDatabase storage.Database) (*gomatrixserverlib.RespPublicRooms, error) { + var response gomatrixserverlib.RespPublicRooms + var limit int16 + var offset int64 + limit = request.Limit + offset, err := strconv.ParseInt(request.Since, 10, 64) + // ParseInt returns 0 and an error when trying to parse an empty string + // In that case, we want to assign 0 so we ignore the error + if err != nil && len(request.Since) > 0 { + util.GetLogger(ctx).WithError(err).Error("strconv.ParseInt failed") + return nil, err + } + + est, err := publicRoomDatabase.CountPublicRooms(ctx) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("publicRoomDatabase.CountPublicRooms failed") + return nil, err + } + response.TotalRoomCountEstimate = int(est) + + if offset > 0 { + response.PrevBatch = strconv.Itoa(int(offset) - 1) + } + nextIndex := int(offset) + int(limit) + if response.TotalRoomCountEstimate > nextIndex { + response.NextBatch = strconv.Itoa(nextIndex) + } + + if response.Chunk, err = publicRoomDatabase.GetPublicRooms( + ctx, offset, limit, request.Filter.SearchTerms, + ); err != nil { + util.GetLogger(ctx).WithError(err).Error("publicRoomDatabase.GetPublicRooms failed") + return nil, err + } + + return &response, nil +} + // fillPublicRoomsReq fills the Limit, Since and Filter attributes of a GET or POST request // on /publicRooms by parsing the incoming HTTP request // Filter is only filled for POST requests -func fillPublicRoomsReq(httpReq *http.Request, request *publicRoomReq) *util.JSONResponse { +func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSONResponse { if httpReq.Method == http.MethodGet { limit, err := strconv.Atoi(httpReq.FormValue("limit")) // Atoi returns 0 and an error when trying to parse an empty string diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index 1e2a3f9bb..399c0cc57 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -20,7 +20,9 @@ import ( "github.com/matrix-org/dendrite/publicroomsapi/consumers" "github.com/matrix-org/dendrite/publicroomsapi/routing" "github.com/matrix-org/dendrite/publicroomsapi/storage" + "github.com/matrix-org/dendrite/publicroomsapi/types" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) @@ -30,6 +32,8 @@ func SetupPublicRoomsAPIComponent( base *basecomponent.BaseDendrite, deviceDB devices.Database, rsQueryAPI roomserverAPI.RoomserverQueryAPI, + fedClient *gomatrixserverlib.FederationClient, + extRoomsProvider types.ExternalPublicRoomsProvider, ) { publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) if err != nil { @@ -43,5 +47,5 @@ func SetupPublicRoomsAPIComponent( logrus.WithError(err).Panic("failed to start public rooms server consumer") } - routing.Setup(base.APIMux, deviceDB, publicRoomsDB) + routing.Setup(base.APIMux, deviceDB, publicRoomsDB, fedClient, extRoomsProvider) } diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index 1953e04fc..321b61b89 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -24,6 +24,8 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/publicroomsapi/directory" "github.com/matrix-org/dendrite/publicroomsapi/storage" + "github.com/matrix-org/dendrite/publicroomsapi/types" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -34,7 +36,10 @@ const pathPrefixR0 = "/_matrix/client/r0" // Due to Setup being used to call many other functions, a gocyclo nolint is // applied: // nolint: gocyclo -func Setup(apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database) { +func Setup( + apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, + fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, +) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() authData := auth.Data{ @@ -64,7 +69,17 @@ func Setup(apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage. ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/publicRooms", common.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse { + if extRoomsProvider != nil { + return directory.GetPostPublicRoomsWithExternal(req, publicRoomsDB, fedClient, extRoomsProvider) + } return directory.GetPostPublicRooms(req, publicRoomsDB) }), ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) + + // Federation - TODO: should this live here or in federation API? It's sure easier if it's here so here it is. + apiMux.Handle("/_matrix/federation/v1/publicRooms", + common.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse { + return directory.GetPostPublicRooms(req, publicRoomsDB) + }), + ).Methods(http.MethodGet) } diff --git a/publicroomsapi/storage/interface.go b/publicroomsapi/storage/interface.go index 3f7e65892..0feca0e20 100644 --- a/publicroomsapi/storage/interface.go +++ b/publicroomsapi/storage/interface.go @@ -18,7 +18,6 @@ import ( "context" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/publicroomsapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -27,7 +26,7 @@ type Database interface { GetRoomVisibility(ctx context.Context, roomID string) (bool, error) SetRoomVisibility(ctx context.Context, visible bool, roomID string) error CountPublicRooms(ctx context.Context) (int64, error) - GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]types.PublicRoom, error) + GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error } diff --git a/publicroomsapi/storage/postgres/public_rooms_table.go b/publicroomsapi/storage/postgres/public_rooms_table.go index 7e606e939..7e31afd2a 100644 --- a/publicroomsapi/storage/postgres/public_rooms_table.go +++ b/publicroomsapi/storage/postgres/public_rooms_table.go @@ -22,9 +22,9 @@ import ( "fmt" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" "github.com/lib/pq" - "github.com/matrix-org/dendrite/publicroomsapi/types" ) var editableAttributes = []string{ @@ -177,7 +177,7 @@ func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64, func (s *publicRoomsStatements) selectPublicRooms( ctx context.Context, offset int64, limit int16, filter string, -) ([]types.PublicRoom, error) { +) ([]gomatrixserverlib.PublicRoom, error) { var rows *sql.Rows var err error @@ -203,17 +203,17 @@ func (s *publicRoomsStatements) selectPublicRooms( } if err != nil { - return []types.PublicRoom{}, nil + return []gomatrixserverlib.PublicRoom{}, nil } defer common.CloseAndLogIfError(ctx, rows, "selectPublicRooms: rows.close() failed") - rooms := []types.PublicRoom{} + rooms := []gomatrixserverlib.PublicRoom{} for rows.Next() { - var r types.PublicRoom + var r gomatrixserverlib.PublicRoom var aliases pq.StringArray err = rows.Scan( - &r.RoomID, &r.NumJoinedMembers, &aliases, &r.CanonicalAlias, + &r.RoomID, &r.JoinedMembersCount, &aliases, &r.CanonicalAlias, &r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL, ) if err != nil { diff --git a/publicroomsapi/storage/postgres/storage.go b/publicroomsapi/storage/postgres/storage.go index 5365c766f..5a4bc8f9a 100644 --- a/publicroomsapi/storage/postgres/storage.go +++ b/publicroomsapi/storage/postgres/storage.go @@ -21,7 +21,6 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/publicroomsapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -85,7 +84,7 @@ func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64 // Returns an error if the retrieval failed. func (d *PublicRoomsServerDatabase) GetPublicRooms( ctx context.Context, offset int64, limit int16, filter string, -) ([]types.PublicRoom, error) { +) ([]gomatrixserverlib.PublicRoom, error) { return d.statements.selectPublicRooms(ctx, offset, limit, filter) } diff --git a/publicroomsapi/storage/sqlite3/public_rooms_table.go b/publicroomsapi/storage/sqlite3/public_rooms_table.go index beba0d694..44679837f 100644 --- a/publicroomsapi/storage/sqlite3/public_rooms_table.go +++ b/publicroomsapi/storage/sqlite3/public_rooms_table.go @@ -22,7 +22,8 @@ import ( "errors" "fmt" - "github.com/matrix-org/dendrite/publicroomsapi/types" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" ) var editableAttributes = []string{ @@ -66,7 +67,7 @@ const selectPublicRoomsWithLimitSQL = "" + "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + " FROM publicroomsapi_public_rooms WHERE visibility = true" + " ORDER BY joined_members DESC" + - " LIMIT $2 OFFSET $1" + " LIMIT $1 OFFSET $2" const selectPublicRoomsWithFilterSQL = "" + "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + @@ -164,7 +165,7 @@ func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64, func (s *publicRoomsStatements) selectPublicRooms( ctx context.Context, offset int64, limit int16, filter string, -) ([]types.PublicRoom, error) { +) ([]gomatrixserverlib.PublicRoom, error) { var rows *sql.Rows var err error @@ -190,16 +191,17 @@ func (s *publicRoomsStatements) selectPublicRooms( } if err != nil { - return []types.PublicRoom{}, nil + return []gomatrixserverlib.PublicRoom{}, nil } + defer common.CloseAndLogIfError(ctx, rows, "selectPublicRooms failed to close rows") - rooms := []types.PublicRoom{} + rooms := []gomatrixserverlib.PublicRoom{} for rows.Next() { - var r types.PublicRoom + var r gomatrixserverlib.PublicRoom var aliasesJSON string err = rows.Scan( - &r.RoomID, &r.NumJoinedMembers, &aliasesJSON, &r.CanonicalAlias, + &r.RoomID, &r.JoinedMembersCount, &aliasesJSON, &r.CanonicalAlias, &r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL, ) if err != nil { diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go index f8ba71a89..80c04cab7 100644 --- a/publicroomsapi/storage/sqlite3/storage.go +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -23,7 +23,6 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/publicroomsapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -87,7 +86,7 @@ func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64 // Returns an error if the retrieval failed. func (d *PublicRoomsServerDatabase) GetPublicRooms( ctx context.Context, offset int64, limit int16, filter string, -) ([]types.PublicRoom, error) { +) ([]gomatrixserverlib.PublicRoom, error) { return d.statements.selectPublicRooms(ctx, offset, limit, filter) } diff --git a/publicroomsapi/types/types.go b/publicroomsapi/types/types.go index c284bcca4..11cb0d204 100644 --- a/publicroomsapi/types/types.go +++ b/publicroomsapi/types/types.go @@ -14,15 +14,11 @@ package types -// PublicRoom represents a local public room -type PublicRoom struct { - RoomID string `json:"room_id"` - Aliases []string `json:"aliases,omitempty"` - CanonicalAlias string `json:"canonical_alias,omitempty"` - Name string `json:"name,omitempty"` - Topic string `json:"topic,omitempty"` - AvatarURL string `json:"avatar_url,omitempty"` - NumJoinedMembers int64 `json:"num_joined_members"` - WorldReadable bool `json:"world_readable"` - GuestCanJoin bool `json:"guest_can_join"` +// ExternalPublicRoomsProvider provides a list of homeservers who should be queried +// periodically for a list of public rooms on their server. +type ExternalPublicRoomsProvider interface { + // The list of homeserver domains to query. These servers will receive a request + // via this API: https://matrix.org/docs/spec/server_server/latest#public-room-directory + // This will be called -on demand- by clients, so cache appropriately! + Homeservers() []string } From ad5849d2224245e1277b8bc7d3e2104ed6061a99 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 19 Mar 2020 12:07:01 +0000 Subject: [PATCH 15/17] HeaderedEvents in sync API (#922) * Use HeaderedEvent in syncapi * Update notifier test * Fix persisting headered event * Clean up unused API function * Fix overshadowed err from linter * Write headered JSON to invites table too * Rename event_json to headered_event_json in syncapi database schemae * Fix invites_table queries * Update QueryRoomVersionCapabilitiesResponse comment * Fix syncapi SQLite --- go.mod | 2 +- go.sum | 6 ++ roomserver/api/query.go | 14 +++-- roomserver/query/query.go | 9 ++- roomserver/storage/postgres/storage.go | 2 +- roomserver/storage/sqlite3/storage.go | 2 +- syncapi/consumers/roomserver.go | 29 +++++----- syncapi/routing/messages.go | 29 ++++++---- syncapi/routing/state.go | 6 +- syncapi/storage/interface.go | 12 ++-- .../postgres/current_room_state_table.go | 40 +++++++------ syncapi/storage/postgres/invites_table.go | 25 ++++++--- .../postgres/output_room_events_table.go | 39 +++++++------ .../output_room_events_topology_table.go | 2 +- syncapi/storage/postgres/syncserver.go | 44 +++++++-------- .../sqlite3/current_room_state_table.go | 40 +++++++------ syncapi/storage/sqlite3/invites_table.go | 25 ++++++--- .../sqlite3/output_room_events_table.go | 36 +++++++----- .../output_room_events_topology_table.go | 2 +- syncapi/storage/sqlite3/syncserver.go | 56 ++++++++++--------- syncapi/sync/notifier.go | 2 +- syncapi/sync/notifier_test.go | 25 ++++++--- syncapi/types/types.go | 2 +- 23 files changed, 260 insertions(+), 189 deletions(-) diff --git a/go.mod b/go.mod index 556a6b8f1..3ceaea528 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd + github.com/matrix-org/gomatrixserverlib v0.0.0-20200318180716-bc4ff56961e2 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 2c51915bb..0f4542b28 100644 --- a/go.sum +++ b/go.sum @@ -270,6 +270,12 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200317114945-9a368ea4620d h1:0G github.com/matrix-org/gomatrixserverlib v0.0.0-20200317114945-9a368ea4620d/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd h1:n95A8YyiCZ8Nu2beqw4akCaPIRrZr/nesHYDZV8WkXI= github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318141718-859b9f256ffd h1:381JBgdNYOQwWelV/LOPqDoUkdSBUCkVZY8Of+n5jAM= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318141718-859b9f256ffd/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318145320-bc896516d72a h1:7+e7ManmyiGNqo06sQIRIoUtoV92XWzSbc0o3e4aTGY= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318145320-bc896516d72a/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318150006-bc27294f9203 h1:7HkL6bF7/M2cYteNFVtvGW5qjD4wHIiR0HsdCm2Rqao= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318150006-bc27294f9203/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 24cca03ec..9003a95a3 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -43,6 +43,8 @@ type QueryLatestEventsAndStateResponse struct { // Does the room exist? // If the room doesn't exist this will be false and LatestEvents will be empty. RoomExists bool `json:"room_exists"` + // The room version of the room. + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` // The latest events in the room. // These are used to set the prev_events when sending an event. LatestEvents []gomatrixserverlib.EventReference `json:"latest_events"` @@ -74,6 +76,8 @@ type QueryStateAfterEventsResponse struct { // Does the room exist on this roomserver? // If the room doesn't exist this will be false and StateEvents will be empty. RoomExists bool `json:"room_exists"` + // The room version of the room. + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` // Do all the previous events exist on this roomserver? // If some of previous events do not exist this will be false and StateEvents will be empty. PrevEventsExist bool `json:"prev_events_exist"` @@ -207,6 +211,8 @@ type QueryStateAndAuthChainResponse struct { // Does the room exist on this roomserver? // If the room doesn't exist this will be false and StateEvents will be empty. RoomExists bool `json:"room_exists"` + // The room version of the room. + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` // Do all the previous events exist on this roomserver? // If some of previous events do not exist this will be false and StateEvents will be empty. PrevEventsExist bool `json:"prev_events_exist"` @@ -249,10 +255,10 @@ type QueryServersInRoomAtEventResponse struct { // QueryRoomVersionCapabilities asks for the default room version type QueryRoomVersionCapabilitiesRequest struct{} -// QueryRoomVersionCapabilitiesResponse is a response to QueryServersInRoomAtEventResponse +// QueryRoomVersionCapabilitiesResponse is a response to QueryRoomVersionCapabilitiesRequest type QueryRoomVersionCapabilitiesResponse struct { - DefaultRoomVersion string `json:"default"` - AvailableRoomVersions map[string]string `json:"available"` + DefaultRoomVersion gomatrixserverlib.RoomVersion `json:"default"` + AvailableRoomVersions map[gomatrixserverlib.RoomVersion]string `json:"available"` } // RoomserverQueryAPI is used to query information from the room server. @@ -536,7 +542,7 @@ func (h *httpRoomserverQueryAPI) QueryServersInRoomAtEvent( return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } -// QueryServersInRoomAtEvent implements RoomServerQueryAPI +// QueryRoomVersionCapabilities implements RoomServerQueryAPI func (h *httpRoomserverQueryAPI) QueryRoomVersionCapabilities( ctx context.Context, request *QueryRoomVersionCapabilitiesRequest, diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 3ab12d60a..e0f385883 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -808,14 +808,13 @@ func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities( request *api.QueryRoomVersionCapabilitiesRequest, response *api.QueryRoomVersionCapabilitiesResponse, ) error { - response.DefaultRoomVersion = string(version.DefaultRoomVersion()) - response.AvailableRoomVersions = make(map[string]string) + response.DefaultRoomVersion = version.DefaultRoomVersion() + response.AvailableRoomVersions = make(map[gomatrixserverlib.RoomVersion]string) for v, desc := range version.SupportedRoomVersions() { - sv := string(v) if desc.Stable { - response.AvailableRoomVersions[sv] = "stable" + response.AvailableRoomVersions[v] = "stable" } else { - response.AvailableRoomVersions[sv] = "unstable" + response.AvailableRoomVersions[v] = "unstable" } } return nil diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index af6afe5c1..9bb6de9d7 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -155,7 +155,7 @@ func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( } // A room version was specified in the event content? if createContent.RoomVersion != nil { - roomVersion = *createContent.RoomVersion + roomVersion = gomatrixserverlib.RoomVersion(*createContent.RoomVersion) } return roomVersion, err } diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index ea926ca66..ae09a88ad 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -184,7 +184,7 @@ func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( } // A room version was specified in the event content? if createContent.RoomVersion != nil { - roomVersion = *createContent.RoomVersion + roomVersion = gomatrixserverlib.RoomVersion(*createContent.RoomVersion) } return roomVersion, err } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 332ac6205..f1e68c262 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -98,10 +98,11 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { func (s *OutputRoomEventConsumer) onNewRoomEvent( ctx context.Context, msg api.OutputNewRoomEvent, ) error { - ev := msg.Event.Event + ev := msg.Event log.WithFields(log.Fields{ - "event_id": ev.EventID(), - "room_id": ev.RoomID(), + "event_id": ev.EventID(), + "room_id": ev.RoomID(), + "room_version": ev.RoomVersion, }).Info("received event from roomserver") addsStateEvents, err := s.lookupStateEvents(msg.AddsStateEventIDs, ev) @@ -153,7 +154,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( func (s *OutputRoomEventConsumer) onNewInviteEvent( ctx context.Context, msg api.OutputNewInviteEvent, ) error { - pduPos, err := s.db.AddInviteEvent(ctx, msg.Event.Event) + pduPos, err := s.db.AddInviteEvent(ctx, msg.Event) if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ @@ -163,7 +164,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent( }).Panicf("roomserver output log: write invite failure") return nil } - s.notifier.OnNewEvent(&msg.Event.Event, "", nil, types.PaginationToken{PDUPosition: pduPos}) + s.notifier.OnNewEvent(&msg.Event, "", nil, types.PaginationToken{PDUPosition: pduPos}) return nil } @@ -186,8 +187,8 @@ func (s *OutputRoomEventConsumer) onRetireInviteEvent( // lookupStateEvents looks up the state events that are added by a new event. func (s *OutputRoomEventConsumer) lookupStateEvents( - addsStateEventIDs []string, event gomatrixserverlib.Event, -) ([]gomatrixserverlib.Event, error) { + addsStateEventIDs []string, event gomatrixserverlib.HeaderedEvent, +) ([]gomatrixserverlib.HeaderedEvent, error) { // Fast path if there aren't any new state events. if len(addsStateEventIDs) == 0 { return nil, nil @@ -195,7 +196,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // Fast path if the only state event added is the event itself. if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() { - return []gomatrixserverlib.Event{event}, nil + return []gomatrixserverlib.HeaderedEvent{event}, nil } // Check if this is re-adding a state events that we previously processed @@ -229,10 +230,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( return nil, err } - for _, headeredEvent := range eventResp.Events { - result = append(result, headeredEvent.Event) - } - + result = append(result, eventResp.Events...) missing = missingEventsFrom(result, addsStateEventIDs) if len(missing) != 0 { @@ -244,7 +242,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( return result, nil } -func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.Event) (gomatrixserverlib.Event, error) { +func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.HeaderedEvent) (gomatrixserverlib.HeaderedEvent, error) { var stateKey string if event.StateKey() == nil { stateKey = "" @@ -269,10 +267,11 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.Event PrevSender: prevEvent.Sender(), } - return event.SetUnsigned(prev) + event.Event, err = event.SetUnsigned(prev) + return event, err } -func missingEventsFrom(events []gomatrixserverlib.Event, required []string) []string { +func missingEventsFrom(events []gomatrixserverlib.HeaderedEvent, required []string) []string { have := map[string]bool{} for _, event := range events { have[event.EventID()] = true diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index e0901a780..cd21c3bd8 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -181,7 +181,7 @@ func (r *messagesReq) retrieveEvents() ( return } - var events []gomatrixserverlib.Event + var events []gomatrixserverlib.HeaderedEvent // There can be two reasons for streamEvents to be empty: either we've // reached the oldest event in the room (or the most recent one, depending @@ -217,7 +217,7 @@ func (r *messagesReq) retrieveEvents() ( } // Convert all of the events into client events. - clientEvents = gomatrixserverlib.ToClientEvents(events, gomatrixserverlib.FormatAll) + clientEvents = gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatAll) // Get the position of the first and the last event in the room's topology. // This position is currently determined by the event's depth, so we could // also use it instead of retrieving from the database. However, if we ever @@ -273,7 +273,7 @@ func (r *messagesReq) retrieveEvents() ( // Returns an error if there was an issue talking with the database or // backfilling. func (r *messagesReq) handleEmptyEventsSlice() ( - events []gomatrixserverlib.Event, err error, + events []gomatrixserverlib.HeaderedEvent, err error, ) { backwardExtremities, err := r.db.BackwardExtremitiesForRoom(r.ctx, r.roomID) @@ -287,7 +287,7 @@ func (r *messagesReq) handleEmptyEventsSlice() ( } else { // If not, it means the slice was empty because we reached the room's // creation, so return an empty slice. - events = []gomatrixserverlib.Event{} + events = []gomatrixserverlib.HeaderedEvent{} } return @@ -299,7 +299,7 @@ func (r *messagesReq) handleEmptyEventsSlice() ( // through backfilling if needed. // Returns an error if there was an issue while backfilling. func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent) ( - events []gomatrixserverlib.Event, err error, + events []gomatrixserverlib.HeaderedEvent, err error, ) { // Check if we have enough events. isSetLargeEnough := true @@ -326,7 +326,7 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent // Backfill is needed if we've reached a backward extremity and need more // events. It's only needed if the direction is backward. if len(backwardExtremities) > 0 && !isSetLargeEnough && r.backwardOrdering { - var pdus []gomatrixserverlib.Event + var pdus []gomatrixserverlib.HeaderedEvent // Only ask the remote server for enough events to reach the limit. pdus, err = r.backfill(backwardExtremities, r.limit-len(streamEvents)) if err != nil { @@ -400,7 +400,7 @@ func (r *messagesReq) containsBackwardExtremity(events []types.StreamEvent) (boo // event, or if there is no remote homeserver to contact. // Returns an error if there was an issue with retrieving the list of servers in // the room or sending the request. -func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserverlib.Event, error) { +func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { // Query the list of servers in the room when one of the backward extremities // was sent. var serversResponse api.QueryServersInRoomAtEventResponse @@ -428,7 +428,7 @@ func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserv } } - pdus := make([]gomatrixserverlib.Event, 0) + pdus := make([]gomatrixserverlib.HeaderedEvent, 0) // If the roomserver responded with at least one server that isn't us, // send it a request for backfill. @@ -440,13 +440,20 @@ func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserv return nil, err } - pdus = txn.PDUs + for _, p := range txn.PDUs { + pdus = append(pdus, p.Headered(gomatrixserverlib.RoomVersionV1)) + } // Store the events in the database, while marking them as unfit to show // up in responses to sync requests. for _, pdu := range pdus { + headered := pdu.Headered(gomatrixserverlib.RoomVersionV1) if _, err = r.db.WriteEvent( - r.ctx, &pdu, []gomatrixserverlib.Event{}, []string{}, []string{}, + r.ctx, + &headered, + []gomatrixserverlib.HeaderedEvent{}, + []string{}, + []string{}, nil, true, ); err != nil { return nil, err @@ -486,7 +493,7 @@ func setToDefault( // timestamp of two Matrix events. // Returns true if the first event happened before the second one, false // otherwise. -func sortEvents(e1 *gomatrixserverlib.Event, e2 *gomatrixserverlib.Event) bool { +func sortEvents(e1 *gomatrixserverlib.HeaderedEvent, e2 *gomatrixserverlib.HeaderedEvent) bool { t := e1.OriginServerTS().Time() return e2.OriginServerTS().Time().After(t) } diff --git a/syncapi/routing/state.go b/syncapi/routing/state.go index 5688086cd..87b6396ac 100644 --- a/syncapi/routing/state.go +++ b/syncapi/routing/state.go @@ -56,7 +56,9 @@ func OnIncomingStateRequest(req *http.Request, db storage.Database, roomID strin // Fill the prev_content and replaces_state keys if necessary for _, event := range stateEvents { stateEvent := stateEventInStateResp{ - ClientEvent: gomatrixserverlib.ToClientEvent(event, gomatrixserverlib.FormatAll), + ClientEvent: gomatrixserverlib.HeaderedToClientEvents( + []gomatrixserverlib.HeaderedEvent{event}, gomatrixserverlib.FormatAll, + )[0], } var prevEventRef types.PrevEventRef if len(event.Unsigned()) > 0 { @@ -113,7 +115,7 @@ func OnIncomingStateTypeRequest(req *http.Request, db storage.Database, roomID s } stateEvent := stateEventInStateResp{ - ClientEvent: gomatrixserverlib.ToClientEvent(*event, gomatrixserverlib.FormatAll), + ClientEvent: gomatrixserverlib.HeaderedToClientEvent(*event, gomatrixserverlib.FormatAll), } return util.JSONResponse{ diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 0abeac34d..b6dc19696 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -29,16 +29,16 @@ import ( type Database interface { common.PartitionStorer AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) - Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) - WriteEvent(context.Context, *gomatrixserverlib.Event, []gomatrixserverlib.Event, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) - GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.Event, error) - GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.Event, err error) + Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) + WriteEvent(context.Context, *gomatrixserverlib.HeaderedEvent, []gomatrixserverlib.HeaderedEvent, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) + GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) + GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) SyncPosition(ctx context.Context) (types.PaginationToken, error) IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error) GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) UpsertAccountData(ctx context.Context, userID, roomID, dataType string) (types.StreamPosition, error) - AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.Event) (types.StreamPosition, error) + AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent) (types.StreamPosition, error) RetireInviteEvent(ctx context.Context, inviteEventID string) error SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) AddTypingUser(userID, roomID string, expireTime *time.Time) types.StreamPosition @@ -48,6 +48,6 @@ type Database interface { EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error) BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) MaxTopologicalPosition(ctx context.Context, roomID string) (types.StreamPosition, error) - StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.Event + StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) } diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 48cc2b5fd..ab8f07b21 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( -- The state_key value for this state event e.g '' state_key TEXT NOT NULL, -- The JSON for the event. Stored as TEXT because this should be valid UTF-8. - event_json TEXT NOT NULL, + headered_event_json TEXT NOT NULL, -- The 'content.membership' value if this event is an m.room.member event. For other -- events, this will be NULL. membership TEXT, @@ -59,10 +59,10 @@ CREATE INDEX IF NOT EXISTS syncapi_membership_idx ON syncapi_current_room_state( ` const upsertRoomStateSQL = "" + - "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, event_json, membership, added_at)" + + "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, headered_event_json, membership, added_at)" + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + " ON CONFLICT ON CONSTRAINT syncapi_room_state_unique" + - " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, event_json = $7, membership = $8, added_at = $9" + " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, headered_event_json = $7, membership = $8, added_at = $9" const deleteRoomStateByEventIDSQL = "" + "DELETE FROM syncapi_current_room_state WHERE event_id = $1" @@ -71,7 +71,7 @@ const selectRoomIDsWithMembershipSQL = "" + "SELECT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2" const selectCurrentStateSQL = "" + - "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1" + + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" + " AND ( $2::text[] IS NULL OR sender = ANY($2) )" + " AND ( $3::text[] IS NULL OR NOT(sender = ANY($3)) )" + " AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" + @@ -83,14 +83,14 @@ const selectJoinedUsersSQL = "" + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" const selectStateEventSQL = "" + - "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" const selectEventsWithEventIDsSQL = "" + // TODO: The session_id and transaction_id blanks are here because otherwise // the rowsToStreamEvents expects there to be exactly five columns. We need to // figure out if these really need to be in the DB, and if so, we need a // better permanent fix for this. - neilalexander, 2 Jan 2020 - "SELECT added_at, event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + + "SELECT added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + " FROM syncapi_current_room_state WHERE event_id = ANY($1)" type currentRoomStateStatements struct { @@ -185,7 +185,7 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( func (s *currentRoomStateStatements) selectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, stateFilter *gomatrixserverlib.StateFilter, -) ([]gomatrixserverlib.Event, error) { +) ([]gomatrixserverlib.HeaderedEvent, error) { stmt := common.TxStmt(txn, s.selectCurrentStateStmt) rows, err := stmt.QueryContext(ctx, roomID, pq.StringArray(stateFilter.Senders), @@ -213,7 +213,7 @@ func (s *currentRoomStateStatements) deleteRoomStateByEventID( func (s *currentRoomStateStatements) upsertRoomState( ctx context.Context, txn *sql.Tx, - event gomatrixserverlib.Event, membership *string, addedAt types.StreamPosition, + event gomatrixserverlib.HeaderedEvent, membership *string, addedAt types.StreamPosition, ) error { // Parse content as JSON and search for an "url" key containsURL := false @@ -223,9 +223,14 @@ func (s *currentRoomStateStatements) upsertRoomState( _, containsURL = content["url"] } + headeredJSON, err := json.Marshal(event) + if err != nil { + return err + } + // upsert state event stmt := common.TxStmt(txn, s.upsertRoomStateStmt) - _, err := stmt.ExecContext( + _, err = stmt.ExecContext( ctx, event.RoomID(), event.EventID(), @@ -233,7 +238,7 @@ func (s *currentRoomStateStatements) upsertRoomState( event.Sender(), containsURL, *event.StateKey(), - event.JSON(), + headeredJSON, membership, addedAt, ) @@ -252,16 +257,16 @@ func (s *currentRoomStateStatements) selectEventsWithEventIDs( return rowsToStreamEvents(rows) } -func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.Event, error) { - result := []gomatrixserverlib.Event{} +func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.HeaderedEvent, error) { + result := []gomatrixserverlib.HeaderedEvent{} for rows.Next() { var eventBytes []byte if err := rows.Scan(&eventBytes); err != nil { return nil, err } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, err } result = append(result, ev) @@ -271,7 +276,7 @@ func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.Event, error) { func (s *currentRoomStateStatements) selectStateEvent( ctx context.Context, roomID, evType, stateKey string, -) (*gomatrixserverlib.Event, error) { +) (*gomatrixserverlib.HeaderedEvent, error) { stmt := s.selectStateEventStmt var res []byte err := stmt.QueryRowContext(ctx, roomID, evType, stateKey).Scan(&res) @@ -281,6 +286,9 @@ func (s *currentRoomStateStatements) selectStateEvent( if err != nil { return nil, err } - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(res, false) + var ev gomatrixserverlib.HeaderedEvent + if err = json.Unmarshal(res, &ev); err != nil { + return nil, err + } return &ev, err } diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go index 6a7f9ef28..ca0c64fb9 100644 --- a/syncapi/storage/postgres/invites_table.go +++ b/syncapi/storage/postgres/invites_table.go @@ -18,6 +18,7 @@ package postgres import ( "context" "database/sql" + "encoding/json" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" @@ -30,7 +31,7 @@ CREATE TABLE IF NOT EXISTS syncapi_invite_events ( event_id TEXT NOT NULL, room_id TEXT NOT NULL, target_user_id TEXT NOT NULL, - event_json TEXT NOT NULL + headered_event_json TEXT NOT NULL ); -- For looking up the invites for a given user. @@ -44,14 +45,14 @@ CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx const insertInviteEventSQL = "" + "INSERT INTO syncapi_invite_events (" + - " room_id, event_id, target_user_id, event_json" + + " room_id, event_id, target_user_id, headered_event_json" + ") VALUES ($1, $2, $3, $4) RETURNING id" const deleteInviteEventSQL = "" + "DELETE FROM syncapi_invite_events WHERE event_id = $1" const selectInviteEventsInRangeSQL = "" + - "SELECT room_id, event_json FROM syncapi_invite_events" + + "SELECT room_id, headered_event_json FROM syncapi_invite_events" + " WHERE target_user_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC" @@ -86,14 +87,20 @@ func (s *inviteEventsStatements) prepare(db *sql.DB) (err error) { } func (s *inviteEventsStatements) insertInviteEvent( - ctx context.Context, inviteEvent gomatrixserverlib.Event, + ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (streamPos types.StreamPosition, err error) { + var headeredJSON []byte + headeredJSON, err = json.Marshal(inviteEvent) + if err != nil { + return + } + err = s.insertInviteEventStmt.QueryRowContext( ctx, inviteEvent.RoomID(), inviteEvent.EventID(), *inviteEvent.StateKey(), - inviteEvent.JSON(), + headeredJSON, ).Scan(&streamPos) return } @@ -109,14 +116,14 @@ func (s *inviteEventsStatements) deleteInviteEvent( // active invites for the target user ID in the supplied range. func (s *inviteEventsStatements) selectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, startPos, endPos types.StreamPosition, -) (map[string]gomatrixserverlib.Event, error) { +) (map[string]gomatrixserverlib.HeaderedEvent, error) { stmt := common.TxStmt(txn, s.selectInviteEventsInRangeStmt) rows, err := stmt.QueryContext(ctx, targetUserID, startPos, endPos) if err != nil { return nil, err } defer common.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") - result := map[string]gomatrixserverlib.Event{} + result := map[string]gomatrixserverlib.HeaderedEvent{} for rows.Next() { var ( roomID string @@ -126,8 +133,8 @@ func (s *inviteEventsStatements) selectInviteEventsInRange( return nil, err } - event, err := gomatrixserverlib.NewEventFromTrustedJSON(eventJSON, false) - if err != nil { + var event gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventJSON, &event); err != nil { return nil, err } diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index e7cf19354..5f9a1d0c6 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -44,8 +44,9 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( event_id TEXT NOT NULL CONSTRAINT syncapi_event_id_idx UNIQUE, -- The 'room_id' key for the event. room_id TEXT NOT NULL, - -- The JSON for the event. Stored as TEXT because this should be valid UTF-8. - event_json TEXT NOT NULL, + -- The headered JSON for the event, containing potentially additional metadata such as + -- the room version. Stored as TEXT because this should be valid UTF-8. + headered_event_json TEXT NOT NULL, -- The event type e.g 'm.room.member'. type TEXT NOT NULL, -- The 'sender' property of the event. @@ -70,26 +71,26 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( const insertEventSQL = "" + "INSERT INTO syncapi_output_room_events (" + - "room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + + "room_id, event_id, headered_event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) " + "ON CONFLICT ON CONSTRAINT syncapi_event_id_idx DO UPDATE SET exclude_from_sync = $11 " + "RETURNING id" const selectEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = ANY($1)" + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = ANY($1)" const selectRecentEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC LIMIT $4" const selectRecentEventsForSyncSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" + " ORDER BY id DESC LIMIT $4" const selectEarlyEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id ASC LIMIT $4" @@ -98,7 +99,7 @@ const selectMaxEventIDSQL = "" + // In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). const selectStateInRangeSQL = "" + - "SELECT id, event_json, exclude_from_sync, add_state_ids, remove_state_ids" + + "SELECT id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2) AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + " AND ( $3::text[] IS NULL OR sender = ANY($3) )" + @@ -203,8 +204,8 @@ func (s *outputRoomEventsStatements) selectStateInRange( } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, nil, err } needSet := stateNeeded[ev.RoomID()] @@ -220,7 +221,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( stateNeeded[ev.RoomID()] = needSet eventIDToEvent[ev.EventID()] = types.StreamEvent{ - Event: ev, + HeaderedEvent: ev, StreamPosition: streamPos, ExcludeFromSync: excludeFromSync, } @@ -248,7 +249,7 @@ func (s *outputRoomEventsStatements) selectMaxEventID( // of the inserted event. func (s *outputRoomEventsStatements) insertEvent( ctx context.Context, txn *sql.Tx, - event *gomatrixserverlib.Event, addState, removeState []string, + event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, transactionID *api.TransactionID, excludeFromSync bool, ) (streamPos types.StreamPosition, err error) { var txnID *string @@ -266,12 +267,18 @@ func (s *outputRoomEventsStatements) insertEvent( _, containsURL = content["url"] } + var headeredJSON []byte + headeredJSON, err = json.Marshal(event) + if err != nil { + return + } + stmt := common.TxStmt(txn, s.insertEventStmt) err = stmt.QueryRowContext( ctx, event.RoomID(), event.EventID(), - event.JSON(), + headeredJSON, event.Type(), event.Sender(), containsURL, @@ -373,8 +380,8 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { return nil, err } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, err } @@ -386,7 +393,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { } result = append(result, types.StreamEvent{ - Event: ev, + HeaderedEvent: ev, StreamPosition: streamPos, TransactionID: transactionID, ExcludeFromSync: excludeFromSync, diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index b114fd43d..280d4ec39 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -104,7 +104,7 @@ func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { // insertEventInTopology inserts the given event in the room's topology, based // on the event's depth. func (s *outputRoomEventsTopologyStatements) insertEventInTopology( - ctx context.Context, event *gomatrixserverlib.Event, + ctx context.Context, event *gomatrixserverlib.HeaderedEvent, ) (err error) { _, err = s.insertEventInTopologyStmt.ExecContext( ctx, event.EventID(), event.Depth(), event.RoomID(), diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index abb5b4a4c..ccf1c5656 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -37,7 +37,7 @@ import ( type stateDelta struct { roomID string - stateEvents []gomatrixserverlib.Event + stateEvents []gomatrixserverlib.HeaderedEvent membership string // The PDU stream position of the latest membership event for this user, if applicable. // Can be 0 if there is no membership event in this delta. @@ -100,7 +100,7 @@ func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[s // If an event is not found in the database then it will be omitted from the list. // Returns an error if there was a problem talking with the database. // Does not include any transaction IDs in the returned events. -func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { +func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) { streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs) if err != nil { return nil, err @@ -111,7 +111,7 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([ return d.StreamEventsToEvents(nil, streamEvents), nil } -func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, ev *gomatrixserverlib.Event) error { +func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent) error { // If the event is already known as a backward extremity, don't consider // it as such anymore now that we have it. isBackwardExtremity, err := d.backwardExtremities.isBackwardExtremity(ctx, ev.RoomID(), ev.EventID()) @@ -155,8 +155,8 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, ev // Returns an error if there was a problem inserting this event. func (d *SyncServerDatasource) WriteEvent( ctx context.Context, - ev *gomatrixserverlib.Event, - addStateEvents []gomatrixserverlib.Event, + ev *gomatrixserverlib.HeaderedEvent, + addStateEvents []gomatrixserverlib.HeaderedEvent, addStateEventIDs, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool, ) (pduPosition types.StreamPosition, returnErr error) { @@ -192,7 +192,7 @@ func (d *SyncServerDatasource) WriteEvent( func (d *SyncServerDatasource) updateRoomState( ctx context.Context, txn *sql.Tx, removedEventIDs []string, - addedEvents []gomatrixserverlib.Event, + addedEvents []gomatrixserverlib.HeaderedEvent, pduPosition types.StreamPosition, ) error { // remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add. @@ -228,7 +228,7 @@ func (d *SyncServerDatasource) updateRoomState( // If there was an issue during the retrieval, returns an error func (d *SyncServerDatasource) GetStateEvent( ctx context.Context, roomID, evType, stateKey string, -) (*gomatrixserverlib.Event, error) { +) (*gomatrixserverlib.HeaderedEvent, error) { return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey) } @@ -237,7 +237,7 @@ func (d *SyncServerDatasource) GetStateEvent( // Returns an error if there was an issue with the retrieval. func (d *SyncServerDatasource) GetStateEventsForRoom( ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter, -) (stateEvents []gomatrixserverlib.Event, err error) { +) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilter) return err @@ -599,7 +599,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // Build up a /sync response. Add joined rooms. for _, roomID := range joinedRoomIDs { - var stateEvents []gomatrixserverlib.Event + var stateEvents []gomatrixserverlib.HeaderedEvent stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, &stateFilter) if err != nil { return @@ -633,9 +633,9 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true - jr.State.Events = gomatrixserverlib.ToClientEvents(stateEvents, gomatrixserverlib.FormatSync) + jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[roomID] = *jr } @@ -707,7 +707,7 @@ func (d *SyncServerDatasource) UpsertAccountData( // If the invite was successfully stored this returns the stream ID it was stored at. // Returns an error if there was a problem communicating with the database. func (d *SyncServerDatasource) AddInviteEvent( - ctx context.Context, inviteEvent gomatrixserverlib.Event, + ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (types.StreamPosition, error) { return d.invites.insertInviteEvent(ctx, inviteEvent) } @@ -758,7 +758,7 @@ func (d *SyncServerDatasource) addInvitesToResponse( for roomID, inviteEvent := range invites { ir := types.NewInviteResponse() ir.InviteState.Events = gomatrixserverlib.ToClientEvents( - []gomatrixserverlib.Event{inviteEvent}, gomatrixserverlib.FormatSync, + []gomatrixserverlib.Event{inviteEvent.Event}, gomatrixserverlib.FormatSync, ) // TODO: add the invite state from the invite event. res.Rooms.Invite[roomID] = *ir @@ -821,9 +821,9 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - jr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[delta.roomID] = *jr case gomatrixserverlib.Leave: fallthrough // transitions to leave are the same as ban @@ -834,9 +834,9 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( lr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - lr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Leave[delta.roomID] = *lr } @@ -1074,7 +1074,7 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( } s := make([]types.StreamEvent, len(allState)) for i := 0; i < len(s); i++ { - s[i] = types.StreamEvent{Event: allState[i], StreamPosition: 0} + s[i] = types.StreamEvent{HeaderedEvent: allState[i], StreamPosition: 0} } return s, nil } @@ -1082,10 +1082,10 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and // matches the streamevent.transactionID device then the transaction ID gets // added to the unsigned section of the output event. -func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.Event { - out := make([]gomatrixserverlib.Event, len(in)) +func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent { + out := make([]gomatrixserverlib.HeaderedEvent, len(in)) for i := 0; i < len(in); i++ { - out[i] = in[i].Event + out[i] = in[i].HeaderedEvent if device != nil && in[i].TransactionID != nil { if device.UserID == in[i].Sender() && device.SessionID == in[i].TransactionID.SessionID { err := out[i].SetUnsignedField( @@ -1105,7 +1105,7 @@ func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in // There may be some overlap where events in stateEvents are already in recentEvents, so filter // them out so we don't include them twice in the /sync response. They should be in recentEvents // only, so clients get to the correct state once they have rolled forward. -func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.Event) []gomatrixserverlib.Event { +func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { for _, recentEv := range recentEvents { if recentEv.StateKey() == nil { continue // not a state event diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 87007f2b0..9fafdbede 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( sender TEXT NOT NULL, contains_url BOOL NOT NULL DEFAULT false, state_key TEXT NOT NULL, - event_json TEXT NOT NULL, + headered_event_json TEXT NOT NULL, membership TEXT, added_at BIGINT, UNIQUE (room_id, type, state_key) @@ -47,10 +47,10 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_s ` const upsertRoomStateSQL = "" + - "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, event_json, membership, added_at)" + + "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, headered_event_json, membership, added_at)" + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + " ON CONFLICT (event_id, room_id, type, sender, contains_url)" + - " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, event_json = $7, membership = $8, added_at = $9" + " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, headered_event_json = $7, membership = $8, added_at = $9" const deleteRoomStateByEventIDSQL = "" + "DELETE FROM syncapi_current_room_state WHERE event_id = $1" @@ -59,7 +59,7 @@ const selectRoomIDsWithMembershipSQL = "" + "SELECT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2" const selectCurrentStateSQL = "" + - "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1" + + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" + " AND ( $2 IS NULL OR sender IN ($2) )" + " AND ( $3 IS NULL OR NOT(sender IN ($3)) )" + " AND ( $4 IS NULL OR type IN ($4) )" + @@ -71,14 +71,14 @@ const selectJoinedUsersSQL = "" + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" const selectStateEventSQL = "" + - "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" const selectEventsWithEventIDsSQL = "" + // TODO: The session_id and transaction_id blanks are here because otherwise // the rowsToStreamEvents expects there to be exactly five columns. We need to // figure out if these really need to be in the DB, and if so, we need a // better permanent fix for this. - neilalexander, 2 Jan 2020 - "SELECT added_at, event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + + "SELECT added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + " FROM syncapi_current_room_state WHERE event_id IN ($1)" type currentRoomStateStatements struct { @@ -171,7 +171,7 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( func (s *currentRoomStateStatements) selectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, -) ([]gomatrixserverlib.Event, error) { +) ([]gomatrixserverlib.HeaderedEvent, error) { stmt := common.TxStmt(txn, s.selectCurrentStateStmt) rows, err := stmt.QueryContext(ctx, roomID, nil, // FIXME: pq.StringArray(stateFilterPart.Senders), @@ -199,7 +199,7 @@ func (s *currentRoomStateStatements) deleteRoomStateByEventID( func (s *currentRoomStateStatements) upsertRoomState( ctx context.Context, txn *sql.Tx, - event gomatrixserverlib.Event, membership *string, addedAt types.StreamPosition, + event gomatrixserverlib.HeaderedEvent, membership *string, addedAt types.StreamPosition, ) error { // Parse content as JSON and search for an "url" key containsURL := false @@ -209,9 +209,14 @@ func (s *currentRoomStateStatements) upsertRoomState( _, containsURL = content["url"] } + headeredJSON, err := json.Marshal(event) + if err != nil { + return err + } + // upsert state event stmt := common.TxStmt(txn, s.upsertRoomStateStmt) - _, err := stmt.ExecContext( + _, err = stmt.ExecContext( ctx, event.RoomID(), event.EventID(), @@ -219,7 +224,7 @@ func (s *currentRoomStateStatements) upsertRoomState( event.Sender(), containsURL, *event.StateKey(), - event.JSON(), + headeredJSON, membership, addedAt, ) @@ -242,16 +247,16 @@ func (s *currentRoomStateStatements) selectEventsWithEventIDs( return rowsToStreamEvents(rows) } -func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.Event, error) { - result := []gomatrixserverlib.Event{} +func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.HeaderedEvent, error) { + result := []gomatrixserverlib.HeaderedEvent{} for rows.Next() { var eventBytes []byte if err := rows.Scan(&eventBytes); err != nil { return nil, err } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, err } result = append(result, ev) @@ -261,7 +266,7 @@ func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.Event, error) { func (s *currentRoomStateStatements) selectStateEvent( ctx context.Context, roomID, evType, stateKey string, -) (*gomatrixserverlib.Event, error) { +) (*gomatrixserverlib.HeaderedEvent, error) { stmt := s.selectStateEventStmt var res []byte err := stmt.QueryRowContext(ctx, roomID, evType, stateKey).Scan(&res) @@ -271,6 +276,9 @@ func (s *currentRoomStateStatements) selectStateEvent( if err != nil { return nil, err } - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(res, false) + var ev gomatrixserverlib.HeaderedEvent + if err = json.Unmarshal(res, &ev); err != nil { + return nil, err + } return &ev, err } diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index df6f1c9a5..22efeaeb0 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -18,6 +18,7 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" @@ -30,7 +31,7 @@ CREATE TABLE IF NOT EXISTS syncapi_invite_events ( event_id TEXT NOT NULL, room_id TEXT NOT NULL, target_user_id TEXT NOT NULL, - event_json TEXT NOT NULL + headered_event_json TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS syncapi_invites_target_user_id_idx ON syncapi_invite_events (target_user_id, id); @@ -39,14 +40,14 @@ CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx ON syncapi_invite_events const insertInviteEventSQL = "" + "INSERT INTO syncapi_invite_events" + - " (id, room_id, event_id, target_user_id, event_json)" + + " (id, room_id, event_id, target_user_id, headered_event_json)" + " VALUES ($1, $2, $3, $4, $5)" const deleteInviteEventSQL = "" + "DELETE FROM syncapi_invite_events WHERE event_id = $1" const selectInviteEventsInRangeSQL = "" + - "SELECT room_id, event_json FROM syncapi_invite_events" + + "SELECT room_id, headered_event_json FROM syncapi_invite_events" + " WHERE target_user_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC" @@ -83,15 +84,21 @@ func (s *inviteEventsStatements) prepare(db *sql.DB, streamID *streamIDStatement } func (s *inviteEventsStatements) insertInviteEvent( - ctx context.Context, txn *sql.Tx, inviteEvent gomatrixserverlib.Event, streamPos types.StreamPosition, + ctx context.Context, txn *sql.Tx, inviteEvent gomatrixserverlib.HeaderedEvent, streamPos types.StreamPosition, ) (err error) { + var headeredJSON []byte + headeredJSON, err = json.Marshal(inviteEvent) + if err != nil { + return + } + _, err = txn.Stmt(s.insertInviteEventStmt).ExecContext( ctx, streamPos, inviteEvent.RoomID(), inviteEvent.EventID(), *inviteEvent.StateKey(), - inviteEvent.JSON(), + headeredJSON, ) return } @@ -107,14 +114,14 @@ func (s *inviteEventsStatements) deleteInviteEvent( // active invites for the target user ID in the supplied range. func (s *inviteEventsStatements) selectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, startPos, endPos types.StreamPosition, -) (map[string]gomatrixserverlib.Event, error) { +) (map[string]gomatrixserverlib.HeaderedEvent, error) { stmt := common.TxStmt(txn, s.selectInviteEventsInRangeStmt) rows, err := stmt.QueryContext(ctx, targetUserID, startPos, endPos) if err != nil { return nil, err } defer common.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") - result := map[string]gomatrixserverlib.Event{} + result := map[string]gomatrixserverlib.HeaderedEvent{} for rows.Next() { var ( roomID string @@ -124,8 +131,8 @@ func (s *inviteEventsStatements) selectInviteEventsInRange( return nil, err } - event, err := gomatrixserverlib.NewEventFromTrustedJSON(eventJSON, false) - if err != nil { + var event gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventJSON, &event); err != nil { return nil, err } diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 5ae28ca3f..08299f64b 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( id INTEGER PRIMARY KEY AUTOINCREMENT, event_id TEXT NOT NULL UNIQUE, room_id TEXT NOT NULL, - event_json TEXT NOT NULL, + headered_event_json TEXT NOT NULL, type TEXT NOT NULL, sender TEXT NOT NULL, contains_url BOOL NOT NULL, @@ -49,25 +49,25 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( const insertEventSQL = "" + "INSERT INTO syncapi_output_room_events (" + - "id, room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + + "id, room_id, event_id, headered_event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) " + "ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = $13" const selectEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = $1" + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = $1" const selectRecentEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC LIMIT $4" const selectRecentEventsForSyncSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" + " ORDER BY id DESC LIMIT $4" const selectEarlyEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id ASC LIMIT $4" @@ -86,7 +86,7 @@ const selectMaxEventIDSQL = "" + $8 = stateFilterPart.Limit, */ const selectStateInRangeSQL = "" + - "SELECT id, event_json, exclude_from_sync, add_state_ids, remove_state_ids" + + "SELECT id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2)" + // old/new pos " AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + @@ -200,8 +200,8 @@ func (s *outputRoomEventsStatements) selectStateInRange( } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, nil, err } needSet := stateNeeded[ev.RoomID()] @@ -217,7 +217,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( stateNeeded[ev.RoomID()] = needSet eventIDToEvent[ev.EventID()] = types.StreamEvent{ - Event: ev, + HeaderedEvent: ev, StreamPosition: streamPos, ExcludeFromSync: excludeFromSync, } @@ -245,7 +245,7 @@ func (s *outputRoomEventsStatements) selectMaxEventID( // of the inserted event. func (s *outputRoomEventsStatements) insertEvent( ctx context.Context, txn *sql.Tx, - event *gomatrixserverlib.Event, addState, removeState []string, + event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, transactionID *api.TransactionID, excludeFromSync bool, ) (streamPos types.StreamPosition, err error) { var txnID *string @@ -263,6 +263,12 @@ func (s *outputRoomEventsStatements) insertEvent( _, containsURL = content["url"] } + var headeredJSON []byte + headeredJSON, err = json.Marshal(event) + if err != nil { + return + } + streamPos, err = s.streamIDStatements.nextStreamID(ctx, txn) if err != nil { return @@ -283,7 +289,7 @@ func (s *outputRoomEventsStatements) insertEvent( streamPos, event.RoomID(), event.EventID(), - event.JSON(), + headeredJSON, event.Type(), event.Sender(), containsURL, @@ -392,8 +398,8 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { return nil, err } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, err } @@ -405,7 +411,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { } result = append(result, types.StreamEvent{ - Event: ev, + HeaderedEvent: ev, StreamPosition: streamPos, TransactionID: transactionID, ExcludeFromSync: excludeFromSync, diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index f7075bd6f..a2944c2f9 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -101,7 +101,7 @@ func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { // insertEventInTopology inserts the given event in the room's topology, based // on the event's depth. func (s *outputRoomEventsTopologyStatements) insertEventInTopology( - ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.Event, + ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, ) (err error) { stmt := common.TxStmt(txn, s.insertEventInTopologyStmt) _, err = stmt.ExecContext( diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 0e84c8c86..221d96728 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -40,7 +40,7 @@ import ( type stateDelta struct { roomID string - stateEvents []gomatrixserverlib.Event + stateEvents []gomatrixserverlib.HeaderedEvent membership string // The PDU stream position of the latest membership event for this user, if applicable. // Can be 0 if there is no membership event in this delta. @@ -126,7 +126,7 @@ func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[s // If an event is not found in the database then it will be omitted from the list. // Returns an error if there was a problem talking with the database. // Does not include any transaction IDs in the returned events. -func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { +func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) { streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs) if err != nil { return nil, err @@ -137,7 +137,7 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([ return d.StreamEventsToEvents(nil, streamEvents), nil } -func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.Event) error { +func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { // If the event is already known as a backward extremity, don't consider // it as such anymore now that we have it. isBackwardExtremity, err := d.backwardExtremities.isBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()) @@ -181,8 +181,8 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx // Returns an error if there was a problem inserting this event. func (d *SyncServerDatasource) WriteEvent( ctx context.Context, - ev *gomatrixserverlib.Event, - addStateEvents []gomatrixserverlib.Event, + ev *gomatrixserverlib.HeaderedEvent, + addStateEvents []gomatrixserverlib.HeaderedEvent, addStateEventIDs, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool, ) (pduPosition types.StreamPosition, returnErr error) { @@ -218,7 +218,7 @@ func (d *SyncServerDatasource) WriteEvent( func (d *SyncServerDatasource) updateRoomState( ctx context.Context, txn *sql.Tx, removedEventIDs []string, - addedEvents []gomatrixserverlib.Event, + addedEvents []gomatrixserverlib.HeaderedEvent, pduPosition types.StreamPosition, ) error { // remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add. @@ -254,7 +254,7 @@ func (d *SyncServerDatasource) updateRoomState( // If there was an issue during the retrieval, returns an error func (d *SyncServerDatasource) GetStateEvent( ctx context.Context, roomID, evType, stateKey string, -) (*gomatrixserverlib.Event, error) { +) (*gomatrixserverlib.HeaderedEvent, error) { return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey) } @@ -263,7 +263,7 @@ func (d *SyncServerDatasource) GetStateEvent( // Returns an error if there was an issue with the retrieval. func (d *SyncServerDatasource) GetStateEventsForRoom( ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, -) (stateEvents []gomatrixserverlib.Event, err error) { +) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilterPart) return err @@ -633,8 +633,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // Build up a /sync response. Add joined rooms. for _, roomID := range joinedRoomIDs { - - var stateEvents []gomatrixserverlib.Event + var stateEvents []gomatrixserverlib.HeaderedEvent stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, &stateFilterPart) if err != nil { return @@ -665,14 +664,13 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // We don't include a device here as we don't need to send down // transaction IDs for complete syncs recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents) - stateEvents = removeDuplicates(stateEvents, recentEvents) jr := types.NewJoinResponse() jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true - jr.State.Events = gomatrixserverlib.ToClientEvents(stateEvents, gomatrixserverlib.FormatSync) + jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[roomID] = *jr } @@ -748,7 +746,7 @@ func (d *SyncServerDatasource) UpsertAccountData( // If the invite was successfully stored this returns the stream ID it was stored at. // Returns an error if there was a problem communicating with the database. func (d *SyncServerDatasource) AddInviteEvent( - ctx context.Context, inviteEvent gomatrixserverlib.Event, + ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (streamPos types.StreamPosition, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { streamPos, err = d.streamID.nextStreamID(ctx, txn) @@ -806,7 +804,7 @@ func (d *SyncServerDatasource) addInvitesToResponse( for roomID, inviteEvent := range invites { ir := types.NewInviteResponse() ir.InviteState.Events = gomatrixserverlib.ToClientEvents( - []gomatrixserverlib.Event{inviteEvent}, gomatrixserverlib.FormatSync, + []gomatrixserverlib.Event{inviteEvent.Event}, gomatrixserverlib.FormatSync, ) // TODO: add the invite state from the invite event. res.Rooms.Invite[roomID] = *ir @@ -858,8 +856,12 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( if err != nil { return err } - recentEvents := d.StreamEventsToEvents(device, recentStreamEvents) - delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) // roll back + headeredRecentEvents := d.StreamEventsToEvents(device, recentStreamEvents) + var recentEvents []gomatrixserverlib.Event + for _, event := range d.StreamEventsToEvents(nil, recentStreamEvents) { + recentEvents = append(recentEvents, event.Event) + } + delta.stateEvents = removeDuplicates(delta.stateEvents, headeredRecentEvents) backwardTopologyPos := d.getBackwardTopologyPos(ctx, txn, recentStreamEvents) switch delta.membership { @@ -871,7 +873,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( ).String() jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - jr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[delta.roomID] = *jr case gomatrixserverlib.Leave: fallthrough // transitions to leave are the same as ban @@ -884,7 +886,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( ).String() lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - lr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Leave[delta.roomID] = *lr } @@ -1013,7 +1015,7 @@ func (d *SyncServerDatasource) getStateDeltas( // dupe join events will result in the entire room state coming down to the client again. This is added in // the 'state' part of the response though, so is transparent modulo bandwidth concerns as it is not added to // the timeline. - if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { + if membership := getMembershipFromEvent(&ev.HeaderedEvent, userID); membership != "" { if membership == gomatrixserverlib.Join { // send full room state down instead of a delta var s []types.StreamEvent @@ -1094,7 +1096,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( for roomID, stateStreamEvents := range state { for _, ev := range stateStreamEvents { - if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { + if membership := getMembershipFromEvent(&ev.HeaderedEvent, userID); membership != "" { if membership != gomatrixserverlib.Join { // We've already added full state for all joined rooms above. deltas = append(deltas, stateDelta{ membership: membership, @@ -1122,7 +1124,7 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( } s := make([]types.StreamEvent, len(allState)) for i := 0; i < len(s); i++ { - s[i] = types.StreamEvent{Event: allState[i], StreamPosition: 0} + s[i] = types.StreamEvent{HeaderedEvent: allState[i], StreamPosition: 0} } return s, nil } @@ -1130,10 +1132,10 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and // matches the streamevent.transactionID device then the transaction ID gets // added to the unsigned section of the output event. -func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.Event { - out := make([]gomatrixserverlib.Event, len(in)) +func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent { + out := make([]gomatrixserverlib.HeaderedEvent, len(in)) for i := 0; i < len(in); i++ { - out[i] = in[i].Event + out[i] = in[i].HeaderedEvent if device != nil && in[i].TransactionID != nil { if device.UserID == in[i].Sender() && device.SessionID == in[i].TransactionID.SessionID { err := out[i].SetUnsignedField( @@ -1153,7 +1155,7 @@ func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in // There may be some overlap where events in stateEvents are already in recentEvents, so filter // them out so we don't include them twice in the /sync response. They should be in recentEvents // only, so clients get to the correct state once they have rolled forward. -func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.Event) []gomatrixserverlib.Event { +func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { for _, recentEv := range recentEvents { if recentEv.StateKey() == nil { continue // not a state event @@ -1177,7 +1179,7 @@ func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.Event) []gom // getMembershipFromEvent returns the value of content.membership iff the event is a state event // with type 'm.room.member' and state_key of userID. Otherwise, an empty string is returned. -func getMembershipFromEvent(ev *gomatrixserverlib.Event, userID string) string { +func getMembershipFromEvent(ev *gomatrixserverlib.HeaderedEvent, userID string) string { if ev.Type() == "m.room.member" && ev.StateKeyEquals(userID) { membership, err := ev.Membership() if err != nil { diff --git a/syncapi/sync/notifier.go b/syncapi/sync/notifier.go index aaee49d34..0d8050112 100644 --- a/syncapi/sync/notifier.go +++ b/syncapi/sync/notifier.go @@ -67,7 +67,7 @@ func NewNotifier(pos types.PaginationToken) *Notifier { // Typically a consumer supplies a posUpdate with the latest sync position for the // event type it handles, leaving other fields as 0. func (n *Notifier) OnNewEvent( - ev *gomatrixserverlib.Event, roomID string, userIDs []string, + ev *gomatrixserverlib.HeaderedEvent, roomID string, userIDs []string, posUpdate types.PaginationToken, ) { // update the current position then notify relevant /sync streams. diff --git a/syncapi/sync/notifier_test.go b/syncapi/sync/notifier_test.go index 350f85594..350d757c6 100644 --- a/syncapi/sync/notifier_test.go +++ b/syncapi/sync/notifier_test.go @@ -16,6 +16,7 @@ package sync import ( "context" + "encoding/json" "fmt" "sync" "testing" @@ -29,9 +30,9 @@ import ( ) var ( - randomMessageEvent gomatrixserverlib.Event - aliceInviteBobEvent gomatrixserverlib.Event - bobLeaveEvent gomatrixserverlib.Event + randomMessageEvent gomatrixserverlib.HeaderedEvent + aliceInviteBobEvent gomatrixserverlib.HeaderedEvent + bobLeaveEvent gomatrixserverlib.HeaderedEvent syncPositionVeryOld types.PaginationToken syncPositionBefore types.PaginationToken syncPositionAfter types.PaginationToken @@ -67,7 +68,8 @@ func init() { syncPositionAfter2.PDUPosition = 13 var err error - randomMessageEvent, err = gomatrixserverlib.NewEventFromTrustedJSON([]byte(`{ + err = json.Unmarshal([]byte(`{ + "_room_version": "1", "type": "m.room.message", "content": { "body": "Hello World", @@ -75,13 +77,15 @@ func init() { }, "sender": "@noone:localhost", "room_id": "`+roomID+`", + "origin": "localhost", "origin_server_ts": 12345, "event_id": "$randomMessageEvent:localhost" - }`), false) + }`), &randomMessageEvent) if err != nil { panic(err) } - aliceInviteBobEvent, err = gomatrixserverlib.NewEventFromTrustedJSON([]byte(`{ + err = json.Unmarshal([]byte(`{ + "_room_version": "1", "type": "m.room.member", "state_key": "`+bob+`", "content": { @@ -89,13 +93,15 @@ func init() { }, "sender": "`+alice+`", "room_id": "`+roomID+`", + "origin": "localhost", "origin_server_ts": 12345, "event_id": "$aliceInviteBobEvent:localhost" - }`), false) + }`), &aliceInviteBobEvent) if err != nil { panic(err) } - bobLeaveEvent, err = gomatrixserverlib.NewEventFromTrustedJSON([]byte(`{ + err = json.Unmarshal([]byte(`{ + "_room_version": "1", "type": "m.room.member", "state_key": "`+bob+`", "content": { @@ -103,9 +109,10 @@ func init() { }, "sender": "`+bob+`", "room_id": "`+roomID+`", + "origin": "localhost", "origin_server_ts": 12345, "event_id": "$bobLeaveEvent:localhost" - }`), false) + }`), &bobLeaveEvent) if err != nil { panic(err) } diff --git a/syncapi/types/types.go b/syncapi/types/types.go index c25a38cdf..718906ecd 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -40,7 +40,7 @@ type StreamPosition int64 // Same as gomatrixserverlib.Event but also has the PDU stream position for this event. type StreamEvent struct { - gomatrixserverlib.Event + gomatrixserverlib.HeaderedEvent StreamPosition StreamPosition TransactionID *api.TransactionID ExcludeFromSync bool From 944d454cb03445e7c173d1b44e4254f425d09b09 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 19 Mar 2020 13:15:35 +0000 Subject: [PATCH 16/17] Fix SQLite problems after sync API changes --- syncapi/storage/sqlite3/syncserver.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 221d96728..a06fc91f5 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -664,6 +664,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // We don't include a device here as we don't need to send down // transaction IDs for complete syncs recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents) + stateEvents = removeDuplicates(stateEvents, recentEvents) jr := types.NewJoinResponse() jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, @@ -803,8 +804,8 @@ func (d *SyncServerDatasource) addInvitesToResponse( } for roomID, inviteEvent := range invites { ir := types.NewInviteResponse() - ir.InviteState.Events = gomatrixserverlib.ToClientEvents( - []gomatrixserverlib.Event{inviteEvent.Event}, gomatrixserverlib.FormatSync, + ir.InviteState.Events = gomatrixserverlib.HeaderedToClientEvents( + []gomatrixserverlib.HeaderedEvent{inviteEvent}, gomatrixserverlib.FormatSync, ) // TODO: add the invite state from the invite event. res.Rooms.Invite[roomID] = *ir @@ -856,12 +857,8 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( if err != nil { return err } - headeredRecentEvents := d.StreamEventsToEvents(device, recentStreamEvents) - var recentEvents []gomatrixserverlib.Event - for _, event := range d.StreamEventsToEvents(nil, recentStreamEvents) { - recentEvents = append(recentEvents, event.Event) - } - delta.stateEvents = removeDuplicates(delta.stateEvents, headeredRecentEvents) + recentEvents := d.StreamEventsToEvents(device, recentStreamEvents) + delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) backwardTopologyPos := d.getBackwardTopologyPos(ctx, txn, recentStreamEvents) switch delta.membership { @@ -871,7 +868,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[delta.roomID] = *jr @@ -884,7 +881,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( lr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Leave[delta.roomID] = *lr From f2030286de4838f26cffbb2fb3f48f850dc7335a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 19 Mar 2020 18:33:04 +0000 Subject: [PATCH 17/17] Room server changes for room versions (#930) * Rearrange state package a bit, add some code to look up the right state resolution algorithm * Remove shared * Add GetRoomVersionForRoomNID * Try to use room version to get correct state resolution algorithm * Fix room joins over federation * nolint resolveConflictsV2 because all attempts to break it up so far just result in it being awfully less obvious how it works * Rename Prepare to NewStateResolution * Update comments * Re-add missing tests --- roomserver/input/events.go | 7 +- roomserver/input/latest_events.go | 6 +- roomserver/query/query.go | 50 +- roomserver/state/database/database.go | 3 + roomserver/state/shared/shared.go | 1 + roomserver/state/state.go | 1102 +++++++++++++++++++- roomserver/state/{v1 => }/state_test.go | 2 +- roomserver/state/v1/state.go | 932 ----------------- roomserver/storage/postgres/rooms_table.go | 14 + roomserver/storage/postgres/storage.go | 8 + roomserver/storage/sqlite3/rooms_table.go | 14 + roomserver/storage/sqlite3/storage.go | 8 + 12 files changed, 1122 insertions(+), 1025 deletions(-) create mode 100644 roomserver/state/shared/shared.go rename roomserver/state/{v1 => }/state_test.go (99%) delete mode 100644 roomserver/state/v1/state.go diff --git a/roomserver/input/events.go b/roomserver/input/events.go index 034b06c11..ab2bbe1cd 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -156,11 +156,8 @@ func calculateAndSetState( stateAtEvent *types.StateAtEvent, event gomatrixserverlib.Event, ) error { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, db) - if err != nil { - return err - } + var err error + roomState := state.NewStateResolution(db) if input.HasState { // We've been told what the state at the event is so we don't need to calculate it. diff --git a/roomserver/input/latest_events.go b/roomserver/input/latest_events.go index 4d75daae9..525a6f518 100644 --- a/roomserver/input/latest_events.go +++ b/roomserver/input/latest_events.go @@ -178,11 +178,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { func (u *latestEventsUpdater) latestState() error { var err error - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, u.db) - if err != nil { - return err - } + roomState := state.NewStateResolution(u.db) latestStateAtEvents := make([]types.StateAtEvent, len(u.latest)) for i := range u.latest { diff --git a/roomserver/query/query.go b/roomserver/query/query.go index e0f385883..2638919ad 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -106,11 +106,14 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( request *api.QueryLatestEventsAndStateRequest, response *api.QueryLatestEventsAndStateResponse, ) error { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) if err != nil { - return err + response.RoomExists = false + return nil } + + roomState := state.NewStateResolution(r.DB) + response.QueryLatestEventsAndStateRequest = *request roomNID, err := r.DB.RoomNID(ctx, request.RoomID) if err != nil { @@ -121,11 +124,6 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( } response.RoomExists = true - roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) - if err != nil { - return err - } - var currentStateSnapshotNID types.StateSnapshotNID response.LatestEvents, currentStateSnapshotNID, response.Depth, err = r.DB.LatestEventIDs(ctx, roomNID) @@ -159,11 +157,14 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( request *api.QueryStateAfterEventsRequest, response *api.QueryStateAfterEventsResponse, ) error { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) if err != nil { - return err + response.RoomExists = false + return nil } + + roomState := state.NewStateResolution(r.DB) + response.QueryStateAfterEventsRequest = *request roomNID, err := r.DB.RoomNID(ctx, request.RoomID) if err != nil { @@ -174,11 +175,6 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( } response.RoomExists = true - roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) - if err != nil { - return err - } - prevStates, err := r.DB.StateAtEventIDs(ctx, request.PrevEventIDs) if err != nil { switch err.(type) { @@ -192,7 +188,7 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( // Look up the currrent state for the requested tuples. stateEntries, err := roomState.LoadStateAfterEventsForStringTuples( - ctx, prevStates, request.StateToFetch, + ctx, roomNID, prevStates, request.StateToFetch, ) if err != nil { return err @@ -358,11 +354,7 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( func (r *RoomserverQueryAPI) getMembershipsBeforeEventNID( ctx context.Context, eventNID types.EventNID, joinedOnly bool, ) ([]types.Event, error) { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) - if err != nil { - return []types.Event{}, err - } + roomState := state.NewStateResolution(r.DB) events := []types.Event{} // Lookup the event NID eIDs, err := r.DB.EventIDs(ctx, []types.EventNID{eventNID}) @@ -464,12 +456,7 @@ func (r *RoomserverQueryAPI) QueryServerAllowedToSeeEvent( func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent( ctx context.Context, eventID string, serverName gomatrixserverlib.ServerName, ) (bool, error) { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) - if err != nil { - return false, err - } - + roomState := state.NewStateResolution(r.DB) stateEntries, err := roomState.LoadStateAtEvent(ctx, eventID) if err != nil { return false, err @@ -689,12 +676,7 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( } func (r *RoomserverQueryAPI) loadStateAtEventIDs(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) - if err != nil { - return nil, err - } - + roomState := state.NewStateResolution(r.DB) prevStates, err := r.DB.StateAtEventIDs(ctx, eventIDs) if err != nil { switch err.(type) { diff --git a/roomserver/state/database/database.go b/roomserver/state/database/database.go index ede6c5ec3..80f1b14f4 100644 --- a/roomserver/state/database/database.go +++ b/roomserver/state/database/database.go @@ -20,6 +20,7 @@ import ( "context" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" ) // A RoomStateDatabase has the storage APIs needed to load state from the database @@ -61,4 +62,6 @@ type RoomStateDatabase interface { Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error) // Look up snapshot NID for an event ID string SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) + // Look up a room version from the room NID. + GetRoomVersionForRoomNID(ctx context.Context, roomNID types.RoomNID) (gomatrixserverlib.RoomVersion, error) } diff --git a/roomserver/state/shared/shared.go b/roomserver/state/shared/shared.go new file mode 100644 index 000000000..a29b5e403 --- /dev/null +++ b/roomserver/state/shared/shared.go @@ -0,0 +1 @@ +package shared diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 687a120e3..b8e3e18a1 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -19,63 +19,1069 @@ package state import ( "context" "errors" + "fmt" + "sort" + "time" "github.com/matrix-org/dendrite/roomserver/state/database" - v1 "github.com/matrix-org/dendrite/roomserver/state/v1" + "github.com/matrix-org/util" + "github.com/prometheus/client_golang/prometheus" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) -type StateResolutionVersion int +type StateResolution struct { + db database.RoomStateDatabase +} -const ( - StateResolutionAlgorithmV1 StateResolutionVersion = iota + 1 - StateResolutionAlgorithmV2 -) - -func GetStateResolutionAlgorithm( - version StateResolutionVersion, db database.RoomStateDatabase, -) (StateResolutionImpl, error) { - switch version { - case StateResolutionAlgorithmV1: - return v1.Prepare(db), nil - default: - return nil, errors.New("unsupported room version") +func NewStateResolution(db database.RoomStateDatabase) StateResolution { + return StateResolution{ + db: db, } } -type StateResolutionImpl interface { - LoadStateAtSnapshot( - ctx context.Context, stateNID types.StateSnapshotNID, - ) ([]types.StateEntry, error) - LoadStateAtEvent( - ctx context.Context, eventID string, - ) ([]types.StateEntry, error) - LoadCombinedStateAfterEvents( - ctx context.Context, prevStates []types.StateAtEvent, - ) ([]types.StateEntry, error) - DifferenceBetweeenStateSnapshots( - ctx context.Context, oldStateNID, newStateNID types.StateSnapshotNID, - ) (removed, added []types.StateEntry, err error) - LoadStateAtSnapshotForStringTuples( - ctx context.Context, - stateNID types.StateSnapshotNID, - stateKeyTuples []gomatrixserverlib.StateKeyTuple, - ) ([]types.StateEntry, error) - LoadStateAfterEventsForStringTuples( - ctx context.Context, - prevStates []types.StateAtEvent, - stateKeyTuples []gomatrixserverlib.StateKeyTuple, - ) ([]types.StateEntry, error) - CalculateAndStoreStateBeforeEvent( - ctx context.Context, - event gomatrixserverlib.Event, - roomNID types.RoomNID, - ) (types.StateSnapshotNID, error) - CalculateAndStoreStateAfterEvents( - ctx context.Context, - roomNID types.RoomNID, - prevStates []types.StateAtEvent, - ) (types.StateSnapshotNID, error) +// LoadStateAtSnapshot loads the full state of a room at a particular snapshot. +// This is typically the state before an event or the current state of a room. +// Returns a sorted list of state entries or an error if there was a problem talking to the database. +func (v StateResolution) LoadStateAtSnapshot( + ctx context.Context, stateNID types.StateSnapshotNID, +) ([]types.StateEntry, error) { + stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) + if err != nil { + return nil, err + } + // We've asked for exactly one snapshot from the db so we should have exactly one entry in the result. + stateBlockNIDList := stateBlockNIDLists[0] + + stateEntryLists, err := v.db.StateEntries(ctx, stateBlockNIDList.StateBlockNIDs) + if err != nil { + return nil, err + } + stateEntriesMap := stateEntryListMap(stateEntryLists) + + // Combine all the state entries for this snapshot. + // The order of state block NIDs in the list tells us the order to combine them in. + var fullState []types.StateEntry + for _, stateBlockNID := range stateBlockNIDList.StateBlockNIDs { + entries, ok := stateEntriesMap.lookup(stateBlockNID) + if !ok { + // This should only get hit if the database is corrupt. + // It should be impossible for an event to reference a NID that doesn't exist + panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) + } + fullState = append(fullState, entries...) + } + + // Stable sort so that the most recent entry for each state key stays + // remains later in the list than the older entries for the same state key. + sort.Stable(stateEntryByStateKeySorter(fullState)) + // Unique returns the last entry and hence the most recent entry for each state key. + fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] + return fullState, nil +} + +// LoadStateAtEvent loads the full state of a room before a particular event. +func (v StateResolution) LoadStateAtEvent( + ctx context.Context, eventID string, +) ([]types.StateEntry, error) { + snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID) + if err != nil { + return nil, err + } + + stateEntries, err := v.LoadStateAtSnapshot(ctx, snapshotNID) + if err != nil { + return nil, err + } + + return stateEntries, nil +} + +// LoadCombinedStateAfterEvents loads a snapshot of the state after each of the events +// and combines those snapshots together into a single list. At this point it is +// possible to run into duplicate (type, state key) tuples. +func (v StateResolution) LoadCombinedStateAfterEvents( + ctx context.Context, prevStates []types.StateAtEvent, +) ([]types.StateEntry, error) { + stateNIDs := make([]types.StateSnapshotNID, len(prevStates)) + for i, state := range prevStates { + stateNIDs[i] = state.BeforeStateSnapshotNID + } + // Fetch the state snapshots for the state before the each prev event from the database. + // Deduplicate the IDs before passing them to the database. + // There could be duplicates because the events could be state events where + // the snapshot of the room state before them was the same. + stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, uniqueStateSnapshotNIDs(stateNIDs)) + if err != nil { + return nil, err + } + + var stateBlockNIDs []types.StateBlockNID + for _, list := range stateBlockNIDLists { + stateBlockNIDs = append(stateBlockNIDs, list.StateBlockNIDs...) + } + // Fetch the state entries that will be combined to create the snapshots. + // Deduplicate the IDs before passing them to the database. + // There could be duplicates because a block of state entries could be reused by + // multiple snapshots. + stateEntryLists, err := v.db.StateEntries(ctx, uniqueStateBlockNIDs(stateBlockNIDs)) + if err != nil { + return nil, err + } + stateBlockNIDsMap := stateBlockNIDListMap(stateBlockNIDLists) + stateEntriesMap := stateEntryListMap(stateEntryLists) + + // Combine the entries from all the snapshots of state after each prev event into a single list. + var combined []types.StateEntry + for _, prevState := range prevStates { + // Grab the list of state data NIDs for this snapshot. + stateBlockNIDs, ok := stateBlockNIDsMap.lookup(prevState.BeforeStateSnapshotNID) + if !ok { + // This should only get hit if the database is corrupt. + // It should be impossible for an event to reference a NID that doesn't exist + panic(fmt.Errorf("Corrupt DB: Missing state snapshot numeric ID %d", prevState.BeforeStateSnapshotNID)) + } + + // Combine all the state entries for this snapshot. + // The order of state block NIDs in the list tells us the order to combine them in. + var fullState []types.StateEntry + for _, stateBlockNID := range stateBlockNIDs { + entries, ok := stateEntriesMap.lookup(stateBlockNID) + if !ok { + // This should only get hit if the database is corrupt. + // It should be impossible for an event to reference a NID that doesn't exist + panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) + } + fullState = append(fullState, entries...) + } + if prevState.IsStateEvent() { + // If the prev event was a state event then add an entry for the event itself + // so that we get the state after the event rather than the state before. + fullState = append(fullState, prevState.StateEntry) + } + + // Stable sort so that the most recent entry for each state key stays + // remains later in the list than the older entries for the same state key. + sort.Stable(stateEntryByStateKeySorter(fullState)) + // Unique returns the last entry and hence the most recent entry for each state key. + fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] + // Add the full state for this StateSnapshotNID. + combined = append(combined, fullState...) + } + return combined, nil +} + +// DifferenceBetweeenStateSnapshots works out which state entries have been added and removed between two snapshots. +func (v StateResolution) DifferenceBetweeenStateSnapshots( + ctx context.Context, oldStateNID, newStateNID types.StateSnapshotNID, +) (removed, added []types.StateEntry, err error) { + if oldStateNID == newStateNID { + // If the snapshot NIDs are the same then nothing has changed + return nil, nil, nil + } + + var oldEntries []types.StateEntry + var newEntries []types.StateEntry + if oldStateNID != 0 { + oldEntries, err = v.LoadStateAtSnapshot(ctx, oldStateNID) + if err != nil { + return nil, nil, err + } + } + if newStateNID != 0 { + newEntries, err = v.LoadStateAtSnapshot(ctx, newStateNID) + if err != nil { + return nil, nil, err + } + } + + var oldI int + var newI int + for { + switch { + case oldI == len(oldEntries): + // We've reached the end of the old entries. + // The rest of the new list must have been newly added. + added = append(added, newEntries[newI:]...) + return + case newI == len(newEntries): + // We've reached the end of the new entries. + // The rest of the old list must be have been removed. + removed = append(removed, oldEntries[oldI:]...) + return + case oldEntries[oldI] == newEntries[newI]: + // The entry is in both lists so skip over it. + oldI++ + newI++ + case oldEntries[oldI].LessThan(newEntries[newI]): + // The lists are sorted so the old entry being less than the new entry means that it only appears in the old list. + removed = append(removed, oldEntries[oldI]) + oldI++ + default: + // Reaching the default case implies that the new entry is less than the old entry. + // Since the lists are sorted this means that it only appears in the new list. + added = append(added, newEntries[newI]) + newI++ + } + } +} + +// LoadStateAtSnapshotForStringTuples loads the state for a list of event type and state key pairs at a snapshot. +// This is used when we only want to load a subset of the room state at a snapshot. +// If there is no entry for a given event type and state key pair then it will be discarded. +// This is typically the state before an event or the current state of a room. +// Returns a sorted list of state entries or an error if there was a problem talking to the database. +func (v StateResolution) LoadStateAtSnapshotForStringTuples( + ctx context.Context, + stateNID types.StateSnapshotNID, + stateKeyTuples []gomatrixserverlib.StateKeyTuple, +) ([]types.StateEntry, error) { + numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples) + if err != nil { + return nil, err + } + return v.loadStateAtSnapshotForNumericTuples(ctx, stateNID, numericTuples) +} + +// stringTuplesToNumericTuples converts the string state key tuples into numeric IDs +// If there isn't a numeric ID for either the event type or the event state key then the tuple is discarded. +// Returns an error if there was a problem talking to the database. +func (v StateResolution) stringTuplesToNumericTuples( + ctx context.Context, + stringTuples []gomatrixserverlib.StateKeyTuple, +) ([]types.StateKeyTuple, error) { + eventTypes := make([]string, len(stringTuples)) + stateKeys := make([]string, len(stringTuples)) + for i := range stringTuples { + eventTypes[i] = stringTuples[i].EventType + stateKeys[i] = stringTuples[i].StateKey + } + eventTypes = util.UniqueStrings(eventTypes) + eventTypeMap, err := v.db.EventTypeNIDs(ctx, eventTypes) + if err != nil { + return nil, err + } + stateKeys = util.UniqueStrings(stateKeys) + stateKeyMap, err := v.db.EventStateKeyNIDs(ctx, stateKeys) + if err != nil { + return nil, err + } + + var result []types.StateKeyTuple + for _, stringTuple := range stringTuples { + var numericTuple types.StateKeyTuple + var ok1, ok2 bool + numericTuple.EventTypeNID, ok1 = eventTypeMap[stringTuple.EventType] + numericTuple.EventStateKeyNID, ok2 = stateKeyMap[stringTuple.StateKey] + // Discard the tuple if there wasn't a numeric ID for either the event type or the state key. + if ok1 && ok2 { + result = append(result, numericTuple) + } + } + + return result, nil +} + +// loadStateAtSnapshotForNumericTuples loads the state for a list of event type and state key pairs at a snapshot. +// This is used when we only want to load a subset of the room state at a snapshot. +// If there is no entry for a given event type and state key pair then it will be discarded. +// This is typically the state before an event or the current state of a room. +// Returns a sorted list of state entries or an error if there was a problem talking to the database. +func (v StateResolution) loadStateAtSnapshotForNumericTuples( + ctx context.Context, + stateNID types.StateSnapshotNID, + stateKeyTuples []types.StateKeyTuple, +) ([]types.StateEntry, error) { + stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) + if err != nil { + return nil, err + } + // We've asked for exactly one snapshot from the db so we should have exactly one entry in the result. + stateBlockNIDList := stateBlockNIDLists[0] + + stateEntryLists, err := v.db.StateEntriesForTuples( + ctx, stateBlockNIDList.StateBlockNIDs, stateKeyTuples, + ) + if err != nil { + return nil, err + } + stateEntriesMap := stateEntryListMap(stateEntryLists) + + // Combine all the state entries for this snapshot. + // The order of state block NIDs in the list tells us the order to combine them in. + var fullState []types.StateEntry + for _, stateBlockNID := range stateBlockNIDList.StateBlockNIDs { + entries, ok := stateEntriesMap.lookup(stateBlockNID) + if !ok { + // If the block is missing from the map it means that none of its entries matched a requested tuple. + // This can happen if the block doesn't contain an update for one of the requested tuples. + // If none of the requested tuples are in the block then it can be safely skipped. + continue + } + fullState = append(fullState, entries...) + } + + // Stable sort so that the most recent entry for each state key stays + // remains later in the list than the older entries for the same state key. + sort.Stable(stateEntryByStateKeySorter(fullState)) + // Unique returns the last entry and hence the most recent entry for each state key. + fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] + return fullState, nil +} + +// LoadStateAfterEventsForStringTuples loads the state for a list of event type +// and state key pairs after list of events. +// This is used when we only want to load a subset of the room state after a list of events. +// If there is no entry for a given event type and state key pair then it will be discarded. +// This is typically the state before an event. +// Returns a sorted list of state entries or an error if there was a problem talking to the database. +func (v StateResolution) LoadStateAfterEventsForStringTuples( + ctx context.Context, roomNID types.RoomNID, + prevStates []types.StateAtEvent, + stateKeyTuples []gomatrixserverlib.StateKeyTuple, +) ([]types.StateEntry, error) { + numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples) + if err != nil { + return nil, err + } + return v.loadStateAfterEventsForNumericTuples(ctx, roomNID, prevStates, numericTuples) +} + +func (v StateResolution) loadStateAfterEventsForNumericTuples( + ctx context.Context, roomNID types.RoomNID, + prevStates []types.StateAtEvent, + stateKeyTuples []types.StateKeyTuple, +) ([]types.StateEntry, error) { + roomVersion, err := v.db.GetRoomVersionForRoomNID(ctx, roomNID) + if err != nil { + return nil, err + } + + if len(prevStates) == 1 { + // Fast path for a single event. + prevState := prevStates[0] + var result []types.StateEntry + result, err = v.loadStateAtSnapshotForNumericTuples( + ctx, prevState.BeforeStateSnapshotNID, stateKeyTuples, + ) + if err != nil { + return nil, err + } + if prevState.IsStateEvent() { + // The result is current the state before the requested event. + // We want the state after the requested event. + // If the requested event was a state event then we need to + // update that key in the result. + // If the requested event wasn't a state event then the state after + // it is the same as the state before it. + set := false + for i := range result { + if result[i].StateKeyTuple == prevState.StateKeyTuple { + result[i] = prevState.StateEntry + set = true + } + } + if !set { // no previous state exists for this event: add new state + result = append(result, prevState.StateEntry) + } + } + return result, nil + } + + // Slow path for more that one event. + // Load the entire state so that we can do conflict resolution if we need to. + // TODO: The are some optimistations we could do here: + // 1) We only need to do conflict resolution if there is a conflict in the + // requested tuples so we might try loading just those tuples and then + // checking for conflicts. + // 2) When there is a conflict we still only need to load the state + // needed to do conflict resolution which would save us having to load + // the full state. + + // TODO: Add metrics for this as it could take a long time for big rooms + // with large conflicts. + fullState, _, _, err := v.calculateStateAfterManyEvents(ctx, roomVersion, prevStates) + if err != nil { + return nil, err + } + + // Sort the full state so we can use it as a map. + sort.Sort(stateEntrySorter(fullState)) + + // Filter the full state down to the required tuples. + var result []types.StateEntry + for _, tuple := range stateKeyTuples { + eventNID, ok := stateEntryMap(fullState).lookup(tuple) + if ok { + result = append(result, types.StateEntry{ + StateKeyTuple: tuple, + EventNID: eventNID, + }) + } + } + sort.Sort(stateEntrySorter(result)) + return result, nil +} + +var calculateStateDurations = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "dendrite", + Subsystem: "roomserver", + Name: "calculate_state_duration_microseconds", + Help: "How long it takes to calculate the state after a list of events", + }, + // Takes two labels: + // algorithm: + // The algorithm used to calculate the state or the step it failed on if it failed. + // Labels starting with "_" are used to indicate when the algorithm fails halfway. + // outcome: + // Whether the state was successfully calculated. + // + // The possible values for algorithm are: + // empty_state -> The list of events was empty so the state is empty. + // no_change -> The state hasn't changed. + // single_delta -> There was a single event added to the state in a way that can be encoded as a single delta + // full_state_no_conflicts -> We created a new copy of the full room state, but didn't enounter any conflicts + // while doing so. + // full_state_with_conflicts -> We created a new copy of the full room state and had to resolve conflicts to do so. + // _load_state_block_nids -> Failed loading the state block nids for a single previous state. + // _load_combined_state -> Failed to load the combined state. + // _resolve_conflicts -> Failed to resolve conflicts. + []string{"algorithm", "outcome"}, +) + +var calculateStatePrevEventLength = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "dendrite", + Subsystem: "roomserver", + Name: "calculate_state_prev_event_length", + Help: "The length of the list of events to calculate the state after", + }, + []string{"algorithm", "outcome"}, +) + +var calculateStateFullStateLength = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "dendrite", + Subsystem: "roomserver", + Name: "calculate_state_full_state_length", + Help: "The length of the full room state.", + }, + []string{"algorithm", "outcome"}, +) + +var calculateStateConflictLength = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "dendrite", + Subsystem: "roomserver", + Name: "calculate_state_conflict_state_length", + Help: "The length of the conflicted room state.", + }, + []string{"algorithm", "outcome"}, +) + +type calculateStateMetrics struct { + algorithm string + startTime time.Time + prevEventLength int + fullStateLength int + conflictLength int +} + +func (c *calculateStateMetrics) stop(stateNID types.StateSnapshotNID, err error) (types.StateSnapshotNID, error) { + var outcome string + if err == nil { + outcome = "success" + } else { + outcome = "failure" + } + endTime := time.Now() + calculateStateDurations.WithLabelValues(c.algorithm, outcome).Observe( + float64(endTime.Sub(c.startTime).Nanoseconds()) / 1000., + ) + calculateStatePrevEventLength.WithLabelValues(c.algorithm, outcome).Observe( + float64(c.prevEventLength), + ) + calculateStateFullStateLength.WithLabelValues(c.algorithm, outcome).Observe( + float64(c.fullStateLength), + ) + calculateStateConflictLength.WithLabelValues(c.algorithm, outcome).Observe( + float64(c.conflictLength), + ) + return stateNID, err +} + +func init() { + prometheus.MustRegister( + calculateStateDurations, calculateStatePrevEventLength, + calculateStateFullStateLength, calculateStateConflictLength, + ) +} + +// CalculateAndStoreStateBeforeEvent calculates a snapshot of the state of a room before an event. +// Stores the snapshot of the state in the database. +// Returns a numeric ID for the snapshot of the state before the event. +func (v StateResolution) CalculateAndStoreStateBeforeEvent( + ctx context.Context, + event gomatrixserverlib.Event, + roomNID types.RoomNID, +) (types.StateSnapshotNID, error) { + // Load the state at the prev events. + prevEventRefs := event.PrevEvents() + prevEventIDs := make([]string, len(prevEventRefs)) + for i := range prevEventRefs { + prevEventIDs[i] = prevEventRefs[i].EventID + } + + prevStates, err := v.db.StateAtEventIDs(ctx, prevEventIDs) + if err != nil { + return 0, err + } + + // The state before this event will be the state after the events that came before it. + return v.CalculateAndStoreStateAfterEvents(ctx, roomNID, prevStates) +} + +// CalculateAndStoreStateAfterEvents finds the room state after the given events. +// Stores the resulting state in the database and returns a numeric ID for that snapshot. +func (v StateResolution) CalculateAndStoreStateAfterEvents( + ctx context.Context, + roomNID types.RoomNID, + prevStates []types.StateAtEvent, +) (types.StateSnapshotNID, error) { + metrics := calculateStateMetrics{startTime: time.Now(), prevEventLength: len(prevStates)} + + if len(prevStates) == 0 { + // 2) There weren't any prev_events for this event so the state is + // empty. + metrics.algorithm = "empty_state" + return metrics.stop(v.db.AddState(ctx, roomNID, nil, nil)) + } + + if len(prevStates) == 1 { + prevState := prevStates[0] + if prevState.EventStateKeyNID == 0 { + // 3) None of the previous events were state events and they all + // have the same state, so this event has exactly the same state + // as the previous events. + // This should be the common case. + metrics.algorithm = "no_change" + return metrics.stop(prevState.BeforeStateSnapshotNID, nil) + } + // The previous event was a state event so we need to store a copy + // of the previous state updated with that event. + stateBlockNIDLists, err := v.db.StateBlockNIDs( + ctx, []types.StateSnapshotNID{prevState.BeforeStateSnapshotNID}, + ) + if err != nil { + metrics.algorithm = "_load_state_blocks" + return metrics.stop(0, err) + } + stateBlockNIDs := stateBlockNIDLists[0].StateBlockNIDs + if len(stateBlockNIDs) < maxStateBlockNIDs { + // 4) The number of state data blocks is small enough that we can just + // add the state event as a block of size one to the end of the blocks. + metrics.algorithm = "single_delta" + return metrics.stop(v.db.AddState( + ctx, roomNID, stateBlockNIDs, []types.StateEntry{prevState.StateEntry}, + )) + } + // If there are too many deltas then we need to calculate the full state + // So fall through to calculateAndStoreStateAfterManyEvents + } + + return v.calculateAndStoreStateAfterManyEvents(ctx, roomNID, prevStates, metrics) +} + +// maxStateBlockNIDs is the maximum number of state data blocks to use to encode a snapshot of room state. +// Increasing this number means that we can encode more of the state changes as simple deltas which means that +// we need fewer entries in the state data table. However making this number bigger will increase the size of +// the rows in the state table itself and will require more index lookups when retrieving a snapshot. +// TODO: Tune this to get the right balance between size and lookup performance. +const maxStateBlockNIDs = 64 + +// calculateAndStoreStateAfterManyEvents finds the room state after the given events. +// This handles the slow path of calculateAndStoreStateAfterEvents for when there is more than one event. +// Stores the resulting state and returns a numeric ID for the snapshot. +func (v StateResolution) calculateAndStoreStateAfterManyEvents( + ctx context.Context, + roomNID types.RoomNID, + prevStates []types.StateAtEvent, + metrics calculateStateMetrics, +) (types.StateSnapshotNID, error) { + roomVersion, err := v.db.GetRoomVersionForRoomNID(ctx, roomNID) + if err != nil { + return metrics.stop(0, err) + } + + state, algorithm, conflictLength, err := + v.calculateStateAfterManyEvents(ctx, roomVersion, prevStates) + metrics.algorithm = algorithm + if err != nil { + return metrics.stop(0, err) + } + + // TODO: Check if we can encode the new state as a delta against the + // previous state. + metrics.conflictLength = conflictLength + metrics.fullStateLength = len(state) + return metrics.stop(v.db.AddState(ctx, roomNID, nil, state)) +} + +func (v StateResolution) calculateStateAfterManyEvents( + ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, + prevStates []types.StateAtEvent, +) (state []types.StateEntry, algorithm string, conflictLength int, err error) { + var combined []types.StateEntry + // Conflict resolution. + // First stage: load the state after each of the prev events. + combined, err = v.LoadCombinedStateAfterEvents(ctx, prevStates) + if err != nil { + algorithm = "_load_combined_state" + return + } + + // Collect all the entries with the same type and key together. + // We don't care about the order here because the conflict resolution + // algorithm doesn't depend on the order of the prev events. + // Remove duplicate entires. + combined = combined[:util.SortAndUnique(stateEntrySorter(combined))] + + // Find the conflicts + conflicts := findDuplicateStateKeys(combined) + + if len(conflicts) > 0 { + conflictLength = len(conflicts) + + // 5) There are conflicting state events, for each conflict workout + // what the appropriate state event is. + + // Work out which entries aren't conflicted. + var notConflicted []types.StateEntry + for _, entry := range combined { + if _, ok := stateEntryMap(conflicts).lookup(entry.StateKeyTuple); !ok { + notConflicted = append(notConflicted, entry) + } + } + + var resolved []types.StateEntry + resolved, err = v.resolveConflicts(ctx, roomVersion, notConflicted, conflicts) + if err != nil { + algorithm = "_resolve_conflicts" + return + } + algorithm = "full_state_with_conflicts" + state = resolved + } else { + algorithm = "full_state_no_conflicts" + // 6) There weren't any conflicts + state = combined + } + return +} + +func (v StateResolution) resolveConflicts( + ctx context.Context, version gomatrixserverlib.RoomVersion, + notConflicted, conflicted []types.StateEntry, +) ([]types.StateEntry, error) { + stateResAlgo, err := version.StateResAlgorithm() + if err != nil { + return nil, err + } + switch stateResAlgo { + case gomatrixserverlib.StateResV1: + return v.resolveConflictsV1(ctx, notConflicted, conflicted) + case gomatrixserverlib.StateResV2: + return v.resolveConflictsV2(ctx, notConflicted, conflicted) + } + return nil, errors.New("unsupported state resolution algorithm") +} + +// resolveConflicts resolves a list of conflicted state entries. It takes two lists. +// The first is a list of all state entries that are not conflicted. +// The second is a list of all state entries that are conflicted +// A state entry is conflicted when there is more than one numeric event ID for the same state key tuple. +// Returns a list that combines the entries without conflicts with the result of state resolution for the entries with conflicts. +// The returned list is sorted by state key tuple. +// Returns an error if there was a problem talking to the database. +func (v StateResolution) resolveConflictsV1( + ctx context.Context, + notConflicted, conflicted []types.StateEntry, +) ([]types.StateEntry, error) { + + // Load the conflicted events + conflictedEvents, eventIDMap, err := v.loadStateEvents(ctx, conflicted) + if err != nil { + return nil, err + } + + // Work out which auth events we need to load. + needed := gomatrixserverlib.StateNeededForAuth(conflictedEvents) + + // Find the numeric IDs for the necessary state keys. + var neededStateKeys []string + neededStateKeys = append(neededStateKeys, needed.Member...) + neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...) + stateKeyNIDMap, err := v.db.EventStateKeyNIDs(ctx, neededStateKeys) + if err != nil { + return nil, err + } + + // Load the necessary auth events. + tuplesNeeded := v.stateKeyTuplesNeeded(stateKeyNIDMap, needed) + var authEntries []types.StateEntry + for _, tuple := range tuplesNeeded { + if eventNID, ok := stateEntryMap(notConflicted).lookup(tuple); ok { + authEntries = append(authEntries, types.StateEntry{ + StateKeyTuple: tuple, + EventNID: eventNID, + }) + } + } + authEvents, _, err := v.loadStateEvents(ctx, authEntries) + if err != nil { + return nil, err + } + + // Resolve the conflicts. + resolvedEvents := gomatrixserverlib.ResolveStateConflicts(conflictedEvents, authEvents) + + // Map from the full events back to numeric state entries. + for _, resolvedEvent := range resolvedEvents { + entry, ok := eventIDMap[resolvedEvent.EventID()] + if !ok { + panic(fmt.Errorf("Missing state entry for event ID %q", resolvedEvent.EventID())) + } + notConflicted = append(notConflicted, entry) + } + + // Sort the result so it can be searched. + sort.Sort(stateEntrySorter(notConflicted)) + return notConflicted, nil +} + +// resolveConflicts resolves a list of conflicted state entries. It takes two lists. +// The first is a list of all state entries that are not conflicted. +// The second is a list of all state entries that are conflicted +// A state entry is conflicted when there is more than one numeric event ID for the same state key tuple. +// Returns a list that combines the entries without conflicts with the result of state resolution for the entries with conflicts. +// The returned list is sorted by state key tuple. +// Returns an error if there was a problem talking to the database. +// nolint:gocyclo +func (v StateResolution) resolveConflictsV2( + ctx context.Context, + notConflicted, conflicted []types.StateEntry, +) ([]types.StateEntry, error) { + eventIDMap := make(map[string]types.StateEntry) + + // Load the conflicted events + conflictedEvents, conflictedEventMap, err := v.loadStateEvents(ctx, conflicted) + if err != nil { + return nil, err + } + for k, v := range conflictedEventMap { + eventIDMap[k] = v + } + + // Load the non-conflicted events + nonConflictedEvents, nonConflictedEventMap, err := v.loadStateEvents(ctx, notConflicted) + if err != nil { + return nil, err + } + for k, v := range nonConflictedEventMap { + eventIDMap[k] = v + } + + // For each conflicted event, we will add a new set of auth events. Auth + // events may be duplicated across these sets but that's OK. + authSets := make(map[string][]gomatrixserverlib.Event) + var authEvents []gomatrixserverlib.Event + var authDifference []gomatrixserverlib.Event + + // For each conflicted event, let's try and get the needed auth events. + for _, conflictedEvent := range conflictedEvents { + // Work out which auth events we need to load. + key := conflictedEvent.EventID() + needed := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{conflictedEvent}) + + // Find the numeric IDs for the necessary state keys. + var neededStateKeys []string + neededStateKeys = append(neededStateKeys, needed.Member...) + neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...) + stateKeyNIDMap, err := v.db.EventStateKeyNIDs(ctx, neededStateKeys) + if err != nil { + return nil, err + } + + // Load the necessary auth events. + tuplesNeeded := v.stateKeyTuplesNeeded(stateKeyNIDMap, needed) + var authEntries []types.StateEntry + for _, tuple := range tuplesNeeded { + if eventNID, ok := stateEntryMap(notConflicted).lookup(tuple); ok { + authEntries = append(authEntries, types.StateEntry{ + StateKeyTuple: tuple, + EventNID: eventNID, + }) + } + } + + // Store the newly found auth events in the auth set for this event. + authSets[key], _, err = v.loadStateEvents(ctx, authEntries) + if err != nil { + return nil, err + } + authEvents = append(authEvents, authSets[key]...) + } + + // This function helps us to work out whether an event exists in one of the + // auth sets. + isInAuthList := func(k string, event gomatrixserverlib.Event) bool { + for _, e := range authSets[k] { + if e.EventID() == event.EventID() { + return true + } + } + return false + } + + // This function works out if an event exists in all of the auth sets. + isInAllAuthLists := func(event gomatrixserverlib.Event) bool { + found := true + for k := range authSets { + found = found && isInAuthList(k, event) + } + return found + } + + // Look through all of the auth events that we've been given and work out if + // there are any events which don't appear in all of the auth sets. If they + // don't then we add them to the auth difference. + for _, event := range authEvents { + if !isInAllAuthLists(event) { + authDifference = append(authDifference, event) + } + } + + // Resolve the conflicts. + resolvedEvents := gomatrixserverlib.ResolveStateConflictsV2( + conflictedEvents, + nonConflictedEvents, + authEvents, + authDifference, + ) + + // Map from the full events back to numeric state entries. + for _, resolvedEvent := range resolvedEvents { + entry, ok := eventIDMap[resolvedEvent.EventID()] + if !ok { + panic(fmt.Errorf("Missing state entry for event ID %q", resolvedEvent.EventID())) + } + notConflicted = append(notConflicted, entry) + } + + // Sort the result so it can be searched. + sort.Sort(stateEntrySorter(notConflicted)) + return notConflicted, nil +} + +// stateKeyTuplesNeeded works out which numeric state key tuples we need to authenticate some events. +func (v StateResolution) stateKeyTuplesNeeded(stateKeyNIDMap map[string]types.EventStateKeyNID, stateNeeded gomatrixserverlib.StateNeeded) []types.StateKeyTuple { + var keyTuples []types.StateKeyTuple + if stateNeeded.Create { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomCreateNID, + EventStateKeyNID: types.EmptyStateKeyNID, + }) + } + if stateNeeded.PowerLevels { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomPowerLevelsNID, + EventStateKeyNID: types.EmptyStateKeyNID, + }) + } + if stateNeeded.JoinRules { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomJoinRulesNID, + EventStateKeyNID: types.EmptyStateKeyNID, + }) + } + for _, member := range stateNeeded.Member { + stateKeyNID, ok := stateKeyNIDMap[member] + if ok { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomMemberNID, + EventStateKeyNID: stateKeyNID, + }) + } + } + for _, token := range stateNeeded.ThirdPartyInvite { + stateKeyNID, ok := stateKeyNIDMap[token] + if ok { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomThirdPartyInviteNID, + EventStateKeyNID: stateKeyNID, + }) + } + } + return keyTuples +} + +// loadStateEvents loads the matrix events for a list of state entries. +// Returns a list of state events in no particular order and a map from string event ID back to state entry. +// The map can be used to recover which numeric state entry a given event is for. +// Returns an error if there was a problem talking to the database. +func (v StateResolution) loadStateEvents( + ctx context.Context, entries []types.StateEntry, +) ([]gomatrixserverlib.Event, map[string]types.StateEntry, error) { + eventNIDs := make([]types.EventNID, len(entries)) + for i := range entries { + eventNIDs[i] = entries[i].EventNID + } + events, err := v.db.Events(ctx, eventNIDs) + if err != nil { + return nil, nil, err + } + eventIDMap := map[string]types.StateEntry{} + result := make([]gomatrixserverlib.Event, len(entries)) + for i := range entries { + event, ok := eventMap(events).lookup(entries[i].EventNID) + if !ok { + panic(fmt.Errorf("Corrupt DB: Missing event numeric ID %d", entries[i].EventNID)) + } + result[i] = event.Event + eventIDMap[event.Event.EventID()] = entries[i] + } + return result, eventIDMap, nil +} + +// findDuplicateStateKeys finds the state entries where the state key tuple appears more than once in a sorted list. +// Returns a sorted list of those state entries. +func findDuplicateStateKeys(a []types.StateEntry) []types.StateEntry { + var result []types.StateEntry + // j is the starting index of a block of entries with the same state key tuple. + j := 0 + for i := 1; i < len(a); i++ { + // Check if the state key tuple matches the start of the block + if a[j].StateKeyTuple != a[i].StateKeyTuple { + // If the state key tuple is different then we've reached the end of a block of duplicates. + // Check if the size of the block is bigger than one. + // If the size is one then there was only a single entry with that state key tuple so we don't add it to the result + if j+1 != i { + // Add the block to the result. + result = append(result, a[j:i]...) + } + // Start a new block for the next state key tuple. + j = i + } + } + // Check if the last block with the same state key tuple had more than one event in it. + if j+1 != len(a) { + result = append(result, a[j:]...) + } + return result +} + +type stateEntrySorter []types.StateEntry + +func (s stateEntrySorter) Len() int { return len(s) } +func (s stateEntrySorter) Less(i, j int) bool { return s[i].LessThan(s[j]) } +func (s stateEntrySorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +type stateBlockNIDListMap []types.StateBlockNIDList + +func (m stateBlockNIDListMap) lookup(stateNID types.StateSnapshotNID) (stateBlockNIDs []types.StateBlockNID, ok bool) { + list := []types.StateBlockNIDList(m) + i := sort.Search(len(list), func(i int) bool { + return list[i].StateSnapshotNID >= stateNID + }) + if i < len(list) && list[i].StateSnapshotNID == stateNID { + ok = true + stateBlockNIDs = list[i].StateBlockNIDs + } + return +} + +type stateEntryListMap []types.StateEntryList + +func (m stateEntryListMap) lookup(stateBlockNID types.StateBlockNID) (stateEntries []types.StateEntry, ok bool) { + list := []types.StateEntryList(m) + i := sort.Search(len(list), func(i int) bool { + return list[i].StateBlockNID >= stateBlockNID + }) + if i < len(list) && list[i].StateBlockNID == stateBlockNID { + ok = true + stateEntries = list[i].StateEntries + } + return +} + +type stateEntryByStateKeySorter []types.StateEntry + +func (s stateEntryByStateKeySorter) Len() int { return len(s) } +func (s stateEntryByStateKeySorter) Less(i, j int) bool { + return s[i].StateKeyTuple.LessThan(s[j].StateKeyTuple) +} +func (s stateEntryByStateKeySorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +type stateNIDSorter []types.StateSnapshotNID + +func (s stateNIDSorter) Len() int { return len(s) } +func (s stateNIDSorter) Less(i, j int) bool { return s[i] < s[j] } +func (s stateNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func uniqueStateSnapshotNIDs(nids []types.StateSnapshotNID) []types.StateSnapshotNID { + return nids[:util.SortAndUnique(stateNIDSorter(nids))] +} + +type stateBlockNIDSorter []types.StateBlockNID + +func (s stateBlockNIDSorter) Len() int { return len(s) } +func (s stateBlockNIDSorter) Less(i, j int) bool { return s[i] < s[j] } +func (s stateBlockNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func uniqueStateBlockNIDs(nids []types.StateBlockNID) []types.StateBlockNID { + return nids[:util.SortAndUnique(stateBlockNIDSorter(nids))] +} + +// Map from event type, state key tuple to numeric event ID. +// Implemented using binary search on a sorted array. +type stateEntryMap []types.StateEntry + +// lookup an entry in the event map. +func (m stateEntryMap) lookup(stateKey types.StateKeyTuple) (eventNID types.EventNID, ok bool) { + // Since the list is sorted we can implement this using binary search. + // This is faster than using a hash map. + // We don't have to worry about pathological cases because the keys are fixed + // size and are controlled by us. + list := []types.StateEntry(m) + i := sort.Search(len(list), func(i int) bool { + return !list[i].StateKeyTuple.LessThan(stateKey) + }) + if i < len(list) && list[i].StateKeyTuple == stateKey { + ok = true + eventNID = list[i].EventNID + } + return +} + +// Map from numeric event ID to event. +// Implemented using binary search on a sorted array. +type eventMap []types.Event + +// lookup an entry in the event map. +func (m eventMap) lookup(eventNID types.EventNID) (event *types.Event, ok bool) { + // Since the list is sorted we can implement this using binary search. + // This is faster than using a hash map. + // We don't have to worry about pathological cases because the keys are fixed + // size are controlled by us. + list := []types.Event(m) + i := sort.Search(len(list), func(i int) bool { + return list[i].EventNID >= eventNID + }) + if i < len(list) && list[i].EventNID == eventNID { + ok = true + event = &list[i] + } + return } diff --git a/roomserver/state/v1/state_test.go b/roomserver/state/state_test.go similarity index 99% rename from roomserver/state/v1/state_test.go rename to roomserver/state/state_test.go index 4dc7e52ec..c57056786 100644 --- a/roomserver/state/v1/state_test.go +++ b/roomserver/state/state_test.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package v1 +package state import ( "testing" diff --git a/roomserver/state/v1/state.go b/roomserver/state/v1/state.go deleted file mode 100644 index 3eb601925..000000000 --- a/roomserver/state/v1/state.go +++ /dev/null @@ -1,932 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package state provides functions for reading state from the database. -// The functions for writing state to the database are the input package. -package v1 - -import ( - "context" - "fmt" - "sort" - "time" - - "github.com/matrix-org/dendrite/roomserver/state/database" - "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - "github.com/prometheus/client_golang/prometheus" -) - -type StateResolutionV1 struct { - db database.RoomStateDatabase -} - -func Prepare(db database.RoomStateDatabase) StateResolutionV1 { - return StateResolutionV1{ - db: db, - } -} - -// LoadStateAtSnapshot loads the full state of a room at a particular snapshot. -// This is typically the state before an event or the current state of a room. -// Returns a sorted list of state entries or an error if there was a problem talking to the database. -func (v StateResolutionV1) LoadStateAtSnapshot( - ctx context.Context, stateNID types.StateSnapshotNID, -) ([]types.StateEntry, error) { - stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) - if err != nil { - return nil, err - } - // We've asked for exactly one snapshot from the db so we should have exactly one entry in the result. - stateBlockNIDList := stateBlockNIDLists[0] - - stateEntryLists, err := v.db.StateEntries(ctx, stateBlockNIDList.StateBlockNIDs) - if err != nil { - return nil, err - } - stateEntriesMap := stateEntryListMap(stateEntryLists) - - // Combine all the state entries for this snapshot. - // The order of state block NIDs in the list tells us the order to combine them in. - var fullState []types.StateEntry - for _, stateBlockNID := range stateBlockNIDList.StateBlockNIDs { - entries, ok := stateEntriesMap.lookup(stateBlockNID) - if !ok { - // This should only get hit if the database is corrupt. - // It should be impossible for an event to reference a NID that doesn't exist - panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) - } - fullState = append(fullState, entries...) - } - - // Stable sort so that the most recent entry for each state key stays - // remains later in the list than the older entries for the same state key. - sort.Stable(stateEntryByStateKeySorter(fullState)) - // Unique returns the last entry and hence the most recent entry for each state key. - fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] - return fullState, nil -} - -// LoadStateAtEvent loads the full state of a room at a particular event. -func (v StateResolutionV1) LoadStateAtEvent( - ctx context.Context, eventID string, -) ([]types.StateEntry, error) { - snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID) - if err != nil { - return nil, err - } - - stateEntries, err := v.LoadStateAtSnapshot(ctx, snapshotNID) - if err != nil { - return nil, err - } - - return stateEntries, nil -} - -// LoadCombinedStateAfterEvents loads a snapshot of the state after each of the events -// and combines those snapshots together into a single list. -func (v StateResolutionV1) LoadCombinedStateAfterEvents( - ctx context.Context, prevStates []types.StateAtEvent, -) ([]types.StateEntry, error) { - stateNIDs := make([]types.StateSnapshotNID, len(prevStates)) - for i, state := range prevStates { - stateNIDs[i] = state.BeforeStateSnapshotNID - } - // Fetch the state snapshots for the state before the each prev event from the database. - // Deduplicate the IDs before passing them to the database. - // There could be duplicates because the events could be state events where - // the snapshot of the room state before them was the same. - stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, uniqueStateSnapshotNIDs(stateNIDs)) - if err != nil { - return nil, err - } - - var stateBlockNIDs []types.StateBlockNID - for _, list := range stateBlockNIDLists { - stateBlockNIDs = append(stateBlockNIDs, list.StateBlockNIDs...) - } - // Fetch the state entries that will be combined to create the snapshots. - // Deduplicate the IDs before passing them to the database. - // There could be duplicates because a block of state entries could be reused by - // multiple snapshots. - stateEntryLists, err := v.db.StateEntries(ctx, uniqueStateBlockNIDs(stateBlockNIDs)) - if err != nil { - return nil, err - } - stateBlockNIDsMap := stateBlockNIDListMap(stateBlockNIDLists) - stateEntriesMap := stateEntryListMap(stateEntryLists) - - // Combine the entries from all the snapshots of state after each prev event into a single list. - var combined []types.StateEntry - for _, prevState := range prevStates { - // Grab the list of state data NIDs for this snapshot. - stateBlockNIDs, ok := stateBlockNIDsMap.lookup(prevState.BeforeStateSnapshotNID) - if !ok { - // This should only get hit if the database is corrupt. - // It should be impossible for an event to reference a NID that doesn't exist - panic(fmt.Errorf("Corrupt DB: Missing state snapshot numeric ID %d", prevState.BeforeStateSnapshotNID)) - } - - // Combine all the state entries for this snapshot. - // The order of state block NIDs in the list tells us the order to combine them in. - var fullState []types.StateEntry - for _, stateBlockNID := range stateBlockNIDs { - entries, ok := stateEntriesMap.lookup(stateBlockNID) - if !ok { - // This should only get hit if the database is corrupt. - // It should be impossible for an event to reference a NID that doesn't exist - panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) - } - fullState = append(fullState, entries...) - } - if prevState.IsStateEvent() { - // If the prev event was a state event then add an entry for the event itself - // so that we get the state after the event rather than the state before. - fullState = append(fullState, prevState.StateEntry) - } - - // Stable sort so that the most recent entry for each state key stays - // remains later in the list than the older entries for the same state key. - sort.Stable(stateEntryByStateKeySorter(fullState)) - // Unique returns the last entry and hence the most recent entry for each state key. - fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] - // Add the full state for this StateSnapshotNID. - combined = append(combined, fullState...) - } - return combined, nil -} - -// DifferenceBetweeenStateSnapshots works out which state entries have been added and removed between two snapshots. -func (v StateResolutionV1) DifferenceBetweeenStateSnapshots( - ctx context.Context, oldStateNID, newStateNID types.StateSnapshotNID, -) (removed, added []types.StateEntry, err error) { - if oldStateNID == newStateNID { - // If the snapshot NIDs are the same then nothing has changed - return nil, nil, nil - } - - var oldEntries []types.StateEntry - var newEntries []types.StateEntry - if oldStateNID != 0 { - oldEntries, err = v.LoadStateAtSnapshot(ctx, oldStateNID) - if err != nil { - return nil, nil, err - } - } - if newStateNID != 0 { - newEntries, err = v.LoadStateAtSnapshot(ctx, newStateNID) - if err != nil { - return nil, nil, err - } - } - - var oldI int - var newI int - for { - switch { - case oldI == len(oldEntries): - // We've reached the end of the old entries. - // The rest of the new list must have been newly added. - added = append(added, newEntries[newI:]...) - return - case newI == len(newEntries): - // We've reached the end of the new entries. - // The rest of the old list must be have been removed. - removed = append(removed, oldEntries[oldI:]...) - return - case oldEntries[oldI] == newEntries[newI]: - // The entry is in both lists so skip over it. - oldI++ - newI++ - case oldEntries[oldI].LessThan(newEntries[newI]): - // The lists are sorted so the old entry being less than the new entry means that it only appears in the old list. - removed = append(removed, oldEntries[oldI]) - oldI++ - default: - // Reaching the default case implies that the new entry is less than the old entry. - // Since the lists are sorted this means that it only appears in the new list. - added = append(added, newEntries[newI]) - newI++ - } - } -} - -// LoadStateAtSnapshotForStringTuples loads the state for a list of event type and state key pairs at a snapshot. -// This is used when we only want to load a subset of the room state at a snapshot. -// If there is no entry for a given event type and state key pair then it will be discarded. -// This is typically the state before an event or the current state of a room. -// Returns a sorted list of state entries or an error if there was a problem talking to the database. -func (v StateResolutionV1) LoadStateAtSnapshotForStringTuples( - ctx context.Context, - stateNID types.StateSnapshotNID, - stateKeyTuples []gomatrixserverlib.StateKeyTuple, -) ([]types.StateEntry, error) { - numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples) - if err != nil { - return nil, err - } - return v.loadStateAtSnapshotForNumericTuples(ctx, stateNID, numericTuples) -} - -// stringTuplesToNumericTuples converts the string state key tuples into numeric IDs -// If there isn't a numeric ID for either the event type or the event state key then the tuple is discarded. -// Returns an error if there was a problem talking to the database. -func (v StateResolutionV1) stringTuplesToNumericTuples( - ctx context.Context, - stringTuples []gomatrixserverlib.StateKeyTuple, -) ([]types.StateKeyTuple, error) { - eventTypes := make([]string, len(stringTuples)) - stateKeys := make([]string, len(stringTuples)) - for i := range stringTuples { - eventTypes[i] = stringTuples[i].EventType - stateKeys[i] = stringTuples[i].StateKey - } - eventTypes = util.UniqueStrings(eventTypes) - eventTypeMap, err := v.db.EventTypeNIDs(ctx, eventTypes) - if err != nil { - return nil, err - } - stateKeys = util.UniqueStrings(stateKeys) - stateKeyMap, err := v.db.EventStateKeyNIDs(ctx, stateKeys) - if err != nil { - return nil, err - } - - var result []types.StateKeyTuple - for _, stringTuple := range stringTuples { - var numericTuple types.StateKeyTuple - var ok1, ok2 bool - numericTuple.EventTypeNID, ok1 = eventTypeMap[stringTuple.EventType] - numericTuple.EventStateKeyNID, ok2 = stateKeyMap[stringTuple.StateKey] - // Discard the tuple if there wasn't a numeric ID for either the event type or the state key. - if ok1 && ok2 { - result = append(result, numericTuple) - } - } - - return result, nil -} - -// loadStateAtSnapshotForNumericTuples loads the state for a list of event type and state key pairs at a snapshot. -// This is used when we only want to load a subset of the room state at a snapshot. -// If there is no entry for a given event type and state key pair then it will be discarded. -// This is typically the state before an event or the current state of a room. -// Returns a sorted list of state entries or an error if there was a problem talking to the database. -func (v StateResolutionV1) loadStateAtSnapshotForNumericTuples( - ctx context.Context, - stateNID types.StateSnapshotNID, - stateKeyTuples []types.StateKeyTuple, -) ([]types.StateEntry, error) { - stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) - if err != nil { - return nil, err - } - // We've asked for exactly one snapshot from the db so we should have exactly one entry in the result. - stateBlockNIDList := stateBlockNIDLists[0] - - stateEntryLists, err := v.db.StateEntriesForTuples( - ctx, stateBlockNIDList.StateBlockNIDs, stateKeyTuples, - ) - if err != nil { - return nil, err - } - stateEntriesMap := stateEntryListMap(stateEntryLists) - - // Combine all the state entries for this snapshot. - // The order of state block NIDs in the list tells us the order to combine them in. - var fullState []types.StateEntry - for _, stateBlockNID := range stateBlockNIDList.StateBlockNIDs { - entries, ok := stateEntriesMap.lookup(stateBlockNID) - if !ok { - // If the block is missing from the map it means that none of its entries matched a requested tuple. - // This can happen if the block doesn't contain an update for one of the requested tuples. - // If none of the requested tuples are in the block then it can be safely skipped. - continue - } - fullState = append(fullState, entries...) - } - - // Stable sort so that the most recent entry for each state key stays - // remains later in the list than the older entries for the same state key. - sort.Stable(stateEntryByStateKeySorter(fullState)) - // Unique returns the last entry and hence the most recent entry for each state key. - fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] - return fullState, nil -} - -// LoadStateAfterEventsForStringTuples loads the state for a list of event type -// and state key pairs after list of events. -// This is used when we only want to load a subset of the room state after a list of events. -// If there is no entry for a given event type and state key pair then it will be discarded. -// This is typically the state before an event. -// Returns a sorted list of state entries or an error if there was a problem talking to the database. -func (v StateResolutionV1) LoadStateAfterEventsForStringTuples( - ctx context.Context, - prevStates []types.StateAtEvent, - stateKeyTuples []gomatrixserverlib.StateKeyTuple, -) ([]types.StateEntry, error) { - numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples) - if err != nil { - return nil, err - } - return v.loadStateAfterEventsForNumericTuples(ctx, prevStates, numericTuples) -} - -func (v StateResolutionV1) loadStateAfterEventsForNumericTuples( - ctx context.Context, - prevStates []types.StateAtEvent, - stateKeyTuples []types.StateKeyTuple, -) ([]types.StateEntry, error) { - if len(prevStates) == 1 { - // Fast path for a single event. - prevState := prevStates[0] - result, err := v.loadStateAtSnapshotForNumericTuples( - ctx, prevState.BeforeStateSnapshotNID, stateKeyTuples, - ) - if err != nil { - return nil, err - } - if prevState.IsStateEvent() { - // The result is current the state before the requested event. - // We want the state after the requested event. - // If the requested event was a state event then we need to - // update that key in the result. - // If the requested event wasn't a state event then the state after - // it is the same as the state before it. - set := false - for i := range result { - if result[i].StateKeyTuple == prevState.StateKeyTuple { - result[i] = prevState.StateEntry - set = true - } - } - if !set { // no previous state exists for this event: add new state - result = append(result, prevState.StateEntry) - } - } - return result, nil - } - - // Slow path for more that one event. - // Load the entire state so that we can do conflict resolution if we need to. - // TODO: The are some optimistations we could do here: - // 1) We only need to do conflict resolution if there is a conflict in the - // requested tuples so we might try loading just those tuples and then - // checking for conflicts. - // 2) When there is a conflict we still only need to load the state - // needed to do conflict resolution which would save us having to load - // the full state. - - // TODO: Add metrics for this as it could take a long time for big rooms - // with large conflicts. - fullState, _, _, err := v.calculateStateAfterManyEvents(ctx, prevStates) - if err != nil { - return nil, err - } - - // Sort the full state so we can use it as a map. - sort.Sort(stateEntrySorter(fullState)) - - // Filter the full state down to the required tuples. - var result []types.StateEntry - for _, tuple := range stateKeyTuples { - eventNID, ok := stateEntryMap(fullState).lookup(tuple) - if ok { - result = append(result, types.StateEntry{ - StateKeyTuple: tuple, - EventNID: eventNID, - }) - } - } - sort.Sort(stateEntrySorter(result)) - return result, nil -} - -var calculateStateDurations = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "dendrite", - Subsystem: "roomserver", - Name: "calculate_state_duration_microseconds", - Help: "How long it takes to calculate the state after a list of events", - }, - // Takes two labels: - // algorithm: - // The algorithm used to calculate the state or the step it failed on if it failed. - // Labels starting with "_" are used to indicate when the algorithm fails halfway. - // outcome: - // Whether the state was successfully calculated. - // - // The possible values for algorithm are: - // empty_state -> The list of events was empty so the state is empty. - // no_change -> The state hasn't changed. - // single_delta -> There was a single event added to the state in a way that can be encoded as a single delta - // full_state_no_conflicts -> We created a new copy of the full room state, but didn't enounter any conflicts - // while doing so. - // full_state_with_conflicts -> We created a new copy of the full room state and had to resolve conflicts to do so. - // _load_state_block_nids -> Failed loading the state block nids for a single previous state. - // _load_combined_state -> Failed to load the combined state. - // _resolve_conflicts -> Failed to resolve conflicts. - []string{"algorithm", "outcome"}, -) - -var calculateStatePrevEventLength = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "dendrite", - Subsystem: "roomserver", - Name: "calculate_state_prev_event_length", - Help: "The length of the list of events to calculate the state after", - }, - []string{"algorithm", "outcome"}, -) - -var calculateStateFullStateLength = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "dendrite", - Subsystem: "roomserver", - Name: "calculate_state_full_state_length", - Help: "The length of the full room state.", - }, - []string{"algorithm", "outcome"}, -) - -var calculateStateConflictLength = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "dendrite", - Subsystem: "roomserver", - Name: "calculate_state_conflict_state_length", - Help: "The length of the conflicted room state.", - }, - []string{"algorithm", "outcome"}, -) - -type calculateStateMetrics struct { - algorithm string - startTime time.Time - prevEventLength int - fullStateLength int - conflictLength int -} - -func (c *calculateStateMetrics) stop(stateNID types.StateSnapshotNID, err error) (types.StateSnapshotNID, error) { - var outcome string - if err == nil { - outcome = "success" - } else { - outcome = "failure" - } - endTime := time.Now() - calculateStateDurations.WithLabelValues(c.algorithm, outcome).Observe( - float64(endTime.Sub(c.startTime).Nanoseconds()) / 1000., - ) - calculateStatePrevEventLength.WithLabelValues(c.algorithm, outcome).Observe( - float64(c.prevEventLength), - ) - calculateStateFullStateLength.WithLabelValues(c.algorithm, outcome).Observe( - float64(c.fullStateLength), - ) - calculateStateConflictLength.WithLabelValues(c.algorithm, outcome).Observe( - float64(c.conflictLength), - ) - return stateNID, err -} - -func init() { - prometheus.MustRegister( - calculateStateDurations, calculateStatePrevEventLength, - calculateStateFullStateLength, calculateStateConflictLength, - ) -} - -// CalculateAndStoreStateBeforeEvent calculates a snapshot of the state of a room before an event. -// Stores the snapshot of the state in the database. -// Returns a numeric ID for the snapshot of the state before the event. -func (v StateResolutionV1) CalculateAndStoreStateBeforeEvent( - ctx context.Context, - event gomatrixserverlib.Event, - roomNID types.RoomNID, -) (types.StateSnapshotNID, error) { - // Load the state at the prev events. - prevEventRefs := event.PrevEvents() - prevEventIDs := make([]string, len(prevEventRefs)) - for i := range prevEventRefs { - prevEventIDs[i] = prevEventRefs[i].EventID - } - - prevStates, err := v.db.StateAtEventIDs(ctx, prevEventIDs) - if err != nil { - return 0, err - } - - // The state before this event will be the state after the events that came before it. - return v.CalculateAndStoreStateAfterEvents(ctx, roomNID, prevStates) -} - -// CalculateAndStoreStateAfterEvents finds the room state after the given events. -// Stores the resulting state in the database and returns a numeric ID for that snapshot. -func (v StateResolutionV1) CalculateAndStoreStateAfterEvents( - ctx context.Context, - roomNID types.RoomNID, - prevStates []types.StateAtEvent, -) (types.StateSnapshotNID, error) { - metrics := calculateStateMetrics{startTime: time.Now(), prevEventLength: len(prevStates)} - - if len(prevStates) == 0 { - // 2) There weren't any prev_events for this event so the state is - // empty. - metrics.algorithm = "empty_state" - return metrics.stop(v.db.AddState(ctx, roomNID, nil, nil)) - } - - if len(prevStates) == 1 { - prevState := prevStates[0] - if prevState.EventStateKeyNID == 0 { - // 3) None of the previous events were state events and they all - // have the same state, so this event has exactly the same state - // as the previous events. - // This should be the common case. - metrics.algorithm = "no_change" - return metrics.stop(prevState.BeforeStateSnapshotNID, nil) - } - // The previous event was a state event so we need to store a copy - // of the previous state updated with that event. - stateBlockNIDLists, err := v.db.StateBlockNIDs( - ctx, []types.StateSnapshotNID{prevState.BeforeStateSnapshotNID}, - ) - if err != nil { - metrics.algorithm = "_load_state_blocks" - return metrics.stop(0, err) - } - stateBlockNIDs := stateBlockNIDLists[0].StateBlockNIDs - if len(stateBlockNIDs) < maxStateBlockNIDs { - // 4) The number of state data blocks is small enough that we can just - // add the state event as a block of size one to the end of the blocks. - metrics.algorithm = "single_delta" - return metrics.stop(v.db.AddState( - ctx, roomNID, stateBlockNIDs, []types.StateEntry{prevState.StateEntry}, - )) - } - // If there are too many deltas then we need to calculate the full state - // So fall through to calculateAndStoreStateAfterManyEvents - } - - return v.calculateAndStoreStateAfterManyEvents(ctx, roomNID, prevStates, metrics) -} - -// maxStateBlockNIDs is the maximum number of state data blocks to use to encode a snapshot of room state. -// Increasing this number means that we can encode more of the state changes as simple deltas which means that -// we need fewer entries in the state data table. However making this number bigger will increase the size of -// the rows in the state table itself and will require more index lookups when retrieving a snapshot. -// TODO: Tune this to get the right balance between size and lookup performance. -const maxStateBlockNIDs = 64 - -// calculateAndStoreStateAfterManyEvents finds the room state after the given events. -// This handles the slow path of calculateAndStoreStateAfterEvents for when there is more than one event. -// Stores the resulting state and returns a numeric ID for the snapshot. -func (v StateResolutionV1) calculateAndStoreStateAfterManyEvents( - ctx context.Context, - roomNID types.RoomNID, - prevStates []types.StateAtEvent, - metrics calculateStateMetrics, -) (types.StateSnapshotNID, error) { - - state, algorithm, conflictLength, err := - v.calculateStateAfterManyEvents(ctx, prevStates) - metrics.algorithm = algorithm - if err != nil { - return metrics.stop(0, err) - } - - // TODO: Check if we can encode the new state as a delta against the - // previous state. - metrics.conflictLength = conflictLength - metrics.fullStateLength = len(state) - return metrics.stop(v.db.AddState(ctx, roomNID, nil, state)) -} - -func (v StateResolutionV1) calculateStateAfterManyEvents( - ctx context.Context, prevStates []types.StateAtEvent, -) (state []types.StateEntry, algorithm string, conflictLength int, err error) { - var combined []types.StateEntry - // Conflict resolution. - // First stage: load the state after each of the prev events. - combined, err = v.LoadCombinedStateAfterEvents(ctx, prevStates) - if err != nil { - algorithm = "_load_combined_state" - return - } - - // Collect all the entries with the same type and key together. - // We don't care about the order here because the conflict resolution - // algorithm doesn't depend on the order of the prev events. - // Remove duplicate entires. - combined = combined[:util.SortAndUnique(stateEntrySorter(combined))] - - // Find the conflicts - conflicts := findDuplicateStateKeys(combined) - - if len(conflicts) > 0 { - conflictLength = len(conflicts) - - // 5) There are conflicting state events, for each conflict workout - // what the appropriate state event is. - - // Work out which entries aren't conflicted. - var notConflicted []types.StateEntry - for _, entry := range combined { - if _, ok := stateEntryMap(conflicts).lookup(entry.StateKeyTuple); !ok { - notConflicted = append(notConflicted, entry) - } - } - - var resolved []types.StateEntry - resolved, err = v.resolveConflicts(ctx, notConflicted, conflicts) - if err != nil { - algorithm = "_resolve_conflicts" - return - } - algorithm = "full_state_with_conflicts" - state = resolved - } else { - algorithm = "full_state_no_conflicts" - // 6) There weren't any conflicts - state = combined - } - return -} - -// resolveConflicts resolves a list of conflicted state entries. It takes two lists. -// The first is a list of all state entries that are not conflicted. -// The second is a list of all state entries that are conflicted -// A state entry is conflicted when there is more than one numeric event ID for the same state key tuple. -// Returns a list that combines the entries without conflicts with the result of state resolution for the entries with conflicts. -// The returned list is sorted by state key tuple. -// Returns an error if there was a problem talking to the database. -func (v StateResolutionV1) resolveConflicts( - ctx context.Context, - notConflicted, conflicted []types.StateEntry, -) ([]types.StateEntry, error) { - - // Load the conflicted events - conflictedEvents, eventIDMap, err := v.loadStateEvents(ctx, conflicted) - if err != nil { - return nil, err - } - - // Work out which auth events we need to load. - needed := gomatrixserverlib.StateNeededForAuth(conflictedEvents) - - // Find the numeric IDs for the necessary state keys. - var neededStateKeys []string - neededStateKeys = append(neededStateKeys, needed.Member...) - neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...) - stateKeyNIDMap, err := v.db.EventStateKeyNIDs(ctx, neededStateKeys) - if err != nil { - return nil, err - } - - // Load the necessary auth events. - tuplesNeeded := v.stateKeyTuplesNeeded(stateKeyNIDMap, needed) - var authEntries []types.StateEntry - for _, tuple := range tuplesNeeded { - if eventNID, ok := stateEntryMap(notConflicted).lookup(tuple); ok { - authEntries = append(authEntries, types.StateEntry{ - StateKeyTuple: tuple, - EventNID: eventNID, - }) - } - } - authEvents, _, err := v.loadStateEvents(ctx, authEntries) - if err != nil { - return nil, err - } - - // Resolve the conflicts. - resolvedEvents := gomatrixserverlib.ResolveStateConflicts(conflictedEvents, authEvents) - - // Map from the full events back to numeric state entries. - for _, resolvedEvent := range resolvedEvents { - entry, ok := eventIDMap[resolvedEvent.EventID()] - if !ok { - panic(fmt.Errorf("Missing state entry for event ID %q", resolvedEvent.EventID())) - } - notConflicted = append(notConflicted, entry) - } - - // Sort the result so it can be searched. - sort.Sort(stateEntrySorter(notConflicted)) - return notConflicted, nil -} - -// stateKeyTuplesNeeded works out which numeric state key tuples we need to authenticate some events. -func (v StateResolutionV1) stateKeyTuplesNeeded(stateKeyNIDMap map[string]types.EventStateKeyNID, stateNeeded gomatrixserverlib.StateNeeded) []types.StateKeyTuple { - var keyTuples []types.StateKeyTuple - if stateNeeded.Create { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomCreateNID, - EventStateKeyNID: types.EmptyStateKeyNID, - }) - } - if stateNeeded.PowerLevels { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomPowerLevelsNID, - EventStateKeyNID: types.EmptyStateKeyNID, - }) - } - if stateNeeded.JoinRules { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomJoinRulesNID, - EventStateKeyNID: types.EmptyStateKeyNID, - }) - } - for _, member := range stateNeeded.Member { - stateKeyNID, ok := stateKeyNIDMap[member] - if ok { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomMemberNID, - EventStateKeyNID: stateKeyNID, - }) - } - } - for _, token := range stateNeeded.ThirdPartyInvite { - stateKeyNID, ok := stateKeyNIDMap[token] - if ok { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomThirdPartyInviteNID, - EventStateKeyNID: stateKeyNID, - }) - } - } - return keyTuples -} - -// loadStateEvents loads the matrix events for a list of state entries. -// Returns a list of state events in no particular order and a map from string event ID back to state entry. -// The map can be used to recover which numeric state entry a given event is for. -// Returns an error if there was a problem talking to the database. -func (v StateResolutionV1) loadStateEvents( - ctx context.Context, entries []types.StateEntry, -) ([]gomatrixserverlib.Event, map[string]types.StateEntry, error) { - eventNIDs := make([]types.EventNID, len(entries)) - for i := range entries { - eventNIDs[i] = entries[i].EventNID - } - events, err := v.db.Events(ctx, eventNIDs) - if err != nil { - return nil, nil, err - } - eventIDMap := map[string]types.StateEntry{} - result := make([]gomatrixserverlib.Event, len(entries)) - for i := range entries { - event, ok := eventMap(events).lookup(entries[i].EventNID) - if !ok { - panic(fmt.Errorf("Corrupt DB: Missing event numeric ID %d", entries[i].EventNID)) - } - result[i] = event.Event - eventIDMap[event.Event.EventID()] = entries[i] - } - return result, eventIDMap, nil -} - -// findDuplicateStateKeys finds the state entries where the state key tuple appears more than once in a sorted list. -// Returns a sorted list of those state entries. -func findDuplicateStateKeys(a []types.StateEntry) []types.StateEntry { - var result []types.StateEntry - // j is the starting index of a block of entries with the same state key tuple. - j := 0 - for i := 1; i < len(a); i++ { - // Check if the state key tuple matches the start of the block - if a[j].StateKeyTuple != a[i].StateKeyTuple { - // If the state key tuple is different then we've reached the end of a block of duplicates. - // Check if the size of the block is bigger than one. - // If the size is one then there was only a single entry with that state key tuple so we don't add it to the result - if j+1 != i { - // Add the block to the result. - result = append(result, a[j:i]...) - } - // Start a new block for the next state key tuple. - j = i - } - } - // Check if the last block with the same state key tuple had more than one event in it. - if j+1 != len(a) { - result = append(result, a[j:]...) - } - return result -} - -type stateEntrySorter []types.StateEntry - -func (s stateEntrySorter) Len() int { return len(s) } -func (s stateEntrySorter) Less(i, j int) bool { return s[i].LessThan(s[j]) } -func (s stateEntrySorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -type stateBlockNIDListMap []types.StateBlockNIDList - -func (m stateBlockNIDListMap) lookup(stateNID types.StateSnapshotNID) (stateBlockNIDs []types.StateBlockNID, ok bool) { - list := []types.StateBlockNIDList(m) - i := sort.Search(len(list), func(i int) bool { - return list[i].StateSnapshotNID >= stateNID - }) - if i < len(list) && list[i].StateSnapshotNID == stateNID { - ok = true - stateBlockNIDs = list[i].StateBlockNIDs - } - return -} - -type stateEntryListMap []types.StateEntryList - -func (m stateEntryListMap) lookup(stateBlockNID types.StateBlockNID) (stateEntries []types.StateEntry, ok bool) { - list := []types.StateEntryList(m) - i := sort.Search(len(list), func(i int) bool { - return list[i].StateBlockNID >= stateBlockNID - }) - if i < len(list) && list[i].StateBlockNID == stateBlockNID { - ok = true - stateEntries = list[i].StateEntries - } - return -} - -type stateEntryByStateKeySorter []types.StateEntry - -func (s stateEntryByStateKeySorter) Len() int { return len(s) } -func (s stateEntryByStateKeySorter) Less(i, j int) bool { - return s[i].StateKeyTuple.LessThan(s[j].StateKeyTuple) -} -func (s stateEntryByStateKeySorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -type stateNIDSorter []types.StateSnapshotNID - -func (s stateNIDSorter) Len() int { return len(s) } -func (s stateNIDSorter) Less(i, j int) bool { return s[i] < s[j] } -func (s stateNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -func uniqueStateSnapshotNIDs(nids []types.StateSnapshotNID) []types.StateSnapshotNID { - return nids[:util.SortAndUnique(stateNIDSorter(nids))] -} - -type stateBlockNIDSorter []types.StateBlockNID - -func (s stateBlockNIDSorter) Len() int { return len(s) } -func (s stateBlockNIDSorter) Less(i, j int) bool { return s[i] < s[j] } -func (s stateBlockNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -func uniqueStateBlockNIDs(nids []types.StateBlockNID) []types.StateBlockNID { - return nids[:util.SortAndUnique(stateBlockNIDSorter(nids))] -} - -// Map from event type, state key tuple to numeric event ID. -// Implemented using binary search on a sorted array. -type stateEntryMap []types.StateEntry - -// lookup an entry in the event map. -func (m stateEntryMap) lookup(stateKey types.StateKeyTuple) (eventNID types.EventNID, ok bool) { - // Since the list is sorted we can implement this using binary search. - // This is faster than using a hash map. - // We don't have to worry about pathological cases because the keys are fixed - // size and are controlled by us. - list := []types.StateEntry(m) - i := sort.Search(len(list), func(i int) bool { - return !list[i].StateKeyTuple.LessThan(stateKey) - }) - if i < len(list) && list[i].StateKeyTuple == stateKey { - ok = true - eventNID = list[i].EventNID - } - return -} - -// Map from numeric event ID to event. -// Implemented using binary search on a sorted array. -type eventMap []types.Event - -// lookup an entry in the event map. -func (m eventMap) lookup(eventNID types.EventNID) (event *types.Event, ok bool) { - // Since the list is sorted we can implement this using binary search. - // This is faster than using a hash map. - // We don't have to worry about pathological cases because the keys are fixed - // size are controlled by us. - list := []types.Event(m) - i := sort.Search(len(list), func(i int) bool { - return list[i].EventNID >= eventNID - }) - if i < len(list) && list[i].EventNID == eventNID { - ok = true - event = &list[i] - } - return -} diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index 6bb96f1de..0fd9d5b53 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -68,6 +68,9 @@ const updateLatestEventNIDsSQL = "" + const selectRoomVersionForRoomIDSQL = "" + "SELECT room_version FROM roomserver_rooms WHERE room_id = $1" +const selectRoomVersionForRoomNIDSQL = "" + + "SELECT room_version FROM roomserver_rooms WHERE room_nid = $1" + type roomStatements struct { insertRoomNIDStmt *sql.Stmt selectRoomNIDStmt *sql.Stmt @@ -75,6 +78,7 @@ type roomStatements struct { selectLatestEventNIDsForUpdateStmt *sql.Stmt updateLatestEventNIDsStmt *sql.Stmt selectRoomVersionForRoomIDStmt *sql.Stmt + selectRoomVersionForRoomNIDStmt *sql.Stmt } func (s *roomStatements) prepare(db *sql.DB) (err error) { @@ -89,6 +93,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { {&s.selectLatestEventNIDsForUpdateStmt, selectLatestEventNIDsForUpdateSQL}, {&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL}, {&s.selectRoomVersionForRoomIDStmt, selectRoomVersionForRoomIDSQL}, + {&s.selectRoomVersionForRoomNIDStmt, selectRoomVersionForRoomNIDSQL}, }.prepare(db) } @@ -173,3 +178,12 @@ func (s *roomStatements) selectRoomVersionForRoomID( err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) return roomVersion, err } + +func (s *roomStatements) selectRoomVersionForRoomNID( + ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, +) (gomatrixserverlib.RoomVersion, error) { + var roomVersion gomatrixserverlib.RoomVersion + stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) + err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) + return roomVersion, err +} diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 9bb6de9d7..084b83ce6 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -747,6 +747,14 @@ func (d *Database) GetRoomVersionForRoom( ) } +func (d *Database) GetRoomVersionForRoomNID( + ctx context.Context, roomNID types.RoomNID, +) (gomatrixserverlib.RoomVersion, error) { + return d.statements.selectRoomVersionForRoomNID( + ctx, nil, roomNID, + ) +} + type transaction struct { ctx context.Context txn *sql.Tx diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index 49fa07ea8..512b98137 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -57,6 +57,9 @@ const updateLatestEventNIDsSQL = "" + const selectRoomVersionForRoomIDSQL = "" + "SELECT room_version FROM roomserver_rooms WHERE room_id = $1" +const selectRoomVersionForRoomNIDSQL = "" + + "SELECT room_version FROM roomserver_rooms WHERE room_nid = $1" + type roomStatements struct { insertRoomNIDStmt *sql.Stmt selectRoomNIDStmt *sql.Stmt @@ -64,6 +67,7 @@ type roomStatements struct { selectLatestEventNIDsForUpdateStmt *sql.Stmt updateLatestEventNIDsStmt *sql.Stmt selectRoomVersionForRoomIDStmt *sql.Stmt + selectRoomVersionForRoomNIDStmt *sql.Stmt } func (s *roomStatements) prepare(db *sql.DB) (err error) { @@ -78,6 +82,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { {&s.selectLatestEventNIDsForUpdateStmt, selectLatestEventNIDsForUpdateSQL}, {&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL}, {&s.selectRoomVersionForRoomIDStmt, selectRoomVersionForRoomIDSQL}, + {&s.selectRoomVersionForRoomNIDStmt, selectRoomVersionForRoomNIDSQL}, }.prepare(db) } @@ -165,3 +170,12 @@ func (s *roomStatements) selectRoomVersionForRoomID( err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) return roomVersion, err } + +func (s *roomStatements) selectRoomVersionForRoomNID( + ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, +) (gomatrixserverlib.RoomVersion, error) { + var roomVersion gomatrixserverlib.RoomVersion + stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) + err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) + return roomVersion, err +} diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index ae09a88ad..28d608ca5 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -901,6 +901,14 @@ func (d *Database) GetRoomVersionForRoom( ) } +func (d *Database) GetRoomVersionForRoomNID( + ctx context.Context, roomNID types.RoomNID, +) (gomatrixserverlib.RoomVersion, error) { + return d.statements.selectRoomVersionForRoomNID( + ctx, nil, roomNID, + ) +} + type transaction struct { ctx context.Context txn *sql.Tx